From 172591d701d5c4079e42ca1b017bfa07fc64d2aa Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Wed, 10 Jun 2026 23:28:34 -0500 Subject: [PATCH 01/77] sdk_v2/cpp/telemetry: extend ITelemetry with typed payloads and trackers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the C# Foundry Core telemetry event taxonomy to C++ in advance of the 1DS bridge. The interface refactor is the source of truth — all backend implementations and call sites are wired against it. ITelemetry additions: - 4 new `Action` values (EpDownloadAttempt, EpDownloadAndRegister, ModelFileDownload, ModelInference). - 4 new payload structs (`EpDownloadAttemptInfo`, `EpDownloadAndRegisterInfo`, `ModelUsageInfo`, `DownloadInfo`). - 3 new virtual methods (`RecordEpDownloadAttempt`, `RecordEpDownloadAndRegister`, `RecordDownload`). - Replaced `RecordModelUsage(model_id, prompt_tokens, completion_tokens, duration_ms)` with `RecordModelUsage(const ModelUsageInfo&)` so callers can populate richer fields (TimeToFirstToken, EP, memory). - Extended `RecordModelId(action, model_id)` to take `status` and `user_agent`; added `RecordException(action, ex, user_agent)` overload. New supporting code (all in `sdk_v2/cpp/src/telemetry/`): - `telemetry_environment.{h,cc}` — `IsCiEnvironment` / `IsTestingMode` / `IsTruthyValue` / `GetEnv`. 13 CI env-var names ported verbatim from neutron-server's `TelemetryEnvironment.cs`. - `telemetry_metadata.{h,cc}` — captures per-process metadata (`app_session_guid`, version, os_name / os_version / cpu_arch, test_mode flag) for stamping on every event. - `ep_download_tracker.{h,cc}` — RAII tracker that emits one `EPDownloadAndRegister` event per bootstrapper, recording stage transitions (initial -> download -> register). Default failure semantics: dtor records remaining stages as `kFailure`; `Done()` records them as `kSkipped` for happy-path early exit. - `download_tracker.{h,cc}` — RAII tracker that emits one `Download` event per `DownloadManager::DownloadModel` call. `TelemetryLogger` (the fallback / mirror) implements every new method, formatting events as `[Telemetry] EventName Field=value ...` at Debug. Test stubs (`NullTelemetry` and the `RecordingTelemetry` used by `telemetry_test.cc`) were updated for the new interface. Build verified RelWithDebInfo --no_telemetry. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/src/telemetry/download_tracker.cc | 29 +++++ sdk_v2/cpp/src/telemetry/download_tracker.h | 58 +++++++++ .../cpp/src/telemetry/ep_download_tracker.cc | 91 +++++++++++++ .../cpp/src/telemetry/ep_download_tracker.h | 76 +++++++++++ sdk_v2/cpp/src/telemetry/telemetry.cc | 8 ++ sdk_v2/cpp/src/telemetry/telemetry.h | 106 +++++++++++++-- .../src/telemetry/telemetry_action_tracker.cc | 4 +- .../src/telemetry/telemetry_environment.cc | 108 ++++++++++++++++ .../cpp/src/telemetry/telemetry_environment.h | 32 +++++ sdk_v2/cpp/src/telemetry/telemetry_logger.cc | 68 ++++++++-- sdk_v2/cpp/src/telemetry/telemetry_logger.h | 24 +++- .../cpp/src/telemetry/telemetry_metadata.cc | 121 ++++++++++++++++++ sdk_v2/cpp/src/telemetry/telemetry_metadata.h | 38 ++++++ sdk_v2/cpp/test/internal_api/null_telemetry.h | 15 ++- .../cpp/test/internal_api/telemetry_test.cc | 76 ++++++----- 15 files changed, 787 insertions(+), 67 deletions(-) create mode 100644 sdk_v2/cpp/src/telemetry/download_tracker.cc create mode 100644 sdk_v2/cpp/src/telemetry/download_tracker.h create mode 100644 sdk_v2/cpp/src/telemetry/ep_download_tracker.cc create mode 100644 sdk_v2/cpp/src/telemetry/ep_download_tracker.h create mode 100644 sdk_v2/cpp/src/telemetry/telemetry_environment.cc create mode 100644 sdk_v2/cpp/src/telemetry/telemetry_environment.h create mode 100644 sdk_v2/cpp/src/telemetry/telemetry_metadata.cc create mode 100644 sdk_v2/cpp/src/telemetry/telemetry_metadata.h diff --git a/sdk_v2/cpp/src/telemetry/download_tracker.cc b/sdk_v2/cpp/src/telemetry/download_tracker.cc new file mode 100644 index 000000000..dbb369c03 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/download_tracker.cc @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/download_tracker.h" + +#include + +namespace fl { + +DownloadTracker::DownloadTracker(std::string model_id, + std::string user_agent, + ITelemetry& telemetry) + : telemetry_(telemetry) { + info_.model_id = std::move(model_id); + info_.user_agent = std::move(user_agent); + info_.status = ActionStatus::kFailure; + download_phase_start_ = std::chrono::steady_clock::now(); +} + +DownloadTracker::~DownloadTracker() { + // Emit the Download event regardless of outcome. The default status is + // kFailure so abrupt exits (exceptions) are recorded as failures. + telemetry_.RecordDownload(info_); +} + +void DownloadTracker::RecordException(const std::exception& exception) { + telemetry_.RecordException(Action::kModelFileDownload, exception, info_.user_agent); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/download_tracker.h b/sdk_v2/cpp/src/telemetry/download_tracker.h new file mode 100644 index 000000000..240bf5b4a --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/download_tracker.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "telemetry/telemetry.h" + +#include +#include + +namespace fl { + +/// RAII tracker for the per-model "Download" event. +/// Caller updates fields as the download progresses; the event is emitted on +/// destruction. Status defaults to kFailure so that abrupt exits (exceptions) +/// are recorded as failures unless the caller calls SetStatus(kSuccess) on the +/// happy path or SetStatus(kSkipped) when the model was already cached. +class DownloadTracker { + public: + DownloadTracker(std::string model_id, + std::string user_agent, + ITelemetry& telemetry); + ~DownloadTracker(); + + // Non-copyable, non-movable + DownloadTracker(const DownloadTracker&) = delete; + DownloadTracker& operator=(const DownloadTracker&) = delete; + + void SetStatus(ActionStatus status) { info_.status = status; } + void SetLockWaitMs(int64_t v) { info_.lock_wait_ms = v; } + void SetEnumerationMs(int64_t v) { info_.enumeration_ms = v; } + void SetTotalSizeBytes(int64_t v) { info_.total_size_bytes = v; } + void SetAlreadyCachedBytes(int64_t v) { info_.already_cached_bytes = v; } + void SetFileCount(int32_t v) { info_.file_count = v; } + void SetSkippedFileCount(int32_t v) { info_.skipped_file_count = v; } + void SetDownloadWaitResult(std::string v) { info_.download_wait_result = std::move(v); } + void SetMaxConcurrency(int32_t v) { info_.max_concurrency = v; } + + /// Start the timer for the download phase. Use after lock wait and + /// enumeration are complete, so download_ms only reflects byte transfer. + void BeginDownloadPhase() { download_phase_start_ = std::chrono::steady_clock::now(); } + + /// Stop the timer for the download phase. The duration is captured into the + /// emitted event. Safe to call multiple times — the last call wins. + void EndDownloadPhase() { + info_.download_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - download_phase_start_) + .count(); + } + + void RecordException(const std::exception& exception); + + private: + ITelemetry& telemetry_; + DownloadInfo info_; + std::chrono::steady_clock::time_point download_phase_start_; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc b/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc new file mode 100644 index 000000000..65d01f34d --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/ep_download_tracker.h" + +#include + +namespace fl { + +namespace { + +int64_t ElapsedMs(std::chrono::steady_clock::time_point start) { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); +} + +} // namespace + +EpDownloadTracker::EpDownloadTracker(std::string provider_name, + std::string user_agent, + ITelemetry& telemetry) + : telemetry_(telemetry), + provider_name_(std::move(provider_name)), + user_agent_(std::move(user_agent)), + stage_start_(std::chrono::steady_clock::now()) { +} + +EpDownloadTracker::~EpDownloadTracker() { + // Mirror neutron-server: if the caller didn't reach Done() or + // RecordRegisterComplete, assume the abrupt exit was an exception path and + // record any unfinished stage as kFailure. + RecordEvent(ActionStatus::kFailure); +} + +void EpDownloadTracker::RecordInitialState(std::string ready_state) { + init_ready_state_ = std::move(ready_state); + stage_ = Stage::Download; + stage_start_ = std::chrono::steady_clock::now(); +} + +void EpDownloadTracker::RecordDownloadComplete(ActionStatus status, std::string ready_state) { + download_duration_ms_ = ElapsedMs(stage_start_); + download_ready_state_ = std::move(ready_state); + download_status_ = status; + stage_ = Stage::Register; + stage_start_ = std::chrono::steady_clock::now(); +} + +void EpDownloadTracker::RecordRegisterComplete(ActionStatus status, std::string ready_state) { + register_duration_ms_ = ElapsedMs(stage_start_); + register_ready_state_ = std::move(ready_state); + register_status_ = status; + stage_ = Stage::Final; +} + +void EpDownloadTracker::Done() { + RecordEvent(ActionStatus::kSkipped); +} + +void EpDownloadTracker::RecordException(const std::exception& ex) { + telemetry_.RecordException(Action::kEpDownloadAndRegister, ex, user_agent_); +} + +void EpDownloadTracker::RecordEvent(ActionStatus incomplete_stage_status) { + if (recorded_event_) { + return; + } + recorded_event_ = true; + + if (stage_ == Stage::Download) { + download_duration_ms_ = ElapsedMs(stage_start_); + download_status_ = incomplete_stage_status; + } else if (stage_ == Stage::Register) { + register_duration_ms_ = ElapsedMs(stage_start_); + register_status_ = incomplete_stage_status; + } + + EpDownloadAndRegisterInfo info; + info.user_agent = user_agent_; + info.provider_name = provider_name_; + info.init_ready_state = init_ready_state_; + info.download_ready_state = download_ready_state_; + info.download_status = download_status_; + info.download_duration_ms = download_duration_ms_; + info.register_ready_state = register_ready_state_; + info.register_status = register_status_; + info.register_duration_ms = register_duration_ms_; + telemetry_.RecordEpDownloadAndRegister(info); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/ep_download_tracker.h b/sdk_v2/cpp/src/telemetry/ep_download_tracker.h new file mode 100644 index 000000000..6ee322f15 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/ep_download_tracker.h @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "telemetry/telemetry.h" + +#include +#include +#include + +namespace fl { + +/// RAII tracker for the per-provider "EPDownloadAndRegister" event. Mirrors the +/// neutron-server EPDownloadTracker. Stages advance Initial -> Download -> +/// Register -> Final. Each stage that the caller doesn't explicitly complete +/// is recorded with one of: +/// * kFailure on destruction (assumed exception path) +/// * kSkipped if Done() is called before destruction (early-exit without +/// exception, e.g. "EP already registered, nothing to download") +class EpDownloadTracker { + public: + EpDownloadTracker(std::string provider_name, + std::string user_agent, + ITelemetry& telemetry); + ~EpDownloadTracker(); + + // Non-copyable, non-movable + EpDownloadTracker(const EpDownloadTracker&) = delete; + EpDownloadTracker& operator=(const EpDownloadTracker&) = delete; + + /// Captures the EP's ready state before the download phase begins. Restarts + /// the stopwatch so subsequent timings measure the download phase only. + void RecordInitialState(std::string ready_state = "N/A"); + + /// Captures the download phase outcome and ready state, restarts the + /// stopwatch for the register phase. + void RecordDownloadComplete(ActionStatus status, std::string ready_state = "N/A"); + + /// Captures the register phase outcome and ready state. Stops the stopwatch. + void RecordRegisterComplete(ActionStatus status, std::string ready_state = "N/A"); + + /// Mark tracking as complete, filling any remaining stages with kSkipped + /// instead of the default kFailure (exception-assumed) status. Call this on + /// happy-path early exits (e.g. "EP already registered"). + void Done(); + + /// Record an exception associated with the bootstrap operation. Does not + /// finalize the EPDownloadAndRegister event — that happens on destruction. + void RecordException(const std::exception& ex); + + private: + enum class Stage { + Initial = 0, + Download = 1, + Register = 2, + Final = 3, + }; + + void RecordEvent(ActionStatus incomplete_stage_status); + + ITelemetry& telemetry_; + std::string provider_name_; + std::string user_agent_; + std::string init_ready_state_ = "N/A"; + std::string download_ready_state_ = "N/A"; + std::string register_ready_state_ = "N/A"; + ActionStatus download_status_ = ActionStatus::kSkipped; + ActionStatus register_status_ = ActionStatus::kSkipped; + int64_t download_duration_ms_ = 0; + int64_t register_duration_ms_ = 0; + std::chrono::steady_clock::time_point stage_start_; + Stage stage_ = Stage::Initial; + bool recorded_event_ = false; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry.cc b/sdk_v2/cpp/src/telemetry/telemetry.cc index 6d8fc06f6..97fc24122 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry.cc @@ -50,6 +50,14 @@ std::string_view ActionToString(Action action) { return "OpenAIResponsesGetInputItems"; case Action::kCoreAudioTranscribe: return "CoreAudioTranscribe"; + case Action::kEpDownloadAttempt: + return "EpDownloadAttempt"; + case Action::kEpDownloadAndRegister: + return "EpDownloadAndRegister"; + case Action::kModelFileDownload: + return "ModelFileDownload"; + case Action::kModelInference: + return "ModelInference"; default: return "Unknown"; } diff --git a/sdk_v2/cpp/src/telemetry/telemetry.h b/sdk_v2/cpp/src/telemetry/telemetry.h index 680dfcd9a..b0f0ed4a9 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.h +++ b/sdk_v2/cpp/src/telemetry/telemetry.h @@ -9,8 +9,9 @@ namespace fl { -/// Telemetry action identifiers matching the C# ITelemetry.Action enum. -// TODO: This is a lie. The enum values don't match. Do they need to? +/// Telemetry action identifiers. The values are stable IDs for log greppability; +/// the over-the-wire field is the human-readable name produced by ActionToString. +/// The 1DS implementation emits the string, so changing numeric values is safe. enum class Action { kInvalid = 0, @@ -46,6 +47,16 @@ enum class Action { // Audio kCoreAudioTranscribe = 400, + + // EP catalog operations + kEpDownloadAttempt = 500, // Wraps the entire DownloadAndRegisterEps call + kEpDownloadAndRegister = 501, // One per-provider attempt within DownloadAndRegisterEps + + // Model file download + kModelFileDownload = 600, // Wraps the per-model DownloadManager flow + + // EP runtime usage (TimeToFirstToken / total tokens / memory) + kModelInference = 700, // The "Model" event in the C# implementation }; /// Status of a tracked telemetry action. @@ -62,9 +73,66 @@ std::string_view ActionToString(Action action); /// Convert ActionStatus to human-readable string. std::string_view ActionStatusToString(ActionStatus status); +/// Payload for the EPDownloadAttempt event — emitted once per DownloadAndRegisterEps call. +struct EpDownloadAttemptInfo { + std::string user_agent; + int attempts = 0; // Total per-provider attempts made + int num_providers = 0; // Number of providers requested + int succeeded = 0; // Number of providers that registered successfully + int failed = 0; // Number of providers that failed + bool resolved = false; // True if at least one provider became Registered + ActionStatus status = ActionStatus::kInvalid; + int64_t duration_ms = 0; +}; + +/// Payload for the EPDownloadAndRegister event — emitted once per provider attempt. +struct EpDownloadAndRegisterInfo { + std::string user_agent; + std::string provider_name; + std::string init_ready_state; // EP state before this call (e.g. "NotPresent") + std::string download_ready_state; // EP state after the download phase (e.g. "Installed") + ActionStatus download_status = ActionStatus::kInvalid; + int64_t download_duration_ms = 0; + std::string register_ready_state; // EP state after the register phase (e.g. "Registered") + ActionStatus register_status = ActionStatus::kInvalid; + int64_t register_duration_ms = 0; +}; + +/// Payload for the Model event — emitted once per inference completion with token / memory metrics. +struct ModelUsageInfo { + std::string model_id; + std::string execution_provider; + std::string user_agent; + int64_t time_to_first_token_ms = 0; + int64_t total_time_ms = 0; + int32_t total_tokens = 0; + int32_t input_token_count = 0; + uint64_t num_messages = 0; + int64_t memory_used_mb = -1; // -1 if not measured + int64_t cpu_time_ms = -1; // -1 if not measured + int64_t gpu_memory_used_mb = -1; // -1 if not measured +}; + +/// Payload for the Download event — emitted once per DownloadManager::DownloadModel call. +struct DownloadInfo { + std::string model_id; + std::string user_agent; + ActionStatus status = ActionStatus::kInvalid; + int64_t lock_wait_ms = 0; + int64_t enumeration_ms = 0; + int64_t download_ms = 0; + int64_t total_size_bytes = 0; + int64_t already_cached_bytes = 0; + int32_t file_count = 0; + int32_t skipped_file_count = 0; + std::string download_wait_result; // e.g. "Completed", "TimedOut", "AlreadyHeld" + int32_t max_concurrency = 0; +}; + /// Abstract telemetry interface. -/// Implementations may send events to a telemetry backend (ETW, OpenTelemetry, etc.) -/// or simply log them. The stub TelemetryLogger logs via ILogger. +/// Implementations may send events to a telemetry backend (1DS, ETW, OpenTelemetry, …) +/// or simply log them. The OneDsTelemetry implementation sends to 1DS; the +/// TelemetryLogger stub formats them to the ILogger sink. class ITelemetry { public: virtual ~ITelemetry() = default; @@ -77,14 +145,30 @@ class ITelemetry { /// Record an exception associated with an action. virtual void RecordException(Action action, const std::exception& exception) = 0; - /// Record model usage metrics after inference. - virtual void RecordModelUsage(const std::string& model_id, - int64_t prompt_tokens, - int64_t completion_tokens, - int64_t duration_ms) = 0; + /// Record an exception associated with an action, with an optional user agent. + /// Default forwards to RecordException(action, exception) for implementations + /// that don't yet propagate the user agent. + virtual void RecordException(Action action, const std::exception& exception, + const std::string& /*user_agent*/) { + RecordException(action, exception); + } + + /// Record model usage metrics after inference (Model event). + virtual void RecordModelUsage(const ModelUsageInfo& info) = 0; + + /// Record which model was used for an action (ModelId event). + virtual void RecordModelId(Action action, const std::string& model_id, + ActionStatus status = ActionStatus::kSuccess, + const std::string& user_agent = "") = 0; + + /// Record the result of a DownloadAndRegisterEps call (EPDownloadAttempt event). + virtual void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) = 0; + + /// Record one EP provider's download+register attempt (EPDownloadAndRegister event). + virtual void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) = 0; - /// Record which model was used for an action. - virtual void RecordModelId(Action action, const std::string& model_id) = 0; + /// Record one model file download (Download event). + virtual void RecordDownload(const DownloadInfo& info) = 0; }; } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc index 7672cb1f4..f30f1246c 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc @@ -19,7 +19,7 @@ ActionTracker::~ActionTracker() { telemetry_.RecordAction(action_, status_, user_agent_, indirect_, duration_ms); if (!model_id_.empty()) { - telemetry_.RecordModelId(action_, model_id_); + telemetry_.RecordModelId(action_, model_id_, status_, user_agent_); } } @@ -28,7 +28,7 @@ void ActionTracker::SetStatus(ActionStatus status) { } void ActionTracker::RecordException(const std::exception& exception) { - telemetry_.RecordException(action_, exception); + telemetry_.RecordException(action_, exception, user_agent_); } void ActionTracker::SetModelId(const std::string& model_id) { diff --git a/sdk_v2/cpp/src/telemetry/telemetry_environment.cc b/sdk_v2/cpp/src/telemetry/telemetry_environment.cc new file mode 100644 index 000000000..a19838ba7 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_environment.cc @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/telemetry_environment.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#endif + +namespace fl { + +namespace { + +// Mirrors neutron-server's CiEnvironmentVariableNames. Keep this in sync if the +// list there changes — telemetry behavior in CI must match across stacks. +constexpr std::array kCiEnvironmentVariableNames = { + "CI", // Generic CI flag used by many providers + "TF_BUILD", // Azure Pipelines + "GITHUB_ACTIONS", // GitHub Actions + "GITLAB_CI", // GitLab CI + "CIRCLECI", // CircleCI + "TRAVIS", // Travis CI + "JENKINS_URL", // Jenkins + "CODEBUILD_BUILD_ID", // AWS CodeBuild + "BUILDKITE", // Buildkite + "TEAMCITY_VERSION", // TeamCity + "APPVEYOR", // AppVeyor + "BITBUCKET_BUILD_NUMBER", // Bitbucket Pipelines + "SYSTEM_TEAMFOUNDATIONCOLLECTIONURI", // Azure DevOps +}; + +bool EqualsIgnoreCase(std::string_view a, std::string_view b) { + if (a.size() != b.size()) { + return false; + } + for (size_t i = 0; i < a.size(); ++i) { + if (std::tolower(static_cast(a[i])) != + std::tolower(static_cast(b[i]))) { + return false; + } + } + return true; +} + +std::string_view Trim(std::string_view s) { + auto is_ws = [](unsigned char c) { return std::isspace(c) != 0; }; + while (!s.empty() && is_ws(static_cast(s.front()))) { + s.remove_prefix(1); + } + while (!s.empty() && is_ws(static_cast(s.back()))) { + s.remove_suffix(1); + } + return s; +} + +} // namespace + +std::string TelemetryEnvironment::GetEnv(const char* name) { +#ifdef _WIN32 + // Use the W variant so we don't depend on the legacy CRT _CRT_SECURE_NO_WARNINGS. + // Env-var values are ASCII for the CI flags we care about; if a value is unicode + // we still get the bytes round-tripped correctly because we only do truthiness checks. + DWORD needed = ::GetEnvironmentVariableA(name, nullptr, 0); + if (needed == 0) { + return {}; + } + std::vector buf(needed); + DWORD written = ::GetEnvironmentVariableA(name, buf.data(), needed); + if (written == 0 || written >= needed) { + return {}; + } + return std::string(buf.data(), written); +#else + const char* value = std::getenv(name); + return value ? std::string(value) : std::string{}; +#endif +} + +bool TelemetryEnvironment::IsTruthyValue(std::string_view value) { + auto trimmed = Trim(value); + if (trimmed.empty()) { + return false; + } + return !EqualsIgnoreCase(trimmed, "0") && + !EqualsIgnoreCase(trimmed, "false") && + !EqualsIgnoreCase(trimmed, "no") && + !EqualsIgnoreCase(trimmed, "off"); +} + +bool TelemetryEnvironment::IsCiEnvironment() { + for (const char* name : kCiEnvironmentVariableNames) { + if (IsTruthyValue(GetEnv(name))) { + return true; + } + } + return false; +} + +bool TelemetryEnvironment::IsTestingMode() { + return IsTruthyValue(GetEnv("FOUNDRY_TESTING_MODE")); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_environment.h b/sdk_v2/cpp/src/telemetry/telemetry_environment.h new file mode 100644 index 000000000..71f3e2a5a --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_environment.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include + +namespace fl { + +/// Static helpers for telemetry runtime gating. Ported from neutron-server's +/// TelemetryEnvironment.cs so the CI suppression behavior matches across stacks. +class TelemetryEnvironment { + public: + /// Returns true if any well-known CI environment variable is set to a truthy + /// value. The set matches neutron-server's TelemetryEnvironment.cs. + /// In CI, OneDsTelemetry skips Initialize entirely — no 1DS events emitted. + static bool IsCiEnvironment(); + + /// Returns true if the FOUNDRY_TESTING_MODE env var is set to a truthy value. + /// Outside CI, this stamps a `test=true` boolean on every emitted event but + /// does not suppress emission. In CI, this is moot — emission is suppressed. + static bool IsTestingMode(); + + /// Truthy-value semantics: a non-empty, non-whitespace string whose trimmed + /// value is not "0", "false", "no", or "off" (case-insensitive). + static bool IsTruthyValue(std::string_view value); + + /// Read an env var (cross-platform). Returns empty string if unset. + static std::string GetEnv(const char* name); +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc index 68d250513..214a0cc6d 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc @@ -13,31 +13,75 @@ TelemetryLogger::TelemetryLogger(const std::string& app_name, ILogger& logger) void TelemetryLogger::RecordAction(Action action, ActionStatus status, const std::string& user_agent, bool indirect, int64_t duration_ms) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] AppName:{} UserAgent:{} Command:{} Status:{} Direct:{} Time:{}ms", + fmt::format("[Telemetry] Action AppName={} UserAgent={} Action={} Status={} Direct={} TimeMs={}", app_name_, user_agent, ActionToString(action), ActionStatusToString(status), !indirect, duration_ms)); } void TelemetryLogger::RecordException(Action action, const std::exception& exception) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] AppName:{} Command:{} Exception:{}", + fmt::format("[Telemetry] Error AppName={} Action={} Exception={}", app_name_, ActionToString(action), exception.what())); } -void TelemetryLogger::RecordModelUsage(const std::string& model_id, - int64_t prompt_tokens, - int64_t completion_tokens, - int64_t duration_ms) { +void TelemetryLogger::RecordException(Action action, const std::exception& exception, + const std::string& user_agent) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] AppName:{} ModelUsage: model={} prompt_tokens={} " - "completion_tokens={} duration={}ms", - app_name_, model_id, prompt_tokens, completion_tokens, duration_ms)); + fmt::format("[Telemetry] Error AppName={} UserAgent={} Action={} Exception={}", + app_name_, user_agent, ActionToString(action), exception.what())); } -void TelemetryLogger::RecordModelId(Action action, const std::string& model_id) { +void TelemetryLogger::RecordModelUsage(const ModelUsageInfo& info) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] AppName:{} Command:{} ModelId:{}", - app_name_, ActionToString(action), model_id)); + fmt::format("[Telemetry] Model AppName={} UserAgent={} ModelId={} EP={} TimeToFirstTokenMs={} " + "TotalTimeMs={} TotalTokens={} InputTokenCount={} NumMessages={} MemoryUsedMB={} " + "CpuTimeMs={} GpuMemoryUsedMB={}", + app_name_, info.user_agent, info.model_id, info.execution_provider, + info.time_to_first_token_ms, info.total_time_ms, info.total_tokens, + info.input_token_count, info.num_messages, info.memory_used_mb, + info.cpu_time_ms, info.gpu_memory_used_mb)); +} + +void TelemetryLogger::RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const std::string& user_agent) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] ModelId AppName={} UserAgent={} Action={} ModelId={} Status={}", + app_name_, user_agent, ActionToString(action), model_id, + ActionStatusToString(status))); +} + +void TelemetryLogger::RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] EPDownloadAttempt AppName={} UserAgent={} Attempts={} " + "NumProviders={} Succeeded={} Failed={} Resolved={} Status={} TimeMs={}", + app_name_, info.user_agent, info.attempts, info.num_providers, + info.succeeded, info.failed, info.resolved, + ActionStatusToString(info.status), info.duration_ms)); +} + +void TelemetryLogger::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] EPDownloadAndRegister AppName={} UserAgent={} Provider={} " + "InitReadyState={} DownloadReadyState={} DownloadStatus={} DownloadTimeMs={} " + "RegisterReadyState={} RegisterStatus={} RegisterTimeMs={}", + app_name_, info.user_agent, info.provider_name, + info.init_ready_state, info.download_ready_state, + ActionStatusToString(info.download_status), info.download_duration_ms, + info.register_ready_state, ActionStatusToString(info.register_status), + info.register_duration_ms)); +} + +void TelemetryLogger::RecordDownload(const DownloadInfo& info) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] Download AppName={} UserAgent={} ModelId={} Status={} " + "LockWaitMs={} EnumerationMs={} DownloadMs={} TotalSizeBytes={} " + "AlreadyCachedBytes={} FileCount={} SkippedFileCount={} " + "DownloadWaitResult={} MaxConcurrency={}", + app_name_, info.user_agent, info.model_id, + ActionStatusToString(info.status), info.lock_wait_ms, + info.enumeration_ms, info.download_ms, info.total_size_bytes, + info.already_cached_bytes, info.file_count, info.skipped_file_count, + info.download_wait_result, info.max_concurrency)); } } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.h b/sdk_v2/cpp/src/telemetry/telemetry_logger.h index 056838846..6982c8166 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.h @@ -9,8 +9,14 @@ namespace fl { -/// Stub ITelemetry implementation that logs telemetry events via ILogger. -/// Used as a fallback when no platform-specific telemetry backend is available. +/// ITelemetry implementation that formats telemetry events to ILogger. +/// Used: +/// 1. As the fallback when no 1DS backend is available (cpp-client-telemetry +/// not configured, or FOUNDRY_LOCAL_USE_TELEMETRY=OFF). +/// 2. Inside OneDsTelemetry to provide local diagnostic logging in addition +/// to the 1DS upload. +/// In both cases, the local logger receives every event regardless of CI / +/// FOUNDRY_TESTING_MODE state — those flags only gate the 1DS upload. class TelemetryLogger : public ITelemetry { public: TelemetryLogger(const std::string& app_name, ILogger& logger); @@ -19,13 +25,17 @@ class TelemetryLogger : public ITelemetry { bool indirect, int64_t duration_ms) override; void RecordException(Action action, const std::exception& exception) override; + void RecordException(Action action, const std::exception& exception, + const std::string& user_agent) override; - void RecordModelUsage(const std::string& model_id, - int64_t prompt_tokens, - int64_t completion_tokens, - int64_t duration_ms) override; + void RecordModelUsage(const ModelUsageInfo& info) override; - void RecordModelId(Action action, const std::string& model_id) override; + void RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const std::string& user_agent) override; + + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; + void RecordDownload(const DownloadInfo& info) override; private: std::string app_name_; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc b/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc new file mode 100644 index 000000000..826690ed1 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/telemetry_metadata.h" + +#include "telemetry/telemetry_environment.h" +#include "version.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#else +#include +#endif + +namespace fl { + +namespace { + +std::string MakeGuidV4Hex() { + // RFC 4122 v4 UUID, hex-encoded with hyphens. We use std::random_device + mt19937_64 + // because we don't need the OS UUID API — this is just a per-process correlation id, + // not a cryptographic identifier. + std::random_device rd; + std::mt19937_64 gen{(static_cast(rd()) << 32) | rd()}; + uint64_t hi = gen(); + uint64_t lo = gen(); + + // Set version (4) and variant (10xx) bits. + hi = (hi & 0xFFFFFFFFFFFF0FFFULL) | 0x0000000000004000ULL; + lo = (lo & 0x3FFFFFFFFFFFFFFFULL) | 0x8000000000000000ULL; + + char buf[37]; + std::snprintf(buf, sizeof(buf), + "%08x-%04x-%04x-%04x-%012llx", + static_cast((hi >> 32) & 0xFFFFFFFFu), + static_cast((hi >> 16) & 0xFFFFu), + static_cast(hi & 0xFFFFu), + static_cast((lo >> 48) & 0xFFFFu), + static_cast(lo & 0x0000FFFFFFFFFFFFULL)); + return std::string(buf); +} + +#ifdef _WIN32 +std::string GetWindowsVersion() { + // GetVersionExA is deprecated and lies for unmanifested apps. The reliable + // approach is to read the build number directly from kernel32 via + // RtlGetVersion, or fall back to the OS version registry. For now, use + // GetVersionEx — the deprecation only affects apps without a manifest, and + // Foundry Local has a manifest declaring Win10 / Win11 compat. + OSVERSIONINFOEXA info{}; + info.dwOSVersionInfoSize = sizeof(info); +#pragma warning(suppress : 4996) // GetVersionExA deprecation + if (::GetVersionExA(reinterpret_cast(&info)) == 0) { + return "unknown"; + } + char buf[64]; + std::snprintf(buf, sizeof(buf), "%lu.%lu.%lu", + info.dwMajorVersion, info.dwMinorVersion, info.dwBuildNumber); + return std::string(buf); +} + +std::string GetCpuArch() { + SYSTEM_INFO si{}; + ::GetNativeSystemInfo(&si); + switch (si.wProcessorArchitecture) { + case PROCESSOR_ARCHITECTURE_AMD64: return "amd64"; + case PROCESSOR_ARCHITECTURE_ARM: return "arm"; + case PROCESSOR_ARCHITECTURE_ARM64: return "arm64"; + case PROCESSOR_ARCHITECTURE_IA64: return "ia64"; + case PROCESSOR_ARCHITECTURE_INTEL: return "x86"; + default: return "unknown"; + } +} +#else +struct PosixOsInfo { + std::string name; + std::string version; + std::string arch; +}; + +PosixOsInfo GetPosixOsInfo() { + PosixOsInfo out{"unknown", "unknown", "unknown"}; + ::utsname u{}; + if (::uname(&u) == 0) { + out.name = u.sysname; + out.version = u.release; + out.arch = u.machine; + } + return out; +} +#endif + +} // namespace + +TelemetryMetadata BuildTelemetryMetadata(std::string app_name) { + TelemetryMetadata m; + m.app_session_guid = MakeGuidV4Hex(); + m.version = FOUNDRY_LOCAL_VERSION; + m.app_name = std::move(app_name); + m.test_mode = TelemetryEnvironment::IsTestingMode(); + +#ifdef _WIN32 + m.os_name = "Windows"; + m.os_version = GetWindowsVersion(); + m.cpu_arch = GetCpuArch(); +#else + auto info = GetPosixOsInfo(); + m.os_name = info.name; + m.os_version = info.version; + m.cpu_arch = info.arch; +#endif + + return m; +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_metadata.h b/sdk_v2/cpp/src/telemetry/telemetry_metadata.h new file mode 100644 index 000000000..58f880c1f --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_metadata.h @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include + +namespace fl { + +/// Process-wide metadata stamped onto every 1DS event as common context. +/// Computed once at startup and cached. Cheap to copy. +struct TelemetryMetadata { + /// Hex-encoded random 128-bit GUID; correlates events from the same process. + /// 1DS interprets the magic field name "UTCReplace_AppSessionGuid" by replacing + /// the value with the OS-supplied app session GUID on Windows, and accepts our + /// random GUID on other platforms. We always provide a value so the event is + /// well-formed even if UTC's magic isn't honored. + std::string app_session_guid; + + /// Foundry Local SDK version (FoundryLocalGetVersionString). + std::string version; + + /// Configured app name (from Configuration::app_name). + std::string app_name; + + /// Free-form "Windows 11 10.0.26100 amd64" / "Linux 6.5.0 x86_64" / "macOS 14.4 arm64". + std::string os_name; // "Windows" / "Linux" / "Darwin" + std::string os_version; // "10.0.26100" / "6.5.0-azure" / "14.4" + std::string cpu_arch; // "amd64" / "arm64" / "x86" / ... + + /// True if FOUNDRY_TESTING_MODE was truthy at startup (stamped per-event as `test`). + bool test_mode = false; +}; + +/// Build the metadata for this process. Reads env vars and OS APIs once. +/// app_name comes from Configuration; version comes from FoundryLocalGetVersionString. +TelemetryMetadata BuildTelemetryMetadata(std::string app_name); + +} // namespace fl diff --git a/sdk_v2/cpp/test/internal_api/null_telemetry.h b/sdk_v2/cpp/test/internal_api/null_telemetry.h index 6d2c11c16..2c70a1b66 100644 --- a/sdk_v2/cpp/test/internal_api/null_telemetry.h +++ b/sdk_v2/cpp/test/internal_api/null_telemetry.h @@ -15,12 +15,17 @@ class NullTelemetry : public ITelemetry { void RecordException(Action /*action*/, const std::exception& /*exception*/) override {} - void RecordModelUsage(const std::string& /*model_id*/, - int64_t /*prompt_tokens*/, - int64_t /*completion_tokens*/, - int64_t /*duration_ms*/) override {} + void RecordModelUsage(const ModelUsageInfo& /*info*/) override {} - void RecordModelId(Action /*action*/, const std::string& /*model_id*/) override {} + void RecordModelId(Action /*action*/, const std::string& /*model_id*/, + ActionStatus /*status*/ = ActionStatus::kSuccess, + const std::string& /*user_agent*/ = "") override {} + + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& /*info*/) override {} + + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& /*info*/) override {} + + void RecordDownload(const DownloadInfo& /*info*/) override {} }; } // namespace fl::test diff --git a/sdk_v2/cpp/test/internal_api/telemetry_test.cc b/sdk_v2/cpp/test/internal_api/telemetry_test.cc index 56683cd68..42fc068f6 100644 --- a/sdk_v2/cpp/test/internal_api/telemetry_test.cc +++ b/sdk_v2/cpp/test/internal_api/telemetry_test.cc @@ -51,29 +51,35 @@ class RecordingTelemetry : public ITelemetry { exception_calls.emplace_back(action, exception.what()); } - void RecordModelUsage(const std::string& model_id, - int64_t prompt_tokens, - int64_t completion_tokens, - int64_t duration_ms) override { - model_usage_calls.push_back( - ModelUsageCall{model_id, prompt_tokens, completion_tokens, duration_ms}); + void RecordModelUsage(const ModelUsageInfo& info) override { + model_usage_calls.push_back(info); } - void RecordModelId(Action action, const std::string& model_id) override { + void RecordModelId(Action action, const std::string& model_id, + ActionStatus /*status*/ = ActionStatus::kSuccess, + const std::string& /*user_agent*/ = "") override { model_id_calls.emplace_back(action, model_id); } - struct ModelUsageCall { - std::string model_id; - int64_t prompt_tokens; - int64_t completion_tokens; - int64_t duration_ms; - }; + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override { + ep_attempt_calls.push_back(info); + } + + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override { + ep_register_calls.push_back(info); + } + + void RecordDownload(const DownloadInfo& info) override { + download_calls.push_back(info); + } std::vector action_calls; std::vector> exception_calls; - std::vector model_usage_calls; + std::vector model_usage_calls; std::vector> model_id_calls; + std::vector ep_attempt_calls; + std::vector ep_register_calls; + std::vector download_calls; }; } // namespace @@ -87,12 +93,12 @@ TEST(TelemetryLoggerTest, RecordActionIncludesConcreteFields) { ASSERT_EQ(logger.entries.size(), 1u); EXPECT_EQ(logger.entries[0].level, LogLevel::Debug); - EXPECT_NE(logger.entries[0].message.find("AppName:foundry-local"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("UserAgent:cli/1.0"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Command:ModelDownload"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Status:Success"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Direct:true"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Time:1234ms"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("AppName=foundry-local"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("UserAgent=cli/1.0"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Action=ModelDownload"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Status=Success"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Direct=true"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("TimeMs=1234"), std::string::npos); } TEST(TelemetryLoggerTest, RecordExceptionAndModelEventsIncludeSpecificValues) { @@ -100,20 +106,30 @@ TEST(TelemetryLoggerTest, RecordExceptionAndModelEventsIncludeSpecificValues) { TelemetryLogger telemetry("foundry-local", logger); telemetry.RecordException(Action::kModelLoad, std::runtime_error("config missing")); - telemetry.RecordModelUsage("phi-3-mini", 17, 31, 250); - telemetry.RecordModelId(Action::kModelLoad, "phi-3-mini"); + + ModelUsageInfo usage; + usage.model_id = "phi-3-mini"; + usage.execution_provider = "CPU"; + usage.user_agent = "cli/1.0"; + usage.total_tokens = 31; + usage.input_token_count = 17; + usage.total_time_ms = 250; + telemetry.RecordModelUsage(usage); + + telemetry.RecordModelId(Action::kModelLoad, "phi-3-mini", ActionStatus::kSuccess, "cli/1.0"); ASSERT_EQ(logger.entries.size(), 3u); - EXPECT_NE(logger.entries[0].message.find("Command:ModelLoad"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Exception:config missing"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Action=ModelLoad"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Exception=config missing"), std::string::npos); - EXPECT_NE(logger.entries[1].message.find("ModelUsage: model=phi-3-mini"), std::string::npos); - EXPECT_NE(logger.entries[1].message.find("prompt_tokens=17"), std::string::npos); - EXPECT_NE(logger.entries[1].message.find("completion_tokens=31"), std::string::npos); - EXPECT_NE(logger.entries[1].message.find("duration=250ms"), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("Model "), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("ModelId=phi-3-mini"), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("InputTokenCount=17"), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("TotalTokens=31"), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("TotalTimeMs=250"), std::string::npos); - EXPECT_NE(logger.entries[2].message.find("Command:ModelLoad"), std::string::npos); - EXPECT_NE(logger.entries[2].message.find("ModelId:phi-3-mini"), std::string::npos); + EXPECT_NE(logger.entries[2].message.find("Action=ModelLoad"), std::string::npos); + EXPECT_NE(logger.entries[2].message.find("ModelId=phi-3-mini"), std::string::npos); } TEST(ActionTrackerTest, DestructorRecordsFailureByDefaultWithoutModelId) { From 339bcd122c06ba96747a3699d6592b7ad8e9da7d Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Wed, 10 Jun 2026 23:29:07 -0500 Subject: [PATCH 02/77] sdk_v2/cpp/telemetry: add OneDsTelemetry 1DS upload bridge OneDsTelemetry is the production ITelemetry implementation. It embeds a TelemetryLogger mirror so every event is also logged locally to ILogger for diagnostics, then conditionally uploads to 1DS through the Microsoft cpp-client-telemetry SDK. Suppression model (three-state): - CI environment detected -> skip `LogManager::Initialize`; local logging still happens, but no upload occurs. - Tenant token empty (build did not pass `-DFOUNDRY_LOCAL_TELEMETRY_TOKEN`) -> same as CI: no upload, only local logging. - Otherwise -> upload. Every event is stamped with `test=true` when `FOUNDRY_TESTING_MODE` is truthy, `test=false` otherwise. Common context, propagated to every event via `ILogger::SetContext`: - `app_name` (from `Configuration::app_name`). - `app_session_guid` (random v4 UUID; on Windows the special `UTCReplace_AppSessionGuid` field also gets the OS app session GUID). - `version` (from the generated `version.h`). - `os_name` / `os_version` / `cpu_arch` (Win: `GetVersionExA` + `GetNativeSystemInfo`; POSIX: `uname`). The `MICROSOFT_KEYWORD_CRITICAL_DATA` (bit 47) policy flag is set on every event via `SetPolicyBitFlags` so the data passes 1DS classifier. Build / packaging: - `vcpkg.json`: `telemetry` feature pulls `cpp-client-telemetry`. The port is pending in microsoft/vcpkg#52316; until that merges, builds must pass `--no_telemetry`. - `CMakeLists.txt`: new `FOUNDRY_LOCAL_USE_TELEMETRY` option (default ON) and `FOUNDRY_LOCAL_TELEMETRY_TOKEN` advanced cache variable. Calls `find_package(MSTelemetry CONFIG REQUIRED)`, includes `one_ds_telemetry.cc` and `configure_file`-generates `one_ds_tenant_token.h` containing the build-time token, then links `MSTelemetry::mat` and defines `FOUNDRY_LOCAL_HAS_1DS=1`. - `build.py`: new `--no_telemetry` and `--telemetry_token` flags; forwards through to CMake / VCPKG_MANIFEST_FEATURES. Build verified RelWithDebInfo --no_telemetry; --use_telemetry will be validated once the vcpkg port lands. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/CMakeLists.txt | 50 ++++ sdk_v2/cpp/build.py | 30 ++- sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc | 250 ++++++++++++++++++ sdk_v2/cpp/src/telemetry/one_ds_telemetry.h | 80 ++++++ .../src/telemetry/one_ds_tenant_token.h.in | 19 ++ sdk_v2/cpp/vcpkg.json | 6 + 6 files changed, 433 insertions(+), 2 deletions(-) create mode 100644 sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc create mode 100644 sdk_v2/cpp/src/telemetry/one_ds_telemetry.h create mode 100644 sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index c203deec6..10013bc16 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -45,8 +45,19 @@ option(FOUNDRY_LOCAL_BUILD_EXAMPLES "Build example programs" ON) option(FOUNDRY_LOCAL_BUILD_TOOLS "Build internal build-time tools (catalog_snapshot, ...)" ON) option(FOUNDRY_LOCAL_BUILD_SERVICE "Build web service support (requires oat++)" ON) option(FOUNDRY_LOCAL_USE_WINML "Use WinML/WindowsAppSDK.ML for OnnxRuntime instead of standalone ORT" OFF) +option(FOUNDRY_LOCAL_USE_TELEMETRY + "Build with 1DS telemetry uploads (requires cpp-client-telemetry vcpkg port)" ON) option(FOUNDRY_LOCAL_ENABLE_ASAN "Enable AddressSanitizer + UndefinedBehaviorSanitizer (Linux only)" OFF) +# 1DS ingestion token. Treat as a secret: pass via CI as +# cmake -DFOUNDRY_LOCAL_TELEMETRY_TOKEN="" +# Defaults to empty so developer builds and forks don't accidentally bind to a +# real tenant. When empty (or when in CI at runtime), OneDsTelemetry initializes +# but does not transmit events. +set(FOUNDRY_LOCAL_TELEMETRY_TOKEN "" + CACHE STRING "1DS / Aria ingestion token baked into the binary. Leave empty in dev builds.") +mark_as_advanced(FOUNDRY_LOCAL_TELEMETRY_TOKEN) + # Android: interactive examples and host tools don't run on device if(ANDROID) set(FOUNDRY_LOCAL_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) @@ -87,6 +98,17 @@ find_package(OnnxRuntime REQUIRED) # Not REQUIRED: builds without it fall back to CPU-only EP detection. find_package(WinMLEpCatalog) +# 1DS C++ client telemetry — provided by the cpp-client-telemetry vcpkg port. +# Enabled via FOUNDRY_LOCAL_USE_TELEMETRY (default ON) and the vcpkg +# manifest feature "telemetry". When off, the ITelemetry interface is satisfied +# by TelemetryLogger only (local diagnostic log only — no upload). +if(FOUNDRY_LOCAL_USE_TELEMETRY) + find_package(MSTelemetry CONFIG REQUIRED) + message(STATUS "1DS telemetry: enabled (cpp-client-telemetry found)") +else() + message(STATUS "1DS telemetry: disabled (FOUNDRY_LOCAL_USE_TELEMETRY=OFF)") +endif() + # -------------------------------------------------------------------------- # Library target # -------------------------------------------------------------------------- @@ -189,7 +211,11 @@ set(FOUNDRY_LOCAL_SOURCES src/service/web_service.cc src/telemetry/telemetry.cc src/telemetry/telemetry_action_tracker.cc + src/telemetry/telemetry_environment.cc src/telemetry/telemetry_logger.cc + src/telemetry/telemetry_metadata.cc + src/telemetry/ep_download_tracker.cc + src/telemetry/download_tracker.cc src/utils.cc src/util/file_lock.cc src/http/http_download.cc @@ -208,6 +234,13 @@ else() list(APPEND FOUNDRY_LOCAL_SOURCES src/catalog/live_catalog_client_stub.cc) endif() +# 1DS bridge — compiled only when the cpp-client-telemetry port is available. +# When disabled, Manager falls back to TelemetryLogger and the OneDsTelemetry +# symbol is never referenced. +if(FOUNDRY_LOCAL_USE_TELEMETRY) + list(APPEND FOUNDRY_LOCAL_SOURCES src/telemetry/one_ds_telemetry.cc) +endif() + # Organize headers into filters matching the directory structure in Visual Studio source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}" PREFIX "Source" FILES ${FOUNDRY_LOCAL_SOURCES}) source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}/include" PREFIX "Public Headers" FILES ${FOUNDRY_LOCAL_PUBLIC_HEADERS}) @@ -275,6 +308,13 @@ function(foundry_local_configure_target TARGET LINK_SCOPE) target_compile_definitions(${TARGET} ${LINK_SCOPE} FOUNDRY_LOCAL_HAS_EP_CATALOG=0) endif() + if(FOUNDRY_LOCAL_USE_TELEMETRY) + target_link_libraries(${TARGET} ${LINK_SCOPE} MSTelemetry::mat) + target_compile_definitions(${TARGET} ${LINK_SCOPE} FOUNDRY_LOCAL_HAS_1DS=1) + else() + target_compile_definitions(${TARGET} ${LINK_SCOPE} FOUNDRY_LOCAL_HAS_1DS=0) + endif() + # WinAppSDK Bootstrap (MddBootstrapInitialize2). Only linked in WinML builds, since # WinML by definition depends on the WindowsAppSDK runtime; non-WinML builds skip it # and the bootstrap call site in Manager::Create is compiled out via the same macro. @@ -303,6 +343,16 @@ configure_file( @ONLY ) +# Generate the 1DS tenant-token header. Always generated (even when +# FOUNDRY_LOCAL_USE_TELEMETRY=OFF) so includes of one_ds_tenant_token.h compile +# in both configurations; consumers must still guard MAT::LogManager calls +# behind FOUNDRY_LOCAL_HAS_1DS. +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/src/telemetry/one_ds_tenant_token.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/generated/one_ds_tenant_token.h" + @ONLY +) + # -------------------------------------------------------------------------- # Object library — compiles all sources once. Both the shared (DLL) and # static library targets re-use these object files, avoiding a double build. diff --git a/sdk_v2/cpp/build.py b/sdk_v2/cpp/build.py index f4e2a6a39..2d7da0e7b 100644 --- a/sdk_v2/cpp/build.py +++ b/sdk_v2/cpp/build.py @@ -172,6 +172,19 @@ class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescript "catalog when --use_winml is set; defaults to the version pinned in " "FindWinMLEpCatalog.cmake).", ) + parser.add_argument( + "--no_telemetry", action="store_true", + help="Skip building the 1DS (cpp-client-telemetry) bridge. Useful while the vcpkg " + "port (microsoft/vcpkg#52316) is unmerged or when building forks that should " + "not link any telemetry transport. Local diagnostic logging via TelemetryLogger " + "still works.", + ) + parser.add_argument( + "--telemetry_token", default=None, type=str, + help="1DS / Aria ingestion token. Baked into the binary at configure time as a " + "secret. Treat with care — do not commit to source. Defaults to empty, which " + "means OneDsTelemetry initializes but skips upload.", + ) # Cross-compilation (mutually exclusive targets) cross_group = parser.add_mutually_exclusive_group() @@ -458,9 +471,22 @@ def configure(args: argparse.Namespace) -> None: f"-DFOUNDRY_LOCAL_BUILD_SERVICE={build_service}", ] - # Enable vcpkg manifest features for tests + # Enable vcpkg manifest features as needed. Multiple features are passed as a + # semicolon-separated list in a single -D flag. + manifest_features = [] if build_tests == "ON": - command += ["-DVCPKG_MANIFEST_FEATURES=tests"] + manifest_features.append("tests") + if not args.no_telemetry: + manifest_features.append("telemetry") + if manifest_features: + command += [f"-DVCPKG_MANIFEST_FEATURES={';'.join(manifest_features)}"] + + if args.no_telemetry: + command += ["-DFOUNDRY_LOCAL_USE_TELEMETRY=OFF"] + else: + command += ["-DFOUNDRY_LOCAL_USE_TELEMETRY=ON"] + if args.telemetry_token is not None: + command += [f"-DFOUNDRY_LOCAL_TELEMETRY_TOKEN={args.telemetry_token}"] if args.use_winml: command += ["-DFOUNDRY_LOCAL_USE_WINML=ON"] diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc new file mode 100644 index 000000000..a07eaf2eb --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc @@ -0,0 +1,250 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// 1DS-backed ITelemetry implementation. Compiled only when FOUNDRY_LOCAL_USE_TELEMETRY=ON +// (which requires the cpp-client-telemetry vcpkg port to be available). + +#include "telemetry/one_ds_telemetry.h" + +#include "telemetry/telemetry_environment.h" + +#include + +// 1DS C++ SDK headers. The vcpkg port ships these via find_package(MSTelemetry CONFIG). +#include +#include +#include + +// The LogManager macro must appear exactly once per binary that uses the +// "v1 classic" LogManager API surface. The macro instantiates the singleton +// statics for our module configuration. +LOGMANAGER_INSTANCE + +namespace fl { + +namespace { + +using ::Microsoft::Applications::Events::LogManager; +using ::Microsoft::Applications::Events::ILogger; +using ::Microsoft::Applications::Events::EventProperties; +using ::Microsoft::Applications::Events::EventPriority; +using ::Microsoft::Applications::Events::PiiKind_None; + +constexpr uint64_t kCriticalData = MICROSOFT_KEYWORD_CRITICAL_DATA; + +void SetCommonContext(ILogger* mat_logger, const TelemetryMetadata& m) { + // Process-wide context — stamped on every event uploaded through this ILogger. + mat_logger->SetContext("AppName", m.app_name); + mat_logger->SetContext("Version", m.version); + // 1DS recognizes UTCReplace_AppSessionGuid as a magic field name on Windows UTC + // and substitutes the OS session GUID. On other platforms we just send the + // GUID we computed at startup — same correlation semantics either way. + mat_logger->SetContext("UTCReplace_AppSessionGuid", m.app_session_guid); + mat_logger->SetContext("OsName", m.os_name); + mat_logger->SetContext("OsVersion", m.os_version); + mat_logger->SetContext("CpuArch", m.cpu_arch); +} + +EventProperties MakeEvent(const char* name, bool test_mode) { + EventProperties ev(name); + ev.SetPriority(EventPriority::EventPriority_Normal); + ev.SetPolicyBitFlags(kCriticalData); + // `test` is stamped on every event so CI/test data is distinguishable in the backend + // when FOUNDRY_TESTING_MODE is set. In CI (IsCiEnvironment()=true) we never reach + // emission at all, so this only ever toggles for explicit test mode. + ev.SetProperty("test", test_mode); + return ev; +} + +void SafeLog(ILogger* mat_logger, EventProperties& ev) { + if (mat_logger != nullptr) { + mat_logger->LogEvent(ev); + } +} + +ILogger* GetMatLogger() { + // LogManager::GetLogger() returns nullptr until Initialize has been called. + return LogManager::GetLogger(); +} + +} // namespace + +OneDsTelemetry::OneDsTelemetry(const std::string& tenant_token, + const std::string& app_name, + ILogger& logger) + : local_log_(app_name, logger), + metadata_(BuildTelemetryMetadata(app_name)), + logger_(logger) { + const bool is_ci = TelemetryEnvironment::IsCiEnvironment(); + if (is_ci) { + logger_.Log(LogLevel::Information, + "[Telemetry] CI environment detected; 1DS upload disabled (events still logged locally)"); + return; + } + if (tenant_token.empty()) { + logger_.Log(LogLevel::Information, + "[Telemetry] Tenant token is empty; 1DS upload disabled (events still logged locally)"); + return; + } + + try { + auto* mat_logger = LogManager::Initialize(tenant_token); + if (mat_logger == nullptr) { + logger_.Log(LogLevel::Warning, + "[Telemetry] LogManager::Initialize returned null; 1DS upload disabled"); + return; + } + SetCommonContext(mat_logger, metadata_); + initialized_.store(true, std::memory_order_release); + logger_.Log(LogLevel::Information, + fmt::format("[Telemetry] 1DS initialized; AppName={} Version={} Os={} {} Arch={} TestMode={}", + metadata_.app_name, metadata_.version, metadata_.os_name, + metadata_.os_version, metadata_.cpu_arch, metadata_.test_mode)); + } catch (const std::exception& ex) { + logger_.Log(LogLevel::Warning, + fmt::format("[Telemetry] LogManager::Initialize threw: {}; 1DS upload disabled", ex.what())); + } catch (...) { + logger_.Log(LogLevel::Warning, + "[Telemetry] LogManager::Initialize threw unknown exception; 1DS upload disabled"); + } +} + +OneDsTelemetry::~OneDsTelemetry() { + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + try { + LogManager::FlushAndTeardown(); + } catch (...) { + // Best-effort: never throw from a destructor. + } +} + +void OneDsTelemetry::RecordAction(Action action, ActionStatus status, const std::string& user_agent, + bool indirect, int64_t duration_ms) { + local_log_.RecordAction(action, status, user_agent, indirect, duration_ms); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("Action", metadata_.test_mode); + ev.SetProperty("Action", std::string(ActionToString(action))); + ev.SetProperty("Status", std::string(ActionStatusToString(status))); + ev.SetProperty("UserAgent", user_agent); + ev.SetProperty("Direct", !indirect); + ev.SetProperty("TimeMs", duration_ms); + SafeLog(GetMatLogger(), ev); +} + +void OneDsTelemetry::RecordException(Action action, const std::exception& exception) { + RecordException(action, exception, std::string{}); +} + +void OneDsTelemetry::RecordException(Action action, const std::exception& exception, + const std::string& user_agent) { + local_log_.RecordException(action, exception, user_agent); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("Error", metadata_.test_mode); + ev.SetProperty("Action", std::string(ActionToString(action))); + ev.SetProperty("UserAgent", user_agent); + ev.SetProperty("ExceptionType", "std::exception"); + ev.SetProperty("ExceptionMessage", std::string(exception.what())); + ev.SetProperty("InnerExceptionType", ""); + ev.SetProperty("InnerExceptionMessage", ""); + ev.SetProperty("StackTrace", ""); + ev.SetProperty("InnerStackTrace", ""); + SafeLog(GetMatLogger(), ev); +} + +void OneDsTelemetry::RecordModelUsage(const ModelUsageInfo& info) { + local_log_.RecordModelUsage(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("Model", metadata_.test_mode); + ev.SetProperty("ModelId", info.model_id); + ev.SetProperty("ExecutionProvider", info.execution_provider); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("TimeToFirstTokenMs", info.time_to_first_token_ms); + ev.SetProperty("TotalTimeMs", info.total_time_ms); + ev.SetProperty("TotalTokens", static_cast(info.total_tokens)); + ev.SetProperty("InputTokenCount", static_cast(info.input_token_count)); + ev.SetProperty("NumMessages", static_cast(info.num_messages)); + ev.SetProperty("MemoryUsedMB", info.memory_used_mb); + ev.SetProperty("CpuTimeMs", info.cpu_time_ms); + ev.SetProperty("GpuMemoryUsedMB", info.gpu_memory_used_mb); + SafeLog(GetMatLogger(), ev); +} + +void OneDsTelemetry::RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const std::string& user_agent) { + local_log_.RecordModelId(action, model_id, status, user_agent); + if (!initialized_.load(std::memory_order_acquire) || model_id.empty()) { + return; + } + auto ev = MakeEvent("ModelId", metadata_.test_mode); + ev.SetProperty("Action", std::string(ActionToString(action))); + ev.SetProperty("ModelId", model_id); + ev.SetProperty("Status", std::string(ActionStatusToString(status))); + ev.SetProperty("UserAgent", user_agent); + SafeLog(GetMatLogger(), ev); +} + +void OneDsTelemetry::RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) { + local_log_.RecordEpDownloadAttempt(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("EPDownloadAttempt", metadata_.test_mode); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("Attempts", static_cast(info.attempts)); + ev.SetProperty("NumProviders", static_cast(info.num_providers)); + ev.SetProperty("Succeeded", static_cast(info.succeeded)); + ev.SetProperty("Failed", static_cast(info.failed)); + ev.SetProperty("Resolved", info.resolved); + ev.SetProperty("Status", std::string(ActionStatusToString(info.status))); + ev.SetProperty("TimeMs", info.duration_ms); + SafeLog(GetMatLogger(), ev); +} + +void OneDsTelemetry::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) { + local_log_.RecordEpDownloadAndRegister(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("EPDownloadAndRegister", metadata_.test_mode); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("ProviderName", info.provider_name); + ev.SetProperty("InitReadyState", info.init_ready_state); + ev.SetProperty("DownloadReadyState", info.download_ready_state); + ev.SetProperty("DownloadStatus", std::string(ActionStatusToString(info.download_status))); + ev.SetProperty("DownloadTimeMs", info.download_duration_ms); + ev.SetProperty("RegisterReadyState", info.register_ready_state); + ev.SetProperty("RegisterStatus", std::string(ActionStatusToString(info.register_status))); + ev.SetProperty("RegisterTimeMs", info.register_duration_ms); + SafeLog(GetMatLogger(), ev); +} + +void OneDsTelemetry::RecordDownload(const DownloadInfo& info) { + local_log_.RecordDownload(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("Download", metadata_.test_mode); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("ModelId", info.model_id); + ev.SetProperty("Status", std::string(ActionStatusToString(info.status))); + ev.SetProperty("LockWaitTimeMs", info.lock_wait_ms); + ev.SetProperty("EnumerationTimeMs", info.enumeration_ms); + ev.SetProperty("DownloadTimeMs", info.download_ms); + ev.SetProperty("TotalSizeBytes", info.total_size_bytes); + ev.SetProperty("AlreadyCachedBytes", info.already_cached_bytes); + ev.SetProperty("FileCount", static_cast(info.file_count)); + ev.SetProperty("SkippedFileCount", static_cast(info.skipped_file_count)); + ev.SetProperty("DownloadWaitResult", info.download_wait_result); + ev.SetProperty("MaxConcurrency", static_cast(info.max_concurrency)); + SafeLog(GetMatLogger(), ev); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h new file mode 100644 index 000000000..31cfbcbc9 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "telemetry/telemetry.h" +#include "telemetry/telemetry_logger.h" +#include "telemetry/telemetry_metadata.h" +#include "logger.h" + +#include +#include +#include + +namespace fl { + +/// 1DS-backed ITelemetry implementation. Built only when the +/// cpp-client-telemetry vcpkg port is available (find_package(MSTelemetry CONFIG) +/// succeeded and FOUNDRY_LOCAL_USE_TELEMETRY=ON). +/// +/// Lifecycle: +/// * Constructor: +/// - If TelemetryEnvironment::IsCiEnvironment() -> CI mode: skip 1DS Initialize +/// entirely. All RecordX calls become no-ops on the 1DS side; the embedded +/// TelemetryLogger still mirrors them to ILogger. +/// - Else if tenant_token is empty -> uninitialized: same behavior as CI mode. +/// Manager normally avoids constructing OneDsTelemetry when the token is +/// empty, but the guard here makes the class robust to misconfiguration. +/// - Else: call MAT::LogManager::Initialize(token), set process-wide common +/// context (app session GUID, version, OS info, app name, test flag). +/// +/// * RecordX: builds an EventProperties with the event-specific fields and the +/// `test` bool, then logs via ILogger->LogEvent. Local mirror via TelemetryLogger +/// always runs first so the diagnostic log is preserved regardless of upload. +/// +/// * Destructor: MAT::LogManager::FlushAndTeardown so events on disk are pushed +/// before the process exits. Safe to call even when initialization was skipped. +/// +/// Thread safety: 1DS LogManager methods are documented as thread-safe. The +/// `initialized_` flag is set once during construction; all other writes happen +/// only in the destructor. +class OneDsTelemetry : public ITelemetry { + public: + /// @param tenant_token 1DS ingestion token. Empty disables uploads (CI-equivalent). + /// @param app_name Configuration::app_name; stamped as AppName on every event. + /// @param logger Diagnostic logger; used by the embedded TelemetryLogger mirror. + OneDsTelemetry(const std::string& tenant_token, + const std::string& app_name, + ILogger& logger); + ~OneDsTelemetry() override; + + // Non-copyable, non-movable (singleton owns LogManager state). + OneDsTelemetry(const OneDsTelemetry&) = delete; + OneDsTelemetry& operator=(const OneDsTelemetry&) = delete; + + void RecordAction(Action action, ActionStatus status, const std::string& user_agent, + bool indirect, int64_t duration_ms) override; + + void RecordException(Action action, const std::exception& exception) override; + void RecordException(Action action, const std::exception& exception, + const std::string& user_agent) override; + + void RecordModelUsage(const ModelUsageInfo& info) override; + void RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const std::string& user_agent) override; + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; + void RecordDownload(const DownloadInfo& info) override; + + /// True if 1DS Initialize succeeded (i.e. events are actually uploaded). + /// False in CI or when the tenant token was empty. + bool IsUploadEnabled() const { return initialized_; } + + private: + TelemetryLogger local_log_; // Always-on local mirror. + TelemetryMetadata metadata_; // Cached at construction. + std::atomic initialized_{false}; // True iff MAT::LogManager::Initialize ran. + ILogger& logger_; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in b/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in new file mode 100644 index 000000000..1417e9281 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +// Auto-generated by CMake from one_ds_tenant_token.h.in — do not edit manually. +// +// The 1DS tenant token is passed at configure time via +// cmake -DFOUNDRY_LOCAL_TELEMETRY_TOKEN="" +// and is intended to be supplied by CI as a secret, similar to how the build +// system supplies ORT paths via -DORT_HOME. The default is an empty string so +// developer builds and forks don't accidentally ship a tenant binding. +// +// When the constant is empty, OneDsTelemetry::Initialize is skipped and the +// Manager falls back to TelemetryLogger (which only writes to the local +// ILogger sink — no upload). +#pragma once + +namespace fl { + +inline constexpr const char* kFoundryLocalTenantToken = "@FOUNDRY_LOCAL_TELEMETRY_TOKEN@"; + +} // namespace fl diff --git a/sdk_v2/cpp/vcpkg.json b/sdk_v2/cpp/vcpkg.json index 781f3f2f0..1905b376b 100644 --- a/sdk_v2/cpp/vcpkg.json +++ b/sdk_v2/cpp/vcpkg.json @@ -20,6 +20,12 @@ "oatpp" ] }, + "telemetry": { + "description": "Build with 1DS (cpp-client-telemetry) telemetry uploads", + "dependencies": [ + "cpp-client-telemetry" + ] + }, "tests": { "description": "Build unit tests", "dependencies": [ From 7b9588a6feac128c50226115832221ceb1f8c406 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Wed, 10 Jun 2026 23:29:42 -0500 Subject: [PATCH 03/77] sdk_v2/cpp: wire ITelemetry into Manager, EpDetector and DownloadManager Plumb the typed telemetry interface through the orchestration layer so that real workloads produce the new EPDownloadAttempt, EPDownloadAndRegister and Download events. Manager: - Construct `telemetry_` before `ep_detector_` so the detector can be handed a non-null sink at ctor time. When `FOUNDRY_LOCAL_HAS_1DS` is defined, the implementation is `OneDsTelemetry` and reads the build-time token from the generated `one_ds_tenant_token.h`; otherwise it falls back to `TelemetryLogger`. - Member-declaration order in `manager.h` reorganised so consumers (`ep_detector_`, `download_manager_`) appear after the providers (`telemetry_`). C++ destroys in reverse declaration order, so this keeps the raw `ITelemetry*` held by `EpDetector` and `DownloadManager` valid for their entire lifetime. - Explicit `~Manager()` Shutdown reset order updated to match: the telemetry sink is reset last (after ep_detector and download_manager). EpDetector: - New optional `ITelemetry* telemetry` ctor arg (default nullptr) so unit tests that instantiate EpDetector directly continue to compile. - `DownloadAndRegisterEps` emits one `EPDownloadAndRegister` event per bootstrapper via `EpDownloadTracker` and a single aggregate `EPDownloadAttempt` event at the end with attempt / success / fail counts and an overall status. - Exceptions thrown from a bootstrapper invoke `EpDownloadTracker::RecordException` before re-throwing so the failure is recorded regardless of how it propagates. DownloadManager: - New optional `ITelemetry* telemetry` ctor arg. When non-null, `DownloadModel` wraps the call with a `DownloadTracker` that captures lock-wait, enumeration and download timings, total bytes, file count, max concurrency and final status (Success / Skipped / Failure). `RecordException` is invoked on the exception path before re-throwing. - `DownloadModel` now takes an optional `user_agent` parameter so HTTP-driven downloads can attribute the event to the calling client. DownloadBlobsToDirectory: - New optional `BlobDownloadStats*` out parameter populated with `total_size_bytes`, `file_count`, `enumeration_ms`, `download_ms`. All existing 4-argument callers (including unit tests) keep working because the parameter defaults to nullptr. Verified RelWithDebInfo --no_telemetry: 756 tests pass; 54 skipped (environmental, missing local test data); 0 functional failures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/src/download/blob_downloader.cc | 26 ++++++- sdk_v2/cpp/src/download/blob_downloader.h | 15 +++- sdk_v2/cpp/src/download/download_manager.cc | 60 ++++++++++++++-- sdk_v2/cpp/src/download/download_manager.h | 14 +++- sdk_v2/cpp/src/ep_detection/ep_detector.cc | 79 ++++++++++++++++++++- sdk_v2/cpp/src/ep_detection/ep_detector.h | 10 ++- sdk_v2/cpp/src/manager.cc | 26 +++++-- sdk_v2/cpp/src/manager.h | 9 ++- 8 files changed, 219 insertions(+), 20 deletions(-) diff --git a/sdk_v2/cpp/src/download/blob_downloader.cc b/sdk_v2/cpp/src/download/blob_downloader.cc index 73b41b173..96471c4a0 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.cc +++ b/sdk_v2/cpp/src/download/blob_downloader.cc @@ -263,7 +263,11 @@ bool EndsWith(const std::string& str, const std::string& suffix) { void DownloadBlobsToDirectory(IBlobDownloader& downloader, const std::string& sas_uri, const std::string& output_directory, - const BlobDownloadOptions& options) { + const BlobDownloadOptions& options, + BlobDownloadStats* stats) { + using clock = std::chrono::steady_clock; + auto enum_start = clock::now(); + // Step 1: Enumerate all blobs auto all_blobs = downloader.ListBlobs(sas_uri); @@ -306,6 +310,11 @@ void DownloadBlobsToDirectory(IBlobDownloader& downloader, blobs_to_download.end()); if (blobs_to_download.empty()) { + if (stats != nullptr) { + stats->enumeration_ms = std::chrono::duration_cast( + clock::now() - enum_start) + .count(); + } return; } @@ -321,6 +330,15 @@ void DownloadBlobsToDirectory(IBlobDownloader& downloader, total_size += blob.content_length; } + if (stats != nullptr) { + stats->file_count = static_cast(blobs_to_download.size()); + stats->total_size_bytes = total_size; + stats->enumeration_ms = std::chrono::duration_cast( + clock::now() - enum_start) + .count(); + } + auto download_start = clock::now(); + // Step 4.5: Emit 0% so callers know the download has started if (options.progress) { int result = options.progress(0.0f); @@ -377,6 +395,12 @@ void DownloadBlobsToDirectory(IBlobDownloader& downloader, if (options.progress) { options.progress(100.0f); } + + if (stats != nullptr) { + stats->download_ms = std::chrono::duration_cast( + clock::now() - download_start) + .count(); + } } } // namespace fl diff --git a/sdk_v2/cpp/src/download/blob_downloader.h b/sdk_v2/cpp/src/download/blob_downloader.h index f43774a16..8047f1c35 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.h +++ b/sdk_v2/cpp/src/download/blob_downloader.h @@ -72,9 +72,22 @@ class AzureBlobDownloader : public IBlobDownloader { /// High-level download function: enumerate, filter, and download all blobs from a SAS URI. /// Handles safetensors optimization, path prefix filtering, and progress reporting. /// Throws fl::Exception on failure. +/// @param stats When non-null, populated with byte/file counts and per-phase timings useful +/// for telemetry. Always populated when the function returns normally; partially +/// populated when it throws (use values only after a clean return). void DownloadBlobsToDirectory(IBlobDownloader& downloader, const std::string& sas_uri, const std::string& output_directory, - const BlobDownloadOptions& options); + const BlobDownloadOptions& options, + struct BlobDownloadStats* stats = nullptr); + +/// Aggregate statistics from one DownloadBlobsToDirectory call. Used by callers to +/// stamp telemetry events with download-shape data (file count, byte volume, timing). +struct BlobDownloadStats { + int64_t total_size_bytes = 0; + int32_t file_count = 0; + int64_t enumeration_ms = 0; // Time spent listing blobs from the SAS URI. + int64_t download_ms = 0; // Time spent transferring blob bytes (excludes enumeration). +}; } // namespace fl diff --git a/sdk_v2/cpp/src/download/download_manager.cc b/sdk_v2/cpp/src/download/download_manager.cc index df5059a35..401e86d0a 100644 --- a/sdk_v2/cpp/src/download/download_manager.cc +++ b/sdk_v2/cpp/src/download/download_manager.cc @@ -3,13 +3,17 @@ #include "download/download_manager.h" #include "download/inference_model_writer.h" #include "exception.h" +#include "telemetry/download_tracker.h" +#include "telemetry/telemetry.h" #include "util/path_safety.h" #include #include +#include #include #include +#include #include #include @@ -151,11 +155,12 @@ std::string SanitizeForPathSegment(std::string_view name) { } // anonymous namespace DownloadManager::DownloadManager(std::string cache_directory, std::string catalog_region, int max_concurrency, - ILogger& logger) + ILogger& logger, ITelemetry* telemetry) : cache_directory_(std::move(cache_directory)), max_concurrency_(max_concurrency), registry_client_(std::make_unique(std::move(catalog_region), logger)), - blob_downloader_(std::make_unique()) {} + blob_downloader_(std::make_unique()), + telemetry_(telemetry) {} DownloadManager::~DownloadManager() = default; @@ -209,13 +214,31 @@ std::string DownloadManager::ComputeModelPath(const ModelInfo& info) const { } std::string DownloadManager::DownloadModel(const ModelInfo& info, - std::function progress_cb) { + std::function progress_cb, + const std::string& user_agent) { + using clock = std::chrono::steady_clock; + auto lock_wait_start = clock::now(); + // Serialize all downloads. Concurrent downloads of the same model would race into // creating the same directory and double-writing inference_model.json; concurrent // downloads of different models would compete for the same per-blob chunk parallelism. // A single global lock keeps the model simple and predictable. std::lock_guard download_guard(download_mutex_); + int64_t lock_wait_ms = std::chrono::duration_cast( + clock::now() - lock_wait_start) + .count(); + + // RAII telemetry tracker — emits a "Download" event on destruction with whatever + // fields have been populated. Default status is kFailure so abrupt exits (exceptions) + // are recorded as failures; the happy path explicitly sets kSuccess / kSkipped. + std::unique_ptr tracker; + if (telemetry_ != nullptr) { + tracker = std::make_unique(info.model_id, user_agent, *telemetry_); + tracker->SetLockWaitMs(lock_wait_ms); + tracker->SetMaxConcurrency(static_cast(max_concurrency_)); + } + auto model_path = ComputeModelPath(info); // Check if already downloaded (before validating URI — cached models don't need one). @@ -230,10 +253,17 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, progress_cb(100.0f); } + if (tracker != nullptr) { + tracker->SetStatus(ActionStatus::kSkipped); + } return ResolveEffectiveModelPath(model_path); } if (info.uri.empty()) { + if (tracker != nullptr) { + auto ex = std::runtime_error("cannot download model: empty URI (asset_id)"); + tracker->RecordException(ex); + } FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "cannot download model: empty URI (asset_id)"); } @@ -273,8 +303,21 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, }; } + if (tracker != nullptr) { + tracker->BeginDownloadPhase(); + } + + BlobDownloadStats stats; DownloadBlobsToDirectory(*blob_downloader_, container.blob_sas_uri, - model_path, download_opts); + model_path, download_opts, &stats); + + if (tracker != nullptr) { + tracker->EndDownloadPhase(); + tracker->SetEnumerationMs(stats.enumeration_ms); + tracker->SetFileCount(stats.file_count); + tracker->SetTotalSizeBytes(stats.total_size_bytes); + tracker->SetDownloadWaitResult("Completed"); + } // Step 3: Write inference_model.json — use model_id (includes version) so the // local model scanner can match it back to catalog entries during startup. @@ -286,8 +329,15 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, // Step 5: Remove download signal — marks download as complete std::filesystem::remove(signal_path); + if (tracker != nullptr) { + tracker->SetStatus(ActionStatus::kSuccess); + } return ResolveEffectiveModelPath(model_path); - } catch (...) { + } catch (const std::exception& e) { + if (tracker != nullptr) { + tracker->SetDownloadWaitResult("Failed"); + tracker->RecordException(e); + } // Leave the signal file in place so the incomplete download is detected throw; } diff --git a/sdk_v2/cpp/src/download/download_manager.h b/sdk_v2/cpp/src/download/download_manager.h index 42a4e69b7..3ab85f02d 100644 --- a/sdk_v2/cpp/src/download/download_manager.h +++ b/sdk_v2/cpp/src/download/download_manager.h @@ -14,6 +14,7 @@ namespace fl { class ILogger; +class ITelemetry; /// Orchestrates the full model download flow: /// 1. Compute local cache path @@ -28,10 +29,14 @@ class DownloadManager { /// @param catalog_region Azure region for the model registry endpoint (e.g. "eastus"). /// @param max_concurrency Per-blob chunk parallelism (default 64). /// @param logger Logger forwarded to the registry client for retry diagnostics. + /// @param telemetry Optional telemetry sink. If non-null, a Download event is emitted + /// per DownloadModel call. nullptr is supported so tests and + /// embedders without telemetry continue to compile and run. DownloadManager(std::string cache_directory, std::string catalog_region, int max_concurrency, - ILogger& logger); + ILogger& logger, + ITelemetry* telemetry = nullptr); ~DownloadManager(); /// Override the model registry client (for testing). @@ -42,9 +47,13 @@ class DownloadManager { /// Download a model to the local cache. /// progress_cb reports 0.0 to 100.0 percentage. + /// user_agent identifies the calling API surface for telemetry attribution; pass an + /// empty string when called outside of an HTTP request context. /// Returns the local path where the model was downloaded. /// Throws fl::Exception on failure. - std::string DownloadModel(const ModelInfo& info, std::function progress_cb = nullptr); + std::string DownloadModel(const ModelInfo& info, + std::function progress_cb = nullptr, + const std::string& user_agent = ""); /// Check if a model is cached locally (directory exists and download is complete). bool IsModelCached(const ModelInfo& info) const; @@ -67,6 +76,7 @@ class DownloadManager { int max_concurrency_; std::unique_ptr registry_client_; std::unique_ptr blob_downloader_; + ITelemetry* telemetry_ = nullptr; /// Serializes all DownloadModel calls. Only one model downloads at a time — simpler /// than per-model locking and avoids contending with the per-blob chunk parallelism diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.cc b/sdk_v2/cpp/src/ep_detection/ep_detector.cc index e086c0b48..ce50eb38d 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.cc @@ -4,21 +4,26 @@ #include "ep_detection/ep_bootstrapper.h" #include "logger.h" +#include "telemetry/ep_download_tracker.h" +#include "telemetry/telemetry.h" #include #include +#include #include namespace fl { EpDetector::EpDetector(const OrtApi& ort_api, OrtEnv& ort_env, std::vector> bootstrappers, - ILogger& logger) + ILogger& logger, + ITelemetry* telemetry) : ort_api_(ort_api), ort_env_(ort_env), bootstrappers_(std::move(bootstrappers)), - logger_(logger) { + logger_(logger), + telemetry_(telemetry) { // Populate both cache vectors exact-sized from bootstrappers_. After this point // size and element addresses (including the EpInfo::name string storage backing // flEpInfo::name) are immutable for the detector's lifetime — only is_registered @@ -134,6 +139,17 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vectorName()); - if (bs->DownloadAndRegister(/*force=*/true, wrapped_cb, logger_)) { + // Per-provider EPDownloadAndRegister event via EpDownloadTracker. When + // telemetry_ is null (e.g. unit tests instantiate EpDetector directly), + // the tracker is skipped — the event has no fallback emitter. + std::unique_ptr tracker; + const bool was_registered_before = bs->IsRegistered(); + if (telemetry_ != nullptr) { + tracker = std::make_unique(bs->Name(), /*user_agent=*/std::string{}, *telemetry_); + tracker->RecordInitialState(was_registered_before ? "Registered" : "NotPresent"); + } + + ++telemetry_attempts; + bool ok = false; + try { + ok = bs->DownloadAndRegister(/*force=*/true, wrapped_cb, logger_); + } catch (const std::exception& ex) { + if (tracker) { + tracker->RecordException(ex); + } + // Re-throw to preserve existing semantics — the wrapper RAII guard above + // resets download_in_progress_; the tracker dtor records the EP event. + throw; + } + + if (ok) { + ++telemetry_succeeded; + telemetry_resolved = true; result.registered_eps.push_back(bs->Name()); // Update cached registration state in place under the cache lock so @@ -178,9 +220,24 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector cache_lock(cache_mutex_); cached_eps_[i].is_registered = true; cached_eps_c_[i].is_registered = true; + + if (tracker) { + tracker->RecordDownloadComplete(ActionStatus::kSuccess, "Installed"); + tracker->RecordRegisterComplete(ActionStatus::kSuccess, "Registered"); + } } else { + ++telemetry_failed; result.failed_eps.push_back(bs->Name()); result.success = false; + if (tracker) { + // The bootstrapper conflated download + register and returned false. + // Record both phases as kFailure so backend dashboards can tell that + // this EP didn't reach Registered, without claiming a specific phase. + tracker->RecordDownloadComplete(ActionStatus::kFailure, + was_registered_before ? "Registered" : "NotPresent"); + tracker->RecordRegisterComplete(ActionStatus::kFailure, + was_registered_before ? "Registered" : "NotPresent"); + } } } @@ -194,6 +251,22 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector( + std::chrono::steady_clock::now() - attempt_start) + .count(); + telemetry_->RecordEpDownloadAttempt(attempt_info); + } + return result; } diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.h b/sdk_v2/cpp/src/ep_detection/ep_detector.h index 254f1e43e..e4c9aceaa 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.h +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.h @@ -23,6 +23,7 @@ struct OrtEnv; namespace fl { class ILogger; +class ITelemetry; /// Interface for detecting available hardware devices and execution providers. class IEpDetector { @@ -71,9 +72,15 @@ class EpDetector : public IEpDetector { /// @param ort_env The ORT environment singleton. /// @param bootstrappers EP bootstrappers for download/registration. /// @param logger Logger instance. + /// @param telemetry Optional telemetry sink. When non-null, DownloadAndRegisterEps + /// emits one EPDownloadAndRegister event per provider via + /// EpDownloadTracker, plus one aggregate EPDownloadAttempt + /// event for the whole call. Pass nullptr in tests / when no + /// telemetry sink is available. EpDetector(const OrtApi& ort_api, OrtEnv& ort_env, std::vector> bootstrappers, - ILogger& logger); + ILogger& logger, + ITelemetry* telemetry = nullptr); ~EpDetector() override = default; // Non-copyable, non-movable (owns bootstrappers and mutex state) @@ -92,6 +99,7 @@ class EpDetector : public IEpDetector { OrtEnv& ort_env_; std::vector> bootstrappers_; ILogger& logger_; + ITelemetry* telemetry_ = nullptr; std::mutex download_mutex_; std::atomic download_in_progress_{false}; mutable std::mutex cache_mutex_; diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index f7c1e4bb1..cf2558785 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -25,6 +25,10 @@ #include "spdlog_logger.h" #include "telemetry/telemetry_action_tracker.h" #include "telemetry/telemetry_logger.h" +#if FOUNDRY_LOCAL_HAS_1DS +#include "one_ds_tenant_token.h" // auto-generated header, in ${CMAKE_BINARY_DIR}/generated +#include "telemetry/one_ds_telemetry.h" +#endif #include "utils.h" #include "winml_bootstrap.h" @@ -262,7 +266,20 @@ Manager::Manager(const Configuration& config) #endif } - ep_detector_ = std::make_unique(*ort_api_, *ort_env_, std::move(bootstrappers), *logger_); + // Telemetry must be constructed before subsystems that emit events so we can + // pass it to them at construction time (e.g. EpDetector emits EPDownloadAttempt + // / EPDownloadAndRegister events from DownloadAndRegisterEps). +#if FOUNDRY_LOCAL_HAS_1DS + // OneDsTelemetry includes a TelemetryLogger mirror, so local diagnostic logging + // is preserved whether or not 1DS upload is enabled. Suppression rules + // (CI environment, empty token) are enforced inside OneDsTelemetry. + telemetry_ = std::make_unique(kFoundryLocalTenantToken, config_.app_name, *logger_); +#else + telemetry_ = std::make_unique(config_.app_name, *logger_); +#endif + + ep_detector_ = std::make_unique(*ort_api_, *ort_env_, std::move(bootstrappers), *logger_, + telemetry_.get()); // Read configurable download concurrency (default 64) int download_concurrency = 64; @@ -282,10 +299,10 @@ Manager::Manager(const Configuration& config) *config_.model_cache_dir, config_.catalog_region.value_or("eastus"), download_concurrency, - *logger_); + *logger_, + telemetry_.get()); model_load_manager_ = std::make_unique(*ep_detector_, *logger_); session_manager_ = std::make_unique(*logger_); - telemetry_ = std::make_unique(config_.app_name, *logger_); catalog_ = std::make_unique( config_.catalog_urls, download_manager_->GetCacheDirectory(), @@ -319,8 +336,9 @@ Manager::~Manager() { model_load_manager_.reset(); download_manager_.reset(); catalog_.reset(); - telemetry_.reset(); + // ep_detector_ holds a raw ITelemetry* — destroy it before telemetry_. ep_detector_.reset(); + telemetry_.reset(); // Unregister EPs we registered, then drop our OrtEnv refcount. Best-effort: // log failures but don't throw from a destructor. diff --git a/sdk_v2/cpp/src/manager.h b/sdk_v2/cpp/src/manager.h index 4b5440db7..187b5b3ff 100644 --- a/sdk_v2/cpp/src/manager.h +++ b/sdk_v2/cpp/src/manager.h @@ -120,9 +120,12 @@ class Manager { // released manually in ~Manager() after all // consumers (sessions, ep_detector_) are gone. // logger_ — everything logs through this, destroyed last + // telemetry_ — used by ep_detector_ and throughout; must + // outlive ep_detector_ because EpDetector holds + // a raw ITelemetry* for emitting EP events // ep_detector_ — detects HW acceleration; holds OrtEnv& (must - // outlive ort_env_ release in ~Manager()) - // telemetry_ — used throughout + // outlive ort_env_ release in ~Manager()) and + // a raw ITelemetry* // catalog_ — owns all Model instances. used by download_manager, model_load_manager, and web service // download_manager_ — uses ModelInfo owned by catalog // model_load_manager_ — holds loaded model state referencing catalog models @@ -135,8 +138,8 @@ class Manager { OrtEnv* ort_env_ = nullptr; std::vector registered_ep_libraries_; std::unique_ptr logger_; - std::unique_ptr ep_detector_; std::unique_ptr telemetry_; + std::unique_ptr ep_detector_; std::unique_ptr catalog_; std::unique_ptr download_manager_; std::unique_ptr model_load_manager_; From bd401e669779a16b71a1f1ac03abc44e4966fbe0 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 15 Jun 2026 11:54:38 -0500 Subject: [PATCH 04/77] sdk_v2/cpp/telemetry: bump vcpkg baseline + fix ILogger collision Now that microsoft/vcpkg#52316 has merged, bump the manifest baseline from 256acc64 to 44819aa2 (current master) so the cpp-client-telemetry 3.10.161.1 port resolves. Also rename the using-declaration in one_ds_telemetry.cc's anonymous namespace from `using ::Microsoft::Applications::Events::ILogger;` to `using MatILogger = ::Microsoft::Applications::Events::ILogger;` so it no longer collides with the local `fl::ILogger` interface from src/logger.h. Three callsites (SetCommonContext, SafeLog, GetMatLogger) updated; the OneDsTelemetry ctor's `ILogger& logger` parameter correctly resolves to fl::ILogger after the rename. Verified with: python sdk_v2/cpp/build.py --use_telemetry --config RelWithDebInfo --skip_examples foundry_local.dll = 12.19 MB (telemetry on, empty token). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc | 8 ++++---- sdk_v2/cpp/vcpkg.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc index a07eaf2eb..72889227b 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc @@ -25,14 +25,14 @@ namespace fl { namespace { using ::Microsoft::Applications::Events::LogManager; -using ::Microsoft::Applications::Events::ILogger; +using MatILogger = ::Microsoft::Applications::Events::ILogger; using ::Microsoft::Applications::Events::EventProperties; using ::Microsoft::Applications::Events::EventPriority; using ::Microsoft::Applications::Events::PiiKind_None; constexpr uint64_t kCriticalData = MICROSOFT_KEYWORD_CRITICAL_DATA; -void SetCommonContext(ILogger* mat_logger, const TelemetryMetadata& m) { +void SetCommonContext(MatILogger* mat_logger, const TelemetryMetadata& m) { // Process-wide context — stamped on every event uploaded through this ILogger. mat_logger->SetContext("AppName", m.app_name); mat_logger->SetContext("Version", m.version); @@ -56,13 +56,13 @@ EventProperties MakeEvent(const char* name, bool test_mode) { return ev; } -void SafeLog(ILogger* mat_logger, EventProperties& ev) { +void SafeLog(MatILogger* mat_logger, EventProperties& ev) { if (mat_logger != nullptr) { mat_logger->LogEvent(ev); } } -ILogger* GetMatLogger() { +MatILogger* GetMatLogger() { // LogManager::GetLogger() returns nullptr until Initialize has been called. return LogManager::GetLogger(); } diff --git a/sdk_v2/cpp/vcpkg.json b/sdk_v2/cpp/vcpkg.json index 1905b376b..fa3c0af16 100644 --- a/sdk_v2/cpp/vcpkg.json +++ b/sdk_v2/cpp/vcpkg.json @@ -2,7 +2,7 @@ "name": "foundry-local", "version-semver": "0.1.0", "description": "Foundry Local C++ SDK", - "builtin-baseline": "256acc64012b23a13041d8705805e1f23b43a024", + "builtin-baseline": "44819aa2a6c10e56065e2b0330e7d6c89d1d2574", "dependencies": [ "azure-storage-blobs-cpp", "ms-gsl", From 979220a879e7b02f858cd03694cf2fcd73e35652 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Tue, 16 Jun 2026 17:24:01 -0500 Subject: [PATCH 05/77] build: enable linker dead-code stripping for foundry_local.dll Add /OPT:REF, /OPT:ICF, and /INCREMENTAL:NO to the foundry_local shared library target for Release and RelWithDebInfo configs. MSVC's /DEBUG flag (needed for PDB generation) silently disables these optimizations unless they are explicitly re-enabled, leaving all unreferenced symbols from statically-linked dependencies (notably cpp-client-telemetry/mat.lib) in the final binary. Also add -Wl,--gc-sections (Linux) and -Wl,-dead_strip (macOS) equivalents for non-Windows builds. Additionally, declare sqlite3 with default-features:false in vcpkg.json to drop the unused json1 extension from the SQLite transitive dependency. Measured impact (RelWithDebInfo, x64-windows, telemetry ON): foundry_local.dll: 12.19 MB -> 4.51 MB (-7.68 MB, -63%) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/CMakeLists.txt | 21 +++++++++++++++++++++ sdk_v2/cpp/vcpkg.json | 3 ++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index 10013bc16..d1fe2fc14 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -420,6 +420,27 @@ if(WIN32) /DELAYLOAD:Microsoft.Windows.AI.MachineLearning.dll ) endif() + + # Strip unreferenced symbols and fold identical COMDATs in Release builds. + # MSVC's /DEBUG (needed for PDBs) silently disables /OPT:REF and /OPT:ICF + # unless they are explicitly re-enabled. /INCREMENTAL:NO is required because + # incremental linking also prevents dead-code elimination. + target_link_options(foundry_local PRIVATE + $<$:/OPT:REF> + $<$:/OPT:ICF> + $<$:/INCREMENTAL:NO> + ) +endif() + +# Strip unreferenced sections on Linux (GCC/Clang) and macOS (Apple ld). +if(UNIX AND NOT APPLE) + target_link_options(foundry_local PRIVATE + $<$:-Wl,--gc-sections> + ) +elseif(APPLE) + target_link_options(foundry_local PRIVATE + $<$:-Wl,-dead_strip> + ) endif() # -------------------------------------------------------------------------- diff --git a/sdk_v2/cpp/vcpkg.json b/sdk_v2/cpp/vcpkg.json index fa3c0af16..43e0b97ad 100644 --- a/sdk_v2/cpp/vcpkg.json +++ b/sdk_v2/cpp/vcpkg.json @@ -7,7 +7,8 @@ "azure-storage-blobs-cpp", "ms-gsl", "nlohmann-json", - "spdlog" + "spdlog", + { "name": "sqlite3", "default-features": false } ], "overrides": [ { "name": "nlohmann-json", "version": "3.11.3" }, From a4838984254031dc6d02e0d26d832925db32c2bd Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 20 Jun 2026 02:18:09 -0500 Subject: [PATCH 06/77] sdk_v2/cpp/telemetry: add InvocationContext (correlation id + indirect) foundation Reworks the telemetry core ahead of broadening route/inference coverage: - New InvocationContext {user_agent, correlation_id, indirect} threaded through the ITelemetry interface in place of the loose user_agent/indirect params. Direct() mints a fresh correlation id; AsIndirect() derives a caused-by child that reuses it. ActionTracker now carries the context and guarantees an id. - Every event (Action/Error/Model/ModelId/Download/EP*) now carries a CorrelationId so all events from one operation can be grouped. The Model event also gains Stream and Direct. - indirect now means "happened as a consequence of another action": the per-provider EPDownloadAndRegister is marked indirect and shares the overall EPDownloadAttempt's correlation id. - Add ActionStatus::kClientError to separate 4xx client rejects from 5xx/internal failures (wired into handlers in a follow-up). - Prune dead Action enum entries (kModelDownload/kModelDelete/kCoreAudioTranscribe) and add kServiceRequestUnmatched for the upcoming router catch-all. - Share the v4 UUID generator (MakeGuidV4Hex) between metadata and correlation. Builds clean (/W4 /WX); telemetry + ActionTracker unit tests pass. --- sdk_v2/cpp/CMakeLists.txt | 1 + sdk_v2/cpp/src/ep_detection/ep_detector.cc | 9 ++- sdk_v2/cpp/src/manager.cc | 3 +- sdk_v2/cpp/src/telemetry/download_tracker.cc | 4 +- .../cpp/src/telemetry/ep_download_tracker.cc | 8 ++- .../cpp/src/telemetry/ep_download_tracker.h | 2 + .../cpp/src/telemetry/invocation_context.cc | 34 +++++++++++ sdk_v2/cpp/src/telemetry/invocation_context.h | 58 +++++++++++++++++++ sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc | 35 ++++++----- sdk_v2/cpp/src/telemetry/one_ds_telemetry.h | 9 ++- sdk_v2/cpp/src/telemetry/telemetry.cc | 10 ++-- sdk_v2/cpp/src/telemetry/telemetry.h | 38 ++++++------ .../src/telemetry/telemetry_action_tracker.cc | 16 +++-- .../src/telemetry/telemetry_action_tracker.h | 12 ++-- sdk_v2/cpp/src/telemetry/telemetry_logger.cc | 51 ++++++++-------- sdk_v2/cpp/src/telemetry/telemetry_logger.h | 9 ++- .../cpp/src/telemetry/telemetry_metadata.cc | 26 +-------- sdk_v2/cpp/test/internal_api/null_telemetry.h | 9 ++- .../cpp/test/internal_api/telemetry_test.cc | 30 +++++----- 19 files changed, 228 insertions(+), 136 deletions(-) create mode 100644 sdk_v2/cpp/src/telemetry/invocation_context.cc create mode 100644 sdk_v2/cpp/src/telemetry/invocation_context.h diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index d1fe2fc14..02423e93f 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -211,6 +211,7 @@ set(FOUNDRY_LOCAL_SOURCES src/service/web_service.cc src/telemetry/telemetry.cc src/telemetry/telemetry_action_tracker.cc + src/telemetry/invocation_context.cc src/telemetry/telemetry_environment.cc src/telemetry/telemetry_logger.cc src/telemetry/telemetry_metadata.cc diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.cc b/sdk_v2/cpp/src/ep_detection/ep_detector.cc index ce50eb38d..027aa3e8a 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.cc @@ -140,8 +140,11 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector tracker; const bool was_registered_before = bs->IsRegistered(); if (telemetry_ != nullptr) { - tracker = std::make_unique(bs->Name(), /*user_agent=*/std::string{}, *telemetry_); + tracker = std::make_unique(bs->Name(), /*user_agent=*/std::string{}, + telemetry_correlation_id, *telemetry_); tracker->RecordInitialState(was_registered_before ? "Registered" : "NotPresent"); } @@ -253,6 +257,7 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vectortelemetry_->RecordAction(Action::kCoreInitialize, ActionStatus::kSuccess, "", false, 0); + created->telemetry_->RecordAction(Action::kCoreInitialize, ActionStatus::kSuccess, + InvocationContext::Direct(), 0); } catch (const std::exception& ex) { created->GetLogger().Log(LogLevel::Error, fmt::format("telemetry RecordAction failed during Create: {}", ex.what())); diff --git a/sdk_v2/cpp/src/telemetry/download_tracker.cc b/sdk_v2/cpp/src/telemetry/download_tracker.cc index dbb369c03..549bf0187 100644 --- a/sdk_v2/cpp/src/telemetry/download_tracker.cc +++ b/sdk_v2/cpp/src/telemetry/download_tracker.cc @@ -12,6 +12,7 @@ DownloadTracker::DownloadTracker(std::string model_id, : telemetry_(telemetry) { info_.model_id = std::move(model_id); info_.user_agent = std::move(user_agent); + info_.correlation_id = MakeGuidV4Hex(); info_.status = ActionStatus::kFailure; download_phase_start_ = std::chrono::steady_clock::now(); } @@ -23,7 +24,8 @@ DownloadTracker::~DownloadTracker() { } void DownloadTracker::RecordException(const std::exception& exception) { - telemetry_.RecordException(Action::kModelFileDownload, exception, info_.user_agent); + telemetry_.RecordException(Action::kModelFileDownload, exception, + InvocationContext{info_.user_agent, info_.correlation_id, /*indirect=*/false}); } } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc b/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc index 65d01f34d..f7a29ea81 100644 --- a/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc +++ b/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc @@ -18,10 +18,12 @@ int64_t ElapsedMs(std::chrono::steady_clock::time_point start) { EpDownloadTracker::EpDownloadTracker(std::string provider_name, std::string user_agent, + std::string correlation_id, ITelemetry& telemetry) : telemetry_(telemetry), provider_name_(std::move(provider_name)), user_agent_(std::move(user_agent)), + correlation_id_(std::move(correlation_id)), stage_start_(std::chrono::steady_clock::now()) { } @@ -58,7 +60,10 @@ void EpDownloadTracker::Done() { } void EpDownloadTracker::RecordException(const std::exception& ex) { - telemetry_.RecordException(Action::kEpDownloadAndRegister, ex, user_agent_); + // The per-provider attempt happens as a consequence of the overall + // DownloadAndRegisterEps call, so it is indirect and shares its correlation id. + telemetry_.RecordException(Action::kEpDownloadAndRegister, ex, + InvocationContext{user_agent_, correlation_id_, /*indirect=*/true}); } void EpDownloadTracker::RecordEvent(ActionStatus incomplete_stage_status) { @@ -77,6 +82,7 @@ void EpDownloadTracker::RecordEvent(ActionStatus incomplete_stage_status) { EpDownloadAndRegisterInfo info; info.user_agent = user_agent_; + info.correlation_id = correlation_id_; info.provider_name = provider_name_; info.init_ready_state = init_ready_state_; info.download_ready_state = download_ready_state_; diff --git a/sdk_v2/cpp/src/telemetry/ep_download_tracker.h b/sdk_v2/cpp/src/telemetry/ep_download_tracker.h index 6ee322f15..7376685bd 100644 --- a/sdk_v2/cpp/src/telemetry/ep_download_tracker.h +++ b/sdk_v2/cpp/src/telemetry/ep_download_tracker.h @@ -21,6 +21,7 @@ class EpDownloadTracker { public: EpDownloadTracker(std::string provider_name, std::string user_agent, + std::string correlation_id, ITelemetry& telemetry); ~EpDownloadTracker(); @@ -61,6 +62,7 @@ class EpDownloadTracker { ITelemetry& telemetry_; std::string provider_name_; std::string user_agent_; + std::string correlation_id_; std::string init_ready_state_ = "N/A"; std::string download_ready_state_ = "N/A"; std::string register_ready_state_ = "N/A"; diff --git a/sdk_v2/cpp/src/telemetry/invocation_context.cc b/sdk_v2/cpp/src/telemetry/invocation_context.cc new file mode 100644 index 000000000..0b4daab04 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/invocation_context.cc @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/invocation_context.h" + +#include +#include +#include + +namespace fl { + +std::string MakeGuidV4Hex() { + // std::random_device + mt19937_64 is enough here — this is a correlation / + // session id, not a cryptographic identifier, so the OS UUID API is overkill. + std::random_device rd; + std::mt19937_64 gen{(static_cast(rd()) << 32) | rd()}; + uint64_t hi = gen(); + uint64_t lo = gen(); + + // Set version (4) and variant (10xx) bits. + hi = (hi & 0xFFFFFFFFFFFF0FFFULL) | 0x0000000000004000ULL; + lo = (lo & 0x3FFFFFFFFFFFFFFFULL) | 0x8000000000000000ULL; + + char buf[37]; + std::snprintf(buf, sizeof(buf), + "%08x-%04x-%04x-%04x-%012llx", + static_cast((hi >> 32) & 0xFFFFFFFFu), + static_cast((hi >> 16) & 0xFFFFu), + static_cast(hi & 0xFFFFu), + static_cast((lo >> 48) & 0xFFFFu), + static_cast(lo & 0x0000FFFFFFFFFFFFULL)); + return std::string(buf); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/invocation_context.h b/sdk_v2/cpp/src/telemetry/invocation_context.h new file mode 100644 index 000000000..8bf30db56 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/invocation_context.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include + +namespace fl { + +/// Generate an RFC 4122 v4 UUID, hex-encoded with hyphens. Not a cryptographic +/// identifier — used for per-operation correlation and per-process session ids. +std::string MakeGuidV4Hex(); + +/// Per-operation telemetry context, threaded from an entry point through every +/// action it triggers. +/// +/// - `user_agent` identifies the calling client (SDK, CLI, Node, browser…). +/// - `correlation_id` groups every event emitted while servicing one logical +/// operation, so the route action, the inference it drives, +/// the Model metrics event and any error can be joined. +/// - `indirect` is true when this action happened as a consequence of +/// another action rather than a direct user/API call (e.g. a +/// session driven by an HTTP route, or a per-provider EP +/// download under an overall attempt). +struct InvocationContext { + std::string user_agent; + std::string correlation_id; + bool indirect = false; + + /// A direct, top-level context with a freshly generated correlation id. + static InvocationContext Direct(std::string user_agent = "") { + InvocationContext ctx; + ctx.user_agent = std::move(user_agent); + ctx.correlation_id = MakeGuidV4Hex(); + ctx.indirect = false; + return ctx; + } + + /// Derive a context for an action triggered by this one: same correlation id + /// and user agent, but marked indirect. + InvocationContext AsIndirect() const { + InvocationContext ctx = *this; + ctx.indirect = true; + return ctx; + } + + /// Guarantee a correlation id is present, generating one when empty. Lets an + /// entry point that received a default-constructed context still group its + /// events (e.g. an SDK caller that didn't build a Direct() context). + InvocationContext& EnsureCorrelationId() { + if (correlation_id.empty()) { + correlation_id = MakeGuidV4Hex(); + } + return *this; + } +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc index 72889227b..324566be7 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc @@ -120,34 +120,32 @@ OneDsTelemetry::~OneDsTelemetry() { } } -void OneDsTelemetry::RecordAction(Action action, ActionStatus status, const std::string& user_agent, - bool indirect, int64_t duration_ms) { - local_log_.RecordAction(action, status, user_agent, indirect, duration_ms); +void OneDsTelemetry::RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t duration_ms) { + local_log_.RecordAction(action, status, context, duration_ms); if (!initialized_.load(std::memory_order_acquire)) { return; } auto ev = MakeEvent("Action", metadata_.test_mode); ev.SetProperty("Action", std::string(ActionToString(action))); ev.SetProperty("Status", std::string(ActionStatusToString(status))); - ev.SetProperty("UserAgent", user_agent); - ev.SetProperty("Direct", !indirect); + ev.SetProperty("UserAgent", context.user_agent); + ev.SetProperty("CorrelationId", context.correlation_id); + ev.SetProperty("Direct", !context.indirect); ev.SetProperty("TimeMs", duration_ms); SafeLog(GetMatLogger(), ev); } -void OneDsTelemetry::RecordException(Action action, const std::exception& exception) { - RecordException(action, exception, std::string{}); -} - void OneDsTelemetry::RecordException(Action action, const std::exception& exception, - const std::string& user_agent) { - local_log_.RecordException(action, exception, user_agent); + const InvocationContext& context) { + local_log_.RecordException(action, exception, context); if (!initialized_.load(std::memory_order_acquire)) { return; } auto ev = MakeEvent("Error", metadata_.test_mode); ev.SetProperty("Action", std::string(ActionToString(action))); - ev.SetProperty("UserAgent", user_agent); + ev.SetProperty("UserAgent", context.user_agent); + ev.SetProperty("CorrelationId", context.correlation_id); ev.SetProperty("ExceptionType", "std::exception"); ev.SetProperty("ExceptionMessage", std::string(exception.what())); ev.SetProperty("InnerExceptionType", ""); @@ -166,6 +164,9 @@ void OneDsTelemetry::RecordModelUsage(const ModelUsageInfo& info) { ev.SetProperty("ModelId", info.model_id); ev.SetProperty("ExecutionProvider", info.execution_provider); ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); + ev.SetProperty("Stream", info.stream); + ev.SetProperty("Direct", !info.indirect); ev.SetProperty("TimeToFirstTokenMs", info.time_to_first_token_ms); ev.SetProperty("TotalTimeMs", info.total_time_ms); ev.SetProperty("TotalTokens", static_cast(info.total_tokens)); @@ -178,8 +179,8 @@ void OneDsTelemetry::RecordModelUsage(const ModelUsageInfo& info) { } void OneDsTelemetry::RecordModelId(Action action, const std::string& model_id, - ActionStatus status, const std::string& user_agent) { - local_log_.RecordModelId(action, model_id, status, user_agent); + ActionStatus status, const InvocationContext& context) { + local_log_.RecordModelId(action, model_id, status, context); if (!initialized_.load(std::memory_order_acquire) || model_id.empty()) { return; } @@ -187,7 +188,8 @@ void OneDsTelemetry::RecordModelId(Action action, const std::string& model_id, ev.SetProperty("Action", std::string(ActionToString(action))); ev.SetProperty("ModelId", model_id); ev.SetProperty("Status", std::string(ActionStatusToString(status))); - ev.SetProperty("UserAgent", user_agent); + ev.SetProperty("UserAgent", context.user_agent); + ev.SetProperty("CorrelationId", context.correlation_id); SafeLog(GetMatLogger(), ev); } @@ -198,6 +200,7 @@ void OneDsTelemetry::RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) } auto ev = MakeEvent("EPDownloadAttempt", metadata_.test_mode); ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); ev.SetProperty("Attempts", static_cast(info.attempts)); ev.SetProperty("NumProviders", static_cast(info.num_providers)); ev.SetProperty("Succeeded", static_cast(info.succeeded)); @@ -215,6 +218,7 @@ void OneDsTelemetry::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo } auto ev = MakeEvent("EPDownloadAndRegister", metadata_.test_mode); ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); ev.SetProperty("ProviderName", info.provider_name); ev.SetProperty("InitReadyState", info.init_ready_state); ev.SetProperty("DownloadReadyState", info.download_ready_state); @@ -233,6 +237,7 @@ void OneDsTelemetry::RecordDownload(const DownloadInfo& info) { } auto ev = MakeEvent("Download", metadata_.test_mode); ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); ev.SetProperty("ModelId", info.model_id); ev.SetProperty("Status", std::string(ActionStatusToString(info.status))); ev.SetProperty("LockWaitTimeMs", info.lock_wait_ms); diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h index 31cfbcbc9..d7f0fe303 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h @@ -52,16 +52,15 @@ class OneDsTelemetry : public ITelemetry { OneDsTelemetry(const OneDsTelemetry&) = delete; OneDsTelemetry& operator=(const OneDsTelemetry&) = delete; - void RecordAction(Action action, ActionStatus status, const std::string& user_agent, - bool indirect, int64_t duration_ms) override; + void RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t duration_ms) override; - void RecordException(Action action, const std::exception& exception) override; void RecordException(Action action, const std::exception& exception, - const std::string& user_agent) override; + const InvocationContext& context) override; void RecordModelUsage(const ModelUsageInfo& info) override; void RecordModelId(Action action, const std::string& model_id, - ActionStatus status, const std::string& user_agent) override; + ActionStatus status, const InvocationContext& context) override; void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; void RecordDownload(const DownloadInfo& info) override; diff --git a/sdk_v2/cpp/src/telemetry/telemetry.cc b/sdk_v2/cpp/src/telemetry/telemetry.cc index 97fc24122..da603efa5 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry.cc @@ -22,10 +22,6 @@ std::string_view ActionToString(Action action) { return "ModelLoad"; case Action::kModelUnload: return "ModelUnload"; - case Action::kModelDownload: - return "ModelDownload"; - case Action::kModelDelete: - return "ModelDelete"; case Action::kModelList: return "ModelList"; case Action::kOpenAIChatCompletions: @@ -48,8 +44,6 @@ std::string_view ActionToString(Action action) { return "OpenAIResponsesDelete"; case Action::kOpenAIResponsesGetInputItems: return "OpenAIResponsesGetInputItems"; - case Action::kCoreAudioTranscribe: - return "CoreAudioTranscribe"; case Action::kEpDownloadAttempt: return "EpDownloadAttempt"; case Action::kEpDownloadAndRegister: @@ -58,6 +52,8 @@ std::string_view ActionToString(Action action) { return "ModelFileDownload"; case Action::kModelInference: return "ModelInference"; + case Action::kServiceRequestUnmatched: + return "ServiceRequestUnmatched"; default: return "Unknown"; } @@ -73,6 +69,8 @@ std::string_view ActionStatusToString(ActionStatus status) { return "Invalid"; case ActionStatus::kSkipped: return "Skipped"; + case ActionStatus::kClientError: + return "ClientError"; default: return "Unknown"; } diff --git a/sdk_v2/cpp/src/telemetry/telemetry.h b/sdk_v2/cpp/src/telemetry/telemetry.h index b0f0ed4a9..681a3cccc 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.h +++ b/sdk_v2/cpp/src/telemetry/telemetry.h @@ -2,6 +2,8 @@ // Licensed under the MIT License. #pragma once +#include "telemetry/invocation_context.h" + #include #include #include @@ -27,8 +29,6 @@ enum class Action { // Model management kModelLoad = 100, kModelUnload = 101, - kModelDownload = 102, - kModelDelete = 103, kModelList = 104, // OpenAI Chat/Audio/Embeddings APIs @@ -45,9 +45,6 @@ enum class Action { kOpenAIResponsesDelete = 303, kOpenAIResponsesGetInputItems = 304, - // Audio - kCoreAudioTranscribe = 400, - // EP catalog operations kEpDownloadAttempt = 500, // Wraps the entire DownloadAndRegisterEps call kEpDownloadAndRegister = 501, // One per-provider attempt within DownloadAndRegisterEps @@ -57,14 +54,18 @@ enum class Action { // EP runtime usage (TimeToFirstToken / total tokens / memory) kModelInference = 700, // The "Model" event in the C# implementation + + // HTTP service plumbing + kServiceRequestUnmatched = 800, // A request reached the service but matched no route / method }; /// Status of a tracked telemetry action. enum class ActionStatus { - kFailure = 0, + kFailure = 0, // Server-side / internal failure (maps to HTTP 5xx) kSuccess, kInvalid, kSkipped, + kClientError, // Rejected due to invalid client input (maps to HTTP 4xx) — not a service fault }; /// Convert Action to human-readable string. @@ -76,6 +77,7 @@ std::string_view ActionStatusToString(ActionStatus status); /// Payload for the EPDownloadAttempt event — emitted once per DownloadAndRegisterEps call. struct EpDownloadAttemptInfo { std::string user_agent; + std::string correlation_id; int attempts = 0; // Total per-provider attempts made int num_providers = 0; // Number of providers requested int succeeded = 0; // Number of providers that registered successfully @@ -88,6 +90,7 @@ struct EpDownloadAttemptInfo { /// Payload for the EPDownloadAndRegister event — emitted once per provider attempt. struct EpDownloadAndRegisterInfo { std::string user_agent; + std::string correlation_id; std::string provider_name; std::string init_ready_state; // EP state before this call (e.g. "NotPresent") std::string download_ready_state; // EP state after the download phase (e.g. "Installed") @@ -103,6 +106,9 @@ struct ModelUsageInfo { std::string model_id; std::string execution_provider; std::string user_agent; + std::string correlation_id; + bool stream = false; // True if the inference was streamed (SSE) vs a single response + bool indirect = false; // True if the inference was driven by another action (e.g. an HTTP route) int64_t time_to_first_token_ms = 0; int64_t total_time_ms = 0; int32_t total_tokens = 0; @@ -117,6 +123,7 @@ struct ModelUsageInfo { struct DownloadInfo { std::string model_id; std::string user_agent; + std::string correlation_id; ActionStatus status = ActionStatus::kInvalid; int64_t lock_wait_ms = 0; int64_t enumeration_ms = 0; @@ -137,29 +144,22 @@ class ITelemetry { public: virtual ~ITelemetry() = default; - /// Record a completed action with timing and status. + /// Record a completed action with timing and status. The context carries the + /// user agent, the correlation id grouping this operation's events, and whether + /// the action was indirect (triggered by another action). virtual void RecordAction(Action action, ActionStatus status, - const std::string& user_agent, - bool indirect, int64_t duration_ms) = 0; + const InvocationContext& context, int64_t duration_ms) = 0; /// Record an exception associated with an action. - virtual void RecordException(Action action, const std::exception& exception) = 0; - - /// Record an exception associated with an action, with an optional user agent. - /// Default forwards to RecordException(action, exception) for implementations - /// that don't yet propagate the user agent. virtual void RecordException(Action action, const std::exception& exception, - const std::string& /*user_agent*/) { - RecordException(action, exception); - } + const InvocationContext& context) = 0; /// Record model usage metrics after inference (Model event). virtual void RecordModelUsage(const ModelUsageInfo& info) = 0; /// Record which model was used for an action (ModelId event). virtual void RecordModelId(Action action, const std::string& model_id, - ActionStatus status = ActionStatus::kSuccess, - const std::string& user_agent = "") = 0; + ActionStatus status, const InvocationContext& context) = 0; /// Record the result of a DownloadAndRegisterEps call (EPDownloadAttempt event). virtual void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) = 0; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc index f30f1246c..54bcd938a 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc @@ -2,24 +2,28 @@ // Licensed under the MIT License. #include "telemetry/telemetry_action_tracker.h" +#include + namespace fl { -ActionTracker::ActionTracker(Action action, ITelemetry& telemetry, const std::string& user_agent, bool indirect) +ActionTracker::ActionTracker(Action action, ITelemetry& telemetry, InvocationContext context) : action_(action), telemetry_(telemetry), - user_agent_(user_agent), - indirect_(indirect), + context_(std::move(context)), start_(std::chrono::steady_clock::now()) { + // Guarantee a correlation id so this action and anything it triggers can be + // grouped, even when the caller passed a default-constructed context. + context_.EnsureCorrelationId(); } ActionTracker::~ActionTracker() { auto end = std::chrono::steady_clock::now(); auto duration_ms = std::chrono::duration_cast(end - start_).count(); - telemetry_.RecordAction(action_, status_, user_agent_, indirect_, duration_ms); + telemetry_.RecordAction(action_, status_, context_, duration_ms); if (!model_id_.empty()) { - telemetry_.RecordModelId(action_, model_id_, status_, user_agent_); + telemetry_.RecordModelId(action_, model_id_, status_, context_); } } @@ -28,7 +32,7 @@ void ActionTracker::SetStatus(ActionStatus status) { } void ActionTracker::RecordException(const std::exception& exception) { - telemetry_.RecordException(action_, exception, user_agent_); + telemetry_.RecordException(action_, exception, context_); } void ActionTracker::SetModelId(const std::string& model_id) { diff --git a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.h b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.h index e06da6130..e19971eb2 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.h @@ -14,9 +14,7 @@ namespace fl { /// Matches the C# ActionTracker (IDisposable) pattern. class ActionTracker { public: - ActionTracker(Action action, ITelemetry& telemetry, - const std::string& user_agent = "", - bool indirect = false); + ActionTracker(Action action, ITelemetry& telemetry, InvocationContext context = {}); ~ActionTracker(); // Non-copyable, non-movable @@ -32,11 +30,15 @@ class ActionTracker { /// Associate a model ID with this action. void SetModelId(const std::string& model_id); + /// This action's context, with its correlation id resolved. Use to derive a + /// child context (Context().AsIndirect()) for any caused-by action so all + /// events from one operation share a correlation id. + const InvocationContext& Context() const { return context_; } + private: Action action_; ITelemetry& telemetry_; - std::string user_agent_; - bool indirect_; + InvocationContext context_; ActionStatus status_ = ActionStatus::kFailure; std::string model_id_; std::chrono::steady_clock::time_point start_; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc index 214a0cc6d..5a5354b93 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc @@ -10,61 +10,60 @@ TelemetryLogger::TelemetryLogger(const std::string& app_name, ILogger& logger) : app_name_(app_name), logger_(logger) { } -void TelemetryLogger::RecordAction(Action action, ActionStatus status, const std::string& user_agent, - bool indirect, int64_t duration_ms) { +void TelemetryLogger::RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t duration_ms) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] Action AppName={} UserAgent={} Action={} Status={} Direct={} TimeMs={}", - app_name_, user_agent, ActionToString(action), - ActionStatusToString(status), !indirect, duration_ms)); -} - -void TelemetryLogger::RecordException(Action action, const std::exception& exception) { - logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] Error AppName={} Action={} Exception={}", - app_name_, ActionToString(action), exception.what())); + fmt::format("[Telemetry] Action AppName={} UserAgent={} CorrelationId={} Action={} Status={} " + "Direct={} TimeMs={}", + app_name_, context.user_agent, context.correlation_id, ActionToString(action), + ActionStatusToString(status), !context.indirect, duration_ms)); } void TelemetryLogger::RecordException(Action action, const std::exception& exception, - const std::string& user_agent) { + const InvocationContext& context) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] Error AppName={} UserAgent={} Action={} Exception={}", - app_name_, user_agent, ActionToString(action), exception.what())); + fmt::format("[Telemetry] Error AppName={} UserAgent={} CorrelationId={} Action={} Exception={}", + app_name_, context.user_agent, context.correlation_id, ActionToString(action), + exception.what())); } void TelemetryLogger::RecordModelUsage(const ModelUsageInfo& info) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] Model AppName={} UserAgent={} ModelId={} EP={} TimeToFirstTokenMs={} " + fmt::format("[Telemetry] Model AppName={} UserAgent={} CorrelationId={} ModelId={} EP={} " + "Stream={} Direct={} TimeToFirstTokenMs={} " "TotalTimeMs={} TotalTokens={} InputTokenCount={} NumMessages={} MemoryUsedMB={} " "CpuTimeMs={} GpuMemoryUsedMB={}", - app_name_, info.user_agent, info.model_id, info.execution_provider, + app_name_, info.user_agent, info.correlation_id, info.model_id, + info.execution_provider, info.stream, !info.indirect, info.time_to_first_token_ms, info.total_time_ms, info.total_tokens, info.input_token_count, info.num_messages, info.memory_used_mb, info.cpu_time_ms, info.gpu_memory_used_mb)); } void TelemetryLogger::RecordModelId(Action action, const std::string& model_id, - ActionStatus status, const std::string& user_agent) { + ActionStatus status, const InvocationContext& context) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] ModelId AppName={} UserAgent={} Action={} ModelId={} Status={}", - app_name_, user_agent, ActionToString(action), model_id, - ActionStatusToString(status))); + fmt::format("[Telemetry] ModelId AppName={} UserAgent={} CorrelationId={} Action={} ModelId={} " + "Status={}", + app_name_, context.user_agent, context.correlation_id, ActionToString(action), + model_id, ActionStatusToString(status))); } void TelemetryLogger::RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] EPDownloadAttempt AppName={} UserAgent={} Attempts={} " + fmt::format("[Telemetry] EPDownloadAttempt AppName={} UserAgent={} CorrelationId={} Attempts={} " "NumProviders={} Succeeded={} Failed={} Resolved={} Status={} TimeMs={}", - app_name_, info.user_agent, info.attempts, info.num_providers, + app_name_, info.user_agent, info.correlation_id, info.attempts, info.num_providers, info.succeeded, info.failed, info.resolved, ActionStatusToString(info.status), info.duration_ms)); } void TelemetryLogger::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] EPDownloadAndRegister AppName={} UserAgent={} Provider={} " + fmt::format("[Telemetry] EPDownloadAndRegister AppName={} UserAgent={} CorrelationId={} Provider={} " "InitReadyState={} DownloadReadyState={} DownloadStatus={} DownloadTimeMs={} " "RegisterReadyState={} RegisterStatus={} RegisterTimeMs={}", - app_name_, info.user_agent, info.provider_name, + app_name_, info.user_agent, info.correlation_id, info.provider_name, info.init_ready_state, info.download_ready_state, ActionStatusToString(info.download_status), info.download_duration_ms, info.register_ready_state, ActionStatusToString(info.register_status), @@ -73,11 +72,11 @@ void TelemetryLogger::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInf void TelemetryLogger::RecordDownload(const DownloadInfo& info) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] Download AppName={} UserAgent={} ModelId={} Status={} " + fmt::format("[Telemetry] Download AppName={} UserAgent={} CorrelationId={} ModelId={} Status={} " "LockWaitMs={} EnumerationMs={} DownloadMs={} TotalSizeBytes={} " "AlreadyCachedBytes={} FileCount={} SkippedFileCount={} " "DownloadWaitResult={} MaxConcurrency={}", - app_name_, info.user_agent, info.model_id, + app_name_, info.user_agent, info.correlation_id, info.model_id, ActionStatusToString(info.status), info.lock_wait_ms, info.enumeration_ms, info.download_ms, info.total_size_bytes, info.already_cached_bytes, info.file_count, info.skipped_file_count, diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.h b/sdk_v2/cpp/src/telemetry/telemetry_logger.h index 6982c8166..8679ef537 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.h @@ -21,17 +21,16 @@ class TelemetryLogger : public ITelemetry { public: TelemetryLogger(const std::string& app_name, ILogger& logger); - void RecordAction(Action action, ActionStatus status, const std::string& user_agent, - bool indirect, int64_t duration_ms) override; + void RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t duration_ms) override; - void RecordException(Action action, const std::exception& exception) override; void RecordException(Action action, const std::exception& exception, - const std::string& user_agent) override; + const InvocationContext& context) override; void RecordModelUsage(const ModelUsageInfo& info) override; void RecordModelId(Action action, const std::string& model_id, - ActionStatus status, const std::string& user_agent) override; + ActionStatus status, const InvocationContext& context) override; void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc b/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc index 826690ed1..9aba4ed5f 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc @@ -2,13 +2,13 @@ // Licensed under the MIT License. #include "telemetry/telemetry_metadata.h" +#include "telemetry/invocation_context.h" #include "telemetry/telemetry_environment.h" #include "version.h" #include #include #include -#include #ifdef _WIN32 #include @@ -21,30 +21,6 @@ namespace fl { namespace { -std::string MakeGuidV4Hex() { - // RFC 4122 v4 UUID, hex-encoded with hyphens. We use std::random_device + mt19937_64 - // because we don't need the OS UUID API — this is just a per-process correlation id, - // not a cryptographic identifier. - std::random_device rd; - std::mt19937_64 gen{(static_cast(rd()) << 32) | rd()}; - uint64_t hi = gen(); - uint64_t lo = gen(); - - // Set version (4) and variant (10xx) bits. - hi = (hi & 0xFFFFFFFFFFFF0FFFULL) | 0x0000000000004000ULL; - lo = (lo & 0x3FFFFFFFFFFFFFFFULL) | 0x8000000000000000ULL; - - char buf[37]; - std::snprintf(buf, sizeof(buf), - "%08x-%04x-%04x-%04x-%012llx", - static_cast((hi >> 32) & 0xFFFFFFFFu), - static_cast((hi >> 16) & 0xFFFFu), - static_cast(hi & 0xFFFFu), - static_cast((lo >> 48) & 0xFFFFu), - static_cast(lo & 0x0000FFFFFFFFFFFFULL)); - return std::string(buf); -} - #ifdef _WIN32 std::string GetWindowsVersion() { // GetVersionExA is deprecated and lies for unmanifested apps. The reliable diff --git a/sdk_v2/cpp/test/internal_api/null_telemetry.h b/sdk_v2/cpp/test/internal_api/null_telemetry.h index 2c70a1b66..c7c5966e8 100644 --- a/sdk_v2/cpp/test/internal_api/null_telemetry.h +++ b/sdk_v2/cpp/test/internal_api/null_telemetry.h @@ -10,16 +10,15 @@ namespace fl::test { class NullTelemetry : public ITelemetry { public: void RecordAction(Action /*action*/, ActionStatus /*status*/, - const std::string& /*user_agent*/, - bool /*indirect*/, int64_t /*duration_ms*/) override {} + const InvocationContext& /*context*/, int64_t /*duration_ms*/) override {} - void RecordException(Action /*action*/, const std::exception& /*exception*/) override {} + void RecordException(Action /*action*/, const std::exception& /*exception*/, + const InvocationContext& /*context*/) override {} void RecordModelUsage(const ModelUsageInfo& /*info*/) override {} void RecordModelId(Action /*action*/, const std::string& /*model_id*/, - ActionStatus /*status*/ = ActionStatus::kSuccess, - const std::string& /*user_agent*/ = "") override {} + ActionStatus /*status*/, const InvocationContext& /*context*/) override {} void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& /*info*/) override {} diff --git a/sdk_v2/cpp/test/internal_api/telemetry_test.cc b/sdk_v2/cpp/test/internal_api/telemetry_test.cc index 42fc068f6..217fe22aa 100644 --- a/sdk_v2/cpp/test/internal_api/telemetry_test.cc +++ b/sdk_v2/cpp/test/internal_api/telemetry_test.cc @@ -42,12 +42,13 @@ struct ActionCall { class RecordingTelemetry : public ITelemetry { public: void RecordAction(Action action, ActionStatus status, - const std::string& user_agent, - bool indirect, int64_t duration_ms) override { - action_calls.push_back(ActionCall{action, status, user_agent, indirect, duration_ms}); + const InvocationContext& context, int64_t duration_ms) override { + action_calls.push_back( + ActionCall{action, status, context.user_agent, context.indirect, duration_ms}); } - void RecordException(Action action, const std::exception& exception) override { + void RecordException(Action action, const std::exception& exception, + const InvocationContext& /*context*/) override { exception_calls.emplace_back(action, exception.what()); } @@ -56,8 +57,7 @@ class RecordingTelemetry : public ITelemetry { } void RecordModelId(Action action, const std::string& model_id, - ActionStatus /*status*/ = ActionStatus::kSuccess, - const std::string& /*user_agent*/ = "") override { + ActionStatus /*status*/, const InvocationContext& /*context*/) override { model_id_calls.emplace_back(action, model_id); } @@ -88,14 +88,15 @@ TEST(TelemetryLoggerTest, RecordActionIncludesConcreteFields) { RecordingLogger logger; TelemetryLogger telemetry("foundry-local", logger); - telemetry.RecordAction(Action::kModelDownload, ActionStatus::kSuccess, - "cli/1.0", false, 1234); + telemetry.RecordAction(Action::kModelFileDownload, ActionStatus::kSuccess, + InvocationContext{"cli/1.0", "corr-1", false}, 1234); ASSERT_EQ(logger.entries.size(), 1u); EXPECT_EQ(logger.entries[0].level, LogLevel::Debug); EXPECT_NE(logger.entries[0].message.find("AppName=foundry-local"), std::string::npos); EXPECT_NE(logger.entries[0].message.find("UserAgent=cli/1.0"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Action=ModelDownload"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("CorrelationId=corr-1"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Action=ModelFileDownload"), std::string::npos); EXPECT_NE(logger.entries[0].message.find("Status=Success"), std::string::npos); EXPECT_NE(logger.entries[0].message.find("Direct=true"), std::string::npos); EXPECT_NE(logger.entries[0].message.find("TimeMs=1234"), std::string::npos); @@ -105,7 +106,7 @@ TEST(TelemetryLoggerTest, RecordExceptionAndModelEventsIncludeSpecificValues) { RecordingLogger logger; TelemetryLogger telemetry("foundry-local", logger); - telemetry.RecordException(Action::kModelLoad, std::runtime_error("config missing")); + telemetry.RecordException(Action::kModelLoad, std::runtime_error("config missing"), InvocationContext{}); ModelUsageInfo usage; usage.model_id = "phi-3-mini"; @@ -116,7 +117,8 @@ TEST(TelemetryLoggerTest, RecordExceptionAndModelEventsIncludeSpecificValues) { usage.total_time_ms = 250; telemetry.RecordModelUsage(usage); - telemetry.RecordModelId(Action::kModelLoad, "phi-3-mini", ActionStatus::kSuccess, "cli/1.0"); + telemetry.RecordModelId(Action::kModelLoad, "phi-3-mini", ActionStatus::kSuccess, + InvocationContext{"cli/1.0", "", false}); ASSERT_EQ(logger.entries.size(), 3u); EXPECT_NE(logger.entries[0].message.find("Action=ModelLoad"), std::string::npos); @@ -136,11 +138,11 @@ TEST(ActionTrackerTest, DestructorRecordsFailureByDefaultWithoutModelId) { RecordingTelemetry telemetry; { - ActionTracker tracker(Action::kModelDelete, telemetry, "cli/2.0", true); + ActionTracker tracker(Action::kModelFileDownload, telemetry, InvocationContext{"cli/2.0", "", true}); } ASSERT_EQ(telemetry.action_calls.size(), 1u); - EXPECT_EQ(telemetry.action_calls[0].action, Action::kModelDelete); + EXPECT_EQ(telemetry.action_calls[0].action, Action::kModelFileDownload); EXPECT_EQ(telemetry.action_calls[0].status, ActionStatus::kFailure); EXPECT_EQ(telemetry.action_calls[0].user_agent, "cli/2.0"); EXPECT_TRUE(telemetry.action_calls[0].indirect); @@ -152,7 +154,7 @@ TEST(ActionTrackerTest, RecordsExceptionSuccessAndModelId) { RecordingTelemetry telemetry; { - ActionTracker tracker(Action::kModelLoad, telemetry, "cli/3.0", false); + ActionTracker tracker(Action::kModelLoad, telemetry, InvocationContext{"cli/3.0", "", false}); tracker.RecordException(std::runtime_error("failed to load")); tracker.SetModelId("phi-3-mini"); tracker.SetStatus(ActionStatus::kSuccess); From a1bb6f97c2827a9890c0bee94f5eb0f3149fa702 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 20 Jun 2026 02:41:35 -0500 Subject: [PATCH 07/77] sdk_v2/cpp/telemetry: emit the Model event + wire the chat route end-to-end - Session::ProcessRequest now emits the previously-dead Model (kModelInference) event once per inference with model id, tokens, duration, stream flag and the caller's correlation id / indirect flag. Adds Session::SetRequestContext so an HTTP route can stage an indirect child context (shared correlation id), and a virtual ExecutionProvider() hook for the EP field. - Chat completions handler: derive a Direct route context from the User-Agent header; map 4xx early-returns (empty body, invalid JSON, model not found/loaded) to kClientError instead of the default kFailure; emit kSessionCreate (indirect) on the HTTP path; stage the indirect session context. - Streaming fix: the route action is no longer recorded (as a premature success with ~0ms) when the handler returns. The route ActionTracker is moved into the streaming thread and records on completion with the real duration and terminal status, so mid-stream failures surface as failures instead of vanishing. Builds clean (/W4 /WX); telemetry/webservice/sse tests pass. --- sdk_v2/cpp/src/inferencing/session/session.cc | 28 +++++++++- sdk_v2/cpp/src/inferencing/session/session.h | 16 ++++++ .../src/service/chat_completions_handler.cc | 53 +++++++++++++++---- .../src/service/chat_completions_handler.h | 9 +++- sdk_v2/cpp/src/service/handler_utils.h | 10 ++++ 5 files changed, 102 insertions(+), 14 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/session/session.cc b/sdk_v2/cpp/src/inferencing/session/session.cc index e8f90ee69..b9afd6e83 100644 --- a/sdk_v2/cpp/src/inferencing/session/session.cc +++ b/sdk_v2/cpp/src/inferencing/session/session.cc @@ -16,6 +16,7 @@ #include +#include #include namespace fl { @@ -101,9 +102,17 @@ void Session::ProcessRequest(const Request& request, Response& response) { lock.lock(); } - ActionTracker tracker(Action::kSessionProcessRequest, telemetry_); + // Use the context the caller staged (an HTTP route stages an indirect child + // with the route's correlation id); otherwise mint a direct context per call + // for direct SDK use. + InvocationContext context = request_context_ ? *request_context_ : InvocationContext::Direct(); + context.EnsureCorrelationId(); + const bool streaming = static_cast(callback_fn_); + + ActionTracker tracker(Action::kSessionProcessRequest, telemetry_, context); tracker.SetModelId(CatalogModel().Id()); + const auto start = std::chrono::steady_clock::now(); try { ProcessRequestImpl(request, response); @@ -112,6 +121,23 @@ void Session::ProcessRequest(const Request& request, Response& response) { tracker.RecordException(ex); throw; } + + // Per-inference Model event — emitted on success with whatever metrics this run + // produced, sharing the action's correlation id and indirect flag. TTFT and + // memory are not surfaced by the generators yet and stay at their unset values. + ModelUsageInfo usage; + usage.model_id = CatalogModel().Id(); + usage.execution_provider = ExecutionProvider(); + usage.user_agent = context.user_agent; + usage.correlation_id = context.correlation_id; + usage.indirect = context.indirect; + usage.stream = streaming; + usage.total_time_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + usage.total_tokens = static_cast(response.usage.total_tokens); + usage.input_token_count = static_cast(response.usage.prompt_tokens); + telemetry_.RecordModelUsage(usage); } } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/session/session.h b/sdk_v2/cpp/src/inferencing/session/session.h index f3ec0e5f1..a92556c16 100644 --- a/sdk_v2/cpp/src/inferencing/session/session.h +++ b/sdk_v2/cpp/src/inferencing/session/session.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,7 @@ #include "inferencing/session/request.h" #include "inferencing/session/response.h" #include "inferencing/session/types.h" +#include "telemetry/invocation_context.h" #include "util/key_value_pairs.h" namespace fl { @@ -100,6 +102,15 @@ class Session { callback_user_data_ = user_data; } + /// Telemetry context for the next ProcessRequest. HTTP handlers set this to an + /// indirect child of the route's context so the kSessionProcessRequest action + /// and the per-inference Model event are marked indirect and share the route's + /// correlation id. When unset (direct SDK use), ProcessRequest mints its own + /// direct context per call. + void SetRequestContext(InvocationContext context) { + request_context_ = std::move(context); + } + protected: Session(const fl::Model& catalog_model, ILogger& logger, ITelemetry& telemetry, bool allow_concurrent_requests = false); @@ -131,6 +142,10 @@ class Session { /// Requests are serialized if the derived class does not opt into concurrency via allow_concurrent_requests_. virtual void ProcessRequestImpl(const Request& request, Response& response) = 0; + /// Execution provider the loaded model runs on (e.g. "CPU", "CUDA"). Surfaced + /// on the per-inference Model telemetry event. Empty when not known. + virtual std::string ExecutionProvider() const { return {}; } + /// Create a per-request callback handler. Returns nullptr if no callback is set. /// The handler is owned by the caller (unique_ptr) and drains+joins on destruction. std::unique_ptr CreateCallbackHandler(const Request& request) { @@ -151,6 +166,7 @@ class Session { KeyValuePairs session_options_; StreamingCallbackFn callback_fn_; void* callback_user_data_ = nullptr; + std::optional request_context_; const bool allow_concurrent_requests_; mutable std::unique_ptr request_mutex_ = std::make_unique(); }; diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.cc b/sdk_v2/cpp/src/service/chat_completions_handler.cc index fcf13fb29..3475c82ac 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.cc +++ b/sdk_v2/cpp/src/service/chat_completions_handler.cc @@ -104,10 +104,12 @@ void ChatCompletionsHandler::BuildOpenAIJsonRequest(const std::string& body, con std::shared_ptr ChatCompletionsHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIChatCompletions, ctx_.telemetry); + auto route_ctx = InvocationContext::Direct(GetUserAgent(request)); + auto tracker = std::make_unique(Action::kOpenAIChatCompletions, ctx_.telemetry, route_ctx); auto body_str = request->readBodyToString(); if (!body_str || body_str->empty()) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Empty request body"); } @@ -117,6 +119,7 @@ std::shared_ptr ChatCompletionsHandler::ha // telemetry. How much do we care about that? Is it worth the double parsing? ChatCompletionRequest req; if (auto err = ParseAndValidateRequest(body_str->c_str(), req)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } @@ -130,10 +133,11 @@ std::shared_ptr ChatCompletionsHandler::ha Model* model = nullptr; GenAIModelInstance* loaded = nullptr; if (auto err = ResolveModel(model_name, model, loaded)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } - tracker.SetModelId(model_name); + tracker->SetModelId(model_name); // 3. Build an OPENAI_JSON-tagged TEXT request item. Request session_request; @@ -145,21 +149,33 @@ std::shared_ptr ChatCompletionsHandler::ha include_usage_in_stream = req.stream_options->include_usage; } + // The session and the inference it drives happen as a consequence of this + // route, so they are indirect and reuse the route's correlation id. + auto session_ctx = route_ctx.AsIndirect(); + // 6. Run inference via ChatSession try { ChatSession session(*model, *loaded, ctx_.logger, ctx_.telemetry); + { + ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); + create_tracker.SetModelId(model_name); + create_tracker.SetStatus(ActionStatus::kSuccess); + } + session.SetRequestContext(session_ctx); if (stream) { - tracker.SetStatus(ActionStatus::kSuccess); - return HandleStreaming(std::move(session), std::move(session_request), include_usage_in_stream); + // The route action is recorded by the streaming thread when the stream + // finishes, so don't set its status here — move the tracker into the thread. + return HandleStreaming(std::move(session), std::move(session_request), include_usage_in_stream, + std::move(tracker)); } else { SessionRegistration reg(ctx_.session_manager, session); auto response = HandleNonStreaming(session, session_request); - tracker.SetStatus(ActionStatus::kSuccess); + tracker->SetStatus(ActionStatus::kSuccess); return response; } } catch (const std::exception& ex) { - tracker.RecordException(ex); + tracker->RecordException(ex); ctx_.logger.Log(LogLevel::Error, fmt::format("Chat completion inference failed: {}", ex.what())); return ErrorResponse(Status::CODE_500, "Inference failed", ex.what()); } @@ -189,15 +205,17 @@ std::shared_ptr ChatCompletionsHandler::Ha } std::shared_ptr ChatCompletionsHandler::HandleStreaming( - ChatSession&& session, Request session_request, bool include_usage) { + ChatSession&& session, Request session_request, bool include_usage, + std::unique_ptr route_tracker) { auto body = std::make_shared(); auto body_ptr = body; auto& logger = ctx_.logger; - auto& tracker = ctx_.thread_tracker; + auto& thread_tracker = ctx_.thread_tracker; std::thread streaming_thread([bg_session = std::move(session), body_ptr, &logger, req = std::move(session_request), - include_usage, &tracker, + include_usage, &thread_tracker, + route_tracker = std::move(route_tracker), &session_manager = ctx_.session_manager]() mutable { SessionRegistration reg(session_manager, bg_session); @@ -248,18 +266,31 @@ std::shared_ptr ChatCompletionsHandler::Ha } body_ptr->Push("data: [DONE]\n\n"); + + // Inference streamed to completion — record the route action as a success. + if (route_tracker) { + route_tracker->SetStatus(ActionStatus::kSuccess); + } } catch (const std::exception& ex) { nlohmann::json err = { {"error", {{"message", ex.what()}, {"type", "server_error"}, {"param", nullptr}, {"code", nullptr}}}, }; body_ptr->Push("data: " + err.dump() + "\n\n"); + + // Mid-stream failure: record the exception. The route action keeps its + // default kFailure status. + if (route_tracker) { + route_tracker->RecordException(ex); + } } body_ptr->Finish(); - tracker.Remove(std::this_thread::get_id()); + // route_tracker is destroyed with this closure once the thread completes, + // recording the route action with the full streaming duration and final status. + thread_tracker.Remove(std::this_thread::get_id()); }); - tracker.Track(std::move(streaming_thread)); + thread_tracker.Track(std::move(streaming_thread)); auto response = oatpp::web::protocol::http::outgoing::Response::createShared( Status::CODE_200, body); diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.h b/sdk_v2/cpp/src/service/chat_completions_handler.h index ed483c4d0..c7225bf77 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.h +++ b/sdk_v2/cpp/src/service/chat_completions_handler.h @@ -16,6 +16,7 @@ struct ChatCompletionRequest; class ChatSession; class Model; class GenAIModelInstance; +class ActionTracker; struct Request; // ======================================================================== @@ -47,9 +48,13 @@ class ChatCompletionsHandler : public HttpRequestHandler { /// Non-streaming inference — processes request, extracts the OPENAI_JSON-tagged TEXT response. std::shared_ptr HandleNonStreaming(ChatSession& session, Request& session_request); - /// Streaming inference — runs inference in background thread with SSE output. + /// Streaming inference — runs inference in a background thread with SSE output. + /// Takes ownership of the route ActionTracker so the route action is recorded + /// when the stream actually finishes (real duration + terminal status), + /// instead of when this handler returns. std::shared_ptr HandleStreaming(ChatSession&& session, Request session_request, - bool include_usage); + bool include_usage, + std::unique_ptr route_tracker); ServiceContext& ctx_; }; diff --git a/sdk_v2/cpp/src/service/handler_utils.h b/sdk_v2/cpp/src/service/handler_utils.h index e25528fa3..7216892ce 100644 --- a/sdk_v2/cpp/src/service/handler_utils.h +++ b/sdk_v2/cpp/src/service/handler_utils.h @@ -62,6 +62,16 @@ inline std::string GenerateCompletionId(const std::string& prefix) { return ss.str(); } +/// Extract the User-Agent header from an incoming request ("" if absent), for +/// attribution on the telemetry events the request drives. +inline std::string GetUserAgent(const std::shared_ptr& request) { + if (!request) { + return {}; + } + auto ua = request->getHeader("User-Agent"); + return ua ? *ua : std::string{}; +} + // ======================================================================== // SSE stream body — feeds token-by-token SSE events to oatpp's chunked // transfer encoding. A producer thread pushes formatted SSE strings into From ea5aebb2e77cf8912c3131a9925c11c2cf8c62d5 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 20 Jun 2026 03:10:41 -0500 Subject: [PATCH 08/77] sdk_v2/cpp/telemetry: wire all service routes (indirect, kClientError, sampling) Extends the chat-route pattern to every handler: - Streaming completion fix also applied to the audio transcriptions route (its ActionTracker moved into the streaming thread). - Embeddings, audio, responses (create + get/list/delete/input_items), and the model load/unload/list/retrieve routes now derive a Direct context from the User-Agent header, map 4xx early-returns to kClientError, and (for the inference routes) emit kSessionCreate and stage an indirect session context. - Instrument GET /models/loaded (kModelList), previously untracked. - Sample GET /status: emit a kServiceStatus action at most once per hour per process so the orchestrator heartbeat doesn't dominate volume. Builds clean (/W4 /WX); telemetry/webservice/sse tests pass. --- .../service/audio_transcriptions_handler.cc | 46 +++++++++++---- .../service/audio_transcriptions_handler.h | 8 ++- sdk_v2/cpp/src/service/embeddings_handler.cc | 17 +++++- sdk_v2/cpp/src/service/models_handlers.cc | 27 +++++++-- sdk_v2/cpp/src/service/responses_handler.cc | 56 +++++++++++++++---- sdk_v2/cpp/src/service/responses_handler.h | 4 +- sdk_v2/cpp/src/service/web_service.cc | 31 +++++++++- sdk_v2/cpp/src/telemetry/telemetry.cc | 2 + sdk_v2/cpp/src/telemetry/telemetry.h | 1 + 9 files changed, 159 insertions(+), 33 deletions(-) diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc index e7b2ac642..c62468500 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc @@ -87,16 +87,19 @@ void AudioTranscriptionsHandler::BuildOpenAIJsonRequest(const std::string& body, std::shared_ptr AudioTranscriptionsHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIAudioTranscribe, ctx_.telemetry); + auto route_ctx = InvocationContext::Direct(GetUserAgent(request)); + auto tracker = std::make_unique(Action::kOpenAIAudioTranscribe, ctx_.telemetry, route_ctx); auto body_str = request->readBodyToString(); if (!body_str || body_str->empty()) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Empty request body"); } // 1. Parse & validate AudioTranscriptionRequest req; if (auto err = ParseAndValidateRequest(body_str->c_str(), req)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } @@ -108,9 +111,11 @@ std::shared_ptr AudioTranscriptionsHandler // 2. Validate file path try { if (!std::filesystem::exists(req.filename)) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Audio file not found", "'" + req.filename + "'"); } } catch (const std::filesystem::filesystem_error& ex) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Invalid file path", ex.what()); } @@ -119,30 +124,40 @@ std::shared_ptr AudioTranscriptionsHandler Model* model = nullptr; GenAIModelInstance* loaded = nullptr; if (auto err = ResolveModel(model_name, model, loaded)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } - tracker.SetModelId(model_name); + tracker->SetModelId(model_name); // 4. Build an OPENAI_JSON-tagged TEXT request item — pass original body directly Request session_request; BuildOpenAIJsonRequest(body_str->c_str(), session_request); + // The session and the inference it drives are indirect children of this route. + auto session_ctx = route_ctx.AsIndirect(); + // 5. Dispatch to streaming or non-streaming try { AudioSession session(*model, *loaded, ctx_.logger, ctx_.telemetry); + { + ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); + create_tracker.SetModelId(model_name); + create_tracker.SetStatus(ActionStatus::kSuccess); + } + session.SetRequestContext(session_ctx); if (stream) { - tracker.SetStatus(ActionStatus::kSuccess); - return HandleStreaming(std::move(session), std::move(session_request)); + // The route action is recorded by the streaming thread on completion. + return HandleStreaming(std::move(session), std::move(session_request), std::move(tracker)); } else { SessionRegistration reg(ctx_.session_manager, session); auto response = HandleNonStreaming(session, session_request); - tracker.SetStatus(ActionStatus::kSuccess); + tracker->SetStatus(ActionStatus::kSuccess); return response; } } catch (const std::exception& ex) { - tracker.RecordException(ex); + tracker->RecordException(ex); ctx_.logger.Log(LogLevel::Error, fmt::format("Audio transcription inference failed: {}", ex.what())); return ErrorResponse(Status::CODE_500, "Inference failed", ex.what()); } catch (...) { @@ -177,14 +192,15 @@ std::shared_ptr AudioTranscriptionsHandler } std::shared_ptr AudioTranscriptionsHandler::HandleStreaming( - AudioSession&& session, Request session_request) { + AudioSession&& session, Request session_request, std::unique_ptr route_tracker) { auto body = std::make_shared(); auto body_ptr = body; auto& logger = ctx_.logger; - auto& tracker = ctx_.thread_tracker; + auto& thread_tracker = ctx_.thread_tracker; std::thread streaming_thread([bg_session = std::move(session), body_ptr, &logger, - req = std::move(session_request), &tracker, + req = std::move(session_request), &thread_tracker, + route_tracker = std::move(route_tracker), &session_manager = ctx_.session_manager]() mutable { SessionRegistration reg(session_manager, bg_session); @@ -216,19 +232,27 @@ std::shared_ptr AudioTranscriptionsHandler // Send terminal event body_ptr->Push("data: [DONE]\n\n"); + + if (route_tracker) { + route_tracker->SetStatus(ActionStatus::kSuccess); + } } catch (const std::exception& ex) { logger.Log(LogLevel::Error, fmt::format("Audio streaming transcription failed: {}", ex.what())); // Push error to stream so client doesn't hang nlohmann::json error = {{"error", {{"message", ex.what()}}}}; body_ptr->Push("data: " + error.dump() + "\n\n"); + + if (route_tracker) { + route_tracker->RecordException(ex); + } } body_ptr->Finish(); - tracker.Remove(std::this_thread::get_id()); + thread_tracker.Remove(std::this_thread::get_id()); }); - tracker.Track(std::move(streaming_thread)); + thread_tracker.Track(std::move(streaming_thread)); auto response = oatpp::web::protocol::http::outgoing::Response::createShared(Status::CODE_200, body); response->putHeader("Content-Type", "text/event-stream"); diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.h b/sdk_v2/cpp/src/service/audio_transcriptions_handler.h index 5dffeaa90..1f68c058c 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.h +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.h @@ -16,6 +16,7 @@ struct AudioTranscriptionRequest; class AudioSession; class Model; class GenAIModelInstance; +class ActionTracker; struct Request; // ======================================================================== @@ -43,8 +44,11 @@ class AudioTranscriptionsHandler : public HttpRequestHandler { /// Non-streaming inference — processes request, extracts the OPENAI_JSON-tagged TEXT response. std::shared_ptr HandleNonStreaming(AudioSession& session, Request& session_request); - /// Streaming inference — runs inference in background thread with SSE output. - std::shared_ptr HandleStreaming(AudioSession&& session, Request session_request); + /// Streaming inference — runs inference in a background thread with SSE output. + /// Takes ownership of the route ActionTracker so the route action records when + /// the stream finishes (real duration + terminal status). + std::shared_ptr HandleStreaming(AudioSession&& session, Request session_request, + std::unique_ptr route_tracker); ServiceContext& ctx_; }; diff --git a/sdk_v2/cpp/src/service/embeddings_handler.cc b/sdk_v2/cpp/src/service/embeddings_handler.cc index b5f734b07..526e9cc51 100644 --- a/sdk_v2/cpp/src/service/embeddings_handler.cc +++ b/sdk_v2/cpp/src/service/embeddings_handler.cc @@ -28,10 +28,12 @@ class EmbeddingsHandler : public HttpRequestHandler { explicit EmbeddingsHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr handle(const std::shared_ptr& request) override { - ActionTracker tracker(Action::kOpenAIEmbeddings, ctx_.telemetry); + auto route_ctx = InvocationContext::Direct(GetUserAgent(request)); + ActionTracker tracker(Action::kOpenAIEmbeddings, ctx_.telemetry, route_ctx); auto body_str = request->readBodyToString(); if (!body_str || body_str->empty()) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Empty request body"); } @@ -41,6 +43,7 @@ class EmbeddingsHandler : public HttpRequestHandler { auto j = nlohmann::json::parse(*body_str); req = j.get(); } catch (const nlohmann::json::exception& ex) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Invalid JSON", ex.what()); } @@ -53,6 +56,7 @@ class EmbeddingsHandler : public HttpRequestHandler { } if (inputs.empty()) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "\"input\" must not be empty"); } @@ -60,19 +64,30 @@ class EmbeddingsHandler : public HttpRequestHandler { std::string model_name = req.model; auto* model = ctx_.catalog.GetModelVariant(model_name); if (!model) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_404, "Model not found", model_name); } auto* loaded = ctx_.model_load_manager.GetLoadedModel(model->Id()); if (!loaded) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Model not loaded", model_name); } tracker.SetModelId(model_name); + // The session and the inference it drives are indirect children of this route. + auto session_ctx = route_ctx.AsIndirect(); + // 4. Create session and process each input try { EmbeddingsSession session(*model, *loaded, ctx_.logger, ctx_.telemetry); + { + ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); + create_tracker.SetModelId(model_name); + create_tracker.SetStatus(ActionStatus::kSuccess); + } + session.SetRequestContext(session_ctx); SessionRegistration reg(ctx_.session_manager, session); Request session_request; diff --git a/sdk_v2/cpp/src/service/models_handlers.cc b/sdk_v2/cpp/src/service/models_handlers.cc index 93988d3b6..b2bb65f51 100644 --- a/sdk_v2/cpp/src/service/models_handlers.cc +++ b/sdk_v2/cpp/src/service/models_handlers.cc @@ -24,7 +24,10 @@ class ListLoadedModelsHandler : public HttpRequestHandler { public: explicit ListLoadedModelsHandler(ServiceContext& ctx) : ctx_(ctx) {} - std::shared_ptr handle(const std::shared_ptr&) override { + std::shared_ptr handle(const std::shared_ptr& request) override { + ActionTracker tracker(Action::kModelList, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); + auto loaded = ctx_.catalog.GetLoadedModels(); nlohmann::json names = nlohmann::json::array(); @@ -32,6 +35,7 @@ class ListLoadedModelsHandler : public HttpRequestHandler { names.push_back(model->Id()); } + tracker.SetStatus(ActionStatus::kSuccess); return JsonResponse(Status::CODE_200, names); } @@ -48,10 +52,12 @@ class LoadModelHandler : public HttpRequestHandler { explicit LoadModelHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr handle(const std::shared_ptr& request) override { - ActionTracker tracker(Action::kModelLoad, ctx_.telemetry); + ActionTracker tracker(Action::kModelLoad, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto name_raw = request->getPathVariable("name"); if (!name_raw) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing model name"); } @@ -59,6 +65,7 @@ class LoadModelHandler : public HttpRequestHandler { auto* model = ctx_.catalog.GetModel(name); if (!model) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + name + "'"); } @@ -69,6 +76,7 @@ class LoadModelHandler : public HttpRequestHandler { } if (!model->IsCached()) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Model not cached", "Model must be downloaded before loading"); } @@ -101,10 +109,12 @@ class UnloadModelHandler : public HttpRequestHandler { explicit UnloadModelHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr handle(const std::shared_ptr& request) override { - ActionTracker tracker(Action::kModelUnload, ctx_.telemetry); + ActionTracker tracker(Action::kModelUnload, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto name_raw = request->getPathVariable("name"); if (!name_raw) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing model name"); } @@ -112,6 +122,7 @@ class UnloadModelHandler : public HttpRequestHandler { auto* model = ctx_.catalog.GetModel(name); if (!model) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + name + "'"); } @@ -149,8 +160,9 @@ class OpenAIListModelsHandler : public HttpRequestHandler { public: explicit OpenAIListModelsHandler(ServiceContext& ctx) : ctx_(ctx) {} - std::shared_ptr handle(const std::shared_ptr&) override { - ActionTracker tracker(Action::kOpenAIModelList, ctx_.telemetry); + std::shared_ptr handle(const std::shared_ptr& request) override { + ActionTracker tracker(Action::kOpenAIModelList, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto models = ctx_.catalog.ListModels(); nlohmann::json data = nlohmann::json::array(); @@ -203,10 +215,12 @@ class OpenAIRetrieveModelHandler : public HttpRequestHandler { explicit OpenAIRetrieveModelHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr handle(const std::shared_ptr& request) override { - ActionTracker tracker(Action::kOpenAIModelRetrieve, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIModelRetrieve, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto name_raw = request->getPathVariable("name"); if (!name_raw) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing model name"); } @@ -214,6 +228,7 @@ class OpenAIRetrieveModelHandler : public HttpRequestHandler { auto* model = ctx_.catalog.GetModelVariant(name); if (!model) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + name + "'"); } diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index 0fdf6d8f7..b343e443b 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -130,10 +130,12 @@ void ResponsesHandler::LoadPreviousContext(const ResponseCreateParams& params, std::shared_ptr ResponsesHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesCreate, ctx_.telemetry); + auto route_ctx = InvocationContext::Direct(GetUserAgent(request)); + auto tracker = std::make_unique(Action::kOpenAIResponsesCreate, ctx_.telemetry, route_ctx); auto body_str = request->readBodyToString(); if (!body_str || body_str->empty()) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Empty request body"); } @@ -141,6 +143,7 @@ std::shared_ptr ResponsesHandler::handle( nlohmann::json req_json; ResponseCreateParams params; if (auto err = ParseAndValidateRequest(body_str->c_str(), req_json, params)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } @@ -155,10 +158,11 @@ std::shared_ptr ResponsesHandler::handle( Model* model = nullptr; GenAIModelInstance* loaded = nullptr; if (auto err = ResolveModel(model_name, model, loaded)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } - tracker.SetModelId(model_name); + tracker->SetModelId(model_name); // 3. Load previous context if chaining via previous_response_id const nlohmann::json* previous_input = nullptr; @@ -193,10 +197,18 @@ std::shared_ptr ResponsesHandler::handle( // happens here in the handler that owns the session lifetime. std::string tools_json = ResponseConverter::ExtractResponsesToolDefinitions(params, session_request); + // The session and the inference it drives happen as a consequence of this + // route, so they are indirect and reuse the route's correlation id. + auto session_ctx = route_ctx.AsIndirect(); + try { if (!session) { + ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); + create_tracker.SetModelId(model_name); session = std::make_unique(*model, *loaded, ctx_.logger, ctx_.telemetry); + create_tracker.SetStatus(ActionStatus::kSuccess); } + session->SetRequestContext(session_ctx); // Sessions can be reused via previous_response_id; clear any stale tool defs from the prior // turn before applying this request's tools so the request stays self-contained. @@ -208,22 +220,23 @@ std::shared_ptr ResponsesHandler::handle( if (params.stream) { ctx_.logger.Log(LogLevel::Debug, fmt::format("Creating streaming response {} for model {}", response_id, model_name)); - tracker.SetStatus(ActionStatus::kSuccess); + // The route action is recorded by the streaming thread when the stream + // finishes — move the tracker in rather than marking success now. return HandleStreaming(std::move(session), std::move(session_request), model_name, - response_id, created_at, params, req_json); + response_id, created_at, params, req_json, std::move(tracker)); } else { ctx_.logger.Log(LogLevel::Debug, fmt::format("Creating response {} for model {}", response_id, model_name)); auto response = HandleNonStreaming(std::move(session), session_request, model_name, response_id, created_at, params, req_json); - tracker.SetStatus(ActionStatus::kSuccess); + tracker->SetStatus(ActionStatus::kSuccess); return response; } } catch (const std::exception& ex) { - tracker.RecordException(ex); + tracker->RecordException(ex); ctx_.logger.Log(LogLevel::Error, fmt::format("Response {} failed: {}", response_id, ex.what())); @@ -280,7 +293,7 @@ std::shared_ptr ResponsesHandler::HandleSt std::unique_ptr session, Request session_request, const std::string& model_name, const std::string& response_id, int64_t created_at, const ResponseCreateParams& params, - const nlohmann::json& req_json) { + const nlohmann::json& req_json, std::unique_ptr route_tracker) { auto body = std::make_shared(); auto initial_response = ResponseConverter::BuildInitialResponseObject(response_id, created_at, model_name, params); @@ -326,6 +339,7 @@ std::shared_ptr ResponsesHandler::HandleSt should_store, &store, req_copy = std::move(req_copy), params_copy = std::move(params_copy), + route_tracker = std::move(route_tracker), &tracker]() mutable { SessionRegistration reg(session_manager, *session); @@ -560,6 +574,11 @@ std::shared_ptr ResponsesHandler::HandleSt session_manager.CheckIn(response_id, std::move(session)); } + // Streamed to completion — record the route action as a success. + if (route_tracker) { + route_tracker->SetStatus(ActionStatus::kSuccess); + } + } catch (const std::exception& ex) { logger.Log(LogLevel::Error, fmt::format("Response {} failed during streaming: {}", response_id, ex.what())); @@ -572,6 +591,11 @@ std::shared_ptr ResponsesHandler::HandleSt failed.sequence_number = seq++; failed.response = error_response; body_ptr->Push("event: response.failed\ndata: " + nlohmann::json(failed).dump() + "\n\n"); + + // Mid-stream failure: record the exception; the route action keeps kFailure. + if (route_tracker) { + route_tracker->RecordException(ex); + } } // Terminal event per spec @@ -598,10 +622,12 @@ GetResponseHandler::GetResponseHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr GetResponseHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesGet, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIResponsesGet, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto id = request->getPathVariable("id"); if (!id) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing response ID"); } @@ -609,6 +635,7 @@ std::shared_ptr GetResponseHandler::handle auto response = ctx_.response_store.Get(id->c_str()); if (!response) { + tracker.SetStatus(ActionStatus::kClientError); nlohmann::json error_body = { {"error", { {"message", "The response '" + std::string(id->c_str()) + "' does not exist."}, @@ -633,7 +660,8 @@ ListResponsesHandler::ListResponsesHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr ListResponsesHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesList, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIResponsesList, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); // Parse query parameters auto limit_str = request->getQueryParameter("limit", "20"); @@ -688,10 +716,12 @@ DeleteResponseHandler::DeleteResponseHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr DeleteResponseHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesDelete, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIResponsesDelete, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto id = request->getPathVariable("id"); if (!id) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing response ID"); } @@ -699,6 +729,7 @@ std::shared_ptr DeleteResponseHandler::han bool deleted = ctx_.response_store.Delete(id->c_str()); if (!deleted) { + tracker.SetStatus(ActionStatus::kClientError); nlohmann::json error_body = { {"error", { {"message", "The response '" + std::string(id->c_str()) + "' does not exist."}, @@ -733,10 +764,12 @@ GetInputItemsHandler::GetInputItemsHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr GetInputItemsHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesGetInputItems, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIResponsesGetInputItems, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto id = request->getPathVariable("id"); if (!id) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing response ID"); } @@ -745,6 +778,7 @@ std::shared_ptr GetInputItemsHandler::hand // Check that response exists auto response = ctx_.response_store.Get(id->c_str()); if (!response) { + tracker.SetStatus(ActionStatus::kClientError); nlohmann::json error_body = { {"error", { {"message", "The response '" + std::string(id->c_str()) + "' does not exist."}, diff --git a/sdk_v2/cpp/src/service/responses_handler.h b/sdk_v2/cpp/src/service/responses_handler.h index da9f5f691..e455d6203 100644 --- a/sdk_v2/cpp/src/service/responses_handler.h +++ b/sdk_v2/cpp/src/service/responses_handler.h @@ -16,6 +16,7 @@ struct Request; class ChatSession; class Model; class GenAIModelInstance; +class ActionTracker; namespace responses { struct ResponseCreateParams; @@ -65,7 +66,8 @@ class ResponsesHandler : public HttpRequestHandler { const std::string& model_name, const std::string& response_id, int64_t created_at, const responses::ResponseCreateParams& params, - const nlohmann::json& req_json); + const nlohmann::json& req_json, + std::unique_ptr route_tracker); ServiceContext& ctx_; }; diff --git a/sdk_v2/cpp/src/service/web_service.cc b/sdk_v2/cpp/src/service/web_service.cc index d2660f41b..5d9161c2d 100644 --- a/sdk_v2/cpp/src/service/web_service.cc +++ b/sdk_v2/cpp/src/service/web_service.cc @@ -13,6 +13,7 @@ #include "service/handler_utils.h" #include "service/models_handlers.h" #include "service/responses_handler.h" +#include "telemetry/telemetry_action_tracker.h" #include #include @@ -23,6 +24,7 @@ #include #include +#include #include #include @@ -134,7 +136,9 @@ class StatusHandler : public HttpRequestHandler { public: explicit StatusHandler(ServiceContext& ctx) : ctx_(ctx) {} - std::shared_ptr handle(const std::shared_ptr&) override { + std::shared_ptr handle(const std::shared_ptr& request) override { + MaybeRecordStatusTelemetry(request); + nlohmann::json body = { {"modelCachePath", ctx_.model_cache_dir}, {"endpoints", ctx_.bound_urls}, @@ -149,7 +153,32 @@ class StatusHandler : public HttpRequestHandler { } private: + // /status is an orchestrator heartbeat that can be polled as often as every + // second, so record it at most once per hour per process — enough to confirm + // the endpoint is being used without letting it dominate telemetry volume. + void MaybeRecordStatusTelemetry(const std::shared_ptr& request) { + constexpr int64_t kIntervalMs = 3'600'000; + constexpr int64_t kNever = INT64_MIN; + const int64_t now_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + + int64_t last = last_status_emit_ms_.load(std::memory_order_relaxed); + if (last != kNever && now_ms - last < kIntervalMs) { + return; + } + // Exactly one caller wins this interval's slot; the rest skip. + if (!last_status_emit_ms_.compare_exchange_strong(last, now_ms, std::memory_order_relaxed)) { + return; + } + + ActionTracker tracker(Action::kServiceStatus, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); + tracker.SetStatus(ActionStatus::kSuccess); + } + ServiceContext& ctx_; + std::atomic last_status_emit_ms_{INT64_MIN}; }; // ======================================================================== diff --git a/sdk_v2/cpp/src/telemetry/telemetry.cc b/sdk_v2/cpp/src/telemetry/telemetry.cc index da603efa5..28b75c27c 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry.cc @@ -54,6 +54,8 @@ std::string_view ActionToString(Action action) { return "ModelInference"; case Action::kServiceRequestUnmatched: return "ServiceRequestUnmatched"; + case Action::kServiceStatus: + return "ServiceStatus"; default: return "Unknown"; } diff --git a/sdk_v2/cpp/src/telemetry/telemetry.h b/sdk_v2/cpp/src/telemetry/telemetry.h index 681a3cccc..4c93282d8 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.h +++ b/sdk_v2/cpp/src/telemetry/telemetry.h @@ -57,6 +57,7 @@ enum class Action { // HTTP service plumbing kServiceRequestUnmatched = 800, // A request reached the service but matched no route / method + kServiceStatus = 801, // GET /status heartbeat (sampled — at most once per hour per process) }; /// Status of a tracked telemetry action. From b93dd8585c42cc2f0b930596ceeb375d9cd68d9b Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 20 Jun 2026 03:22:16 -0500 Subject: [PATCH 09/77] sdk_v2/cpp/telemetry: add router catch-all for unmatched requests + tests - UnmatchedRouteInterceptor: a request interceptor that, before routing, checks the router for a matching route. When none matches (unknown path or wrong method) it records a kServiceRequestUnmatched action (kClientError) and replies 404, so requests that reach the service but no handler are no longer invisible. - Tests (WebServiceTelemetryTest, using a capturing ITelemetry + empty catalog, no real model required): * unmatched route records kServiceRequestUnmatched with the right status, user agent, Direct flag and a correlation id; * an empty chat-completions body records kClientError (not kFailure) and emits no Model event; * three rapid GET /status calls record kServiceStatus exactly once (hourly sampling). Builds clean (/W4 /WX); 72 telemetry/webservice/sse tests pass. --- sdk_v2/cpp/src/service/web_service.cc | 49 +++++++ .../cpp/test/internal_api/web_service_test.cc | 133 ++++++++++++++++++ 2 files changed, 182 insertions(+) diff --git a/sdk_v2/cpp/src/service/web_service.cc b/sdk_v2/cpp/src/service/web_service.cc index 5d9161c2d..fcd8e236e 100644 --- a/sdk_v2/cpp/src/service/web_service.cc +++ b/sdk_v2/cpp/src/service/web_service.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -201,6 +202,49 @@ class ShutdownHandler : public HttpRequestHandler { std::function shutdown_fn_; }; +// ======================================================================== +// UnmatchedRouteInterceptor +// +// oatpp routes straight to handlers, so requests that match no route (unknown +// path or wrong method) never reach an ActionTracker and would be invisible to +// telemetry. This request interceptor runs before routing: when the router has +// no route for the request it records a kServiceRequestUnmatched action and +// replies 404 itself; otherwise it returns nullptr and normal routing proceeds. +// ======================================================================== + +class UnmatchedRouteInterceptor : public oatpp::web::server::interceptor::RequestInterceptor { + public: + UnmatchedRouteInterceptor(std::shared_ptr router, ITelemetry& telemetry) + : router_(std::move(router)), telemetry_(telemetry) {} + + std::shared_ptr intercept(const std::shared_ptr& request) override { + const auto& line = request->getStartingLine(); + if (router_->getRoute(line.method, line.path)) { + return nullptr; // a handler will serve this request + } + + std::string user_agent; + if (auto ua = request->getHeader("User-Agent")) { + user_agent = *ua; + } + { + ActionTracker tracker(Action::kServiceRequestUnmatched, telemetry_, + InvocationContext::Direct(user_agent)); + tracker.SetStatus(ActionStatus::kClientError); + } + + nlohmann::json body = { + {"error", {{"message", "Not found"}, {"type", "invalid_request_error"}, + {"param", nullptr}, {"code", nullptr}}}, + }; + return JsonResponse(Status::CODE_404, body); + } + + private: + std::shared_ptr router_; + ITelemetry& telemetry_; +}; + // ======================================================================== // WebService implementation // ======================================================================== @@ -284,6 +328,11 @@ std::vector WebService::Start(const std::vector& endpo impl_->connection_handler = oatpp::web::server::HttpConnectionHandler::createShared(impl_->router); + // Record requests that match no route (unknown path / wrong method), which + // otherwise bypass every handler's ActionTracker. + impl_->connection_handler->addRequestInterceptor( + std::make_shared(impl_->router, ctx.telemetry)); + std::vector bound_urls; for (const auto& endpoint : endpoints) { diff --git a/sdk_v2/cpp/test/internal_api/web_service_test.cc b/sdk_v2/cpp/test/internal_api/web_service_test.cc index de7a40521..c4f3b921e 100644 --- a/sdk_v2/cpp/test/internal_api/web_service_test.cc +++ b/sdk_v2/cpp/test/internal_api/web_service_test.cc @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -49,6 +50,58 @@ std::string TestHttpDelete(const std::string& url, const std::string& user_agent return http::HttpDelete(url, user_agent, true); } +// Telemetry sink that records every event for assertions. +class CapturingTelemetry : public ITelemetry { + public: + struct ActionCall { + Action action; + ActionStatus status; + std::string user_agent; + std::string correlation_id; + bool indirect; + }; + + void RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t /*duration_ms*/) override { + std::lock_guard lock(mutex_); + actions.push_back({action, status, context.user_agent, context.correlation_id, context.indirect}); + } + void RecordException(Action, const std::exception&, const InvocationContext&) override {} + void RecordModelUsage(const ModelUsageInfo& info) override { + std::lock_guard lock(mutex_); + model_usages.push_back(info); + } + void RecordModelId(Action, const std::string&, ActionStatus, const InvocationContext&) override {} + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo&) override {} + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo&) override {} + void RecordDownload(const DownloadInfo&) override {} + + int Count(Action action) { + std::lock_guard lock(mutex_); + int n = 0; + for (const auto& c : actions) { + if (c.action == action) { + ++n; + } + } + return n; + } + + std::optional Find(Action action) { + std::lock_guard lock(mutex_); + for (const auto& c : actions) { + if (c.action == action) { + return c; + } + } + return std::nullopt; + } + + std::vector actions; + std::vector model_usages; + std::mutex mutex_; +}; + } // namespace // ======================================================================== @@ -476,6 +529,86 @@ TEST(WebServiceEmptyCatalogTest, LoadedModelsReturnsEmptyArray) { service.Stop(); } +// ======================================================================== +// Telemetry behaviors — capture events and assert coverage / classification +// ======================================================================== + +TEST(WebServiceTelemetryTest, UnmatchedRouteRecordsServiceRequestUnmatched) { + test::MockCatalog catalog; + StderrLogger logger; + test::CpuOnlyEpDetector ep_detector; + ModelLoadManager model_load_manager(ep_detector, logger); + SessionManager session_manager(logger); + CapturingTelemetry telemetry; + + WebService service(catalog, logger, "/tmp/test", model_load_manager, session_manager, telemetry, []() {}); + auto urls = service.Start({"http://127.0.0.1:0"}); + + // Unknown path: the interceptor replies 404, so TestHttpGet throws on the non-2xx. + try { + TestHttpGet(urls[0] + "/no/such/route", "probe-agent/9"); + } catch (...) { + } + + service.Stop(); + + ASSERT_EQ(telemetry.Count(Action::kServiceRequestUnmatched), 1); + auto call = telemetry.Find(Action::kServiceRequestUnmatched); + ASSERT_TRUE(call.has_value()); + EXPECT_EQ(call->status, ActionStatus::kClientError); + EXPECT_EQ(call->user_agent, "probe-agent/9"); + EXPECT_FALSE(call->indirect); + EXPECT_FALSE(call->correlation_id.empty()); +} + +TEST(WebServiceTelemetryTest, EmptyChatBodyRecordsClientErrorNotFailure) { + test::MockCatalog catalog; + StderrLogger logger; + test::CpuOnlyEpDetector ep_detector; + ModelLoadManager model_load_manager(ep_detector, logger); + SessionManager session_manager(logger); + CapturingTelemetry telemetry; + + WebService service(catalog, logger, "/tmp/test", model_load_manager, session_manager, telemetry, []() {}); + auto urls = service.Start({"http://127.0.0.1:0"}); + + // Empty body is rejected (400) before any model resolution. + try { + TestHttpPost(urls[0] + "/v1/chat/completions", ""); + } catch (...) { + } + + service.Stop(); + + auto call = telemetry.Find(Action::kOpenAIChatCompletions); + ASSERT_TRUE(call.has_value()) << "chat completions route action was not recorded"; + EXPECT_EQ(call->status, ActionStatus::kClientError); + EXPECT_FALSE(call->indirect); + // A 4xx reject performs no inference, so there is no Model event. + EXPECT_TRUE(telemetry.model_usages.empty()); +} + +TEST(WebServiceTelemetryTest, StatusEndpointIsSampledHourly) { + test::MockCatalog catalog; + StderrLogger logger; + test::CpuOnlyEpDetector ep_detector; + ModelLoadManager model_load_manager(ep_detector, logger); + SessionManager session_manager(logger); + CapturingTelemetry telemetry; + + WebService service(catalog, logger, "/tmp/test", model_load_manager, session_manager, telemetry, []() {}); + auto urls = service.Start({"http://127.0.0.1:0"}); + + // Three rapid heartbeats — only the first inside the hourly window is recorded. + TestHttpGet(urls[0] + "/status"); + TestHttpGet(urls[0] + "/status"); + TestHttpGet(urls[0] + "/status"); + + service.Stop(); + + EXPECT_EQ(telemetry.Count(Action::kServiceStatus), 1); +} + // ======================================================================== // Streaming validation tests — same errors apply with stream=true // ======================================================================== From 87a2686a2c06882fa87007c67b30b47e3f3cd507 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 20 Jun 2026 11:51:08 -0500 Subject: [PATCH 10/77] sdk_v2/cpp/telemetry: track Azure Catalog accesses (CatalogFetch event) Every access to a model catalog source is now recorded, with success/failure: - New CatalogFetch typed event carrying operation ("FetchAll" or the cached-id "FetchByIds" lookup), endpoint/region/format parsed from the catalog URL, status, duration, model count, error message, and a correlation id shared across the accesses of one refresh. - FetchAllModelInfosWithCachedModels gains optional telemetry params (defaulted, so the snapshot tool and existing tests are unchanged) and emits an event for the primary fetch and, when it runs, the secondary cached-id lookup. - AzureModelCatalog parses the catalog URL into endpoint/region/format (https://ai.azure.com/api/eastus/ux/v1.0 -> {ai.azure.com, eastus, ux/v1.0}; "static" for the embedded snapshot) and threads an ITelemetry from Manager. - Tests assert the primary and secondary accesses are each tracked with the right operation, status, endpoint, correlation id and model count. Builds clean (/W4 /WX); catalog + telemetry + webservice tests pass. --- sdk_v2/cpp/src/catalog/azure_model_catalog.cc | 88 ++++++++++++++++++- sdk_v2/cpp/src/catalog/azure_model_catalog.h | 6 +- sdk_v2/cpp/src/catalog/catalog_client.cc | 44 +++++++++- sdk_v2/cpp/src/catalog/catalog_client.h | 12 ++- sdk_v2/cpp/src/manager.cc | 3 +- sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc | 19 ++++ sdk_v2/cpp/src/telemetry/one_ds_telemetry.h | 1 + sdk_v2/cpp/src/telemetry/telemetry.h | 18 ++++ sdk_v2/cpp/src/telemetry/telemetry_logger.cc | 9 ++ sdk_v2/cpp/src/telemetry/telemetry_logger.h | 1 + sdk_v2/cpp/test/internal_api/null_telemetry.h | 2 + .../test/internal_api/static_catalog_test.cc | 68 ++++++++++++++ .../cpp/test/internal_api/telemetry_test.cc | 5 ++ .../cpp/test/internal_api/web_service_test.cc | 5 ++ 14 files changed, 271 insertions(+), 10 deletions(-) diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index 61da8ffd7..7fe6f7dea 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -6,6 +6,8 @@ #include "catalog/local_model_scanner.h" #include "model.h" #include "model_info.h" +#include "telemetry/invocation_context.h" +#include "telemetry/telemetry.h" #include "utils.h" #include @@ -13,19 +15,87 @@ namespace fl { +namespace { + +/// Split a catalog URL into structured telemetry dimensions. The Azure Foundry +/// catalog URL looks like "https://ai.azure.com/api//", e.g. +/// "https://ai.azure.com/api/eastus/ux/v1.0" -> {ai.azure.com, eastus, ux/v1.0}. +/// Custom URLs that don't follow the "/api//" convention keep an empty +/// region and put the whole path in `format`. The embedded snapshot is "static". +struct ParsedCatalogUrl { + std::string endpoint; + std::string region; + std::string format; +}; + +ParsedCatalogUrl ParseCatalogUrl(const std::string& url) { + if (url == "static") { + return {"static", "", ""}; + } + + std::string rest = url; + if (auto scheme = rest.find("://"); scheme != std::string::npos) { + rest = rest.substr(scheme + 3); + } + + ParsedCatalogUrl out; + std::string path; + if (auto slash = rest.find('/'); slash == std::string::npos) { + out.endpoint = rest; + } else { + out.endpoint = rest.substr(0, slash); + path = rest.substr(slash + 1); + } + + if (auto q = path.find_first_of("?#"); q != std::string::npos) { + path = path.substr(0, q); + } + + std::vector segments; + size_t pos = 0; + while (pos < path.size()) { + auto next = path.find('/', pos); + if (next == std::string::npos) { + next = path.size(); + } + if (next > pos) { + segments.push_back(path.substr(pos, next - pos)); + } + pos = next + 1; + } + + size_t format_start = 0; + if (segments.size() >= 2 && segments[0] == "api") { + out.region = segments[1]; + format_start = 2; + } + for (size_t i = format_start; i < segments.size(); ++i) { + if (!out.format.empty()) { + out.format += '/'; + } + out.format += segments[i]; + } + + return out; +} + +} // namespace + AzureModelCatalog::AzureModelCatalog(std::vector>> catalog_urls, std::string cache_dir, ModelFactory model_factory, const IEpDetector& ep_detector, ILogger& logger, - bool cache_only) + bool cache_only, + ITelemetry* telemetry) : BaseModelCatalog(catalog_urls.empty() ? kDefaultCatalogUrl : catalog_urls.front().first, logger), catalog_urls_(std::move(catalog_urls)), cache_dir_(std::move(cache_dir)), model_factory_(std::move(model_factory)), ep_detector_(ep_detector), logger_(logger), - cache_only_(cache_only) { + cache_only_(cache_only), + telemetry_(telemetry) { logger_.Log(LogLevel::Information, fmt::format("Created AzureModelCatalog. Cache directory: {}", cache_dir_)); @@ -65,6 +135,9 @@ std::vector AzureModelCatalog::FetchModels() const { std::vector models; const std::string& cache_dir = cache_dir_; + // One correlation id groups every catalog access made by this refresh. + const std::string correlation_id = MakeGuidV4Hex(); + logger_.Log(LogLevel::Information, "Getting latest info from the Azure catalog and for locally cached models."); @@ -83,7 +156,16 @@ std::vector AzureModelCatalog::FetchModels() const { // Preserve byte-identical behavior for the "no override" case (previously stored as ""), // while letting callers explicitly request "" as a real filter override. auto client = MakeCatalogClient(url, filter.value_or(""), ep_detector_, logger_, cache_dir); - auto model_infos = FetchAllModelInfosWithCachedModels(*client, cached_model_ids, logger_); + + auto parsed = ParseCatalogUrl(url); + CatalogFetchInfo base_info; + base_info.endpoint = parsed.endpoint; + base_info.region = parsed.region; + base_info.format = parsed.format; + base_info.correlation_id = correlation_id; + + auto model_infos = FetchAllModelInfosWithCachedModels(*client, cached_model_ids, logger_, + telemetry_, &base_info); for (const auto& info : model_infos) { // Check if the model is locally cached and pass the path if so. diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.h b/sdk_v2/cpp/src/catalog/azure_model_catalog.h index dbc7502a6..60e45cad2 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.h +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.h @@ -14,6 +14,8 @@ namespace fl { +class ITelemetry; // forward declaration + /// Azure-specific catalog. Fetches from Azure Foundry catalog API, /// scans local cache, merges results. /// Maps to C# AzureModelCatalog. @@ -26,7 +28,8 @@ class AzureModelCatalog : public BaseModelCatalog { ModelFactory model_factory, const IEpDetector& ep_detector, ILogger& logger, - bool cache_only = false); + bool cache_only = false, + ITelemetry* telemetry = nullptr); ~AzureModelCatalog() override; protected: @@ -48,6 +51,7 @@ class AzureModelCatalog : public BaseModelCatalog { const IEpDetector& ep_detector_; ILogger& logger_; bool cache_only_; + ITelemetry* telemetry_; }; } // namespace fl diff --git a/sdk_v2/cpp/src/catalog/catalog_client.cc b/sdk_v2/cpp/src/catalog/catalog_client.cc index 31ae5c3ab..a841066dd 100644 --- a/sdk_v2/cpp/src/catalog/catalog_client.cc +++ b/sdk_v2/cpp/src/catalog/catalog_client.cc @@ -2,12 +2,14 @@ // Licensed under the MIT License. #include "catalog/catalog_client.h" #include "ep_detection/ep_detector.h" +#include "telemetry/telemetry.h" #include "utils.h" #include #include +#include #include #include @@ -46,9 +48,40 @@ std::unique_ptr MakeCatalogClient( std::vector FetchAllModelInfosWithCachedModels( ICatalogClient& client, const std::vector& cached_model_ids, - ILogger& logger) { - // Step 1: Fetch latest catalog models (existing flow). - auto result = client.FetchAllModelInfos(); + ILogger& logger, + ITelemetry* telemetry, + const CatalogFetchInfo* base_info) { + auto now = [] { return std::chrono::steady_clock::now(); }; + auto elapsed_ms = [](std::chrono::steady_clock::time_point start) { + return std::chrono::duration_cast(std::chrono::steady_clock::now() - start) + .count(); + }; + auto emit = [&](const std::string& operation, ActionStatus status, int64_t duration_ms, + int32_t model_count, const std::string& error) { + if (telemetry == nullptr || base_info == nullptr) { + return; + } + CatalogFetchInfo info = *base_info; + info.operation = operation; + info.status = status; + info.duration_ms = duration_ms; + info.model_count = model_count; + info.error_message = error; + telemetry->RecordCatalogFetch(info); + }; + + // Step 1: Fetch latest catalog models (the primary catalog access). + std::vector result; + { + const auto start = now(); + try { + result = client.FetchAllModelInfos(); + } catch (const std::exception& ex) { + emit("FetchAll", ActionStatus::kFailure, elapsed_ms(start), 0, ex.what()); + throw; + } + emit("FetchAll", ActionStatus::kSuccess, elapsed_ms(start), static_cast(result.size()), ""); + } if (cached_model_ids.empty()) { return result; @@ -69,17 +102,22 @@ std::vector FetchAllModelInfosWithCachedModels( // Step 3: Look up unresolved IDs from the catalog (older versions, etc.) if (!unresolved_ids.empty()) { + const auto start = now(); try { auto additional = client.FetchModelsByIds(unresolved_ids); + const auto additional_count = static_cast(additional.size()); for (auto& info : additional) { resolved_ids.insert(info.model_id); result.push_back(std::move(info)); } + emit("FetchByIds", ActionStatus::kSuccess, elapsed_ms(start), additional_count, ""); } catch (const std::exception& ex) { logger.Log(LogLevel::Warning, fmt::format("catalog: failed to fetch cached model IDs — {}", ex.what())); + emit("FetchByIds", ActionStatus::kFailure, elapsed_ms(start), 0, ex.what()); } catch (...) { logger.Log(LogLevel::Warning, "catalog: failed to fetch cached model IDs — unknown error"); + emit("FetchByIds", ActionStatus::kFailure, elapsed_ms(start), 0, "unknown error"); } // Step 4: Create basic entries for any IDs still unresolved (BYO models). diff --git a/sdk_v2/cpp/src/catalog/catalog_client.h b/sdk_v2/cpp/src/catalog/catalog_client.h index d560cbe59..e7f1bebb2 100644 --- a/sdk_v2/cpp/src/catalog/catalog_client.h +++ b/sdk_v2/cpp/src/catalog/catalog_client.h @@ -12,6 +12,9 @@ namespace fl { +class ITelemetry; // forward declaration +struct CatalogFetchInfo; // forward declaration + /// Abstract catalog client. Two implementations exist: the live Azure catalog /// client (private repo) and a snapshot-based static client (public repo). class ICatalogClient { @@ -30,11 +33,16 @@ class ICatalogClient { }; /// Production helper that combines a catalog fetch with locally cached model -/// resolution and BYO synthesis. +/// resolution and BYO synthesis. When `telemetry` and `base_info` are provided, +/// emits a CatalogFetch event for the primary fetch and (if it runs) the +/// cached-id lookup, copying the endpoint/region/format/correlation fields from +/// `base_info` and filling in operation/status/duration/model_count/error. std::vector FetchAllModelInfosWithCachedModels( ICatalogClient& client, const std::vector& cached_model_ids, - ILogger& logger); + ILogger& logger, + ITelemetry* telemetry = nullptr, + const CatalogFetchInfo* base_info = nullptr); /// Construct a catalog client. Dispatches based on `base_url`: /// - "static" -> returns a client backed by the embedded snapshot. Ignores diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index c907c4516..73d5097e9 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -310,7 +310,8 @@ Manager::Manager(const Configuration& config) return CreateModel(std::move(info), std::move(local_path)); }, *ep_detector_, *logger_, - config_.external_service_url.has_value()); + config_.external_service_url.has_value(), + telemetry_.get()); } Manager::~Manager() { diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc index 324566be7..84b6d6a7b 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc @@ -252,4 +252,23 @@ void OneDsTelemetry::RecordDownload(const DownloadInfo& info) { SafeLog(GetMatLogger(), ev); } +void OneDsTelemetry::RecordCatalogFetch(const CatalogFetchInfo& info) { + local_log_.RecordCatalogFetch(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("CatalogFetch", metadata_.test_mode); + ev.SetProperty("Operation", info.operation); + ev.SetProperty("Endpoint", info.endpoint); + ev.SetProperty("Region", info.region); + ev.SetProperty("Format", info.format); + ev.SetProperty("Status", std::string(ActionStatusToString(info.status))); + ev.SetProperty("TimeMs", info.duration_ms); + ev.SetProperty("ModelCount", static_cast(info.model_count)); + ev.SetProperty("ErrorMessage", info.error_message); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); + SafeLog(GetMatLogger(), ev); +} + } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h index d7f0fe303..13339f6b2 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h @@ -64,6 +64,7 @@ class OneDsTelemetry : public ITelemetry { void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; void RecordDownload(const DownloadInfo& info) override; + void RecordCatalogFetch(const CatalogFetchInfo& info) override; /// True if 1DS Initialize succeeded (i.e. events are actually uploaded). /// False in CI or when the tenant token was empty. diff --git a/sdk_v2/cpp/src/telemetry/telemetry.h b/sdk_v2/cpp/src/telemetry/telemetry.h index 4c93282d8..c99d863fd 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.h +++ b/sdk_v2/cpp/src/telemetry/telemetry.h @@ -137,6 +137,21 @@ struct DownloadInfo { int32_t max_concurrency = 0; }; +/// Payload for the CatalogFetch event — emitted once per access to a model +/// catalog source (the live Azure catalog or the embedded static snapshot). +struct CatalogFetchInfo { + std::string operation; // "FetchAll" (full catalog) or "FetchByIds" (cached-id lookup) + std::string endpoint; // catalog host (e.g. "ai.azure.com"), or "static" for the embedded snapshot + std::string region; // region parsed from the catalog URL (e.g. "eastus"); empty if not present + std::string format; // catalog API path/version after the region (e.g. "ux/v1.0") + ActionStatus status = ActionStatus::kInvalid; + int64_t duration_ms = 0; + int32_t model_count = 0; // models returned by this access + std::string error_message; // populated on failure + std::string user_agent; + std::string correlation_id; // shared across the accesses of one catalog refresh +}; + /// Abstract telemetry interface. /// Implementations may send events to a telemetry backend (1DS, ETW, OpenTelemetry, …) /// or simply log them. The OneDsTelemetry implementation sends to 1DS; the @@ -170,6 +185,9 @@ class ITelemetry { /// Record one model file download (Download event). virtual void RecordDownload(const DownloadInfo& info) = 0; + + /// Record one access to a model catalog source (CatalogFetch event). + virtual void RecordCatalogFetch(const CatalogFetchInfo& info) = 0; }; } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc index 5a5354b93..a48885b27 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc @@ -83,4 +83,13 @@ void TelemetryLogger::RecordDownload(const DownloadInfo& info) { info.download_wait_result, info.max_concurrency)); } +void TelemetryLogger::RecordCatalogFetch(const CatalogFetchInfo& info) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] CatalogFetch AppName={} Operation={} Endpoint={} Region={} Format={} " + "Status={} TimeMs={} ModelCount={} Error={} UserAgent={} CorrelationId={}", + app_name_, info.operation, info.endpoint, info.region, info.format, + ActionStatusToString(info.status), info.duration_ms, info.model_count, + info.error_message, info.user_agent, info.correlation_id)); +} + } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.h b/sdk_v2/cpp/src/telemetry/telemetry_logger.h index 8679ef537..da3928f86 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.h @@ -35,6 +35,7 @@ class TelemetryLogger : public ITelemetry { void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; void RecordDownload(const DownloadInfo& info) override; + void RecordCatalogFetch(const CatalogFetchInfo& info) override; private: std::string app_name_; diff --git a/sdk_v2/cpp/test/internal_api/null_telemetry.h b/sdk_v2/cpp/test/internal_api/null_telemetry.h index c7c5966e8..c0a122fe6 100644 --- a/sdk_v2/cpp/test/internal_api/null_telemetry.h +++ b/sdk_v2/cpp/test/internal_api/null_telemetry.h @@ -25,6 +25,8 @@ class NullTelemetry : public ITelemetry { void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& /*info*/) override {} void RecordDownload(const DownloadInfo& /*info*/) override {} + + void RecordCatalogFetch(const CatalogFetchInfo& /*info*/) override {} }; } // namespace fl::test diff --git a/sdk_v2/cpp/test/internal_api/static_catalog_test.cc b/sdk_v2/cpp/test/internal_api/static_catalog_test.cc index 068599d4e..b2dcacafb 100644 --- a/sdk_v2/cpp/test/internal_api/static_catalog_test.cc +++ b/sdk_v2/cpp/test/internal_api/static_catalog_test.cc @@ -19,11 +19,27 @@ #include "internal_api/test_helpers.h" #include "logger.h" #include "model_info.h" +#include "telemetry/telemetry.h" using namespace fl; namespace { +// Telemetry sink that captures CatalogFetch events for assertions. +class CatalogRecordingTelemetry : public ITelemetry { + public: + void RecordAction(Action, ActionStatus, const InvocationContext&, int64_t) override {} + void RecordException(Action, const std::exception&, const InvocationContext&) override {} + void RecordModelUsage(const ModelUsageInfo&) override {} + void RecordModelId(Action, const std::string&, ActionStatus, const InvocationContext&) override {} + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo&) override {} + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo&) override {} + void RecordDownload(const DownloadInfo&) override {} + void RecordCatalogFetch(const CatalogFetchInfo& info) override { fetches.push_back(info); } + + std::vector fetches; +}; + // Returns all models from the static client for a given EP detector. std::vector FetchStaticModels(const IEpDetector& ep_detector) { StderrLogger logger; @@ -174,6 +190,58 @@ TEST(StaticCatalogClientTest, BYOModel_SynthesizedWhenNotInSnapshot) { EXPECT_EQ(byo->string_properties.at(FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR), "Local"); } +// The primary catalog access records a CatalogFetch(FetchAll) event with the +// passed-through endpoint/correlation and the returned model count. +TEST(StaticCatalogClientTest, CatalogFetchTelemetryRecordsPrimaryAccess) { + StderrLogger logger; + fl::test::CpuOnlyEpDetector ep; + auto client = MakeCatalogClient("static", "", ep, logger, ""); + + CatalogRecordingTelemetry telemetry; + CatalogFetchInfo base; + base.endpoint = "static"; + base.correlation_id = "corr-cat"; + + auto result = FetchAllModelInfosWithCachedModels(*client, {}, logger, &telemetry, &base); + + ASSERT_EQ(telemetry.fetches.size(), 1u); + const auto& fetch = telemetry.fetches[0]; + EXPECT_EQ(fetch.operation, "FetchAll"); + EXPECT_EQ(fetch.status, ActionStatus::kSuccess); + EXPECT_EQ(fetch.endpoint, "static"); + EXPECT_EQ(fetch.correlation_id, "corr-cat"); + EXPECT_EQ(fetch.model_count, static_cast(result.size())); + EXPECT_GT(fetch.model_count, 0) << "static snapshot should return models"; +} + +// A cached id absent from the snapshot triggers the secondary FetchModelsByIds +// access, which is tracked as its own CatalogFetch(FetchByIds) event. +TEST(StaticCatalogClientTest, CatalogFetchTelemetryRecordsSecondaryAccess) { + StderrLogger logger; + fl::test::CpuOnlyEpDetector ep; + auto client = MakeCatalogClient("static", "", ep, logger, ""); + + CatalogRecordingTelemetry telemetry; + CatalogFetchInfo base; + base.endpoint = "static"; + base.correlation_id = "corr-cat"; + + FetchAllModelInfosWithCachedModels(*client, {"my-custom-model:0"}, logger, &telemetry, &base); + + int fetch_all = 0; + int fetch_by_ids = 0; + for (const auto& f : telemetry.fetches) { + if (f.operation == "FetchAll") { + ++fetch_all; + } else if (f.operation == "FetchByIds") { + ++fetch_by_ids; + } + EXPECT_EQ(f.correlation_id, "corr-cat"); + } + EXPECT_EQ(fetch_all, 1); + EXPECT_EQ(fetch_by_ids, 1) << "the cached-id lookup access should be tracked even when it returns nothing"; +} + // Verify that CatalogModelToModelInfo handles mixed-case device strings and bool tags // case-insensitively. Lives in static_catalog_test.cc (always compiled) so we have // coverage in public-repo builds where azure_catalog_test.cc is excluded. diff --git a/sdk_v2/cpp/test/internal_api/telemetry_test.cc b/sdk_v2/cpp/test/internal_api/telemetry_test.cc index 217fe22aa..cd0d3f57c 100644 --- a/sdk_v2/cpp/test/internal_api/telemetry_test.cc +++ b/sdk_v2/cpp/test/internal_api/telemetry_test.cc @@ -73,6 +73,10 @@ class RecordingTelemetry : public ITelemetry { download_calls.push_back(info); } + void RecordCatalogFetch(const CatalogFetchInfo& info) override { + catalog_fetch_calls.push_back(info); + } + std::vector action_calls; std::vector> exception_calls; std::vector model_usage_calls; @@ -80,6 +84,7 @@ class RecordingTelemetry : public ITelemetry { std::vector ep_attempt_calls; std::vector ep_register_calls; std::vector download_calls; + std::vector catalog_fetch_calls; }; } // namespace diff --git a/sdk_v2/cpp/test/internal_api/web_service_test.cc b/sdk_v2/cpp/test/internal_api/web_service_test.cc index c4f3b921e..d18cc476c 100644 --- a/sdk_v2/cpp/test/internal_api/web_service_test.cc +++ b/sdk_v2/cpp/test/internal_api/web_service_test.cc @@ -75,6 +75,10 @@ class CapturingTelemetry : public ITelemetry { void RecordEpDownloadAttempt(const EpDownloadAttemptInfo&) override {} void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo&) override {} void RecordDownload(const DownloadInfo&) override {} + void RecordCatalogFetch(const CatalogFetchInfo& info) override { + std::lock_guard lock(mutex_); + catalog_fetches.push_back(info); + } int Count(Action action) { std::lock_guard lock(mutex_); @@ -99,6 +103,7 @@ class CapturingTelemetry : public ITelemetry { std::vector actions; std::vector model_usages; + std::vector catalog_fetches; std::mutex mutex_; }; From efc7f1634fdd520d5c63e5840156c4fda3b0e13c Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 21 Jun 2026 19:07:33 -0500 Subject: [PATCH 11/77] sdk_v2/cpp/telemetry: plain AppSessionGuid + ext.app.sesId usage sessions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Drop the UTCReplace_ magic from the per-process correlation GUID: it only fires on the Windows UTC transmission path (not our direct upload, and never off-Windows), so it was dead weight. The field is now plainly "AppSessionGuid" — a stable per-process correlation id on every platform. - Add ITelemetry::StartSession()/EndSession() (default no-op; OneDsTelemetry maps them to 1DS LogSession(Started/Ended), TelemetryLogger logs them). Manager opens a session when the web service starts and closes it on stop, so events carry the standard, cross-platform usage-session id (ext.app.sesId) and the backend gets session duration. This is additive — it complements the per-run AppSessionGuid and the per-operation CorrelationId. Builds clean (/W4 /WX); telemetry/webservice/catalog tests pass. --- sdk_v2/cpp/src/manager.cc | 4 +++ sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc | 35 ++++++++++++++++--- sdk_v2/cpp/src/telemetry/one_ds_telemetry.h | 2 ++ sdk_v2/cpp/src/telemetry/telemetry.h | 7 ++++ sdk_v2/cpp/src/telemetry/telemetry_logger.cc | 8 +++++ sdk_v2/cpp/src/telemetry/telemetry_logger.h | 2 ++ sdk_v2/cpp/src/telemetry/telemetry_metadata.h | 10 +++--- 7 files changed, 59 insertions(+), 9 deletions(-) diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index 73d5097e9..d09d745e8 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -472,6 +472,9 @@ void Manager::StartWebService() { bound_urls_ = web_service_->Start(endpoints); web_service_running_ = true; + // Open an app-usage session for the lifetime of the running service so events + // carry ext.app.sesId and the backend gets session duration. + telemetry_->StartSession(); tracker.SetStatus(ActionStatus::kSuccess); #else FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -501,6 +504,7 @@ void Manager::StopWebService() { web_service_.reset(); web_service_running_ = false; bound_urls_.clear(); + telemetry_->EndSession(); tracker.SetStatus(ActionStatus::kSuccess); #else FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc index 84b6d6a7b..d46840744 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc @@ -29,6 +29,7 @@ using MatILogger = ::Microsoft::Applications::Events::ILogger; using ::Microsoft::Applications::Events::EventProperties; using ::Microsoft::Applications::Events::EventPriority; using ::Microsoft::Applications::Events::PiiKind_None; +using ::Microsoft::Applications::Events::SessionState; constexpr uint64_t kCriticalData = MICROSOFT_KEYWORD_CRITICAL_DATA; @@ -36,10 +37,10 @@ void SetCommonContext(MatILogger* mat_logger, const TelemetryMetadata& m) { // Process-wide context — stamped on every event uploaded through this ILogger. mat_logger->SetContext("AppName", m.app_name); mat_logger->SetContext("Version", m.version); - // 1DS recognizes UTCReplace_AppSessionGuid as a magic field name on Windows UTC - // and substitutes the OS session GUID. On other platforms we just send the - // GUID we computed at startup — same correlation semantics either way. - mat_logger->SetContext("UTCReplace_AppSessionGuid", m.app_session_guid); + // Per-process correlation GUID, generated at startup. Lets the backend group + // every event from one FL run. (This is distinct from ext.app.sesId, the SDK's + // rotating usage-session id set via LogSession.) + mat_logger->SetContext("AppSessionGuid", m.app_session_guid); mat_logger->SetContext("OsName", m.os_name); mat_logger->SetContext("OsVersion", m.os_version); mat_logger->SetContext("CpuArch", m.cpu_arch); @@ -271,4 +272,30 @@ void OneDsTelemetry::RecordCatalogFetch(const CatalogFetchInfo& info) { SafeLog(GetMatLogger(), ev); } +void OneDsTelemetry::StartSession() { + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto* mat_logger = GetMatLogger(); + if (mat_logger == nullptr) { + return; + } + // LogSession(Started) opens an app-usage session; the SDK stamps ext.app.sesId + // on subsequent events and records session duration on End. + auto ev = MakeEvent("Session", metadata_.test_mode); + mat_logger->LogSession(SessionState::Session_Started, ev); +} + +void OneDsTelemetry::EndSession() { + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto* mat_logger = GetMatLogger(); + if (mat_logger == nullptr) { + return; + } + auto ev = MakeEvent("Session", metadata_.test_mode); + mat_logger->LogSession(SessionState::Session_Ended, ev); +} + } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h index 13339f6b2..c1af838bd 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h @@ -65,6 +65,8 @@ class OneDsTelemetry : public ITelemetry { void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; void RecordDownload(const DownloadInfo& info) override; void RecordCatalogFetch(const CatalogFetchInfo& info) override; + void StartSession() override; + void EndSession() override; /// True if 1DS Initialize succeeded (i.e. events are actually uploaded). /// False in CI or when the tenant token was empty. diff --git a/sdk_v2/cpp/src/telemetry/telemetry.h b/sdk_v2/cpp/src/telemetry/telemetry.h index c99d863fd..91ee2e4b2 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.h +++ b/sdk_v2/cpp/src/telemetry/telemetry.h @@ -188,6 +188,13 @@ class ITelemetry { /// Record one access to a model catalog source (CatalogFetch event). virtual void RecordCatalogFetch(const CatalogFetchInfo& info) = 0; + + /// Mark the start / end of an app-usage session via 1DS LogSession. Between + /// these the SDK stamps a session id (ext.app.sesId) on events and emits a + /// session start/end with duration — standard, cross-platform engagement + /// sessions. Default no-op for the non-1DS implementations. + virtual void StartSession() {} + virtual void EndSession() {} }; } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc index a48885b27..9b594770d 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc @@ -92,4 +92,12 @@ void TelemetryLogger::RecordCatalogFetch(const CatalogFetchInfo& info) { info.error_message, info.user_agent, info.correlation_id)); } +void TelemetryLogger::StartSession() { + logger_.Log(LogLevel::Debug, fmt::format("[Telemetry] SessionStart AppName={}", app_name_)); +} + +void TelemetryLogger::EndSession() { + logger_.Log(LogLevel::Debug, fmt::format("[Telemetry] SessionEnd AppName={}", app_name_)); +} + } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.h b/sdk_v2/cpp/src/telemetry/telemetry_logger.h index da3928f86..b07273987 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.h @@ -36,6 +36,8 @@ class TelemetryLogger : public ITelemetry { void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; void RecordDownload(const DownloadInfo& info) override; void RecordCatalogFetch(const CatalogFetchInfo& info) override; + void StartSession() override; + void EndSession() override; private: std::string app_name_; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_metadata.h b/sdk_v2/cpp/src/telemetry/telemetry_metadata.h index 58f880c1f..8d47fc6e2 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_metadata.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_metadata.h @@ -9,11 +9,11 @@ namespace fl { /// Process-wide metadata stamped onto every 1DS event as common context. /// Computed once at startup and cached. Cheap to copy. struct TelemetryMetadata { - /// Hex-encoded random 128-bit GUID; correlates events from the same process. - /// 1DS interprets the magic field name "UTCReplace_AppSessionGuid" by replacing - /// the value with the OS-supplied app session GUID on Windows, and accepts our - /// random GUID on other platforms. We always provide a value so the event is - /// well-formed even if UTC's magic isn't honored. + /// Hex-encoded random 128-bit GUID, generated once at startup. Stamped on + /// every event as `AppSessionGuid` so the backend can group all events from a + /// single FL process run. This is a stable per-process correlation id and is + /// distinct from the SDK's rotating usage-session id (ext.app.sesId), which is + /// driven separately via LogSession(Started/Ended). std::string app_session_guid; /// Foundry Local SDK version (FoundryLocalGetVersionString). From 554b25b2d1ebb66b5137a34903a48afdabf7800f Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Wed, 10 Jun 2026 23:28:34 -0500 Subject: [PATCH 12/77] sdk_v2/cpp/telemetry: extend ITelemetry with typed payloads and trackers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the C# Foundry Core telemetry event taxonomy to C++ in advance of the 1DS bridge. The interface refactor is the source of truth — all backend implementations and call sites are wired against it. ITelemetry additions: - 4 new `Action` values (EpDownloadAttempt, EpDownloadAndRegister, ModelFileDownload, ModelInference). - 4 new payload structs (`EpDownloadAttemptInfo`, `EpDownloadAndRegisterInfo`, `ModelUsageInfo`, `DownloadInfo`). - 3 new virtual methods (`RecordEpDownloadAttempt`, `RecordEpDownloadAndRegister`, `RecordDownload`). - Replaced `RecordModelUsage(model_id, prompt_tokens, completion_tokens, duration_ms)` with `RecordModelUsage(const ModelUsageInfo&)` so callers can populate richer fields (TimeToFirstToken, EP, memory). - Extended `RecordModelId(action, model_id)` to take `status` and `user_agent`; added `RecordException(action, ex, user_agent)` overload. New supporting code (all in `sdk_v2/cpp/src/telemetry/`): - `telemetry_environment.{h,cc}` — `IsCiEnvironment` / `IsTestingMode` / `IsTruthyValue` / `GetEnv`. 13 CI env-var names ported verbatim from neutron-server's `TelemetryEnvironment.cs`. - `telemetry_metadata.{h,cc}` — captures per-process metadata (`app_session_guid`, version, os_name / os_version / cpu_arch, test_mode flag) for stamping on every event. - `ep_download_tracker.{h,cc}` — RAII tracker that emits one `EPDownloadAndRegister` event per bootstrapper, recording stage transitions (initial -> download -> register). Default failure semantics: dtor records remaining stages as `kFailure`; `Done()` records them as `kSkipped` for happy-path early exit. - `download_tracker.{h,cc}` — RAII tracker that emits one `Download` event per `DownloadManager::DownloadModel` call. `TelemetryLogger` (the fallback / mirror) implements every new method, formatting events as `[Telemetry] EventName Field=value ...` at Debug. Test stubs (`NullTelemetry` and the `RecordingTelemetry` used by `telemetry_test.cc`) were updated for the new interface. Build verified RelWithDebInfo --no_telemetry. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/src/telemetry/download_tracker.cc | 29 +++++ sdk_v2/cpp/src/telemetry/download_tracker.h | 58 +++++++++ .../cpp/src/telemetry/ep_download_tracker.cc | 91 +++++++++++++ .../cpp/src/telemetry/ep_download_tracker.h | 76 +++++++++++ sdk_v2/cpp/src/telemetry/telemetry.cc | 8 ++ sdk_v2/cpp/src/telemetry/telemetry.h | 106 +++++++++++++-- .../src/telemetry/telemetry_action_tracker.cc | 4 +- .../src/telemetry/telemetry_environment.cc | 108 ++++++++++++++++ .../cpp/src/telemetry/telemetry_environment.h | 32 +++++ sdk_v2/cpp/src/telemetry/telemetry_logger.cc | 68 ++++++++-- sdk_v2/cpp/src/telemetry/telemetry_logger.h | 24 +++- .../cpp/src/telemetry/telemetry_metadata.cc | 121 ++++++++++++++++++ sdk_v2/cpp/src/telemetry/telemetry_metadata.h | 38 ++++++ sdk_v2/cpp/test/internal_api/null_telemetry.h | 15 ++- .../cpp/test/internal_api/telemetry_test.cc | 76 ++++++----- 15 files changed, 787 insertions(+), 67 deletions(-) create mode 100644 sdk_v2/cpp/src/telemetry/download_tracker.cc create mode 100644 sdk_v2/cpp/src/telemetry/download_tracker.h create mode 100644 sdk_v2/cpp/src/telemetry/ep_download_tracker.cc create mode 100644 sdk_v2/cpp/src/telemetry/ep_download_tracker.h create mode 100644 sdk_v2/cpp/src/telemetry/telemetry_environment.cc create mode 100644 sdk_v2/cpp/src/telemetry/telemetry_environment.h create mode 100644 sdk_v2/cpp/src/telemetry/telemetry_metadata.cc create mode 100644 sdk_v2/cpp/src/telemetry/telemetry_metadata.h diff --git a/sdk_v2/cpp/src/telemetry/download_tracker.cc b/sdk_v2/cpp/src/telemetry/download_tracker.cc new file mode 100644 index 000000000..dbb369c03 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/download_tracker.cc @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/download_tracker.h" + +#include + +namespace fl { + +DownloadTracker::DownloadTracker(std::string model_id, + std::string user_agent, + ITelemetry& telemetry) + : telemetry_(telemetry) { + info_.model_id = std::move(model_id); + info_.user_agent = std::move(user_agent); + info_.status = ActionStatus::kFailure; + download_phase_start_ = std::chrono::steady_clock::now(); +} + +DownloadTracker::~DownloadTracker() { + // Emit the Download event regardless of outcome. The default status is + // kFailure so abrupt exits (exceptions) are recorded as failures. + telemetry_.RecordDownload(info_); +} + +void DownloadTracker::RecordException(const std::exception& exception) { + telemetry_.RecordException(Action::kModelFileDownload, exception, info_.user_agent); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/download_tracker.h b/sdk_v2/cpp/src/telemetry/download_tracker.h new file mode 100644 index 000000000..240bf5b4a --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/download_tracker.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "telemetry/telemetry.h" + +#include +#include + +namespace fl { + +/// RAII tracker for the per-model "Download" event. +/// Caller updates fields as the download progresses; the event is emitted on +/// destruction. Status defaults to kFailure so that abrupt exits (exceptions) +/// are recorded as failures unless the caller calls SetStatus(kSuccess) on the +/// happy path or SetStatus(kSkipped) when the model was already cached. +class DownloadTracker { + public: + DownloadTracker(std::string model_id, + std::string user_agent, + ITelemetry& telemetry); + ~DownloadTracker(); + + // Non-copyable, non-movable + DownloadTracker(const DownloadTracker&) = delete; + DownloadTracker& operator=(const DownloadTracker&) = delete; + + void SetStatus(ActionStatus status) { info_.status = status; } + void SetLockWaitMs(int64_t v) { info_.lock_wait_ms = v; } + void SetEnumerationMs(int64_t v) { info_.enumeration_ms = v; } + void SetTotalSizeBytes(int64_t v) { info_.total_size_bytes = v; } + void SetAlreadyCachedBytes(int64_t v) { info_.already_cached_bytes = v; } + void SetFileCount(int32_t v) { info_.file_count = v; } + void SetSkippedFileCount(int32_t v) { info_.skipped_file_count = v; } + void SetDownloadWaitResult(std::string v) { info_.download_wait_result = std::move(v); } + void SetMaxConcurrency(int32_t v) { info_.max_concurrency = v; } + + /// Start the timer for the download phase. Use after lock wait and + /// enumeration are complete, so download_ms only reflects byte transfer. + void BeginDownloadPhase() { download_phase_start_ = std::chrono::steady_clock::now(); } + + /// Stop the timer for the download phase. The duration is captured into the + /// emitted event. Safe to call multiple times — the last call wins. + void EndDownloadPhase() { + info_.download_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - download_phase_start_) + .count(); + } + + void RecordException(const std::exception& exception); + + private: + ITelemetry& telemetry_; + DownloadInfo info_; + std::chrono::steady_clock::time_point download_phase_start_; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc b/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc new file mode 100644 index 000000000..65d01f34d --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/ep_download_tracker.h" + +#include + +namespace fl { + +namespace { + +int64_t ElapsedMs(std::chrono::steady_clock::time_point start) { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); +} + +} // namespace + +EpDownloadTracker::EpDownloadTracker(std::string provider_name, + std::string user_agent, + ITelemetry& telemetry) + : telemetry_(telemetry), + provider_name_(std::move(provider_name)), + user_agent_(std::move(user_agent)), + stage_start_(std::chrono::steady_clock::now()) { +} + +EpDownloadTracker::~EpDownloadTracker() { + // Mirror neutron-server: if the caller didn't reach Done() or + // RecordRegisterComplete, assume the abrupt exit was an exception path and + // record any unfinished stage as kFailure. + RecordEvent(ActionStatus::kFailure); +} + +void EpDownloadTracker::RecordInitialState(std::string ready_state) { + init_ready_state_ = std::move(ready_state); + stage_ = Stage::Download; + stage_start_ = std::chrono::steady_clock::now(); +} + +void EpDownloadTracker::RecordDownloadComplete(ActionStatus status, std::string ready_state) { + download_duration_ms_ = ElapsedMs(stage_start_); + download_ready_state_ = std::move(ready_state); + download_status_ = status; + stage_ = Stage::Register; + stage_start_ = std::chrono::steady_clock::now(); +} + +void EpDownloadTracker::RecordRegisterComplete(ActionStatus status, std::string ready_state) { + register_duration_ms_ = ElapsedMs(stage_start_); + register_ready_state_ = std::move(ready_state); + register_status_ = status; + stage_ = Stage::Final; +} + +void EpDownloadTracker::Done() { + RecordEvent(ActionStatus::kSkipped); +} + +void EpDownloadTracker::RecordException(const std::exception& ex) { + telemetry_.RecordException(Action::kEpDownloadAndRegister, ex, user_agent_); +} + +void EpDownloadTracker::RecordEvent(ActionStatus incomplete_stage_status) { + if (recorded_event_) { + return; + } + recorded_event_ = true; + + if (stage_ == Stage::Download) { + download_duration_ms_ = ElapsedMs(stage_start_); + download_status_ = incomplete_stage_status; + } else if (stage_ == Stage::Register) { + register_duration_ms_ = ElapsedMs(stage_start_); + register_status_ = incomplete_stage_status; + } + + EpDownloadAndRegisterInfo info; + info.user_agent = user_agent_; + info.provider_name = provider_name_; + info.init_ready_state = init_ready_state_; + info.download_ready_state = download_ready_state_; + info.download_status = download_status_; + info.download_duration_ms = download_duration_ms_; + info.register_ready_state = register_ready_state_; + info.register_status = register_status_; + info.register_duration_ms = register_duration_ms_; + telemetry_.RecordEpDownloadAndRegister(info); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/ep_download_tracker.h b/sdk_v2/cpp/src/telemetry/ep_download_tracker.h new file mode 100644 index 000000000..6ee322f15 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/ep_download_tracker.h @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "telemetry/telemetry.h" + +#include +#include +#include + +namespace fl { + +/// RAII tracker for the per-provider "EPDownloadAndRegister" event. Mirrors the +/// neutron-server EPDownloadTracker. Stages advance Initial -> Download -> +/// Register -> Final. Each stage that the caller doesn't explicitly complete +/// is recorded with one of: +/// * kFailure on destruction (assumed exception path) +/// * kSkipped if Done() is called before destruction (early-exit without +/// exception, e.g. "EP already registered, nothing to download") +class EpDownloadTracker { + public: + EpDownloadTracker(std::string provider_name, + std::string user_agent, + ITelemetry& telemetry); + ~EpDownloadTracker(); + + // Non-copyable, non-movable + EpDownloadTracker(const EpDownloadTracker&) = delete; + EpDownloadTracker& operator=(const EpDownloadTracker&) = delete; + + /// Captures the EP's ready state before the download phase begins. Restarts + /// the stopwatch so subsequent timings measure the download phase only. + void RecordInitialState(std::string ready_state = "N/A"); + + /// Captures the download phase outcome and ready state, restarts the + /// stopwatch for the register phase. + void RecordDownloadComplete(ActionStatus status, std::string ready_state = "N/A"); + + /// Captures the register phase outcome and ready state. Stops the stopwatch. + void RecordRegisterComplete(ActionStatus status, std::string ready_state = "N/A"); + + /// Mark tracking as complete, filling any remaining stages with kSkipped + /// instead of the default kFailure (exception-assumed) status. Call this on + /// happy-path early exits (e.g. "EP already registered"). + void Done(); + + /// Record an exception associated with the bootstrap operation. Does not + /// finalize the EPDownloadAndRegister event — that happens on destruction. + void RecordException(const std::exception& ex); + + private: + enum class Stage { + Initial = 0, + Download = 1, + Register = 2, + Final = 3, + }; + + void RecordEvent(ActionStatus incomplete_stage_status); + + ITelemetry& telemetry_; + std::string provider_name_; + std::string user_agent_; + std::string init_ready_state_ = "N/A"; + std::string download_ready_state_ = "N/A"; + std::string register_ready_state_ = "N/A"; + ActionStatus download_status_ = ActionStatus::kSkipped; + ActionStatus register_status_ = ActionStatus::kSkipped; + int64_t download_duration_ms_ = 0; + int64_t register_duration_ms_ = 0; + std::chrono::steady_clock::time_point stage_start_; + Stage stage_ = Stage::Initial; + bool recorded_event_ = false; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry.cc b/sdk_v2/cpp/src/telemetry/telemetry.cc index 6d8fc06f6..97fc24122 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry.cc @@ -50,6 +50,14 @@ std::string_view ActionToString(Action action) { return "OpenAIResponsesGetInputItems"; case Action::kCoreAudioTranscribe: return "CoreAudioTranscribe"; + case Action::kEpDownloadAttempt: + return "EpDownloadAttempt"; + case Action::kEpDownloadAndRegister: + return "EpDownloadAndRegister"; + case Action::kModelFileDownload: + return "ModelFileDownload"; + case Action::kModelInference: + return "ModelInference"; default: return "Unknown"; } diff --git a/sdk_v2/cpp/src/telemetry/telemetry.h b/sdk_v2/cpp/src/telemetry/telemetry.h index 680dfcd9a..b0f0ed4a9 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.h +++ b/sdk_v2/cpp/src/telemetry/telemetry.h @@ -9,8 +9,9 @@ namespace fl { -/// Telemetry action identifiers matching the C# ITelemetry.Action enum. -// TODO: This is a lie. The enum values don't match. Do they need to? +/// Telemetry action identifiers. The values are stable IDs for log greppability; +/// the over-the-wire field is the human-readable name produced by ActionToString. +/// The 1DS implementation emits the string, so changing numeric values is safe. enum class Action { kInvalid = 0, @@ -46,6 +47,16 @@ enum class Action { // Audio kCoreAudioTranscribe = 400, + + // EP catalog operations + kEpDownloadAttempt = 500, // Wraps the entire DownloadAndRegisterEps call + kEpDownloadAndRegister = 501, // One per-provider attempt within DownloadAndRegisterEps + + // Model file download + kModelFileDownload = 600, // Wraps the per-model DownloadManager flow + + // EP runtime usage (TimeToFirstToken / total tokens / memory) + kModelInference = 700, // The "Model" event in the C# implementation }; /// Status of a tracked telemetry action. @@ -62,9 +73,66 @@ std::string_view ActionToString(Action action); /// Convert ActionStatus to human-readable string. std::string_view ActionStatusToString(ActionStatus status); +/// Payload for the EPDownloadAttempt event — emitted once per DownloadAndRegisterEps call. +struct EpDownloadAttemptInfo { + std::string user_agent; + int attempts = 0; // Total per-provider attempts made + int num_providers = 0; // Number of providers requested + int succeeded = 0; // Number of providers that registered successfully + int failed = 0; // Number of providers that failed + bool resolved = false; // True if at least one provider became Registered + ActionStatus status = ActionStatus::kInvalid; + int64_t duration_ms = 0; +}; + +/// Payload for the EPDownloadAndRegister event — emitted once per provider attempt. +struct EpDownloadAndRegisterInfo { + std::string user_agent; + std::string provider_name; + std::string init_ready_state; // EP state before this call (e.g. "NotPresent") + std::string download_ready_state; // EP state after the download phase (e.g. "Installed") + ActionStatus download_status = ActionStatus::kInvalid; + int64_t download_duration_ms = 0; + std::string register_ready_state; // EP state after the register phase (e.g. "Registered") + ActionStatus register_status = ActionStatus::kInvalid; + int64_t register_duration_ms = 0; +}; + +/// Payload for the Model event — emitted once per inference completion with token / memory metrics. +struct ModelUsageInfo { + std::string model_id; + std::string execution_provider; + std::string user_agent; + int64_t time_to_first_token_ms = 0; + int64_t total_time_ms = 0; + int32_t total_tokens = 0; + int32_t input_token_count = 0; + uint64_t num_messages = 0; + int64_t memory_used_mb = -1; // -1 if not measured + int64_t cpu_time_ms = -1; // -1 if not measured + int64_t gpu_memory_used_mb = -1; // -1 if not measured +}; + +/// Payload for the Download event — emitted once per DownloadManager::DownloadModel call. +struct DownloadInfo { + std::string model_id; + std::string user_agent; + ActionStatus status = ActionStatus::kInvalid; + int64_t lock_wait_ms = 0; + int64_t enumeration_ms = 0; + int64_t download_ms = 0; + int64_t total_size_bytes = 0; + int64_t already_cached_bytes = 0; + int32_t file_count = 0; + int32_t skipped_file_count = 0; + std::string download_wait_result; // e.g. "Completed", "TimedOut", "AlreadyHeld" + int32_t max_concurrency = 0; +}; + /// Abstract telemetry interface. -/// Implementations may send events to a telemetry backend (ETW, OpenTelemetry, etc.) -/// or simply log them. The stub TelemetryLogger logs via ILogger. +/// Implementations may send events to a telemetry backend (1DS, ETW, OpenTelemetry, …) +/// or simply log them. The OneDsTelemetry implementation sends to 1DS; the +/// TelemetryLogger stub formats them to the ILogger sink. class ITelemetry { public: virtual ~ITelemetry() = default; @@ -77,14 +145,30 @@ class ITelemetry { /// Record an exception associated with an action. virtual void RecordException(Action action, const std::exception& exception) = 0; - /// Record model usage metrics after inference. - virtual void RecordModelUsage(const std::string& model_id, - int64_t prompt_tokens, - int64_t completion_tokens, - int64_t duration_ms) = 0; + /// Record an exception associated with an action, with an optional user agent. + /// Default forwards to RecordException(action, exception) for implementations + /// that don't yet propagate the user agent. + virtual void RecordException(Action action, const std::exception& exception, + const std::string& /*user_agent*/) { + RecordException(action, exception); + } + + /// Record model usage metrics after inference (Model event). + virtual void RecordModelUsage(const ModelUsageInfo& info) = 0; + + /// Record which model was used for an action (ModelId event). + virtual void RecordModelId(Action action, const std::string& model_id, + ActionStatus status = ActionStatus::kSuccess, + const std::string& user_agent = "") = 0; + + /// Record the result of a DownloadAndRegisterEps call (EPDownloadAttempt event). + virtual void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) = 0; + + /// Record one EP provider's download+register attempt (EPDownloadAndRegister event). + virtual void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) = 0; - /// Record which model was used for an action. - virtual void RecordModelId(Action action, const std::string& model_id) = 0; + /// Record one model file download (Download event). + virtual void RecordDownload(const DownloadInfo& info) = 0; }; } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc index 7672cb1f4..f30f1246c 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc @@ -19,7 +19,7 @@ ActionTracker::~ActionTracker() { telemetry_.RecordAction(action_, status_, user_agent_, indirect_, duration_ms); if (!model_id_.empty()) { - telemetry_.RecordModelId(action_, model_id_); + telemetry_.RecordModelId(action_, model_id_, status_, user_agent_); } } @@ -28,7 +28,7 @@ void ActionTracker::SetStatus(ActionStatus status) { } void ActionTracker::RecordException(const std::exception& exception) { - telemetry_.RecordException(action_, exception); + telemetry_.RecordException(action_, exception, user_agent_); } void ActionTracker::SetModelId(const std::string& model_id) { diff --git a/sdk_v2/cpp/src/telemetry/telemetry_environment.cc b/sdk_v2/cpp/src/telemetry/telemetry_environment.cc new file mode 100644 index 000000000..a19838ba7 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_environment.cc @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/telemetry_environment.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#endif + +namespace fl { + +namespace { + +// Mirrors neutron-server's CiEnvironmentVariableNames. Keep this in sync if the +// list there changes — telemetry behavior in CI must match across stacks. +constexpr std::array kCiEnvironmentVariableNames = { + "CI", // Generic CI flag used by many providers + "TF_BUILD", // Azure Pipelines + "GITHUB_ACTIONS", // GitHub Actions + "GITLAB_CI", // GitLab CI + "CIRCLECI", // CircleCI + "TRAVIS", // Travis CI + "JENKINS_URL", // Jenkins + "CODEBUILD_BUILD_ID", // AWS CodeBuild + "BUILDKITE", // Buildkite + "TEAMCITY_VERSION", // TeamCity + "APPVEYOR", // AppVeyor + "BITBUCKET_BUILD_NUMBER", // Bitbucket Pipelines + "SYSTEM_TEAMFOUNDATIONCOLLECTIONURI", // Azure DevOps +}; + +bool EqualsIgnoreCase(std::string_view a, std::string_view b) { + if (a.size() != b.size()) { + return false; + } + for (size_t i = 0; i < a.size(); ++i) { + if (std::tolower(static_cast(a[i])) != + std::tolower(static_cast(b[i]))) { + return false; + } + } + return true; +} + +std::string_view Trim(std::string_view s) { + auto is_ws = [](unsigned char c) { return std::isspace(c) != 0; }; + while (!s.empty() && is_ws(static_cast(s.front()))) { + s.remove_prefix(1); + } + while (!s.empty() && is_ws(static_cast(s.back()))) { + s.remove_suffix(1); + } + return s; +} + +} // namespace + +std::string TelemetryEnvironment::GetEnv(const char* name) { +#ifdef _WIN32 + // Use the W variant so we don't depend on the legacy CRT _CRT_SECURE_NO_WARNINGS. + // Env-var values are ASCII for the CI flags we care about; if a value is unicode + // we still get the bytes round-tripped correctly because we only do truthiness checks. + DWORD needed = ::GetEnvironmentVariableA(name, nullptr, 0); + if (needed == 0) { + return {}; + } + std::vector buf(needed); + DWORD written = ::GetEnvironmentVariableA(name, buf.data(), needed); + if (written == 0 || written >= needed) { + return {}; + } + return std::string(buf.data(), written); +#else + const char* value = std::getenv(name); + return value ? std::string(value) : std::string{}; +#endif +} + +bool TelemetryEnvironment::IsTruthyValue(std::string_view value) { + auto trimmed = Trim(value); + if (trimmed.empty()) { + return false; + } + return !EqualsIgnoreCase(trimmed, "0") && + !EqualsIgnoreCase(trimmed, "false") && + !EqualsIgnoreCase(trimmed, "no") && + !EqualsIgnoreCase(trimmed, "off"); +} + +bool TelemetryEnvironment::IsCiEnvironment() { + for (const char* name : kCiEnvironmentVariableNames) { + if (IsTruthyValue(GetEnv(name))) { + return true; + } + } + return false; +} + +bool TelemetryEnvironment::IsTestingMode() { + return IsTruthyValue(GetEnv("FOUNDRY_TESTING_MODE")); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_environment.h b/sdk_v2/cpp/src/telemetry/telemetry_environment.h new file mode 100644 index 000000000..71f3e2a5a --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_environment.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include + +namespace fl { + +/// Static helpers for telemetry runtime gating. Ported from neutron-server's +/// TelemetryEnvironment.cs so the CI suppression behavior matches across stacks. +class TelemetryEnvironment { + public: + /// Returns true if any well-known CI environment variable is set to a truthy + /// value. The set matches neutron-server's TelemetryEnvironment.cs. + /// In CI, OneDsTelemetry skips Initialize entirely — no 1DS events emitted. + static bool IsCiEnvironment(); + + /// Returns true if the FOUNDRY_TESTING_MODE env var is set to a truthy value. + /// Outside CI, this stamps a `test=true` boolean on every emitted event but + /// does not suppress emission. In CI, this is moot — emission is suppressed. + static bool IsTestingMode(); + + /// Truthy-value semantics: a non-empty, non-whitespace string whose trimmed + /// value is not "0", "false", "no", or "off" (case-insensitive). + static bool IsTruthyValue(std::string_view value); + + /// Read an env var (cross-platform). Returns empty string if unset. + static std::string GetEnv(const char* name); +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc index 68d250513..214a0cc6d 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc @@ -13,31 +13,75 @@ TelemetryLogger::TelemetryLogger(const std::string& app_name, ILogger& logger) void TelemetryLogger::RecordAction(Action action, ActionStatus status, const std::string& user_agent, bool indirect, int64_t duration_ms) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] AppName:{} UserAgent:{} Command:{} Status:{} Direct:{} Time:{}ms", + fmt::format("[Telemetry] Action AppName={} UserAgent={} Action={} Status={} Direct={} TimeMs={}", app_name_, user_agent, ActionToString(action), ActionStatusToString(status), !indirect, duration_ms)); } void TelemetryLogger::RecordException(Action action, const std::exception& exception) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] AppName:{} Command:{} Exception:{}", + fmt::format("[Telemetry] Error AppName={} Action={} Exception={}", app_name_, ActionToString(action), exception.what())); } -void TelemetryLogger::RecordModelUsage(const std::string& model_id, - int64_t prompt_tokens, - int64_t completion_tokens, - int64_t duration_ms) { +void TelemetryLogger::RecordException(Action action, const std::exception& exception, + const std::string& user_agent) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] AppName:{} ModelUsage: model={} prompt_tokens={} " - "completion_tokens={} duration={}ms", - app_name_, model_id, prompt_tokens, completion_tokens, duration_ms)); + fmt::format("[Telemetry] Error AppName={} UserAgent={} Action={} Exception={}", + app_name_, user_agent, ActionToString(action), exception.what())); } -void TelemetryLogger::RecordModelId(Action action, const std::string& model_id) { +void TelemetryLogger::RecordModelUsage(const ModelUsageInfo& info) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] AppName:{} Command:{} ModelId:{}", - app_name_, ActionToString(action), model_id)); + fmt::format("[Telemetry] Model AppName={} UserAgent={} ModelId={} EP={} TimeToFirstTokenMs={} " + "TotalTimeMs={} TotalTokens={} InputTokenCount={} NumMessages={} MemoryUsedMB={} " + "CpuTimeMs={} GpuMemoryUsedMB={}", + app_name_, info.user_agent, info.model_id, info.execution_provider, + info.time_to_first_token_ms, info.total_time_ms, info.total_tokens, + info.input_token_count, info.num_messages, info.memory_used_mb, + info.cpu_time_ms, info.gpu_memory_used_mb)); +} + +void TelemetryLogger::RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const std::string& user_agent) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] ModelId AppName={} UserAgent={} Action={} ModelId={} Status={}", + app_name_, user_agent, ActionToString(action), model_id, + ActionStatusToString(status))); +} + +void TelemetryLogger::RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] EPDownloadAttempt AppName={} UserAgent={} Attempts={} " + "NumProviders={} Succeeded={} Failed={} Resolved={} Status={} TimeMs={}", + app_name_, info.user_agent, info.attempts, info.num_providers, + info.succeeded, info.failed, info.resolved, + ActionStatusToString(info.status), info.duration_ms)); +} + +void TelemetryLogger::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] EPDownloadAndRegister AppName={} UserAgent={} Provider={} " + "InitReadyState={} DownloadReadyState={} DownloadStatus={} DownloadTimeMs={} " + "RegisterReadyState={} RegisterStatus={} RegisterTimeMs={}", + app_name_, info.user_agent, info.provider_name, + info.init_ready_state, info.download_ready_state, + ActionStatusToString(info.download_status), info.download_duration_ms, + info.register_ready_state, ActionStatusToString(info.register_status), + info.register_duration_ms)); +} + +void TelemetryLogger::RecordDownload(const DownloadInfo& info) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] Download AppName={} UserAgent={} ModelId={} Status={} " + "LockWaitMs={} EnumerationMs={} DownloadMs={} TotalSizeBytes={} " + "AlreadyCachedBytes={} FileCount={} SkippedFileCount={} " + "DownloadWaitResult={} MaxConcurrency={}", + app_name_, info.user_agent, info.model_id, + ActionStatusToString(info.status), info.lock_wait_ms, + info.enumeration_ms, info.download_ms, info.total_size_bytes, + info.already_cached_bytes, info.file_count, info.skipped_file_count, + info.download_wait_result, info.max_concurrency)); } } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.h b/sdk_v2/cpp/src/telemetry/telemetry_logger.h index 056838846..6982c8166 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.h @@ -9,8 +9,14 @@ namespace fl { -/// Stub ITelemetry implementation that logs telemetry events via ILogger. -/// Used as a fallback when no platform-specific telemetry backend is available. +/// ITelemetry implementation that formats telemetry events to ILogger. +/// Used: +/// 1. As the fallback when no 1DS backend is available (cpp-client-telemetry +/// not configured, or FOUNDRY_LOCAL_USE_TELEMETRY=OFF). +/// 2. Inside OneDsTelemetry to provide local diagnostic logging in addition +/// to the 1DS upload. +/// In both cases, the local logger receives every event regardless of CI / +/// FOUNDRY_TESTING_MODE state — those flags only gate the 1DS upload. class TelemetryLogger : public ITelemetry { public: TelemetryLogger(const std::string& app_name, ILogger& logger); @@ -19,13 +25,17 @@ class TelemetryLogger : public ITelemetry { bool indirect, int64_t duration_ms) override; void RecordException(Action action, const std::exception& exception) override; + void RecordException(Action action, const std::exception& exception, + const std::string& user_agent) override; - void RecordModelUsage(const std::string& model_id, - int64_t prompt_tokens, - int64_t completion_tokens, - int64_t duration_ms) override; + void RecordModelUsage(const ModelUsageInfo& info) override; - void RecordModelId(Action action, const std::string& model_id) override; + void RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const std::string& user_agent) override; + + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; + void RecordDownload(const DownloadInfo& info) override; private: std::string app_name_; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc b/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc new file mode 100644 index 000000000..826690ed1 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/telemetry_metadata.h" + +#include "telemetry/telemetry_environment.h" +#include "version.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#else +#include +#endif + +namespace fl { + +namespace { + +std::string MakeGuidV4Hex() { + // RFC 4122 v4 UUID, hex-encoded with hyphens. We use std::random_device + mt19937_64 + // because we don't need the OS UUID API — this is just a per-process correlation id, + // not a cryptographic identifier. + std::random_device rd; + std::mt19937_64 gen{(static_cast(rd()) << 32) | rd()}; + uint64_t hi = gen(); + uint64_t lo = gen(); + + // Set version (4) and variant (10xx) bits. + hi = (hi & 0xFFFFFFFFFFFF0FFFULL) | 0x0000000000004000ULL; + lo = (lo & 0x3FFFFFFFFFFFFFFFULL) | 0x8000000000000000ULL; + + char buf[37]; + std::snprintf(buf, sizeof(buf), + "%08x-%04x-%04x-%04x-%012llx", + static_cast((hi >> 32) & 0xFFFFFFFFu), + static_cast((hi >> 16) & 0xFFFFu), + static_cast(hi & 0xFFFFu), + static_cast((lo >> 48) & 0xFFFFu), + static_cast(lo & 0x0000FFFFFFFFFFFFULL)); + return std::string(buf); +} + +#ifdef _WIN32 +std::string GetWindowsVersion() { + // GetVersionExA is deprecated and lies for unmanifested apps. The reliable + // approach is to read the build number directly from kernel32 via + // RtlGetVersion, or fall back to the OS version registry. For now, use + // GetVersionEx — the deprecation only affects apps without a manifest, and + // Foundry Local has a manifest declaring Win10 / Win11 compat. + OSVERSIONINFOEXA info{}; + info.dwOSVersionInfoSize = sizeof(info); +#pragma warning(suppress : 4996) // GetVersionExA deprecation + if (::GetVersionExA(reinterpret_cast(&info)) == 0) { + return "unknown"; + } + char buf[64]; + std::snprintf(buf, sizeof(buf), "%lu.%lu.%lu", + info.dwMajorVersion, info.dwMinorVersion, info.dwBuildNumber); + return std::string(buf); +} + +std::string GetCpuArch() { + SYSTEM_INFO si{}; + ::GetNativeSystemInfo(&si); + switch (si.wProcessorArchitecture) { + case PROCESSOR_ARCHITECTURE_AMD64: return "amd64"; + case PROCESSOR_ARCHITECTURE_ARM: return "arm"; + case PROCESSOR_ARCHITECTURE_ARM64: return "arm64"; + case PROCESSOR_ARCHITECTURE_IA64: return "ia64"; + case PROCESSOR_ARCHITECTURE_INTEL: return "x86"; + default: return "unknown"; + } +} +#else +struct PosixOsInfo { + std::string name; + std::string version; + std::string arch; +}; + +PosixOsInfo GetPosixOsInfo() { + PosixOsInfo out{"unknown", "unknown", "unknown"}; + ::utsname u{}; + if (::uname(&u) == 0) { + out.name = u.sysname; + out.version = u.release; + out.arch = u.machine; + } + return out; +} +#endif + +} // namespace + +TelemetryMetadata BuildTelemetryMetadata(std::string app_name) { + TelemetryMetadata m; + m.app_session_guid = MakeGuidV4Hex(); + m.version = FOUNDRY_LOCAL_VERSION; + m.app_name = std::move(app_name); + m.test_mode = TelemetryEnvironment::IsTestingMode(); + +#ifdef _WIN32 + m.os_name = "Windows"; + m.os_version = GetWindowsVersion(); + m.cpu_arch = GetCpuArch(); +#else + auto info = GetPosixOsInfo(); + m.os_name = info.name; + m.os_version = info.version; + m.cpu_arch = info.arch; +#endif + + return m; +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_metadata.h b/sdk_v2/cpp/src/telemetry/telemetry_metadata.h new file mode 100644 index 000000000..58f880c1f --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_metadata.h @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include + +namespace fl { + +/// Process-wide metadata stamped onto every 1DS event as common context. +/// Computed once at startup and cached. Cheap to copy. +struct TelemetryMetadata { + /// Hex-encoded random 128-bit GUID; correlates events from the same process. + /// 1DS interprets the magic field name "UTCReplace_AppSessionGuid" by replacing + /// the value with the OS-supplied app session GUID on Windows, and accepts our + /// random GUID on other platforms. We always provide a value so the event is + /// well-formed even if UTC's magic isn't honored. + std::string app_session_guid; + + /// Foundry Local SDK version (FoundryLocalGetVersionString). + std::string version; + + /// Configured app name (from Configuration::app_name). + std::string app_name; + + /// Free-form "Windows 11 10.0.26100 amd64" / "Linux 6.5.0 x86_64" / "macOS 14.4 arm64". + std::string os_name; // "Windows" / "Linux" / "Darwin" + std::string os_version; // "10.0.26100" / "6.5.0-azure" / "14.4" + std::string cpu_arch; // "amd64" / "arm64" / "x86" / ... + + /// True if FOUNDRY_TESTING_MODE was truthy at startup (stamped per-event as `test`). + bool test_mode = false; +}; + +/// Build the metadata for this process. Reads env vars and OS APIs once. +/// app_name comes from Configuration; version comes from FoundryLocalGetVersionString. +TelemetryMetadata BuildTelemetryMetadata(std::string app_name); + +} // namespace fl diff --git a/sdk_v2/cpp/test/internal_api/null_telemetry.h b/sdk_v2/cpp/test/internal_api/null_telemetry.h index 6d2c11c16..2c70a1b66 100644 --- a/sdk_v2/cpp/test/internal_api/null_telemetry.h +++ b/sdk_v2/cpp/test/internal_api/null_telemetry.h @@ -15,12 +15,17 @@ class NullTelemetry : public ITelemetry { void RecordException(Action /*action*/, const std::exception& /*exception*/) override {} - void RecordModelUsage(const std::string& /*model_id*/, - int64_t /*prompt_tokens*/, - int64_t /*completion_tokens*/, - int64_t /*duration_ms*/) override {} + void RecordModelUsage(const ModelUsageInfo& /*info*/) override {} - void RecordModelId(Action /*action*/, const std::string& /*model_id*/) override {} + void RecordModelId(Action /*action*/, const std::string& /*model_id*/, + ActionStatus /*status*/ = ActionStatus::kSuccess, + const std::string& /*user_agent*/ = "") override {} + + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& /*info*/) override {} + + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& /*info*/) override {} + + void RecordDownload(const DownloadInfo& /*info*/) override {} }; } // namespace fl::test diff --git a/sdk_v2/cpp/test/internal_api/telemetry_test.cc b/sdk_v2/cpp/test/internal_api/telemetry_test.cc index 56683cd68..42fc068f6 100644 --- a/sdk_v2/cpp/test/internal_api/telemetry_test.cc +++ b/sdk_v2/cpp/test/internal_api/telemetry_test.cc @@ -51,29 +51,35 @@ class RecordingTelemetry : public ITelemetry { exception_calls.emplace_back(action, exception.what()); } - void RecordModelUsage(const std::string& model_id, - int64_t prompt_tokens, - int64_t completion_tokens, - int64_t duration_ms) override { - model_usage_calls.push_back( - ModelUsageCall{model_id, prompt_tokens, completion_tokens, duration_ms}); + void RecordModelUsage(const ModelUsageInfo& info) override { + model_usage_calls.push_back(info); } - void RecordModelId(Action action, const std::string& model_id) override { + void RecordModelId(Action action, const std::string& model_id, + ActionStatus /*status*/ = ActionStatus::kSuccess, + const std::string& /*user_agent*/ = "") override { model_id_calls.emplace_back(action, model_id); } - struct ModelUsageCall { - std::string model_id; - int64_t prompt_tokens; - int64_t completion_tokens; - int64_t duration_ms; - }; + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override { + ep_attempt_calls.push_back(info); + } + + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override { + ep_register_calls.push_back(info); + } + + void RecordDownload(const DownloadInfo& info) override { + download_calls.push_back(info); + } std::vector action_calls; std::vector> exception_calls; - std::vector model_usage_calls; + std::vector model_usage_calls; std::vector> model_id_calls; + std::vector ep_attempt_calls; + std::vector ep_register_calls; + std::vector download_calls; }; } // namespace @@ -87,12 +93,12 @@ TEST(TelemetryLoggerTest, RecordActionIncludesConcreteFields) { ASSERT_EQ(logger.entries.size(), 1u); EXPECT_EQ(logger.entries[0].level, LogLevel::Debug); - EXPECT_NE(logger.entries[0].message.find("AppName:foundry-local"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("UserAgent:cli/1.0"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Command:ModelDownload"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Status:Success"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Direct:true"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Time:1234ms"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("AppName=foundry-local"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("UserAgent=cli/1.0"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Action=ModelDownload"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Status=Success"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Direct=true"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("TimeMs=1234"), std::string::npos); } TEST(TelemetryLoggerTest, RecordExceptionAndModelEventsIncludeSpecificValues) { @@ -100,20 +106,30 @@ TEST(TelemetryLoggerTest, RecordExceptionAndModelEventsIncludeSpecificValues) { TelemetryLogger telemetry("foundry-local", logger); telemetry.RecordException(Action::kModelLoad, std::runtime_error("config missing")); - telemetry.RecordModelUsage("phi-3-mini", 17, 31, 250); - telemetry.RecordModelId(Action::kModelLoad, "phi-3-mini"); + + ModelUsageInfo usage; + usage.model_id = "phi-3-mini"; + usage.execution_provider = "CPU"; + usage.user_agent = "cli/1.0"; + usage.total_tokens = 31; + usage.input_token_count = 17; + usage.total_time_ms = 250; + telemetry.RecordModelUsage(usage); + + telemetry.RecordModelId(Action::kModelLoad, "phi-3-mini", ActionStatus::kSuccess, "cli/1.0"); ASSERT_EQ(logger.entries.size(), 3u); - EXPECT_NE(logger.entries[0].message.find("Command:ModelLoad"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Exception:config missing"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Action=ModelLoad"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Exception=config missing"), std::string::npos); - EXPECT_NE(logger.entries[1].message.find("ModelUsage: model=phi-3-mini"), std::string::npos); - EXPECT_NE(logger.entries[1].message.find("prompt_tokens=17"), std::string::npos); - EXPECT_NE(logger.entries[1].message.find("completion_tokens=31"), std::string::npos); - EXPECT_NE(logger.entries[1].message.find("duration=250ms"), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("Model "), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("ModelId=phi-3-mini"), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("InputTokenCount=17"), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("TotalTokens=31"), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("TotalTimeMs=250"), std::string::npos); - EXPECT_NE(logger.entries[2].message.find("Command:ModelLoad"), std::string::npos); - EXPECT_NE(logger.entries[2].message.find("ModelId:phi-3-mini"), std::string::npos); + EXPECT_NE(logger.entries[2].message.find("Action=ModelLoad"), std::string::npos); + EXPECT_NE(logger.entries[2].message.find("ModelId=phi-3-mini"), std::string::npos); } TEST(ActionTrackerTest, DestructorRecordsFailureByDefaultWithoutModelId) { From 9435ff47728f95b2c29f74ae9eb7abd08534a532 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Wed, 10 Jun 2026 23:29:07 -0500 Subject: [PATCH 13/77] sdk_v2/cpp/telemetry: add OneDsTelemetry 1DS upload bridge OneDsTelemetry is the production ITelemetry implementation. It embeds a TelemetryLogger mirror so every event is also logged locally to ILogger for diagnostics, then conditionally uploads to 1DS through the Microsoft cpp-client-telemetry SDK. Suppression model (three-state): - CI environment detected -> skip `LogManager::Initialize`; local logging still happens, but no upload occurs. - Tenant token empty (build did not pass `-DFOUNDRY_LOCAL_TELEMETRY_TOKEN`) -> same as CI: no upload, only local logging. - Otherwise -> upload. Every event is stamped with `test=true` when `FOUNDRY_TESTING_MODE` is truthy, `test=false` otherwise. Common context, propagated to every event via `ILogger::SetContext`: - `app_name` (from `Configuration::app_name`). - `app_session_guid` (random v4 UUID; on Windows the special `UTCReplace_AppSessionGuid` field also gets the OS app session GUID). - `version` (from the generated `version.h`). - `os_name` / `os_version` / `cpu_arch` (Win: `GetVersionExA` + `GetNativeSystemInfo`; POSIX: `uname`). The `MICROSOFT_KEYWORD_CRITICAL_DATA` (bit 47) policy flag is set on every event via `SetPolicyBitFlags` so the data passes 1DS classifier. Build / packaging: - `vcpkg.json`: `telemetry` feature pulls `cpp-client-telemetry`. The port is pending in microsoft/vcpkg#52316; until that merges, builds must pass `--no_telemetry`. - `CMakeLists.txt`: new `FOUNDRY_LOCAL_USE_TELEMETRY` option (default ON) and `FOUNDRY_LOCAL_TELEMETRY_TOKEN` advanced cache variable. Calls `find_package(MSTelemetry CONFIG REQUIRED)`, includes `one_ds_telemetry.cc` and `configure_file`-generates `one_ds_tenant_token.h` containing the build-time token, then links `MSTelemetry::mat` and defines `FOUNDRY_LOCAL_HAS_1DS=1`. - `build.py`: new `--no_telemetry` and `--telemetry_token` flags; forwards through to CMake / VCPKG_MANIFEST_FEATURES. Build verified RelWithDebInfo --no_telemetry; --use_telemetry will be validated once the vcpkg port lands. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/CMakeLists.txt | 50 ++++ sdk_v2/cpp/build.py | 30 ++- sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc | 250 ++++++++++++++++++ sdk_v2/cpp/src/telemetry/one_ds_telemetry.h | 80 ++++++ .../src/telemetry/one_ds_tenant_token.h.in | 19 ++ sdk_v2/cpp/vcpkg.json | 6 + 6 files changed, 433 insertions(+), 2 deletions(-) create mode 100644 sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc create mode 100644 sdk_v2/cpp/src/telemetry/one_ds_telemetry.h create mode 100644 sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index 0e6eb8133..aa7dc93e5 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -43,8 +43,19 @@ endif() option(FOUNDRY_LOCAL_BUILD_TESTS "Build unit tests" ON) option(FOUNDRY_LOCAL_BUILD_EXAMPLES "Build example programs" ON) option(FOUNDRY_LOCAL_BUILD_SERVICE "Build web service support (requires oat++)" ON) +option(FOUNDRY_LOCAL_USE_TELEMETRY + "Build with 1DS telemetry uploads (requires cpp-client-telemetry vcpkg port)" ON) option(FOUNDRY_LOCAL_ENABLE_ASAN "Enable AddressSanitizer + UndefinedBehaviorSanitizer (Linux only)" OFF) +# 1DS ingestion token. Treat as a secret: pass via CI as +# cmake -DFOUNDRY_LOCAL_TELEMETRY_TOKEN="" +# Defaults to empty so developer builds and forks don't accidentally bind to a +# real tenant. When empty (or when in CI at runtime), OneDsTelemetry initializes +# but does not transmit events. +set(FOUNDRY_LOCAL_TELEMETRY_TOKEN "" + CACHE STRING "1DS / Aria ingestion token baked into the binary. Leave empty in dev builds.") +mark_as_advanced(FOUNDRY_LOCAL_TELEMETRY_TOKEN) + # Android: interactive examples and host tools don't run on device if(ANDROID) set(FOUNDRY_LOCAL_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) @@ -87,6 +98,17 @@ if(WIN32) find_package(WinMLEpCatalog) endif() +# 1DS C++ client telemetry — provided by the cpp-client-telemetry vcpkg port. +# Enabled via FOUNDRY_LOCAL_USE_TELEMETRY (default ON) and the vcpkg +# manifest feature "telemetry". When off, the ITelemetry interface is satisfied +# by TelemetryLogger only (local diagnostic log only — no upload). +if(FOUNDRY_LOCAL_USE_TELEMETRY) + find_package(MSTelemetry CONFIG REQUIRED) + message(STATUS "1DS telemetry: enabled (cpp-client-telemetry found)") +else() + message(STATUS "1DS telemetry: disabled (FOUNDRY_LOCAL_USE_TELEMETRY=OFF)") +endif() + # -------------------------------------------------------------------------- # Library target # -------------------------------------------------------------------------- @@ -192,7 +214,11 @@ set(FOUNDRY_LOCAL_SOURCES src/service/web_service.cc src/telemetry/telemetry.cc src/telemetry/telemetry_action_tracker.cc + src/telemetry/telemetry_environment.cc src/telemetry/telemetry_logger.cc + src/telemetry/telemetry_metadata.cc + src/telemetry/ep_download_tracker.cc + src/telemetry/download_tracker.cc src/utils.cc src/util/file_lock.cc src/http/http_download.cc @@ -204,6 +230,13 @@ set(FOUNDRY_LOCAL_SOURCES ${FOUNDRY_LOCAL_INTERNAL_HEADERS} ) +# 1DS bridge — compiled only when the cpp-client-telemetry port is available. +# When disabled, Manager falls back to TelemetryLogger and the OneDsTelemetry +# symbol is never referenced. +if(FOUNDRY_LOCAL_USE_TELEMETRY) + list(APPEND FOUNDRY_LOCAL_SOURCES src/telemetry/one_ds_telemetry.cc) +endif() + # Organize headers into filters matching the directory structure in Visual Studio source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}" PREFIX "Source" FILES ${FOUNDRY_LOCAL_SOURCES}) source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}/include" PREFIX "Public Headers" FILES ${FOUNDRY_LOCAL_PUBLIC_HEADERS}) @@ -269,6 +302,13 @@ function(foundry_local_configure_target TARGET LINK_SCOPE) else() target_compile_definitions(${TARGET} ${LINK_SCOPE} FOUNDRY_LOCAL_HAS_EP_CATALOG=0) endif() + + if(FOUNDRY_LOCAL_USE_TELEMETRY) + target_link_libraries(${TARGET} ${LINK_SCOPE} MSTelemetry::mat) + target_compile_definitions(${TARGET} ${LINK_SCOPE} FOUNDRY_LOCAL_HAS_1DS=1) + else() + target_compile_definitions(${TARGET} ${LINK_SCOPE} FOUNDRY_LOCAL_HAS_1DS=0) + endif() endfunction() # -------------------------------------------------------------------------- @@ -287,6 +327,16 @@ configure_file( @ONLY ) +# Generate the 1DS tenant-token header. Always generated (even when +# FOUNDRY_LOCAL_USE_TELEMETRY=OFF) so includes of one_ds_tenant_token.h compile +# in both configurations; consumers must still guard MAT::LogManager calls +# behind FOUNDRY_LOCAL_HAS_1DS. +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/src/telemetry/one_ds_tenant_token.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/generated/one_ds_tenant_token.h" + @ONLY +) + # -------------------------------------------------------------------------- # Object library — compiles all sources once. Both the shared (DLL) and # static library targets re-use these object files, avoiding a double build. diff --git a/sdk_v2/cpp/build.py b/sdk_v2/cpp/build.py index b1274cd60..be56d826d 100644 --- a/sdk_v2/cpp/build.py +++ b/sdk_v2/cpp/build.py @@ -165,6 +165,19 @@ class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescript help="Override the Microsoft.Windows.AI.MachineLearning NuGet version for the WinML EP " "catalog (Windows only). Defaults to the version pinned in deps_versions.json.", ) + parser.add_argument( + "--no_telemetry", action="store_true", + help="Skip building the 1DS (cpp-client-telemetry) bridge. Useful while the vcpkg " + "port (microsoft/vcpkg#52316) is unmerged or when building forks that should " + "not link any telemetry transport. Local diagnostic logging via TelemetryLogger " + "still works.", + ) + parser.add_argument( + "--telemetry_token", default=None, type=str, + help="1DS / Aria ingestion token. Baked into the binary at configure time as a " + "secret. Treat with care — do not commit to source. Defaults to empty, which " + "means OneDsTelemetry initializes but skips upload.", + ) # Cross-compilation (mutually exclusive targets) cross_group = parser.add_mutually_exclusive_group() @@ -451,9 +464,22 @@ def configure(args: argparse.Namespace) -> None: f"-DFOUNDRY_LOCAL_BUILD_SERVICE={build_service}", ] - # Enable vcpkg manifest features for tests + # Enable vcpkg manifest features as needed. Multiple features are passed as a + # semicolon-separated list in a single -D flag. + manifest_features = [] if build_tests == "ON": - command += ["-DVCPKG_MANIFEST_FEATURES=tests"] + manifest_features.append("tests") + if not args.no_telemetry: + manifest_features.append("telemetry") + if manifest_features: + command += [f"-DVCPKG_MANIFEST_FEATURES={';'.join(manifest_features)}"] + + if args.no_telemetry: + command += ["-DFOUNDRY_LOCAL_USE_TELEMETRY=OFF"] + else: + command += ["-DFOUNDRY_LOCAL_USE_TELEMETRY=ON"] + if args.telemetry_token is not None: + command += [f"-DFOUNDRY_LOCAL_TELEMETRY_TOKEN={args.telemetry_token}"] # WinML EP catalog is enabled automatically on Windows by CMake. Allow an # optional version override for the Microsoft.Windows.AI.MachineLearning NuGet. diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc new file mode 100644 index 000000000..a07eaf2eb --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc @@ -0,0 +1,250 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// 1DS-backed ITelemetry implementation. Compiled only when FOUNDRY_LOCAL_USE_TELEMETRY=ON +// (which requires the cpp-client-telemetry vcpkg port to be available). + +#include "telemetry/one_ds_telemetry.h" + +#include "telemetry/telemetry_environment.h" + +#include + +// 1DS C++ SDK headers. The vcpkg port ships these via find_package(MSTelemetry CONFIG). +#include +#include +#include + +// The LogManager macro must appear exactly once per binary that uses the +// "v1 classic" LogManager API surface. The macro instantiates the singleton +// statics for our module configuration. +LOGMANAGER_INSTANCE + +namespace fl { + +namespace { + +using ::Microsoft::Applications::Events::LogManager; +using ::Microsoft::Applications::Events::ILogger; +using ::Microsoft::Applications::Events::EventProperties; +using ::Microsoft::Applications::Events::EventPriority; +using ::Microsoft::Applications::Events::PiiKind_None; + +constexpr uint64_t kCriticalData = MICROSOFT_KEYWORD_CRITICAL_DATA; + +void SetCommonContext(ILogger* mat_logger, const TelemetryMetadata& m) { + // Process-wide context — stamped on every event uploaded through this ILogger. + mat_logger->SetContext("AppName", m.app_name); + mat_logger->SetContext("Version", m.version); + // 1DS recognizes UTCReplace_AppSessionGuid as a magic field name on Windows UTC + // and substitutes the OS session GUID. On other platforms we just send the + // GUID we computed at startup — same correlation semantics either way. + mat_logger->SetContext("UTCReplace_AppSessionGuid", m.app_session_guid); + mat_logger->SetContext("OsName", m.os_name); + mat_logger->SetContext("OsVersion", m.os_version); + mat_logger->SetContext("CpuArch", m.cpu_arch); +} + +EventProperties MakeEvent(const char* name, bool test_mode) { + EventProperties ev(name); + ev.SetPriority(EventPriority::EventPriority_Normal); + ev.SetPolicyBitFlags(kCriticalData); + // `test` is stamped on every event so CI/test data is distinguishable in the backend + // when FOUNDRY_TESTING_MODE is set. In CI (IsCiEnvironment()=true) we never reach + // emission at all, so this only ever toggles for explicit test mode. + ev.SetProperty("test", test_mode); + return ev; +} + +void SafeLog(ILogger* mat_logger, EventProperties& ev) { + if (mat_logger != nullptr) { + mat_logger->LogEvent(ev); + } +} + +ILogger* GetMatLogger() { + // LogManager::GetLogger() returns nullptr until Initialize has been called. + return LogManager::GetLogger(); +} + +} // namespace + +OneDsTelemetry::OneDsTelemetry(const std::string& tenant_token, + const std::string& app_name, + ILogger& logger) + : local_log_(app_name, logger), + metadata_(BuildTelemetryMetadata(app_name)), + logger_(logger) { + const bool is_ci = TelemetryEnvironment::IsCiEnvironment(); + if (is_ci) { + logger_.Log(LogLevel::Information, + "[Telemetry] CI environment detected; 1DS upload disabled (events still logged locally)"); + return; + } + if (tenant_token.empty()) { + logger_.Log(LogLevel::Information, + "[Telemetry] Tenant token is empty; 1DS upload disabled (events still logged locally)"); + return; + } + + try { + auto* mat_logger = LogManager::Initialize(tenant_token); + if (mat_logger == nullptr) { + logger_.Log(LogLevel::Warning, + "[Telemetry] LogManager::Initialize returned null; 1DS upload disabled"); + return; + } + SetCommonContext(mat_logger, metadata_); + initialized_.store(true, std::memory_order_release); + logger_.Log(LogLevel::Information, + fmt::format("[Telemetry] 1DS initialized; AppName={} Version={} Os={} {} Arch={} TestMode={}", + metadata_.app_name, metadata_.version, metadata_.os_name, + metadata_.os_version, metadata_.cpu_arch, metadata_.test_mode)); + } catch (const std::exception& ex) { + logger_.Log(LogLevel::Warning, + fmt::format("[Telemetry] LogManager::Initialize threw: {}; 1DS upload disabled", ex.what())); + } catch (...) { + logger_.Log(LogLevel::Warning, + "[Telemetry] LogManager::Initialize threw unknown exception; 1DS upload disabled"); + } +} + +OneDsTelemetry::~OneDsTelemetry() { + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + try { + LogManager::FlushAndTeardown(); + } catch (...) { + // Best-effort: never throw from a destructor. + } +} + +void OneDsTelemetry::RecordAction(Action action, ActionStatus status, const std::string& user_agent, + bool indirect, int64_t duration_ms) { + local_log_.RecordAction(action, status, user_agent, indirect, duration_ms); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("Action", metadata_.test_mode); + ev.SetProperty("Action", std::string(ActionToString(action))); + ev.SetProperty("Status", std::string(ActionStatusToString(status))); + ev.SetProperty("UserAgent", user_agent); + ev.SetProperty("Direct", !indirect); + ev.SetProperty("TimeMs", duration_ms); + SafeLog(GetMatLogger(), ev); +} + +void OneDsTelemetry::RecordException(Action action, const std::exception& exception) { + RecordException(action, exception, std::string{}); +} + +void OneDsTelemetry::RecordException(Action action, const std::exception& exception, + const std::string& user_agent) { + local_log_.RecordException(action, exception, user_agent); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("Error", metadata_.test_mode); + ev.SetProperty("Action", std::string(ActionToString(action))); + ev.SetProperty("UserAgent", user_agent); + ev.SetProperty("ExceptionType", "std::exception"); + ev.SetProperty("ExceptionMessage", std::string(exception.what())); + ev.SetProperty("InnerExceptionType", ""); + ev.SetProperty("InnerExceptionMessage", ""); + ev.SetProperty("StackTrace", ""); + ev.SetProperty("InnerStackTrace", ""); + SafeLog(GetMatLogger(), ev); +} + +void OneDsTelemetry::RecordModelUsage(const ModelUsageInfo& info) { + local_log_.RecordModelUsage(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("Model", metadata_.test_mode); + ev.SetProperty("ModelId", info.model_id); + ev.SetProperty("ExecutionProvider", info.execution_provider); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("TimeToFirstTokenMs", info.time_to_first_token_ms); + ev.SetProperty("TotalTimeMs", info.total_time_ms); + ev.SetProperty("TotalTokens", static_cast(info.total_tokens)); + ev.SetProperty("InputTokenCount", static_cast(info.input_token_count)); + ev.SetProperty("NumMessages", static_cast(info.num_messages)); + ev.SetProperty("MemoryUsedMB", info.memory_used_mb); + ev.SetProperty("CpuTimeMs", info.cpu_time_ms); + ev.SetProperty("GpuMemoryUsedMB", info.gpu_memory_used_mb); + SafeLog(GetMatLogger(), ev); +} + +void OneDsTelemetry::RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const std::string& user_agent) { + local_log_.RecordModelId(action, model_id, status, user_agent); + if (!initialized_.load(std::memory_order_acquire) || model_id.empty()) { + return; + } + auto ev = MakeEvent("ModelId", metadata_.test_mode); + ev.SetProperty("Action", std::string(ActionToString(action))); + ev.SetProperty("ModelId", model_id); + ev.SetProperty("Status", std::string(ActionStatusToString(status))); + ev.SetProperty("UserAgent", user_agent); + SafeLog(GetMatLogger(), ev); +} + +void OneDsTelemetry::RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) { + local_log_.RecordEpDownloadAttempt(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("EPDownloadAttempt", metadata_.test_mode); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("Attempts", static_cast(info.attempts)); + ev.SetProperty("NumProviders", static_cast(info.num_providers)); + ev.SetProperty("Succeeded", static_cast(info.succeeded)); + ev.SetProperty("Failed", static_cast(info.failed)); + ev.SetProperty("Resolved", info.resolved); + ev.SetProperty("Status", std::string(ActionStatusToString(info.status))); + ev.SetProperty("TimeMs", info.duration_ms); + SafeLog(GetMatLogger(), ev); +} + +void OneDsTelemetry::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) { + local_log_.RecordEpDownloadAndRegister(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("EPDownloadAndRegister", metadata_.test_mode); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("ProviderName", info.provider_name); + ev.SetProperty("InitReadyState", info.init_ready_state); + ev.SetProperty("DownloadReadyState", info.download_ready_state); + ev.SetProperty("DownloadStatus", std::string(ActionStatusToString(info.download_status))); + ev.SetProperty("DownloadTimeMs", info.download_duration_ms); + ev.SetProperty("RegisterReadyState", info.register_ready_state); + ev.SetProperty("RegisterStatus", std::string(ActionStatusToString(info.register_status))); + ev.SetProperty("RegisterTimeMs", info.register_duration_ms); + SafeLog(GetMatLogger(), ev); +} + +void OneDsTelemetry::RecordDownload(const DownloadInfo& info) { + local_log_.RecordDownload(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("Download", metadata_.test_mode); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("ModelId", info.model_id); + ev.SetProperty("Status", std::string(ActionStatusToString(info.status))); + ev.SetProperty("LockWaitTimeMs", info.lock_wait_ms); + ev.SetProperty("EnumerationTimeMs", info.enumeration_ms); + ev.SetProperty("DownloadTimeMs", info.download_ms); + ev.SetProperty("TotalSizeBytes", info.total_size_bytes); + ev.SetProperty("AlreadyCachedBytes", info.already_cached_bytes); + ev.SetProperty("FileCount", static_cast(info.file_count)); + ev.SetProperty("SkippedFileCount", static_cast(info.skipped_file_count)); + ev.SetProperty("DownloadWaitResult", info.download_wait_result); + ev.SetProperty("MaxConcurrency", static_cast(info.max_concurrency)); + SafeLog(GetMatLogger(), ev); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h new file mode 100644 index 000000000..31cfbcbc9 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "telemetry/telemetry.h" +#include "telemetry/telemetry_logger.h" +#include "telemetry/telemetry_metadata.h" +#include "logger.h" + +#include +#include +#include + +namespace fl { + +/// 1DS-backed ITelemetry implementation. Built only when the +/// cpp-client-telemetry vcpkg port is available (find_package(MSTelemetry CONFIG) +/// succeeded and FOUNDRY_LOCAL_USE_TELEMETRY=ON). +/// +/// Lifecycle: +/// * Constructor: +/// - If TelemetryEnvironment::IsCiEnvironment() -> CI mode: skip 1DS Initialize +/// entirely. All RecordX calls become no-ops on the 1DS side; the embedded +/// TelemetryLogger still mirrors them to ILogger. +/// - Else if tenant_token is empty -> uninitialized: same behavior as CI mode. +/// Manager normally avoids constructing OneDsTelemetry when the token is +/// empty, but the guard here makes the class robust to misconfiguration. +/// - Else: call MAT::LogManager::Initialize(token), set process-wide common +/// context (app session GUID, version, OS info, app name, test flag). +/// +/// * RecordX: builds an EventProperties with the event-specific fields and the +/// `test` bool, then logs via ILogger->LogEvent. Local mirror via TelemetryLogger +/// always runs first so the diagnostic log is preserved regardless of upload. +/// +/// * Destructor: MAT::LogManager::FlushAndTeardown so events on disk are pushed +/// before the process exits. Safe to call even when initialization was skipped. +/// +/// Thread safety: 1DS LogManager methods are documented as thread-safe. The +/// `initialized_` flag is set once during construction; all other writes happen +/// only in the destructor. +class OneDsTelemetry : public ITelemetry { + public: + /// @param tenant_token 1DS ingestion token. Empty disables uploads (CI-equivalent). + /// @param app_name Configuration::app_name; stamped as AppName on every event. + /// @param logger Diagnostic logger; used by the embedded TelemetryLogger mirror. + OneDsTelemetry(const std::string& tenant_token, + const std::string& app_name, + ILogger& logger); + ~OneDsTelemetry() override; + + // Non-copyable, non-movable (singleton owns LogManager state). + OneDsTelemetry(const OneDsTelemetry&) = delete; + OneDsTelemetry& operator=(const OneDsTelemetry&) = delete; + + void RecordAction(Action action, ActionStatus status, const std::string& user_agent, + bool indirect, int64_t duration_ms) override; + + void RecordException(Action action, const std::exception& exception) override; + void RecordException(Action action, const std::exception& exception, + const std::string& user_agent) override; + + void RecordModelUsage(const ModelUsageInfo& info) override; + void RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const std::string& user_agent) override; + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; + void RecordDownload(const DownloadInfo& info) override; + + /// True if 1DS Initialize succeeded (i.e. events are actually uploaded). + /// False in CI or when the tenant token was empty. + bool IsUploadEnabled() const { return initialized_; } + + private: + TelemetryLogger local_log_; // Always-on local mirror. + TelemetryMetadata metadata_; // Cached at construction. + std::atomic initialized_{false}; // True iff MAT::LogManager::Initialize ran. + ILogger& logger_; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in b/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in new file mode 100644 index 000000000..1417e9281 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +// Auto-generated by CMake from one_ds_tenant_token.h.in — do not edit manually. +// +// The 1DS tenant token is passed at configure time via +// cmake -DFOUNDRY_LOCAL_TELEMETRY_TOKEN="" +// and is intended to be supplied by CI as a secret, similar to how the build +// system supplies ORT paths via -DORT_HOME. The default is an empty string so +// developer builds and forks don't accidentally ship a tenant binding. +// +// When the constant is empty, OneDsTelemetry::Initialize is skipped and the +// Manager falls back to TelemetryLogger (which only writes to the local +// ILogger sink — no upload). +#pragma once + +namespace fl { + +inline constexpr const char* kFoundryLocalTenantToken = "@FOUNDRY_LOCAL_TELEMETRY_TOKEN@"; + +} // namespace fl diff --git a/sdk_v2/cpp/vcpkg.json b/sdk_v2/cpp/vcpkg.json index 8a3d1037e..e174be8d9 100644 --- a/sdk_v2/cpp/vcpkg.json +++ b/sdk_v2/cpp/vcpkg.json @@ -32,6 +32,12 @@ "oatpp" ] }, + "telemetry": { + "description": "Build with 1DS (cpp-client-telemetry) telemetry uploads", + "dependencies": [ + "cpp-client-telemetry" + ] + }, "tests": { "description": "Build unit tests", "dependencies": [ From d38f2f39cb8df8f6bcf4d0ea33e3949086f7ac8f Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Wed, 10 Jun 2026 23:29:42 -0500 Subject: [PATCH 14/77] sdk_v2/cpp: wire ITelemetry into Manager, EpDetector and DownloadManager Plumb the typed telemetry interface through the orchestration layer so that real workloads produce the new EPDownloadAttempt, EPDownloadAndRegister and Download events. Manager: - Construct `telemetry_` before `ep_detector_` so the detector can be handed a non-null sink at ctor time. When `FOUNDRY_LOCAL_HAS_1DS` is defined, the implementation is `OneDsTelemetry` and reads the build-time token from the generated `one_ds_tenant_token.h`; otherwise it falls back to `TelemetryLogger`. - Member-declaration order in `manager.h` reorganised so consumers (`ep_detector_`, `download_manager_`) appear after the providers (`telemetry_`). C++ destroys in reverse declaration order, so this keeps the raw `ITelemetry*` held by `EpDetector` and `DownloadManager` valid for their entire lifetime. - Explicit `~Manager()` Shutdown reset order updated to match: the telemetry sink is reset last (after ep_detector and download_manager). EpDetector: - New optional `ITelemetry* telemetry` ctor arg (default nullptr) so unit tests that instantiate EpDetector directly continue to compile. - `DownloadAndRegisterEps` emits one `EPDownloadAndRegister` event per bootstrapper via `EpDownloadTracker` and a single aggregate `EPDownloadAttempt` event at the end with attempt / success / fail counts and an overall status. - Exceptions thrown from a bootstrapper invoke `EpDownloadTracker::RecordException` before re-throwing so the failure is recorded regardless of how it propagates. DownloadManager: - New optional `ITelemetry* telemetry` ctor arg. When non-null, `DownloadModel` wraps the call with a `DownloadTracker` that captures lock-wait, enumeration and download timings, total bytes, file count, max concurrency and final status (Success / Skipped / Failure). `RecordException` is invoked on the exception path before re-throwing. - `DownloadModel` now takes an optional `user_agent` parameter so HTTP-driven downloads can attribute the event to the calling client. DownloadBlobsToDirectory: - New optional `BlobDownloadStats*` out parameter populated with `total_size_bytes`, `file_count`, `enumeration_ms`, `download_ms`. All existing 4-argument callers (including unit tests) keep working because the parameter defaults to nullptr. Verified RelWithDebInfo --no_telemetry: 756 tests pass; 54 skipped (environmental, missing local test data); 0 functional failures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/src/download/blob_downloader.cc | 26 ++++++- sdk_v2/cpp/src/download/blob_downloader.h | 15 +++- sdk_v2/cpp/src/download/download_manager.cc | 60 ++++++++++++++-- sdk_v2/cpp/src/download/download_manager.h | 14 +++- sdk_v2/cpp/src/ep_detection/ep_detector.cc | 79 ++++++++++++++++++++- sdk_v2/cpp/src/ep_detection/ep_detector.h | 10 ++- sdk_v2/cpp/src/manager.cc | 26 +++++-- sdk_v2/cpp/src/manager.h | 9 ++- 8 files changed, 219 insertions(+), 20 deletions(-) diff --git a/sdk_v2/cpp/src/download/blob_downloader.cc b/sdk_v2/cpp/src/download/blob_downloader.cc index c6a5e701a..e5411d4db 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.cc +++ b/sdk_v2/cpp/src/download/blob_downloader.cc @@ -443,7 +443,11 @@ bool IsDownloadNeeded(const BlobItemInfo& blob, const std::string& local_path) { void DownloadBlobsToDirectory(IBlobDownloader& downloader, const std::string& sas_uri, const std::string& output_directory, - const BlobDownloadOptions& options) { + const BlobDownloadOptions& options, + BlobDownloadStats* stats) { + using clock = std::chrono::steady_clock; + auto enum_start = clock::now(); + // Step 1: Enumerate all blobs auto all_blobs = downloader.ListBlobs(sas_uri); @@ -486,6 +490,11 @@ void DownloadBlobsToDirectory(IBlobDownloader& downloader, blobs_to_download.end()); if (blobs_to_download.empty()) { + if (stats != nullptr) { + stats->enumeration_ms = std::chrono::duration_cast( + clock::now() - enum_start) + .count(); + } return; } @@ -502,6 +511,15 @@ void DownloadBlobsToDirectory(IBlobDownloader& downloader, total_size += blob.content_length; } + if (stats != nullptr) { + stats->file_count = static_cast(blobs_to_download.size()); + stats->total_size_bytes = total_size; + stats->enumeration_ms = std::chrono::duration_cast( + clock::now() - enum_start) + .count(); + } + auto download_start = clock::now(); + // Step 5: Skip blobs already present at the expected size. Their bytes // count toward "downloaded" so the percentage stays accurate when this is a // resume of a partially-completed download. @@ -596,6 +614,12 @@ void DownloadBlobsToDirectory(IBlobDownloader& downloader, if (options.progress) { options.progress(100.0f); } + + if (stats != nullptr) { + stats->download_ms = std::chrono::duration_cast( + clock::now() - download_start) + .count(); + } } } // namespace fl diff --git a/sdk_v2/cpp/src/download/blob_downloader.h b/sdk_v2/cpp/src/download/blob_downloader.h index 5d6849a6e..66aadab89 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.h +++ b/sdk_v2/cpp/src/download/blob_downloader.h @@ -121,9 +121,22 @@ class AzureBlobDownloader : public IBlobDownloader { /// High-level download function: enumerate, filter, and download all blobs from a SAS URI. /// Handles safetensors optimization, path prefix filtering, and progress reporting. /// Throws fl::Exception on failure. +/// @param stats When non-null, populated with byte/file counts and per-phase timings useful +/// for telemetry. Always populated when the function returns normally; partially +/// populated when it throws (use values only after a clean return). void DownloadBlobsToDirectory(IBlobDownloader& downloader, const std::string& sas_uri, const std::string& output_directory, - const BlobDownloadOptions& options); + const BlobDownloadOptions& options, + struct BlobDownloadStats* stats = nullptr); + +/// Aggregate statistics from one DownloadBlobsToDirectory call. Used by callers to +/// stamp telemetry events with download-shape data (file count, byte volume, timing). +struct BlobDownloadStats { + int64_t total_size_bytes = 0; + int32_t file_count = 0; + int64_t enumeration_ms = 0; // Time spent listing blobs from the SAS URI. + int64_t download_ms = 0; // Time spent transferring blob bytes (excludes enumeration). +}; } // namespace fl diff --git a/sdk_v2/cpp/src/download/download_manager.cc b/sdk_v2/cpp/src/download/download_manager.cc index c4f9dc569..82d304d6b 100644 --- a/sdk_v2/cpp/src/download/download_manager.cc +++ b/sdk_v2/cpp/src/download/download_manager.cc @@ -6,6 +6,8 @@ #include "exception.h" #include "log_level.h" #include "logger.h" +#include "telemetry/download_tracker.h" +#include "telemetry/telemetry.h" #include "util/path_safety.h" #include "util/region_fallback.h" #include "utils.h" @@ -13,6 +15,7 @@ #include #include +#include #include #include #include @@ -177,14 +180,15 @@ std::string ResolveRegion(const std::string& config_region, const ModelInfo& inf } // anonymous namespace DownloadManager::DownloadManager(std::string cache_directory, std::string_view catalog_region, int max_concurrency, - ILogger& logger, bool disable_region_fallback) + ILogger& logger, bool disable_region_fallback, ITelemetry* telemetry) : cache_directory_(std::move(cache_directory)), config_region_(NormalizeConfiguredRegion(catalog_region)), max_concurrency_(max_concurrency), logger_(logger), registry_client_(std::make_unique( kDefaultRegistryRegion, logger, std::make_unique(logger, !disable_region_fallback))), - blob_downloader_(std::make_unique(logger)) {} + blob_downloader_(std::make_unique(logger)), + telemetry_(telemetry) {} DownloadManager::~DownloadManager() = default; @@ -238,12 +242,31 @@ std::string DownloadManager::ComputeModelPath(const ModelInfo& info) const { } std::string DownloadManager::DownloadModel(const ModelInfo& info, - std::function progress_cb) { + std::function progress_cb, + const std::string& user_agent) { + using clock = std::chrono::steady_clock; + auto lock_wait_start = clock::now(); + // Serialize all model downloads in this process: only one runs at a time, so it // gets the full network and disk instead of competing with another download. // The cross-process file lock taken below extends the guarantee across every // process and app that shares this cache directory. std::unique_lock download_guard(download_mutex_); + + int64_t lock_wait_ms = std::chrono::duration_cast( + clock::now() - lock_wait_start) + .count(); + + // RAII telemetry tracker — emits a "Download" event on destruction with whatever + // fields have been populated. Default status is kFailure so abrupt exits (exceptions) + // are recorded as failures; the happy path explicitly sets kSuccess / kSkipped. + std::unique_ptr tracker; + if (telemetry_ != nullptr) { + tracker = std::make_unique(info.model_id, user_agent, *telemetry_); + tracker->SetLockWaitMs(lock_wait_ms); + tracker->SetMaxConcurrency(static_cast(max_concurrency_)); + } + auto model_path = ComputeModelPath(info); // Fast path: serve the cache without taking the cross-process lock. @@ -258,10 +281,17 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, progress_cb(100.0f); } + if (tracker != nullptr) { + tracker->SetStatus(ActionStatus::kSkipped); + } return ResolveEffectiveModelPath(model_path); } if (info.uri.empty()) { + if (tracker != nullptr) { + auto ex = std::runtime_error("cannot download model: empty URI (asset_id)"); + tracker->RecordException(ex); + } FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "cannot download model: empty URI (asset_id)"); } @@ -349,8 +379,21 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, }; } + if (tracker != nullptr) { + tracker->BeginDownloadPhase(); + } + + BlobDownloadStats stats; DownloadBlobsToDirectory(*blob_downloader_, container.blob_sas_uri, - model_path, download_opts); + model_path, download_opts, &stats); + + if (tracker != nullptr) { + tracker->EndDownloadPhase(); + tracker->SetEnumerationMs(stats.enumeration_ms); + tracker->SetFileCount(stats.file_count); + tracker->SetTotalSizeBytes(stats.total_size_bytes); + tracker->SetDownloadWaitResult("Completed"); + } // Step 3: Write inference_model.json — use model_id (includes version) so the // local model scanner can match it back to catalog entries during startup. @@ -362,8 +405,15 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, // Step 5: Remove download signal — marks download as complete std::filesystem::remove(signal_path); + if (tracker != nullptr) { + tracker->SetStatus(ActionStatus::kSuccess); + } return ResolveEffectiveModelPath(model_path); - } catch (...) { + } catch (const std::exception& e) { + if (tracker != nullptr) { + tracker->SetDownloadWaitResult("Failed"); + tracker->RecordException(e); + } // Leave the signal file in place so the incomplete download is detected throw; } diff --git a/sdk_v2/cpp/src/download/download_manager.h b/sdk_v2/cpp/src/download/download_manager.h index 7099dcb8a..82ca3bb70 100644 --- a/sdk_v2/cpp/src/download/download_manager.h +++ b/sdk_v2/cpp/src/download/download_manager.h @@ -15,6 +15,7 @@ namespace fl { class ILogger; +class ITelemetry; /// Orchestrates the full model download flow: /// 1. Compute local cache path @@ -32,11 +33,15 @@ class DownloadManager { /// @param logger Logger forwarded to the registry client for retry diagnostics. /// @param disable_region_fallback When true, the registry uses a single region attempt /// with no cross-region fallback. + /// @param telemetry Optional telemetry sink. If non-null, a Download event is emitted + /// per DownloadModel call. nullptr is supported so tests and + /// embedders without telemetry continue to compile and run. DownloadManager(std::string cache_directory, std::string_view catalog_region, int max_concurrency, ILogger& logger, - bool disable_region_fallback = false); + bool disable_region_fallback = false, + ITelemetry* telemetry = nullptr); ~DownloadManager(); /// Override the model registry client (for testing). @@ -47,9 +52,13 @@ class DownloadManager { /// Download a model to the local cache. /// progress_cb reports 0.0 to 100.0 percentage. + /// user_agent identifies the calling API surface for telemetry attribution; pass an + /// empty string when called outside of an HTTP request context. /// Returns the local path where the model was downloaded. /// Throws fl::Exception on failure. - std::string DownloadModel(const ModelInfo& info, std::function progress_cb = nullptr); + std::string DownloadModel(const ModelInfo& info, + std::function progress_cb = nullptr, + const std::string& user_agent = ""); /// Check if a model is cached locally (directory exists and download is complete). bool IsModelCached(const ModelInfo& info) const; @@ -77,6 +86,7 @@ class DownloadManager { ILogger& logger_; std::unique_ptr registry_client_; std::unique_ptr blob_downloader_; + ITelemetry* telemetry_ = nullptr; /// Serializes all model downloads in this process: only one runs at a time, so /// each gets the full network/disk instead of competing with another download. diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.cc b/sdk_v2/cpp/src/ep_detection/ep_detector.cc index f60e6adf9..d26ab7662 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.cc @@ -4,21 +4,26 @@ #include "ep_detection/ep_bootstrapper.h" #include "logger.h" +#include "telemetry/ep_download_tracker.h" +#include "telemetry/telemetry.h" #include #include +#include #include namespace fl { EpDetector::EpDetector(const OrtApi& ort_api, OrtEnv& ort_env, std::vector> bootstrappers, - ILogger& logger) + ILogger& logger, + ITelemetry* telemetry) : ort_api_(ort_api), ort_env_(ort_env), bootstrappers_(std::move(bootstrappers)), - logger_(logger) { + logger_(logger), + telemetry_(telemetry) { // Populate both cache vectors exact-sized from bootstrappers_. After this point // size and element addresses (including the EpInfo::name string storage backing // flEpInfo::name) are immutable for the detector's lifetime — only is_registered @@ -134,6 +139,17 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vectorName()); + // Per-provider EPDownloadAndRegister event via EpDownloadTracker. When + // telemetry_ is null (e.g. unit tests instantiate EpDetector directly), + // the tracker is skipped — the event has no fallback emitter. + std::unique_ptr tracker; + const bool was_registered_before = bs->IsRegistered(); + if (telemetry_ != nullptr) { + tracker = std::make_unique(bs->Name(), /*user_agent=*/std::string{}, *telemetry_); + tracker->RecordInitialState(was_registered_before ? "Registered" : "NotPresent"); + } + + ++telemetry_attempts; // Reuse previously downloaded EP packages unless the caller explicitly asks // for a forced refresh. Downloading every time made the bootstrapper // re-fetch and re-register EPs on every invocation. - if (bs->DownloadAndRegister(/*force=*/false, wrapped_cb, logger_)) { + bool ok = false; + try { + ok = bs->DownloadAndRegister(/*force=*/false, wrapped_cb, logger_); + } catch (const std::exception& ex) { + if (tracker) { + tracker->RecordException(ex); + } + // Re-throw to preserve existing semantics — the wrapper RAII guard above + // resets download_in_progress_; the tracker dtor records the EP event. + throw; + } + + if (ok) { + ++telemetry_succeeded; + telemetry_resolved = true; result.registered_eps.push_back(bs->Name()); // Update cached registration state in place under the cache lock so @@ -181,9 +223,24 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector cache_lock(cache_mutex_); cached_eps_[i].is_registered = true; cached_eps_c_[i].is_registered = true; + + if (tracker) { + tracker->RecordDownloadComplete(ActionStatus::kSuccess, "Installed"); + tracker->RecordRegisterComplete(ActionStatus::kSuccess, "Registered"); + } } else { + ++telemetry_failed; result.failed_eps.push_back(bs->Name()); result.success = false; + if (tracker) { + // The bootstrapper conflated download + register and returned false. + // Record both phases as kFailure so backend dashboards can tell that + // this EP didn't reach Registered, without claiming a specific phase. + tracker->RecordDownloadComplete(ActionStatus::kFailure, + was_registered_before ? "Registered" : "NotPresent"); + tracker->RecordRegisterComplete(ActionStatus::kFailure, + was_registered_before ? "Registered" : "NotPresent"); + } } } @@ -197,6 +254,22 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector( + std::chrono::steady_clock::now() - attempt_start) + .count(); + telemetry_->RecordEpDownloadAttempt(attempt_info); + } + return result; } diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.h b/sdk_v2/cpp/src/ep_detection/ep_detector.h index 254f1e43e..e4c9aceaa 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.h +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.h @@ -23,6 +23,7 @@ struct OrtEnv; namespace fl { class ILogger; +class ITelemetry; /// Interface for detecting available hardware devices and execution providers. class IEpDetector { @@ -71,9 +72,15 @@ class EpDetector : public IEpDetector { /// @param ort_env The ORT environment singleton. /// @param bootstrappers EP bootstrappers for download/registration. /// @param logger Logger instance. + /// @param telemetry Optional telemetry sink. When non-null, DownloadAndRegisterEps + /// emits one EPDownloadAndRegister event per provider via + /// EpDownloadTracker, plus one aggregate EPDownloadAttempt + /// event for the whole call. Pass nullptr in tests / when no + /// telemetry sink is available. EpDetector(const OrtApi& ort_api, OrtEnv& ort_env, std::vector> bootstrappers, - ILogger& logger); + ILogger& logger, + ITelemetry* telemetry = nullptr); ~EpDetector() override = default; // Non-copyable, non-movable (owns bootstrappers and mutex state) @@ -92,6 +99,7 @@ class EpDetector : public IEpDetector { OrtEnv& ort_env_; std::vector> bootstrappers_; ILogger& logger_; + ITelemetry* telemetry_ = nullptr; std::mutex download_mutex_; std::atomic download_in_progress_{false}; mutable std::mutex cache_mutex_; diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index 23c6ccb3f..faaa63f15 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -25,6 +25,10 @@ #include "telemetry/telemetry_action_tracker.h" #include "telemetry/telemetry_logger.h" #include "util/string_utils.h" +#if FOUNDRY_LOCAL_HAS_1DS +#include "one_ds_tenant_token.h" // auto-generated header, in ${CMAKE_BINARY_DIR}/generated +#include "telemetry/one_ds_telemetry.h" +#endif #include "utils.h" #if FOUNDRY_LOCAL_HAS_EP_CATALOG @@ -295,7 +299,20 @@ Manager::Manager(const Configuration& config) const auto webgpu_ep_dir = cache_dir / "webgpu-ep"; bootstrappers.push_back(std::make_unique(webgpu_ep_dir.string(), register_ep)); - ep_detector_ = std::make_unique(*ort_api_, *ort_env_, std::move(bootstrappers), *logger_); + // Telemetry must be constructed before subsystems that emit events so we can + // pass it to them at construction time (e.g. EpDetector emits EPDownloadAttempt + // / EPDownloadAndRegister events from DownloadAndRegisterEps). +#if FOUNDRY_LOCAL_HAS_1DS + // OneDsTelemetry includes a TelemetryLogger mirror, so local diagnostic logging + // is preserved whether or not 1DS upload is enabled. Suppression rules + // (CI environment, empty token) are enforced inside OneDsTelemetry. + telemetry_ = std::make_unique(kFoundryLocalTenantToken, config_.app_name, *logger_); +#else + telemetry_ = std::make_unique(config_.app_name, *logger_); +#endif + + ep_detector_ = std::make_unique(*ort_api_, *ort_env_, std::move(bootstrappers), *logger_, + telemetry_.get()); // Read configurable download concurrency (default 64) int download_concurrency = 64; @@ -320,10 +337,10 @@ Manager::Manager(const Configuration& config) config_.catalog_region.value_or("auto"), download_concurrency, *logger_, - disable_region_fallback); + disable_region_fallback, + telemetry_.get()); model_load_manager_ = std::make_unique(*ep_detector_, *logger_); session_manager_ = std::make_unique(*logger_); - telemetry_ = std::make_unique(config_.app_name, *logger_); catalog_ = std::make_unique( config_.catalog_urls, download_manager_->GetCacheDirectory(), @@ -359,8 +376,9 @@ Manager::~Manager() { model_load_manager_.reset(); download_manager_.reset(); catalog_.reset(); - telemetry_.reset(); + // ep_detector_ holds a raw ITelemetry* — destroy it before telemetry_. ep_detector_.reset(); + telemetry_.reset(); // Unregister EPs we registered, then drop our OrtEnv refcount. Best-effort: // log failures but don't throw from a destructor. diff --git a/sdk_v2/cpp/src/manager.h b/sdk_v2/cpp/src/manager.h index 4b5440db7..187b5b3ff 100644 --- a/sdk_v2/cpp/src/manager.h +++ b/sdk_v2/cpp/src/manager.h @@ -120,9 +120,12 @@ class Manager { // released manually in ~Manager() after all // consumers (sessions, ep_detector_) are gone. // logger_ — everything logs through this, destroyed last + // telemetry_ — used by ep_detector_ and throughout; must + // outlive ep_detector_ because EpDetector holds + // a raw ITelemetry* for emitting EP events // ep_detector_ — detects HW acceleration; holds OrtEnv& (must - // outlive ort_env_ release in ~Manager()) - // telemetry_ — used throughout + // outlive ort_env_ release in ~Manager()) and + // a raw ITelemetry* // catalog_ — owns all Model instances. used by download_manager, model_load_manager, and web service // download_manager_ — uses ModelInfo owned by catalog // model_load_manager_ — holds loaded model state referencing catalog models @@ -135,8 +138,8 @@ class Manager { OrtEnv* ort_env_ = nullptr; std::vector registered_ep_libraries_; std::unique_ptr logger_; - std::unique_ptr ep_detector_; std::unique_ptr telemetry_; + std::unique_ptr ep_detector_; std::unique_ptr catalog_; std::unique_ptr download_manager_; std::unique_ptr model_load_manager_; From df59a13ad50f06800cb3939f3c979a6ea2455262 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 15 Jun 2026 11:54:38 -0500 Subject: [PATCH 15/77] sdk_v2/cpp/telemetry: bump vcpkg baseline + fix ILogger collision Now that microsoft/vcpkg#52316 has merged, bump the manifest baseline from 256acc64 to 44819aa2 (current master) so the cpp-client-telemetry 3.10.161.1 port resolves. Also rename the using-declaration in one_ds_telemetry.cc's anonymous namespace from `using ::Microsoft::Applications::Events::ILogger;` to `using MatILogger = ::Microsoft::Applications::Events::ILogger;` so it no longer collides with the local `fl::ILogger` interface from src/logger.h. Three callsites (SetCommonContext, SafeLog, GetMatLogger) updated; the OneDsTelemetry ctor's `ILogger& logger` parameter correctly resolves to fl::ILogger after the rename. Verified with: python sdk_v2/cpp/build.py --use_telemetry --config RelWithDebInfo --skip_examples foundry_local.dll = 12.19 MB (telemetry on, empty token). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc | 8 ++++---- sdk_v2/cpp/vcpkg.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc index a07eaf2eb..72889227b 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc @@ -25,14 +25,14 @@ namespace fl { namespace { using ::Microsoft::Applications::Events::LogManager; -using ::Microsoft::Applications::Events::ILogger; +using MatILogger = ::Microsoft::Applications::Events::ILogger; using ::Microsoft::Applications::Events::EventProperties; using ::Microsoft::Applications::Events::EventPriority; using ::Microsoft::Applications::Events::PiiKind_None; constexpr uint64_t kCriticalData = MICROSOFT_KEYWORD_CRITICAL_DATA; -void SetCommonContext(ILogger* mat_logger, const TelemetryMetadata& m) { +void SetCommonContext(MatILogger* mat_logger, const TelemetryMetadata& m) { // Process-wide context — stamped on every event uploaded through this ILogger. mat_logger->SetContext("AppName", m.app_name); mat_logger->SetContext("Version", m.version); @@ -56,13 +56,13 @@ EventProperties MakeEvent(const char* name, bool test_mode) { return ev; } -void SafeLog(ILogger* mat_logger, EventProperties& ev) { +void SafeLog(MatILogger* mat_logger, EventProperties& ev) { if (mat_logger != nullptr) { mat_logger->LogEvent(ev); } } -ILogger* GetMatLogger() { +MatILogger* GetMatLogger() { // LogManager::GetLogger() returns nullptr until Initialize has been called. return LogManager::GetLogger(); } diff --git a/sdk_v2/cpp/vcpkg.json b/sdk_v2/cpp/vcpkg.json index e174be8d9..a4d0d00da 100644 --- a/sdk_v2/cpp/vcpkg.json +++ b/sdk_v2/cpp/vcpkg.json @@ -2,7 +2,7 @@ "name": "foundry-local", "version-semver": "0.1.0", "description": "Foundry Local C++ SDK", - "builtin-baseline": "256acc64012b23a13041d8705805e1f23b43a024", + "builtin-baseline": "44819aa2a6c10e56065e2b0330e7d6c89d1d2574", "dependencies": [ "azure-storage-blobs-cpp", { From 753a10d838c708984a56ff5082fc6af06b13d7e2 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Tue, 16 Jun 2026 17:24:01 -0500 Subject: [PATCH 16/77] build: enable linker dead-code stripping for foundry_local.dll Add /OPT:REF, /OPT:ICF, and /INCREMENTAL:NO to the foundry_local shared library target for Release and RelWithDebInfo configs. MSVC's /DEBUG flag (needed for PDB generation) silently disables these optimizations unless they are explicitly re-enabled, leaving all unreferenced symbols from statically-linked dependencies (notably cpp-client-telemetry/mat.lib) in the final binary. Also add -Wl,--gc-sections (Linux) and -Wl,-dead_strip (macOS) equivalents for non-Windows builds. Additionally, declare sqlite3 with default-features:false in vcpkg.json to drop the unused json1 extension from the SQLite transitive dependency. Measured impact (RelWithDebInfo, x64-windows, telemetry ON): foundry_local.dll: 12.19 MB -> 4.51 MB (-7.68 MB, -63%) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/CMakeLists.txt | 21 +++++++++++++++++++++ sdk_v2/cpp/vcpkg.json | 3 ++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index aa7dc93e5..ac7455f3d 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -401,6 +401,27 @@ if(WIN32) /DELAYLOAD:Microsoft.Windows.AI.MachineLearning.dll ) endif() + + # Strip unreferenced symbols and fold identical COMDATs in Release builds. + # MSVC's /DEBUG (needed for PDBs) silently disables /OPT:REF and /OPT:ICF + # unless they are explicitly re-enabled. /INCREMENTAL:NO is required because + # incremental linking also prevents dead-code elimination. + target_link_options(foundry_local PRIVATE + $<$:/OPT:REF> + $<$:/OPT:ICF> + $<$:/INCREMENTAL:NO> + ) +endif() + +# Strip unreferenced sections on Linux (GCC/Clang) and macOS (Apple ld). +if(UNIX AND NOT APPLE) + target_link_options(foundry_local PRIVATE + $<$:-Wl,--gc-sections> + ) +elseif(APPLE) + target_link_options(foundry_local PRIVATE + $<$:-Wl,-dead_strip> + ) endif() # -------------------------------------------------------------------------- diff --git a/sdk_v2/cpp/vcpkg.json b/sdk_v2/cpp/vcpkg.json index a4d0d00da..241b39995 100644 --- a/sdk_v2/cpp/vcpkg.json +++ b/sdk_v2/cpp/vcpkg.json @@ -19,7 +19,8 @@ }, "ms-gsl", "nlohmann-json", - "spdlog" + "spdlog", + { "name": "sqlite3", "default-features": false } ], "overrides": [ { "name": "nlohmann-json", "version": "3.11.3" }, From 91e9a0b575a37a16af8e0bccd3a8bb347c158df8 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 20 Jun 2026 02:18:09 -0500 Subject: [PATCH 17/77] sdk_v2/cpp/telemetry: add InvocationContext (correlation id + indirect) foundation Reworks the telemetry core ahead of broadening route/inference coverage: - New InvocationContext {user_agent, correlation_id, indirect} threaded through the ITelemetry interface in place of the loose user_agent/indirect params. Direct() mints a fresh correlation id; AsIndirect() derives a caused-by child that reuses it. ActionTracker now carries the context and guarantees an id. - Every event (Action/Error/Model/ModelId/Download/EP*) now carries a CorrelationId so all events from one operation can be grouped. The Model event also gains Stream and Direct. - indirect now means "happened as a consequence of another action": the per-provider EPDownloadAndRegister is marked indirect and shares the overall EPDownloadAttempt's correlation id. - Add ActionStatus::kClientError to separate 4xx client rejects from 5xx/internal failures (wired into handlers in a follow-up). - Prune dead Action enum entries (kModelDownload/kModelDelete/kCoreAudioTranscribe) and add kServiceRequestUnmatched for the upcoming router catch-all. - Share the v4 UUID generator (MakeGuidV4Hex) between metadata and correlation. Builds clean (/W4 /WX); telemetry + ActionTracker unit tests pass. --- sdk_v2/cpp/CMakeLists.txt | 1 + sdk_v2/cpp/src/ep_detection/ep_detector.cc | 9 ++- sdk_v2/cpp/src/manager.cc | 3 +- sdk_v2/cpp/src/telemetry/download_tracker.cc | 4 +- .../cpp/src/telemetry/ep_download_tracker.cc | 8 ++- .../cpp/src/telemetry/ep_download_tracker.h | 2 + .../cpp/src/telemetry/invocation_context.cc | 34 +++++++++++ sdk_v2/cpp/src/telemetry/invocation_context.h | 58 +++++++++++++++++++ sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc | 35 ++++++----- sdk_v2/cpp/src/telemetry/one_ds_telemetry.h | 9 ++- sdk_v2/cpp/src/telemetry/telemetry.cc | 10 ++-- sdk_v2/cpp/src/telemetry/telemetry.h | 38 ++++++------ .../src/telemetry/telemetry_action_tracker.cc | 16 +++-- .../src/telemetry/telemetry_action_tracker.h | 12 ++-- sdk_v2/cpp/src/telemetry/telemetry_logger.cc | 51 ++++++++-------- sdk_v2/cpp/src/telemetry/telemetry_logger.h | 9 ++- .../cpp/src/telemetry/telemetry_metadata.cc | 26 +-------- sdk_v2/cpp/test/internal_api/null_telemetry.h | 9 ++- .../cpp/test/internal_api/telemetry_test.cc | 30 +++++----- 19 files changed, 228 insertions(+), 136 deletions(-) create mode 100644 sdk_v2/cpp/src/telemetry/invocation_context.cc create mode 100644 sdk_v2/cpp/src/telemetry/invocation_context.h diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index ac7455f3d..b41fed370 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -214,6 +214,7 @@ set(FOUNDRY_LOCAL_SOURCES src/service/web_service.cc src/telemetry/telemetry.cc src/telemetry/telemetry_action_tracker.cc + src/telemetry/invocation_context.cc src/telemetry/telemetry_environment.cc src/telemetry/telemetry_logger.cc src/telemetry/telemetry_metadata.cc diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.cc b/sdk_v2/cpp/src/ep_detection/ep_detector.cc index d26ab7662..c454d1406 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.cc @@ -140,8 +140,11 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector tracker; const bool was_registered_before = bs->IsRegistered(); if (telemetry_ != nullptr) { - tracker = std::make_unique(bs->Name(), /*user_agent=*/std::string{}, *telemetry_); + tracker = std::make_unique(bs->Name(), /*user_agent=*/std::string{}, + telemetry_correlation_id, *telemetry_); tracker->RecordInitialState(was_registered_before ? "Registered" : "NotPresent"); } @@ -256,6 +260,7 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vectortelemetry_->RecordAction(Action::kCoreInitialize, ActionStatus::kSuccess, "", false, 0); + created->telemetry_->RecordAction(Action::kCoreInitialize, ActionStatus::kSuccess, + InvocationContext::Direct(), 0); } catch (const std::exception& ex) { created->GetLogger().Log(LogLevel::Error, fmt::format("telemetry RecordAction failed during Create: {}", ex.what())); diff --git a/sdk_v2/cpp/src/telemetry/download_tracker.cc b/sdk_v2/cpp/src/telemetry/download_tracker.cc index dbb369c03..549bf0187 100644 --- a/sdk_v2/cpp/src/telemetry/download_tracker.cc +++ b/sdk_v2/cpp/src/telemetry/download_tracker.cc @@ -12,6 +12,7 @@ DownloadTracker::DownloadTracker(std::string model_id, : telemetry_(telemetry) { info_.model_id = std::move(model_id); info_.user_agent = std::move(user_agent); + info_.correlation_id = MakeGuidV4Hex(); info_.status = ActionStatus::kFailure; download_phase_start_ = std::chrono::steady_clock::now(); } @@ -23,7 +24,8 @@ DownloadTracker::~DownloadTracker() { } void DownloadTracker::RecordException(const std::exception& exception) { - telemetry_.RecordException(Action::kModelFileDownload, exception, info_.user_agent); + telemetry_.RecordException(Action::kModelFileDownload, exception, + InvocationContext{info_.user_agent, info_.correlation_id, /*indirect=*/false}); } } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc b/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc index 65d01f34d..f7a29ea81 100644 --- a/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc +++ b/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc @@ -18,10 +18,12 @@ int64_t ElapsedMs(std::chrono::steady_clock::time_point start) { EpDownloadTracker::EpDownloadTracker(std::string provider_name, std::string user_agent, + std::string correlation_id, ITelemetry& telemetry) : telemetry_(telemetry), provider_name_(std::move(provider_name)), user_agent_(std::move(user_agent)), + correlation_id_(std::move(correlation_id)), stage_start_(std::chrono::steady_clock::now()) { } @@ -58,7 +60,10 @@ void EpDownloadTracker::Done() { } void EpDownloadTracker::RecordException(const std::exception& ex) { - telemetry_.RecordException(Action::kEpDownloadAndRegister, ex, user_agent_); + // The per-provider attempt happens as a consequence of the overall + // DownloadAndRegisterEps call, so it is indirect and shares its correlation id. + telemetry_.RecordException(Action::kEpDownloadAndRegister, ex, + InvocationContext{user_agent_, correlation_id_, /*indirect=*/true}); } void EpDownloadTracker::RecordEvent(ActionStatus incomplete_stage_status) { @@ -77,6 +82,7 @@ void EpDownloadTracker::RecordEvent(ActionStatus incomplete_stage_status) { EpDownloadAndRegisterInfo info; info.user_agent = user_agent_; + info.correlation_id = correlation_id_; info.provider_name = provider_name_; info.init_ready_state = init_ready_state_; info.download_ready_state = download_ready_state_; diff --git a/sdk_v2/cpp/src/telemetry/ep_download_tracker.h b/sdk_v2/cpp/src/telemetry/ep_download_tracker.h index 6ee322f15..7376685bd 100644 --- a/sdk_v2/cpp/src/telemetry/ep_download_tracker.h +++ b/sdk_v2/cpp/src/telemetry/ep_download_tracker.h @@ -21,6 +21,7 @@ class EpDownloadTracker { public: EpDownloadTracker(std::string provider_name, std::string user_agent, + std::string correlation_id, ITelemetry& telemetry); ~EpDownloadTracker(); @@ -61,6 +62,7 @@ class EpDownloadTracker { ITelemetry& telemetry_; std::string provider_name_; std::string user_agent_; + std::string correlation_id_; std::string init_ready_state_ = "N/A"; std::string download_ready_state_ = "N/A"; std::string register_ready_state_ = "N/A"; diff --git a/sdk_v2/cpp/src/telemetry/invocation_context.cc b/sdk_v2/cpp/src/telemetry/invocation_context.cc new file mode 100644 index 000000000..0b4daab04 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/invocation_context.cc @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/invocation_context.h" + +#include +#include +#include + +namespace fl { + +std::string MakeGuidV4Hex() { + // std::random_device + mt19937_64 is enough here — this is a correlation / + // session id, not a cryptographic identifier, so the OS UUID API is overkill. + std::random_device rd; + std::mt19937_64 gen{(static_cast(rd()) << 32) | rd()}; + uint64_t hi = gen(); + uint64_t lo = gen(); + + // Set version (4) and variant (10xx) bits. + hi = (hi & 0xFFFFFFFFFFFF0FFFULL) | 0x0000000000004000ULL; + lo = (lo & 0x3FFFFFFFFFFFFFFFULL) | 0x8000000000000000ULL; + + char buf[37]; + std::snprintf(buf, sizeof(buf), + "%08x-%04x-%04x-%04x-%012llx", + static_cast((hi >> 32) & 0xFFFFFFFFu), + static_cast((hi >> 16) & 0xFFFFu), + static_cast(hi & 0xFFFFu), + static_cast((lo >> 48) & 0xFFFFu), + static_cast(lo & 0x0000FFFFFFFFFFFFULL)); + return std::string(buf); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/invocation_context.h b/sdk_v2/cpp/src/telemetry/invocation_context.h new file mode 100644 index 000000000..8bf30db56 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/invocation_context.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include + +namespace fl { + +/// Generate an RFC 4122 v4 UUID, hex-encoded with hyphens. Not a cryptographic +/// identifier — used for per-operation correlation and per-process session ids. +std::string MakeGuidV4Hex(); + +/// Per-operation telemetry context, threaded from an entry point through every +/// action it triggers. +/// +/// - `user_agent` identifies the calling client (SDK, CLI, Node, browser…). +/// - `correlation_id` groups every event emitted while servicing one logical +/// operation, so the route action, the inference it drives, +/// the Model metrics event and any error can be joined. +/// - `indirect` is true when this action happened as a consequence of +/// another action rather than a direct user/API call (e.g. a +/// session driven by an HTTP route, or a per-provider EP +/// download under an overall attempt). +struct InvocationContext { + std::string user_agent; + std::string correlation_id; + bool indirect = false; + + /// A direct, top-level context with a freshly generated correlation id. + static InvocationContext Direct(std::string user_agent = "") { + InvocationContext ctx; + ctx.user_agent = std::move(user_agent); + ctx.correlation_id = MakeGuidV4Hex(); + ctx.indirect = false; + return ctx; + } + + /// Derive a context for an action triggered by this one: same correlation id + /// and user agent, but marked indirect. + InvocationContext AsIndirect() const { + InvocationContext ctx = *this; + ctx.indirect = true; + return ctx; + } + + /// Guarantee a correlation id is present, generating one when empty. Lets an + /// entry point that received a default-constructed context still group its + /// events (e.g. an SDK caller that didn't build a Direct() context). + InvocationContext& EnsureCorrelationId() { + if (correlation_id.empty()) { + correlation_id = MakeGuidV4Hex(); + } + return *this; + } +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc index 72889227b..324566be7 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc @@ -120,34 +120,32 @@ OneDsTelemetry::~OneDsTelemetry() { } } -void OneDsTelemetry::RecordAction(Action action, ActionStatus status, const std::string& user_agent, - bool indirect, int64_t duration_ms) { - local_log_.RecordAction(action, status, user_agent, indirect, duration_ms); +void OneDsTelemetry::RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t duration_ms) { + local_log_.RecordAction(action, status, context, duration_ms); if (!initialized_.load(std::memory_order_acquire)) { return; } auto ev = MakeEvent("Action", metadata_.test_mode); ev.SetProperty("Action", std::string(ActionToString(action))); ev.SetProperty("Status", std::string(ActionStatusToString(status))); - ev.SetProperty("UserAgent", user_agent); - ev.SetProperty("Direct", !indirect); + ev.SetProperty("UserAgent", context.user_agent); + ev.SetProperty("CorrelationId", context.correlation_id); + ev.SetProperty("Direct", !context.indirect); ev.SetProperty("TimeMs", duration_ms); SafeLog(GetMatLogger(), ev); } -void OneDsTelemetry::RecordException(Action action, const std::exception& exception) { - RecordException(action, exception, std::string{}); -} - void OneDsTelemetry::RecordException(Action action, const std::exception& exception, - const std::string& user_agent) { - local_log_.RecordException(action, exception, user_agent); + const InvocationContext& context) { + local_log_.RecordException(action, exception, context); if (!initialized_.load(std::memory_order_acquire)) { return; } auto ev = MakeEvent("Error", metadata_.test_mode); ev.SetProperty("Action", std::string(ActionToString(action))); - ev.SetProperty("UserAgent", user_agent); + ev.SetProperty("UserAgent", context.user_agent); + ev.SetProperty("CorrelationId", context.correlation_id); ev.SetProperty("ExceptionType", "std::exception"); ev.SetProperty("ExceptionMessage", std::string(exception.what())); ev.SetProperty("InnerExceptionType", ""); @@ -166,6 +164,9 @@ void OneDsTelemetry::RecordModelUsage(const ModelUsageInfo& info) { ev.SetProperty("ModelId", info.model_id); ev.SetProperty("ExecutionProvider", info.execution_provider); ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); + ev.SetProperty("Stream", info.stream); + ev.SetProperty("Direct", !info.indirect); ev.SetProperty("TimeToFirstTokenMs", info.time_to_first_token_ms); ev.SetProperty("TotalTimeMs", info.total_time_ms); ev.SetProperty("TotalTokens", static_cast(info.total_tokens)); @@ -178,8 +179,8 @@ void OneDsTelemetry::RecordModelUsage(const ModelUsageInfo& info) { } void OneDsTelemetry::RecordModelId(Action action, const std::string& model_id, - ActionStatus status, const std::string& user_agent) { - local_log_.RecordModelId(action, model_id, status, user_agent); + ActionStatus status, const InvocationContext& context) { + local_log_.RecordModelId(action, model_id, status, context); if (!initialized_.load(std::memory_order_acquire) || model_id.empty()) { return; } @@ -187,7 +188,8 @@ void OneDsTelemetry::RecordModelId(Action action, const std::string& model_id, ev.SetProperty("Action", std::string(ActionToString(action))); ev.SetProperty("ModelId", model_id); ev.SetProperty("Status", std::string(ActionStatusToString(status))); - ev.SetProperty("UserAgent", user_agent); + ev.SetProperty("UserAgent", context.user_agent); + ev.SetProperty("CorrelationId", context.correlation_id); SafeLog(GetMatLogger(), ev); } @@ -198,6 +200,7 @@ void OneDsTelemetry::RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) } auto ev = MakeEvent("EPDownloadAttempt", metadata_.test_mode); ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); ev.SetProperty("Attempts", static_cast(info.attempts)); ev.SetProperty("NumProviders", static_cast(info.num_providers)); ev.SetProperty("Succeeded", static_cast(info.succeeded)); @@ -215,6 +218,7 @@ void OneDsTelemetry::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo } auto ev = MakeEvent("EPDownloadAndRegister", metadata_.test_mode); ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); ev.SetProperty("ProviderName", info.provider_name); ev.SetProperty("InitReadyState", info.init_ready_state); ev.SetProperty("DownloadReadyState", info.download_ready_state); @@ -233,6 +237,7 @@ void OneDsTelemetry::RecordDownload(const DownloadInfo& info) { } auto ev = MakeEvent("Download", metadata_.test_mode); ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); ev.SetProperty("ModelId", info.model_id); ev.SetProperty("Status", std::string(ActionStatusToString(info.status))); ev.SetProperty("LockWaitTimeMs", info.lock_wait_ms); diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h index 31cfbcbc9..d7f0fe303 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h @@ -52,16 +52,15 @@ class OneDsTelemetry : public ITelemetry { OneDsTelemetry(const OneDsTelemetry&) = delete; OneDsTelemetry& operator=(const OneDsTelemetry&) = delete; - void RecordAction(Action action, ActionStatus status, const std::string& user_agent, - bool indirect, int64_t duration_ms) override; + void RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t duration_ms) override; - void RecordException(Action action, const std::exception& exception) override; void RecordException(Action action, const std::exception& exception, - const std::string& user_agent) override; + const InvocationContext& context) override; void RecordModelUsage(const ModelUsageInfo& info) override; void RecordModelId(Action action, const std::string& model_id, - ActionStatus status, const std::string& user_agent) override; + ActionStatus status, const InvocationContext& context) override; void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; void RecordDownload(const DownloadInfo& info) override; diff --git a/sdk_v2/cpp/src/telemetry/telemetry.cc b/sdk_v2/cpp/src/telemetry/telemetry.cc index 97fc24122..da603efa5 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry.cc @@ -22,10 +22,6 @@ std::string_view ActionToString(Action action) { return "ModelLoad"; case Action::kModelUnload: return "ModelUnload"; - case Action::kModelDownload: - return "ModelDownload"; - case Action::kModelDelete: - return "ModelDelete"; case Action::kModelList: return "ModelList"; case Action::kOpenAIChatCompletions: @@ -48,8 +44,6 @@ std::string_view ActionToString(Action action) { return "OpenAIResponsesDelete"; case Action::kOpenAIResponsesGetInputItems: return "OpenAIResponsesGetInputItems"; - case Action::kCoreAudioTranscribe: - return "CoreAudioTranscribe"; case Action::kEpDownloadAttempt: return "EpDownloadAttempt"; case Action::kEpDownloadAndRegister: @@ -58,6 +52,8 @@ std::string_view ActionToString(Action action) { return "ModelFileDownload"; case Action::kModelInference: return "ModelInference"; + case Action::kServiceRequestUnmatched: + return "ServiceRequestUnmatched"; default: return "Unknown"; } @@ -73,6 +69,8 @@ std::string_view ActionStatusToString(ActionStatus status) { return "Invalid"; case ActionStatus::kSkipped: return "Skipped"; + case ActionStatus::kClientError: + return "ClientError"; default: return "Unknown"; } diff --git a/sdk_v2/cpp/src/telemetry/telemetry.h b/sdk_v2/cpp/src/telemetry/telemetry.h index b0f0ed4a9..681a3cccc 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.h +++ b/sdk_v2/cpp/src/telemetry/telemetry.h @@ -2,6 +2,8 @@ // Licensed under the MIT License. #pragma once +#include "telemetry/invocation_context.h" + #include #include #include @@ -27,8 +29,6 @@ enum class Action { // Model management kModelLoad = 100, kModelUnload = 101, - kModelDownload = 102, - kModelDelete = 103, kModelList = 104, // OpenAI Chat/Audio/Embeddings APIs @@ -45,9 +45,6 @@ enum class Action { kOpenAIResponsesDelete = 303, kOpenAIResponsesGetInputItems = 304, - // Audio - kCoreAudioTranscribe = 400, - // EP catalog operations kEpDownloadAttempt = 500, // Wraps the entire DownloadAndRegisterEps call kEpDownloadAndRegister = 501, // One per-provider attempt within DownloadAndRegisterEps @@ -57,14 +54,18 @@ enum class Action { // EP runtime usage (TimeToFirstToken / total tokens / memory) kModelInference = 700, // The "Model" event in the C# implementation + + // HTTP service plumbing + kServiceRequestUnmatched = 800, // A request reached the service but matched no route / method }; /// Status of a tracked telemetry action. enum class ActionStatus { - kFailure = 0, + kFailure = 0, // Server-side / internal failure (maps to HTTP 5xx) kSuccess, kInvalid, kSkipped, + kClientError, // Rejected due to invalid client input (maps to HTTP 4xx) — not a service fault }; /// Convert Action to human-readable string. @@ -76,6 +77,7 @@ std::string_view ActionStatusToString(ActionStatus status); /// Payload for the EPDownloadAttempt event — emitted once per DownloadAndRegisterEps call. struct EpDownloadAttemptInfo { std::string user_agent; + std::string correlation_id; int attempts = 0; // Total per-provider attempts made int num_providers = 0; // Number of providers requested int succeeded = 0; // Number of providers that registered successfully @@ -88,6 +90,7 @@ struct EpDownloadAttemptInfo { /// Payload for the EPDownloadAndRegister event — emitted once per provider attempt. struct EpDownloadAndRegisterInfo { std::string user_agent; + std::string correlation_id; std::string provider_name; std::string init_ready_state; // EP state before this call (e.g. "NotPresent") std::string download_ready_state; // EP state after the download phase (e.g. "Installed") @@ -103,6 +106,9 @@ struct ModelUsageInfo { std::string model_id; std::string execution_provider; std::string user_agent; + std::string correlation_id; + bool stream = false; // True if the inference was streamed (SSE) vs a single response + bool indirect = false; // True if the inference was driven by another action (e.g. an HTTP route) int64_t time_to_first_token_ms = 0; int64_t total_time_ms = 0; int32_t total_tokens = 0; @@ -117,6 +123,7 @@ struct ModelUsageInfo { struct DownloadInfo { std::string model_id; std::string user_agent; + std::string correlation_id; ActionStatus status = ActionStatus::kInvalid; int64_t lock_wait_ms = 0; int64_t enumeration_ms = 0; @@ -137,29 +144,22 @@ class ITelemetry { public: virtual ~ITelemetry() = default; - /// Record a completed action with timing and status. + /// Record a completed action with timing and status. The context carries the + /// user agent, the correlation id grouping this operation's events, and whether + /// the action was indirect (triggered by another action). virtual void RecordAction(Action action, ActionStatus status, - const std::string& user_agent, - bool indirect, int64_t duration_ms) = 0; + const InvocationContext& context, int64_t duration_ms) = 0; /// Record an exception associated with an action. - virtual void RecordException(Action action, const std::exception& exception) = 0; - - /// Record an exception associated with an action, with an optional user agent. - /// Default forwards to RecordException(action, exception) for implementations - /// that don't yet propagate the user agent. virtual void RecordException(Action action, const std::exception& exception, - const std::string& /*user_agent*/) { - RecordException(action, exception); - } + const InvocationContext& context) = 0; /// Record model usage metrics after inference (Model event). virtual void RecordModelUsage(const ModelUsageInfo& info) = 0; /// Record which model was used for an action (ModelId event). virtual void RecordModelId(Action action, const std::string& model_id, - ActionStatus status = ActionStatus::kSuccess, - const std::string& user_agent = "") = 0; + ActionStatus status, const InvocationContext& context) = 0; /// Record the result of a DownloadAndRegisterEps call (EPDownloadAttempt event). virtual void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) = 0; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc index f30f1246c..54bcd938a 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc @@ -2,24 +2,28 @@ // Licensed under the MIT License. #include "telemetry/telemetry_action_tracker.h" +#include + namespace fl { -ActionTracker::ActionTracker(Action action, ITelemetry& telemetry, const std::string& user_agent, bool indirect) +ActionTracker::ActionTracker(Action action, ITelemetry& telemetry, InvocationContext context) : action_(action), telemetry_(telemetry), - user_agent_(user_agent), - indirect_(indirect), + context_(std::move(context)), start_(std::chrono::steady_clock::now()) { + // Guarantee a correlation id so this action and anything it triggers can be + // grouped, even when the caller passed a default-constructed context. + context_.EnsureCorrelationId(); } ActionTracker::~ActionTracker() { auto end = std::chrono::steady_clock::now(); auto duration_ms = std::chrono::duration_cast(end - start_).count(); - telemetry_.RecordAction(action_, status_, user_agent_, indirect_, duration_ms); + telemetry_.RecordAction(action_, status_, context_, duration_ms); if (!model_id_.empty()) { - telemetry_.RecordModelId(action_, model_id_, status_, user_agent_); + telemetry_.RecordModelId(action_, model_id_, status_, context_); } } @@ -28,7 +32,7 @@ void ActionTracker::SetStatus(ActionStatus status) { } void ActionTracker::RecordException(const std::exception& exception) { - telemetry_.RecordException(action_, exception, user_agent_); + telemetry_.RecordException(action_, exception, context_); } void ActionTracker::SetModelId(const std::string& model_id) { diff --git a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.h b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.h index e06da6130..e19971eb2 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.h @@ -14,9 +14,7 @@ namespace fl { /// Matches the C# ActionTracker (IDisposable) pattern. class ActionTracker { public: - ActionTracker(Action action, ITelemetry& telemetry, - const std::string& user_agent = "", - bool indirect = false); + ActionTracker(Action action, ITelemetry& telemetry, InvocationContext context = {}); ~ActionTracker(); // Non-copyable, non-movable @@ -32,11 +30,15 @@ class ActionTracker { /// Associate a model ID with this action. void SetModelId(const std::string& model_id); + /// This action's context, with its correlation id resolved. Use to derive a + /// child context (Context().AsIndirect()) for any caused-by action so all + /// events from one operation share a correlation id. + const InvocationContext& Context() const { return context_; } + private: Action action_; ITelemetry& telemetry_; - std::string user_agent_; - bool indirect_; + InvocationContext context_; ActionStatus status_ = ActionStatus::kFailure; std::string model_id_; std::chrono::steady_clock::time_point start_; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc index 214a0cc6d..5a5354b93 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc @@ -10,61 +10,60 @@ TelemetryLogger::TelemetryLogger(const std::string& app_name, ILogger& logger) : app_name_(app_name), logger_(logger) { } -void TelemetryLogger::RecordAction(Action action, ActionStatus status, const std::string& user_agent, - bool indirect, int64_t duration_ms) { +void TelemetryLogger::RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t duration_ms) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] Action AppName={} UserAgent={} Action={} Status={} Direct={} TimeMs={}", - app_name_, user_agent, ActionToString(action), - ActionStatusToString(status), !indirect, duration_ms)); -} - -void TelemetryLogger::RecordException(Action action, const std::exception& exception) { - logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] Error AppName={} Action={} Exception={}", - app_name_, ActionToString(action), exception.what())); + fmt::format("[Telemetry] Action AppName={} UserAgent={} CorrelationId={} Action={} Status={} " + "Direct={} TimeMs={}", + app_name_, context.user_agent, context.correlation_id, ActionToString(action), + ActionStatusToString(status), !context.indirect, duration_ms)); } void TelemetryLogger::RecordException(Action action, const std::exception& exception, - const std::string& user_agent) { + const InvocationContext& context) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] Error AppName={} UserAgent={} Action={} Exception={}", - app_name_, user_agent, ActionToString(action), exception.what())); + fmt::format("[Telemetry] Error AppName={} UserAgent={} CorrelationId={} Action={} Exception={}", + app_name_, context.user_agent, context.correlation_id, ActionToString(action), + exception.what())); } void TelemetryLogger::RecordModelUsage(const ModelUsageInfo& info) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] Model AppName={} UserAgent={} ModelId={} EP={} TimeToFirstTokenMs={} " + fmt::format("[Telemetry] Model AppName={} UserAgent={} CorrelationId={} ModelId={} EP={} " + "Stream={} Direct={} TimeToFirstTokenMs={} " "TotalTimeMs={} TotalTokens={} InputTokenCount={} NumMessages={} MemoryUsedMB={} " "CpuTimeMs={} GpuMemoryUsedMB={}", - app_name_, info.user_agent, info.model_id, info.execution_provider, + app_name_, info.user_agent, info.correlation_id, info.model_id, + info.execution_provider, info.stream, !info.indirect, info.time_to_first_token_ms, info.total_time_ms, info.total_tokens, info.input_token_count, info.num_messages, info.memory_used_mb, info.cpu_time_ms, info.gpu_memory_used_mb)); } void TelemetryLogger::RecordModelId(Action action, const std::string& model_id, - ActionStatus status, const std::string& user_agent) { + ActionStatus status, const InvocationContext& context) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] ModelId AppName={} UserAgent={} Action={} ModelId={} Status={}", - app_name_, user_agent, ActionToString(action), model_id, - ActionStatusToString(status))); + fmt::format("[Telemetry] ModelId AppName={} UserAgent={} CorrelationId={} Action={} ModelId={} " + "Status={}", + app_name_, context.user_agent, context.correlation_id, ActionToString(action), + model_id, ActionStatusToString(status))); } void TelemetryLogger::RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] EPDownloadAttempt AppName={} UserAgent={} Attempts={} " + fmt::format("[Telemetry] EPDownloadAttempt AppName={} UserAgent={} CorrelationId={} Attempts={} " "NumProviders={} Succeeded={} Failed={} Resolved={} Status={} TimeMs={}", - app_name_, info.user_agent, info.attempts, info.num_providers, + app_name_, info.user_agent, info.correlation_id, info.attempts, info.num_providers, info.succeeded, info.failed, info.resolved, ActionStatusToString(info.status), info.duration_ms)); } void TelemetryLogger::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] EPDownloadAndRegister AppName={} UserAgent={} Provider={} " + fmt::format("[Telemetry] EPDownloadAndRegister AppName={} UserAgent={} CorrelationId={} Provider={} " "InitReadyState={} DownloadReadyState={} DownloadStatus={} DownloadTimeMs={} " "RegisterReadyState={} RegisterStatus={} RegisterTimeMs={}", - app_name_, info.user_agent, info.provider_name, + app_name_, info.user_agent, info.correlation_id, info.provider_name, info.init_ready_state, info.download_ready_state, ActionStatusToString(info.download_status), info.download_duration_ms, info.register_ready_state, ActionStatusToString(info.register_status), @@ -73,11 +72,11 @@ void TelemetryLogger::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInf void TelemetryLogger::RecordDownload(const DownloadInfo& info) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] Download AppName={} UserAgent={} ModelId={} Status={} " + fmt::format("[Telemetry] Download AppName={} UserAgent={} CorrelationId={} ModelId={} Status={} " "LockWaitMs={} EnumerationMs={} DownloadMs={} TotalSizeBytes={} " "AlreadyCachedBytes={} FileCount={} SkippedFileCount={} " "DownloadWaitResult={} MaxConcurrency={}", - app_name_, info.user_agent, info.model_id, + app_name_, info.user_agent, info.correlation_id, info.model_id, ActionStatusToString(info.status), info.lock_wait_ms, info.enumeration_ms, info.download_ms, info.total_size_bytes, info.already_cached_bytes, info.file_count, info.skipped_file_count, diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.h b/sdk_v2/cpp/src/telemetry/telemetry_logger.h index 6982c8166..8679ef537 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.h @@ -21,17 +21,16 @@ class TelemetryLogger : public ITelemetry { public: TelemetryLogger(const std::string& app_name, ILogger& logger); - void RecordAction(Action action, ActionStatus status, const std::string& user_agent, - bool indirect, int64_t duration_ms) override; + void RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t duration_ms) override; - void RecordException(Action action, const std::exception& exception) override; void RecordException(Action action, const std::exception& exception, - const std::string& user_agent) override; + const InvocationContext& context) override; void RecordModelUsage(const ModelUsageInfo& info) override; void RecordModelId(Action action, const std::string& model_id, - ActionStatus status, const std::string& user_agent) override; + ActionStatus status, const InvocationContext& context) override; void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc b/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc index 826690ed1..9aba4ed5f 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc @@ -2,13 +2,13 @@ // Licensed under the MIT License. #include "telemetry/telemetry_metadata.h" +#include "telemetry/invocation_context.h" #include "telemetry/telemetry_environment.h" #include "version.h" #include #include #include -#include #ifdef _WIN32 #include @@ -21,30 +21,6 @@ namespace fl { namespace { -std::string MakeGuidV4Hex() { - // RFC 4122 v4 UUID, hex-encoded with hyphens. We use std::random_device + mt19937_64 - // because we don't need the OS UUID API — this is just a per-process correlation id, - // not a cryptographic identifier. - std::random_device rd; - std::mt19937_64 gen{(static_cast(rd()) << 32) | rd()}; - uint64_t hi = gen(); - uint64_t lo = gen(); - - // Set version (4) and variant (10xx) bits. - hi = (hi & 0xFFFFFFFFFFFF0FFFULL) | 0x0000000000004000ULL; - lo = (lo & 0x3FFFFFFFFFFFFFFFULL) | 0x8000000000000000ULL; - - char buf[37]; - std::snprintf(buf, sizeof(buf), - "%08x-%04x-%04x-%04x-%012llx", - static_cast((hi >> 32) & 0xFFFFFFFFu), - static_cast((hi >> 16) & 0xFFFFu), - static_cast(hi & 0xFFFFu), - static_cast((lo >> 48) & 0xFFFFu), - static_cast(lo & 0x0000FFFFFFFFFFFFULL)); - return std::string(buf); -} - #ifdef _WIN32 std::string GetWindowsVersion() { // GetVersionExA is deprecated and lies for unmanifested apps. The reliable diff --git a/sdk_v2/cpp/test/internal_api/null_telemetry.h b/sdk_v2/cpp/test/internal_api/null_telemetry.h index 2c70a1b66..c7c5966e8 100644 --- a/sdk_v2/cpp/test/internal_api/null_telemetry.h +++ b/sdk_v2/cpp/test/internal_api/null_telemetry.h @@ -10,16 +10,15 @@ namespace fl::test { class NullTelemetry : public ITelemetry { public: void RecordAction(Action /*action*/, ActionStatus /*status*/, - const std::string& /*user_agent*/, - bool /*indirect*/, int64_t /*duration_ms*/) override {} + const InvocationContext& /*context*/, int64_t /*duration_ms*/) override {} - void RecordException(Action /*action*/, const std::exception& /*exception*/) override {} + void RecordException(Action /*action*/, const std::exception& /*exception*/, + const InvocationContext& /*context*/) override {} void RecordModelUsage(const ModelUsageInfo& /*info*/) override {} void RecordModelId(Action /*action*/, const std::string& /*model_id*/, - ActionStatus /*status*/ = ActionStatus::kSuccess, - const std::string& /*user_agent*/ = "") override {} + ActionStatus /*status*/, const InvocationContext& /*context*/) override {} void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& /*info*/) override {} diff --git a/sdk_v2/cpp/test/internal_api/telemetry_test.cc b/sdk_v2/cpp/test/internal_api/telemetry_test.cc index 42fc068f6..217fe22aa 100644 --- a/sdk_v2/cpp/test/internal_api/telemetry_test.cc +++ b/sdk_v2/cpp/test/internal_api/telemetry_test.cc @@ -42,12 +42,13 @@ struct ActionCall { class RecordingTelemetry : public ITelemetry { public: void RecordAction(Action action, ActionStatus status, - const std::string& user_agent, - bool indirect, int64_t duration_ms) override { - action_calls.push_back(ActionCall{action, status, user_agent, indirect, duration_ms}); + const InvocationContext& context, int64_t duration_ms) override { + action_calls.push_back( + ActionCall{action, status, context.user_agent, context.indirect, duration_ms}); } - void RecordException(Action action, const std::exception& exception) override { + void RecordException(Action action, const std::exception& exception, + const InvocationContext& /*context*/) override { exception_calls.emplace_back(action, exception.what()); } @@ -56,8 +57,7 @@ class RecordingTelemetry : public ITelemetry { } void RecordModelId(Action action, const std::string& model_id, - ActionStatus /*status*/ = ActionStatus::kSuccess, - const std::string& /*user_agent*/ = "") override { + ActionStatus /*status*/, const InvocationContext& /*context*/) override { model_id_calls.emplace_back(action, model_id); } @@ -88,14 +88,15 @@ TEST(TelemetryLoggerTest, RecordActionIncludesConcreteFields) { RecordingLogger logger; TelemetryLogger telemetry("foundry-local", logger); - telemetry.RecordAction(Action::kModelDownload, ActionStatus::kSuccess, - "cli/1.0", false, 1234); + telemetry.RecordAction(Action::kModelFileDownload, ActionStatus::kSuccess, + InvocationContext{"cli/1.0", "corr-1", false}, 1234); ASSERT_EQ(logger.entries.size(), 1u); EXPECT_EQ(logger.entries[0].level, LogLevel::Debug); EXPECT_NE(logger.entries[0].message.find("AppName=foundry-local"), std::string::npos); EXPECT_NE(logger.entries[0].message.find("UserAgent=cli/1.0"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Action=ModelDownload"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("CorrelationId=corr-1"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Action=ModelFileDownload"), std::string::npos); EXPECT_NE(logger.entries[0].message.find("Status=Success"), std::string::npos); EXPECT_NE(logger.entries[0].message.find("Direct=true"), std::string::npos); EXPECT_NE(logger.entries[0].message.find("TimeMs=1234"), std::string::npos); @@ -105,7 +106,7 @@ TEST(TelemetryLoggerTest, RecordExceptionAndModelEventsIncludeSpecificValues) { RecordingLogger logger; TelemetryLogger telemetry("foundry-local", logger); - telemetry.RecordException(Action::kModelLoad, std::runtime_error("config missing")); + telemetry.RecordException(Action::kModelLoad, std::runtime_error("config missing"), InvocationContext{}); ModelUsageInfo usage; usage.model_id = "phi-3-mini"; @@ -116,7 +117,8 @@ TEST(TelemetryLoggerTest, RecordExceptionAndModelEventsIncludeSpecificValues) { usage.total_time_ms = 250; telemetry.RecordModelUsage(usage); - telemetry.RecordModelId(Action::kModelLoad, "phi-3-mini", ActionStatus::kSuccess, "cli/1.0"); + telemetry.RecordModelId(Action::kModelLoad, "phi-3-mini", ActionStatus::kSuccess, + InvocationContext{"cli/1.0", "", false}); ASSERT_EQ(logger.entries.size(), 3u); EXPECT_NE(logger.entries[0].message.find("Action=ModelLoad"), std::string::npos); @@ -136,11 +138,11 @@ TEST(ActionTrackerTest, DestructorRecordsFailureByDefaultWithoutModelId) { RecordingTelemetry telemetry; { - ActionTracker tracker(Action::kModelDelete, telemetry, "cli/2.0", true); + ActionTracker tracker(Action::kModelFileDownload, telemetry, InvocationContext{"cli/2.0", "", true}); } ASSERT_EQ(telemetry.action_calls.size(), 1u); - EXPECT_EQ(telemetry.action_calls[0].action, Action::kModelDelete); + EXPECT_EQ(telemetry.action_calls[0].action, Action::kModelFileDownload); EXPECT_EQ(telemetry.action_calls[0].status, ActionStatus::kFailure); EXPECT_EQ(telemetry.action_calls[0].user_agent, "cli/2.0"); EXPECT_TRUE(telemetry.action_calls[0].indirect); @@ -152,7 +154,7 @@ TEST(ActionTrackerTest, RecordsExceptionSuccessAndModelId) { RecordingTelemetry telemetry; { - ActionTracker tracker(Action::kModelLoad, telemetry, "cli/3.0", false); + ActionTracker tracker(Action::kModelLoad, telemetry, InvocationContext{"cli/3.0", "", false}); tracker.RecordException(std::runtime_error("failed to load")); tracker.SetModelId("phi-3-mini"); tracker.SetStatus(ActionStatus::kSuccess); From 139e97cfffcb13dbef5c32890debe0dd2294d28f Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 20 Jun 2026 02:41:35 -0500 Subject: [PATCH 18/77] sdk_v2/cpp/telemetry: emit the Model event + wire the chat route end-to-end - Session::ProcessRequest now emits the previously-dead Model (kModelInference) event once per inference with model id, tokens, duration, stream flag and the caller's correlation id / indirect flag. Adds Session::SetRequestContext so an HTTP route can stage an indirect child context (shared correlation id), and a virtual ExecutionProvider() hook for the EP field. - Chat completions handler: derive a Direct route context from the User-Agent header; map 4xx early-returns (empty body, invalid JSON, model not found/loaded) to kClientError instead of the default kFailure; emit kSessionCreate (indirect) on the HTTP path; stage the indirect session context. - Streaming fix: the route action is no longer recorded (as a premature success with ~0ms) when the handler returns. The route ActionTracker is moved into the streaming thread and records on completion with the real duration and terminal status, so mid-stream failures surface as failures instead of vanishing. Builds clean (/W4 /WX); telemetry/webservice/sse tests pass. --- sdk_v2/cpp/src/inferencing/session/session.cc | 28 +++++++++- sdk_v2/cpp/src/inferencing/session/session.h | 16 ++++++ .../src/service/chat_completions_handler.cc | 53 +++++++++++++++---- .../src/service/chat_completions_handler.h | 9 +++- sdk_v2/cpp/src/service/handler_utils.h | 10 ++++ 5 files changed, 102 insertions(+), 14 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/session/session.cc b/sdk_v2/cpp/src/inferencing/session/session.cc index e8f90ee69..b9afd6e83 100644 --- a/sdk_v2/cpp/src/inferencing/session/session.cc +++ b/sdk_v2/cpp/src/inferencing/session/session.cc @@ -16,6 +16,7 @@ #include +#include #include namespace fl { @@ -101,9 +102,17 @@ void Session::ProcessRequest(const Request& request, Response& response) { lock.lock(); } - ActionTracker tracker(Action::kSessionProcessRequest, telemetry_); + // Use the context the caller staged (an HTTP route stages an indirect child + // with the route's correlation id); otherwise mint a direct context per call + // for direct SDK use. + InvocationContext context = request_context_ ? *request_context_ : InvocationContext::Direct(); + context.EnsureCorrelationId(); + const bool streaming = static_cast(callback_fn_); + + ActionTracker tracker(Action::kSessionProcessRequest, telemetry_, context); tracker.SetModelId(CatalogModel().Id()); + const auto start = std::chrono::steady_clock::now(); try { ProcessRequestImpl(request, response); @@ -112,6 +121,23 @@ void Session::ProcessRequest(const Request& request, Response& response) { tracker.RecordException(ex); throw; } + + // Per-inference Model event — emitted on success with whatever metrics this run + // produced, sharing the action's correlation id and indirect flag. TTFT and + // memory are not surfaced by the generators yet and stay at their unset values. + ModelUsageInfo usage; + usage.model_id = CatalogModel().Id(); + usage.execution_provider = ExecutionProvider(); + usage.user_agent = context.user_agent; + usage.correlation_id = context.correlation_id; + usage.indirect = context.indirect; + usage.stream = streaming; + usage.total_time_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + usage.total_tokens = static_cast(response.usage.total_tokens); + usage.input_token_count = static_cast(response.usage.prompt_tokens); + telemetry_.RecordModelUsage(usage); } } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/session/session.h b/sdk_v2/cpp/src/inferencing/session/session.h index f3ec0e5f1..a92556c16 100644 --- a/sdk_v2/cpp/src/inferencing/session/session.h +++ b/sdk_v2/cpp/src/inferencing/session/session.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,7 @@ #include "inferencing/session/request.h" #include "inferencing/session/response.h" #include "inferencing/session/types.h" +#include "telemetry/invocation_context.h" #include "util/key_value_pairs.h" namespace fl { @@ -100,6 +102,15 @@ class Session { callback_user_data_ = user_data; } + /// Telemetry context for the next ProcessRequest. HTTP handlers set this to an + /// indirect child of the route's context so the kSessionProcessRequest action + /// and the per-inference Model event are marked indirect and share the route's + /// correlation id. When unset (direct SDK use), ProcessRequest mints its own + /// direct context per call. + void SetRequestContext(InvocationContext context) { + request_context_ = std::move(context); + } + protected: Session(const fl::Model& catalog_model, ILogger& logger, ITelemetry& telemetry, bool allow_concurrent_requests = false); @@ -131,6 +142,10 @@ class Session { /// Requests are serialized if the derived class does not opt into concurrency via allow_concurrent_requests_. virtual void ProcessRequestImpl(const Request& request, Response& response) = 0; + /// Execution provider the loaded model runs on (e.g. "CPU", "CUDA"). Surfaced + /// on the per-inference Model telemetry event. Empty when not known. + virtual std::string ExecutionProvider() const { return {}; } + /// Create a per-request callback handler. Returns nullptr if no callback is set. /// The handler is owned by the caller (unique_ptr) and drains+joins on destruction. std::unique_ptr CreateCallbackHandler(const Request& request) { @@ -151,6 +166,7 @@ class Session { KeyValuePairs session_options_; StreamingCallbackFn callback_fn_; void* callback_user_data_ = nullptr; + std::optional request_context_; const bool allow_concurrent_requests_; mutable std::unique_ptr request_mutex_ = std::make_unique(); }; diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.cc b/sdk_v2/cpp/src/service/chat_completions_handler.cc index fcf13fb29..3475c82ac 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.cc +++ b/sdk_v2/cpp/src/service/chat_completions_handler.cc @@ -104,10 +104,12 @@ void ChatCompletionsHandler::BuildOpenAIJsonRequest(const std::string& body, con std::shared_ptr ChatCompletionsHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIChatCompletions, ctx_.telemetry); + auto route_ctx = InvocationContext::Direct(GetUserAgent(request)); + auto tracker = std::make_unique(Action::kOpenAIChatCompletions, ctx_.telemetry, route_ctx); auto body_str = request->readBodyToString(); if (!body_str || body_str->empty()) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Empty request body"); } @@ -117,6 +119,7 @@ std::shared_ptr ChatCompletionsHandler::ha // telemetry. How much do we care about that? Is it worth the double parsing? ChatCompletionRequest req; if (auto err = ParseAndValidateRequest(body_str->c_str(), req)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } @@ -130,10 +133,11 @@ std::shared_ptr ChatCompletionsHandler::ha Model* model = nullptr; GenAIModelInstance* loaded = nullptr; if (auto err = ResolveModel(model_name, model, loaded)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } - tracker.SetModelId(model_name); + tracker->SetModelId(model_name); // 3. Build an OPENAI_JSON-tagged TEXT request item. Request session_request; @@ -145,21 +149,33 @@ std::shared_ptr ChatCompletionsHandler::ha include_usage_in_stream = req.stream_options->include_usage; } + // The session and the inference it drives happen as a consequence of this + // route, so they are indirect and reuse the route's correlation id. + auto session_ctx = route_ctx.AsIndirect(); + // 6. Run inference via ChatSession try { ChatSession session(*model, *loaded, ctx_.logger, ctx_.telemetry); + { + ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); + create_tracker.SetModelId(model_name); + create_tracker.SetStatus(ActionStatus::kSuccess); + } + session.SetRequestContext(session_ctx); if (stream) { - tracker.SetStatus(ActionStatus::kSuccess); - return HandleStreaming(std::move(session), std::move(session_request), include_usage_in_stream); + // The route action is recorded by the streaming thread when the stream + // finishes, so don't set its status here — move the tracker into the thread. + return HandleStreaming(std::move(session), std::move(session_request), include_usage_in_stream, + std::move(tracker)); } else { SessionRegistration reg(ctx_.session_manager, session); auto response = HandleNonStreaming(session, session_request); - tracker.SetStatus(ActionStatus::kSuccess); + tracker->SetStatus(ActionStatus::kSuccess); return response; } } catch (const std::exception& ex) { - tracker.RecordException(ex); + tracker->RecordException(ex); ctx_.logger.Log(LogLevel::Error, fmt::format("Chat completion inference failed: {}", ex.what())); return ErrorResponse(Status::CODE_500, "Inference failed", ex.what()); } @@ -189,15 +205,17 @@ std::shared_ptr ChatCompletionsHandler::Ha } std::shared_ptr ChatCompletionsHandler::HandleStreaming( - ChatSession&& session, Request session_request, bool include_usage) { + ChatSession&& session, Request session_request, bool include_usage, + std::unique_ptr route_tracker) { auto body = std::make_shared(); auto body_ptr = body; auto& logger = ctx_.logger; - auto& tracker = ctx_.thread_tracker; + auto& thread_tracker = ctx_.thread_tracker; std::thread streaming_thread([bg_session = std::move(session), body_ptr, &logger, req = std::move(session_request), - include_usage, &tracker, + include_usage, &thread_tracker, + route_tracker = std::move(route_tracker), &session_manager = ctx_.session_manager]() mutable { SessionRegistration reg(session_manager, bg_session); @@ -248,18 +266,31 @@ std::shared_ptr ChatCompletionsHandler::Ha } body_ptr->Push("data: [DONE]\n\n"); + + // Inference streamed to completion — record the route action as a success. + if (route_tracker) { + route_tracker->SetStatus(ActionStatus::kSuccess); + } } catch (const std::exception& ex) { nlohmann::json err = { {"error", {{"message", ex.what()}, {"type", "server_error"}, {"param", nullptr}, {"code", nullptr}}}, }; body_ptr->Push("data: " + err.dump() + "\n\n"); + + // Mid-stream failure: record the exception. The route action keeps its + // default kFailure status. + if (route_tracker) { + route_tracker->RecordException(ex); + } } body_ptr->Finish(); - tracker.Remove(std::this_thread::get_id()); + // route_tracker is destroyed with this closure once the thread completes, + // recording the route action with the full streaming duration and final status. + thread_tracker.Remove(std::this_thread::get_id()); }); - tracker.Track(std::move(streaming_thread)); + thread_tracker.Track(std::move(streaming_thread)); auto response = oatpp::web::protocol::http::outgoing::Response::createShared( Status::CODE_200, body); diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.h b/sdk_v2/cpp/src/service/chat_completions_handler.h index ed483c4d0..c7225bf77 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.h +++ b/sdk_v2/cpp/src/service/chat_completions_handler.h @@ -16,6 +16,7 @@ struct ChatCompletionRequest; class ChatSession; class Model; class GenAIModelInstance; +class ActionTracker; struct Request; // ======================================================================== @@ -47,9 +48,13 @@ class ChatCompletionsHandler : public HttpRequestHandler { /// Non-streaming inference — processes request, extracts the OPENAI_JSON-tagged TEXT response. std::shared_ptr HandleNonStreaming(ChatSession& session, Request& session_request); - /// Streaming inference — runs inference in background thread with SSE output. + /// Streaming inference — runs inference in a background thread with SSE output. + /// Takes ownership of the route ActionTracker so the route action is recorded + /// when the stream actually finishes (real duration + terminal status), + /// instead of when this handler returns. std::shared_ptr HandleStreaming(ChatSession&& session, Request session_request, - bool include_usage); + bool include_usage, + std::unique_ptr route_tracker); ServiceContext& ctx_; }; diff --git a/sdk_v2/cpp/src/service/handler_utils.h b/sdk_v2/cpp/src/service/handler_utils.h index e25528fa3..7216892ce 100644 --- a/sdk_v2/cpp/src/service/handler_utils.h +++ b/sdk_v2/cpp/src/service/handler_utils.h @@ -62,6 +62,16 @@ inline std::string GenerateCompletionId(const std::string& prefix) { return ss.str(); } +/// Extract the User-Agent header from an incoming request ("" if absent), for +/// attribution on the telemetry events the request drives. +inline std::string GetUserAgent(const std::shared_ptr& request) { + if (!request) { + return {}; + } + auto ua = request->getHeader("User-Agent"); + return ua ? *ua : std::string{}; +} + // ======================================================================== // SSE stream body — feeds token-by-token SSE events to oatpp's chunked // transfer encoding. A producer thread pushes formatted SSE strings into From 4668b6628e64ade9ce8dd0f3a34f5e06686a2644 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 20 Jun 2026 03:10:41 -0500 Subject: [PATCH 19/77] sdk_v2/cpp/telemetry: wire all service routes (indirect, kClientError, sampling) Extends the chat-route pattern to every handler: - Streaming completion fix also applied to the audio transcriptions route (its ActionTracker moved into the streaming thread). - Embeddings, audio, responses (create + get/list/delete/input_items), and the model load/unload/list/retrieve routes now derive a Direct context from the User-Agent header, map 4xx early-returns to kClientError, and (for the inference routes) emit kSessionCreate and stage an indirect session context. - Instrument GET /models/loaded (kModelList), previously untracked. - Sample GET /status: emit a kServiceStatus action at most once per hour per process so the orchestrator heartbeat doesn't dominate volume. Builds clean (/W4 /WX); telemetry/webservice/sse tests pass. --- .../service/audio_transcriptions_handler.cc | 46 +++++++++++---- .../service/audio_transcriptions_handler.h | 8 ++- sdk_v2/cpp/src/service/embeddings_handler.cc | 17 +++++- sdk_v2/cpp/src/service/models_handlers.cc | 27 +++++++-- sdk_v2/cpp/src/service/responses_handler.cc | 56 +++++++++++++++---- sdk_v2/cpp/src/service/responses_handler.h | 4 +- sdk_v2/cpp/src/service/web_service.cc | 31 +++++++++- sdk_v2/cpp/src/telemetry/telemetry.cc | 2 + sdk_v2/cpp/src/telemetry/telemetry.h | 1 + 9 files changed, 159 insertions(+), 33 deletions(-) diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc index e7b2ac642..c62468500 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc @@ -87,16 +87,19 @@ void AudioTranscriptionsHandler::BuildOpenAIJsonRequest(const std::string& body, std::shared_ptr AudioTranscriptionsHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIAudioTranscribe, ctx_.telemetry); + auto route_ctx = InvocationContext::Direct(GetUserAgent(request)); + auto tracker = std::make_unique(Action::kOpenAIAudioTranscribe, ctx_.telemetry, route_ctx); auto body_str = request->readBodyToString(); if (!body_str || body_str->empty()) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Empty request body"); } // 1. Parse & validate AudioTranscriptionRequest req; if (auto err = ParseAndValidateRequest(body_str->c_str(), req)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } @@ -108,9 +111,11 @@ std::shared_ptr AudioTranscriptionsHandler // 2. Validate file path try { if (!std::filesystem::exists(req.filename)) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Audio file not found", "'" + req.filename + "'"); } } catch (const std::filesystem::filesystem_error& ex) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Invalid file path", ex.what()); } @@ -119,30 +124,40 @@ std::shared_ptr AudioTranscriptionsHandler Model* model = nullptr; GenAIModelInstance* loaded = nullptr; if (auto err = ResolveModel(model_name, model, loaded)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } - tracker.SetModelId(model_name); + tracker->SetModelId(model_name); // 4. Build an OPENAI_JSON-tagged TEXT request item — pass original body directly Request session_request; BuildOpenAIJsonRequest(body_str->c_str(), session_request); + // The session and the inference it drives are indirect children of this route. + auto session_ctx = route_ctx.AsIndirect(); + // 5. Dispatch to streaming or non-streaming try { AudioSession session(*model, *loaded, ctx_.logger, ctx_.telemetry); + { + ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); + create_tracker.SetModelId(model_name); + create_tracker.SetStatus(ActionStatus::kSuccess); + } + session.SetRequestContext(session_ctx); if (stream) { - tracker.SetStatus(ActionStatus::kSuccess); - return HandleStreaming(std::move(session), std::move(session_request)); + // The route action is recorded by the streaming thread on completion. + return HandleStreaming(std::move(session), std::move(session_request), std::move(tracker)); } else { SessionRegistration reg(ctx_.session_manager, session); auto response = HandleNonStreaming(session, session_request); - tracker.SetStatus(ActionStatus::kSuccess); + tracker->SetStatus(ActionStatus::kSuccess); return response; } } catch (const std::exception& ex) { - tracker.RecordException(ex); + tracker->RecordException(ex); ctx_.logger.Log(LogLevel::Error, fmt::format("Audio transcription inference failed: {}", ex.what())); return ErrorResponse(Status::CODE_500, "Inference failed", ex.what()); } catch (...) { @@ -177,14 +192,15 @@ std::shared_ptr AudioTranscriptionsHandler } std::shared_ptr AudioTranscriptionsHandler::HandleStreaming( - AudioSession&& session, Request session_request) { + AudioSession&& session, Request session_request, std::unique_ptr route_tracker) { auto body = std::make_shared(); auto body_ptr = body; auto& logger = ctx_.logger; - auto& tracker = ctx_.thread_tracker; + auto& thread_tracker = ctx_.thread_tracker; std::thread streaming_thread([bg_session = std::move(session), body_ptr, &logger, - req = std::move(session_request), &tracker, + req = std::move(session_request), &thread_tracker, + route_tracker = std::move(route_tracker), &session_manager = ctx_.session_manager]() mutable { SessionRegistration reg(session_manager, bg_session); @@ -216,19 +232,27 @@ std::shared_ptr AudioTranscriptionsHandler // Send terminal event body_ptr->Push("data: [DONE]\n\n"); + + if (route_tracker) { + route_tracker->SetStatus(ActionStatus::kSuccess); + } } catch (const std::exception& ex) { logger.Log(LogLevel::Error, fmt::format("Audio streaming transcription failed: {}", ex.what())); // Push error to stream so client doesn't hang nlohmann::json error = {{"error", {{"message", ex.what()}}}}; body_ptr->Push("data: " + error.dump() + "\n\n"); + + if (route_tracker) { + route_tracker->RecordException(ex); + } } body_ptr->Finish(); - tracker.Remove(std::this_thread::get_id()); + thread_tracker.Remove(std::this_thread::get_id()); }); - tracker.Track(std::move(streaming_thread)); + thread_tracker.Track(std::move(streaming_thread)); auto response = oatpp::web::protocol::http::outgoing::Response::createShared(Status::CODE_200, body); response->putHeader("Content-Type", "text/event-stream"); diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.h b/sdk_v2/cpp/src/service/audio_transcriptions_handler.h index 5dffeaa90..1f68c058c 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.h +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.h @@ -16,6 +16,7 @@ struct AudioTranscriptionRequest; class AudioSession; class Model; class GenAIModelInstance; +class ActionTracker; struct Request; // ======================================================================== @@ -43,8 +44,11 @@ class AudioTranscriptionsHandler : public HttpRequestHandler { /// Non-streaming inference — processes request, extracts the OPENAI_JSON-tagged TEXT response. std::shared_ptr HandleNonStreaming(AudioSession& session, Request& session_request); - /// Streaming inference — runs inference in background thread with SSE output. - std::shared_ptr HandleStreaming(AudioSession&& session, Request session_request); + /// Streaming inference — runs inference in a background thread with SSE output. + /// Takes ownership of the route ActionTracker so the route action records when + /// the stream finishes (real duration + terminal status). + std::shared_ptr HandleStreaming(AudioSession&& session, Request session_request, + std::unique_ptr route_tracker); ServiceContext& ctx_; }; diff --git a/sdk_v2/cpp/src/service/embeddings_handler.cc b/sdk_v2/cpp/src/service/embeddings_handler.cc index b5f734b07..526e9cc51 100644 --- a/sdk_v2/cpp/src/service/embeddings_handler.cc +++ b/sdk_v2/cpp/src/service/embeddings_handler.cc @@ -28,10 +28,12 @@ class EmbeddingsHandler : public HttpRequestHandler { explicit EmbeddingsHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr handle(const std::shared_ptr& request) override { - ActionTracker tracker(Action::kOpenAIEmbeddings, ctx_.telemetry); + auto route_ctx = InvocationContext::Direct(GetUserAgent(request)); + ActionTracker tracker(Action::kOpenAIEmbeddings, ctx_.telemetry, route_ctx); auto body_str = request->readBodyToString(); if (!body_str || body_str->empty()) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Empty request body"); } @@ -41,6 +43,7 @@ class EmbeddingsHandler : public HttpRequestHandler { auto j = nlohmann::json::parse(*body_str); req = j.get(); } catch (const nlohmann::json::exception& ex) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Invalid JSON", ex.what()); } @@ -53,6 +56,7 @@ class EmbeddingsHandler : public HttpRequestHandler { } if (inputs.empty()) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "\"input\" must not be empty"); } @@ -60,19 +64,30 @@ class EmbeddingsHandler : public HttpRequestHandler { std::string model_name = req.model; auto* model = ctx_.catalog.GetModelVariant(model_name); if (!model) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_404, "Model not found", model_name); } auto* loaded = ctx_.model_load_manager.GetLoadedModel(model->Id()); if (!loaded) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Model not loaded", model_name); } tracker.SetModelId(model_name); + // The session and the inference it drives are indirect children of this route. + auto session_ctx = route_ctx.AsIndirect(); + // 4. Create session and process each input try { EmbeddingsSession session(*model, *loaded, ctx_.logger, ctx_.telemetry); + { + ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); + create_tracker.SetModelId(model_name); + create_tracker.SetStatus(ActionStatus::kSuccess); + } + session.SetRequestContext(session_ctx); SessionRegistration reg(ctx_.session_manager, session); Request session_request; diff --git a/sdk_v2/cpp/src/service/models_handlers.cc b/sdk_v2/cpp/src/service/models_handlers.cc index 93988d3b6..b2bb65f51 100644 --- a/sdk_v2/cpp/src/service/models_handlers.cc +++ b/sdk_v2/cpp/src/service/models_handlers.cc @@ -24,7 +24,10 @@ class ListLoadedModelsHandler : public HttpRequestHandler { public: explicit ListLoadedModelsHandler(ServiceContext& ctx) : ctx_(ctx) {} - std::shared_ptr handle(const std::shared_ptr&) override { + std::shared_ptr handle(const std::shared_ptr& request) override { + ActionTracker tracker(Action::kModelList, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); + auto loaded = ctx_.catalog.GetLoadedModels(); nlohmann::json names = nlohmann::json::array(); @@ -32,6 +35,7 @@ class ListLoadedModelsHandler : public HttpRequestHandler { names.push_back(model->Id()); } + tracker.SetStatus(ActionStatus::kSuccess); return JsonResponse(Status::CODE_200, names); } @@ -48,10 +52,12 @@ class LoadModelHandler : public HttpRequestHandler { explicit LoadModelHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr handle(const std::shared_ptr& request) override { - ActionTracker tracker(Action::kModelLoad, ctx_.telemetry); + ActionTracker tracker(Action::kModelLoad, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto name_raw = request->getPathVariable("name"); if (!name_raw) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing model name"); } @@ -59,6 +65,7 @@ class LoadModelHandler : public HttpRequestHandler { auto* model = ctx_.catalog.GetModel(name); if (!model) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + name + "'"); } @@ -69,6 +76,7 @@ class LoadModelHandler : public HttpRequestHandler { } if (!model->IsCached()) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Model not cached", "Model must be downloaded before loading"); } @@ -101,10 +109,12 @@ class UnloadModelHandler : public HttpRequestHandler { explicit UnloadModelHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr handle(const std::shared_ptr& request) override { - ActionTracker tracker(Action::kModelUnload, ctx_.telemetry); + ActionTracker tracker(Action::kModelUnload, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto name_raw = request->getPathVariable("name"); if (!name_raw) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing model name"); } @@ -112,6 +122,7 @@ class UnloadModelHandler : public HttpRequestHandler { auto* model = ctx_.catalog.GetModel(name); if (!model) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + name + "'"); } @@ -149,8 +160,9 @@ class OpenAIListModelsHandler : public HttpRequestHandler { public: explicit OpenAIListModelsHandler(ServiceContext& ctx) : ctx_(ctx) {} - std::shared_ptr handle(const std::shared_ptr&) override { - ActionTracker tracker(Action::kOpenAIModelList, ctx_.telemetry); + std::shared_ptr handle(const std::shared_ptr& request) override { + ActionTracker tracker(Action::kOpenAIModelList, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto models = ctx_.catalog.ListModels(); nlohmann::json data = nlohmann::json::array(); @@ -203,10 +215,12 @@ class OpenAIRetrieveModelHandler : public HttpRequestHandler { explicit OpenAIRetrieveModelHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr handle(const std::shared_ptr& request) override { - ActionTracker tracker(Action::kOpenAIModelRetrieve, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIModelRetrieve, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto name_raw = request->getPathVariable("name"); if (!name_raw) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing model name"); } @@ -214,6 +228,7 @@ class OpenAIRetrieveModelHandler : public HttpRequestHandler { auto* model = ctx_.catalog.GetModelVariant(name); if (!model) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + name + "'"); } diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index 0fdf6d8f7..b343e443b 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -130,10 +130,12 @@ void ResponsesHandler::LoadPreviousContext(const ResponseCreateParams& params, std::shared_ptr ResponsesHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesCreate, ctx_.telemetry); + auto route_ctx = InvocationContext::Direct(GetUserAgent(request)); + auto tracker = std::make_unique(Action::kOpenAIResponsesCreate, ctx_.telemetry, route_ctx); auto body_str = request->readBodyToString(); if (!body_str || body_str->empty()) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Empty request body"); } @@ -141,6 +143,7 @@ std::shared_ptr ResponsesHandler::handle( nlohmann::json req_json; ResponseCreateParams params; if (auto err = ParseAndValidateRequest(body_str->c_str(), req_json, params)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } @@ -155,10 +158,11 @@ std::shared_ptr ResponsesHandler::handle( Model* model = nullptr; GenAIModelInstance* loaded = nullptr; if (auto err = ResolveModel(model_name, model, loaded)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } - tracker.SetModelId(model_name); + tracker->SetModelId(model_name); // 3. Load previous context if chaining via previous_response_id const nlohmann::json* previous_input = nullptr; @@ -193,10 +197,18 @@ std::shared_ptr ResponsesHandler::handle( // happens here in the handler that owns the session lifetime. std::string tools_json = ResponseConverter::ExtractResponsesToolDefinitions(params, session_request); + // The session and the inference it drives happen as a consequence of this + // route, so they are indirect and reuse the route's correlation id. + auto session_ctx = route_ctx.AsIndirect(); + try { if (!session) { + ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); + create_tracker.SetModelId(model_name); session = std::make_unique(*model, *loaded, ctx_.logger, ctx_.telemetry); + create_tracker.SetStatus(ActionStatus::kSuccess); } + session->SetRequestContext(session_ctx); // Sessions can be reused via previous_response_id; clear any stale tool defs from the prior // turn before applying this request's tools so the request stays self-contained. @@ -208,22 +220,23 @@ std::shared_ptr ResponsesHandler::handle( if (params.stream) { ctx_.logger.Log(LogLevel::Debug, fmt::format("Creating streaming response {} for model {}", response_id, model_name)); - tracker.SetStatus(ActionStatus::kSuccess); + // The route action is recorded by the streaming thread when the stream + // finishes — move the tracker in rather than marking success now. return HandleStreaming(std::move(session), std::move(session_request), model_name, - response_id, created_at, params, req_json); + response_id, created_at, params, req_json, std::move(tracker)); } else { ctx_.logger.Log(LogLevel::Debug, fmt::format("Creating response {} for model {}", response_id, model_name)); auto response = HandleNonStreaming(std::move(session), session_request, model_name, response_id, created_at, params, req_json); - tracker.SetStatus(ActionStatus::kSuccess); + tracker->SetStatus(ActionStatus::kSuccess); return response; } } catch (const std::exception& ex) { - tracker.RecordException(ex); + tracker->RecordException(ex); ctx_.logger.Log(LogLevel::Error, fmt::format("Response {} failed: {}", response_id, ex.what())); @@ -280,7 +293,7 @@ std::shared_ptr ResponsesHandler::HandleSt std::unique_ptr session, Request session_request, const std::string& model_name, const std::string& response_id, int64_t created_at, const ResponseCreateParams& params, - const nlohmann::json& req_json) { + const nlohmann::json& req_json, std::unique_ptr route_tracker) { auto body = std::make_shared(); auto initial_response = ResponseConverter::BuildInitialResponseObject(response_id, created_at, model_name, params); @@ -326,6 +339,7 @@ std::shared_ptr ResponsesHandler::HandleSt should_store, &store, req_copy = std::move(req_copy), params_copy = std::move(params_copy), + route_tracker = std::move(route_tracker), &tracker]() mutable { SessionRegistration reg(session_manager, *session); @@ -560,6 +574,11 @@ std::shared_ptr ResponsesHandler::HandleSt session_manager.CheckIn(response_id, std::move(session)); } + // Streamed to completion — record the route action as a success. + if (route_tracker) { + route_tracker->SetStatus(ActionStatus::kSuccess); + } + } catch (const std::exception& ex) { logger.Log(LogLevel::Error, fmt::format("Response {} failed during streaming: {}", response_id, ex.what())); @@ -572,6 +591,11 @@ std::shared_ptr ResponsesHandler::HandleSt failed.sequence_number = seq++; failed.response = error_response; body_ptr->Push("event: response.failed\ndata: " + nlohmann::json(failed).dump() + "\n\n"); + + // Mid-stream failure: record the exception; the route action keeps kFailure. + if (route_tracker) { + route_tracker->RecordException(ex); + } } // Terminal event per spec @@ -598,10 +622,12 @@ GetResponseHandler::GetResponseHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr GetResponseHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesGet, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIResponsesGet, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto id = request->getPathVariable("id"); if (!id) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing response ID"); } @@ -609,6 +635,7 @@ std::shared_ptr GetResponseHandler::handle auto response = ctx_.response_store.Get(id->c_str()); if (!response) { + tracker.SetStatus(ActionStatus::kClientError); nlohmann::json error_body = { {"error", { {"message", "The response '" + std::string(id->c_str()) + "' does not exist."}, @@ -633,7 +660,8 @@ ListResponsesHandler::ListResponsesHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr ListResponsesHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesList, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIResponsesList, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); // Parse query parameters auto limit_str = request->getQueryParameter("limit", "20"); @@ -688,10 +716,12 @@ DeleteResponseHandler::DeleteResponseHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr DeleteResponseHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesDelete, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIResponsesDelete, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto id = request->getPathVariable("id"); if (!id) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing response ID"); } @@ -699,6 +729,7 @@ std::shared_ptr DeleteResponseHandler::han bool deleted = ctx_.response_store.Delete(id->c_str()); if (!deleted) { + tracker.SetStatus(ActionStatus::kClientError); nlohmann::json error_body = { {"error", { {"message", "The response '" + std::string(id->c_str()) + "' does not exist."}, @@ -733,10 +764,12 @@ GetInputItemsHandler::GetInputItemsHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr GetInputItemsHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesGetInputItems, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIResponsesGetInputItems, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto id = request->getPathVariable("id"); if (!id) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing response ID"); } @@ -745,6 +778,7 @@ std::shared_ptr GetInputItemsHandler::hand // Check that response exists auto response = ctx_.response_store.Get(id->c_str()); if (!response) { + tracker.SetStatus(ActionStatus::kClientError); nlohmann::json error_body = { {"error", { {"message", "The response '" + std::string(id->c_str()) + "' does not exist."}, diff --git a/sdk_v2/cpp/src/service/responses_handler.h b/sdk_v2/cpp/src/service/responses_handler.h index da9f5f691..e455d6203 100644 --- a/sdk_v2/cpp/src/service/responses_handler.h +++ b/sdk_v2/cpp/src/service/responses_handler.h @@ -16,6 +16,7 @@ struct Request; class ChatSession; class Model; class GenAIModelInstance; +class ActionTracker; namespace responses { struct ResponseCreateParams; @@ -65,7 +66,8 @@ class ResponsesHandler : public HttpRequestHandler { const std::string& model_name, const std::string& response_id, int64_t created_at, const responses::ResponseCreateParams& params, - const nlohmann::json& req_json); + const nlohmann::json& req_json, + std::unique_ptr route_tracker); ServiceContext& ctx_; }; diff --git a/sdk_v2/cpp/src/service/web_service.cc b/sdk_v2/cpp/src/service/web_service.cc index 683ff7817..5a6e014b7 100644 --- a/sdk_v2/cpp/src/service/web_service.cc +++ b/sdk_v2/cpp/src/service/web_service.cc @@ -13,6 +13,7 @@ #include "service/handler_utils.h" #include "service/models_handlers.h" #include "service/responses_handler.h" +#include "telemetry/telemetry_action_tracker.h" #include #include @@ -23,6 +24,7 @@ #include #include +#include #include #include @@ -134,7 +136,9 @@ class StatusHandler : public HttpRequestHandler { public: explicit StatusHandler(ServiceContext& ctx) : ctx_(ctx) {} - std::shared_ptr handle(const std::shared_ptr&) override { + std::shared_ptr handle(const std::shared_ptr& request) override { + MaybeRecordStatusTelemetry(request); + nlohmann::json body = { {"modelCachePath", ctx_.model_cache_dir}, {"endpoints", ctx_.bound_urls}, @@ -149,7 +153,32 @@ class StatusHandler : public HttpRequestHandler { } private: + // /status is an orchestrator heartbeat that can be polled as often as every + // second, so record it at most once per hour per process — enough to confirm + // the endpoint is being used without letting it dominate telemetry volume. + void MaybeRecordStatusTelemetry(const std::shared_ptr& request) { + constexpr int64_t kIntervalMs = 3'600'000; + constexpr int64_t kNever = INT64_MIN; + const int64_t now_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + + int64_t last = last_status_emit_ms_.load(std::memory_order_relaxed); + if (last != kNever && now_ms - last < kIntervalMs) { + return; + } + // Exactly one caller wins this interval's slot; the rest skip. + if (!last_status_emit_ms_.compare_exchange_strong(last, now_ms, std::memory_order_relaxed)) { + return; + } + + ActionTracker tracker(Action::kServiceStatus, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); + tracker.SetStatus(ActionStatus::kSuccess); + } + ServiceContext& ctx_; + std::atomic last_status_emit_ms_{INT64_MIN}; }; // ======================================================================== diff --git a/sdk_v2/cpp/src/telemetry/telemetry.cc b/sdk_v2/cpp/src/telemetry/telemetry.cc index da603efa5..28b75c27c 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry.cc @@ -54,6 +54,8 @@ std::string_view ActionToString(Action action) { return "ModelInference"; case Action::kServiceRequestUnmatched: return "ServiceRequestUnmatched"; + case Action::kServiceStatus: + return "ServiceStatus"; default: return "Unknown"; } diff --git a/sdk_v2/cpp/src/telemetry/telemetry.h b/sdk_v2/cpp/src/telemetry/telemetry.h index 681a3cccc..4c93282d8 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.h +++ b/sdk_v2/cpp/src/telemetry/telemetry.h @@ -57,6 +57,7 @@ enum class Action { // HTTP service plumbing kServiceRequestUnmatched = 800, // A request reached the service but matched no route / method + kServiceStatus = 801, // GET /status heartbeat (sampled — at most once per hour per process) }; /// Status of a tracked telemetry action. From 2b406e1c1d5258f42996308762c93a2377f91d32 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 20 Jun 2026 03:22:16 -0500 Subject: [PATCH 20/77] sdk_v2/cpp/telemetry: add router catch-all for unmatched requests + tests - UnmatchedRouteInterceptor: a request interceptor that, before routing, checks the router for a matching route. When none matches (unknown path or wrong method) it records a kServiceRequestUnmatched action (kClientError) and replies 404, so requests that reach the service but no handler are no longer invisible. - Tests (WebServiceTelemetryTest, using a capturing ITelemetry + empty catalog, no real model required): * unmatched route records kServiceRequestUnmatched with the right status, user agent, Direct flag and a correlation id; * an empty chat-completions body records kClientError (not kFailure) and emits no Model event; * three rapid GET /status calls record kServiceStatus exactly once (hourly sampling). Builds clean (/W4 /WX); 72 telemetry/webservice/sse tests pass. --- sdk_v2/cpp/src/service/web_service.cc | 49 +++++++ .../cpp/test/internal_api/web_service_test.cc | 133 ++++++++++++++++++ 2 files changed, 182 insertions(+) diff --git a/sdk_v2/cpp/src/service/web_service.cc b/sdk_v2/cpp/src/service/web_service.cc index 5a6e014b7..e49fbad8f 100644 --- a/sdk_v2/cpp/src/service/web_service.cc +++ b/sdk_v2/cpp/src/service/web_service.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -201,6 +202,49 @@ class ShutdownHandler : public HttpRequestHandler { std::function shutdown_fn_; }; +// ======================================================================== +// UnmatchedRouteInterceptor +// +// oatpp routes straight to handlers, so requests that match no route (unknown +// path or wrong method) never reach an ActionTracker and would be invisible to +// telemetry. This request interceptor runs before routing: when the router has +// no route for the request it records a kServiceRequestUnmatched action and +// replies 404 itself; otherwise it returns nullptr and normal routing proceeds. +// ======================================================================== + +class UnmatchedRouteInterceptor : public oatpp::web::server::interceptor::RequestInterceptor { + public: + UnmatchedRouteInterceptor(std::shared_ptr router, ITelemetry& telemetry) + : router_(std::move(router)), telemetry_(telemetry) {} + + std::shared_ptr intercept(const std::shared_ptr& request) override { + const auto& line = request->getStartingLine(); + if (router_->getRoute(line.method, line.path)) { + return nullptr; // a handler will serve this request + } + + std::string user_agent; + if (auto ua = request->getHeader("User-Agent")) { + user_agent = *ua; + } + { + ActionTracker tracker(Action::kServiceRequestUnmatched, telemetry_, + InvocationContext::Direct(user_agent)); + tracker.SetStatus(ActionStatus::kClientError); + } + + nlohmann::json body = { + {"error", {{"message", "Not found"}, {"type", "invalid_request_error"}, + {"param", nullptr}, {"code", nullptr}}}, + }; + return JsonResponse(Status::CODE_404, body); + } + + private: + std::shared_ptr router_; + ITelemetry& telemetry_; +}; + // ======================================================================== // WebService implementation // ======================================================================== @@ -301,6 +345,11 @@ std::vector WebService::Start(const std::vector& endpo impl_->connection_handler = oatpp::web::server::HttpConnectionHandler::createShared(impl_->router); + // Record requests that match no route (unknown path / wrong method), which + // otherwise bypass every handler's ActionTracker. + impl_->connection_handler->addRequestInterceptor( + std::make_shared(impl_->router, ctx.telemetry)); + std::vector bound_urls; for (const auto& endpoint : endpoints) { diff --git a/sdk_v2/cpp/test/internal_api/web_service_test.cc b/sdk_v2/cpp/test/internal_api/web_service_test.cc index 4b33be0a3..68bb270ad 100644 --- a/sdk_v2/cpp/test/internal_api/web_service_test.cc +++ b/sdk_v2/cpp/test/internal_api/web_service_test.cc @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -58,6 +59,58 @@ std::string TestHttpDelete(const std::string& url, const std::string& user_agent return http::HttpDelete(url, options); } +// Telemetry sink that records every event for assertions. +class CapturingTelemetry : public ITelemetry { + public: + struct ActionCall { + Action action; + ActionStatus status; + std::string user_agent; + std::string correlation_id; + bool indirect; + }; + + void RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t /*duration_ms*/) override { + std::lock_guard lock(mutex_); + actions.push_back({action, status, context.user_agent, context.correlation_id, context.indirect}); + } + void RecordException(Action, const std::exception&, const InvocationContext&) override {} + void RecordModelUsage(const ModelUsageInfo& info) override { + std::lock_guard lock(mutex_); + model_usages.push_back(info); + } + void RecordModelId(Action, const std::string&, ActionStatus, const InvocationContext&) override {} + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo&) override {} + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo&) override {} + void RecordDownload(const DownloadInfo&) override {} + + int Count(Action action) { + std::lock_guard lock(mutex_); + int n = 0; + for (const auto& c : actions) { + if (c.action == action) { + ++n; + } + } + return n; + } + + std::optional Find(Action action) { + std::lock_guard lock(mutex_); + for (const auto& c : actions) { + if (c.action == action) { + return c; + } + } + return std::nullopt; + } + + std::vector actions; + std::vector model_usages; + std::mutex mutex_; +}; + } // namespace // ======================================================================== @@ -485,6 +538,86 @@ TEST(WebServiceEmptyCatalogTest, LoadedModelsReturnsEmptyArray) { service.Stop(); } +// ======================================================================== +// Telemetry behaviors — capture events and assert coverage / classification +// ======================================================================== + +TEST(WebServiceTelemetryTest, UnmatchedRouteRecordsServiceRequestUnmatched) { + test::MockCatalog catalog; + StderrLogger logger; + test::CpuOnlyEpDetector ep_detector; + ModelLoadManager model_load_manager(ep_detector, logger); + SessionManager session_manager(logger); + CapturingTelemetry telemetry; + + WebService service(catalog, logger, "/tmp/test", model_load_manager, session_manager, telemetry, []() {}); + auto urls = service.Start({"http://127.0.0.1:0"}); + + // Unknown path: the interceptor replies 404, so TestHttpGet throws on the non-2xx. + try { + TestHttpGet(urls[0] + "/no/such/route", "probe-agent/9"); + } catch (...) { + } + + service.Stop(); + + ASSERT_EQ(telemetry.Count(Action::kServiceRequestUnmatched), 1); + auto call = telemetry.Find(Action::kServiceRequestUnmatched); + ASSERT_TRUE(call.has_value()); + EXPECT_EQ(call->status, ActionStatus::kClientError); + EXPECT_EQ(call->user_agent, "probe-agent/9"); + EXPECT_FALSE(call->indirect); + EXPECT_FALSE(call->correlation_id.empty()); +} + +TEST(WebServiceTelemetryTest, EmptyChatBodyRecordsClientErrorNotFailure) { + test::MockCatalog catalog; + StderrLogger logger; + test::CpuOnlyEpDetector ep_detector; + ModelLoadManager model_load_manager(ep_detector, logger); + SessionManager session_manager(logger); + CapturingTelemetry telemetry; + + WebService service(catalog, logger, "/tmp/test", model_load_manager, session_manager, telemetry, []() {}); + auto urls = service.Start({"http://127.0.0.1:0"}); + + // Empty body is rejected (400) before any model resolution. + try { + TestHttpPost(urls[0] + "/v1/chat/completions", ""); + } catch (...) { + } + + service.Stop(); + + auto call = telemetry.Find(Action::kOpenAIChatCompletions); + ASSERT_TRUE(call.has_value()) << "chat completions route action was not recorded"; + EXPECT_EQ(call->status, ActionStatus::kClientError); + EXPECT_FALSE(call->indirect); + // A 4xx reject performs no inference, so there is no Model event. + EXPECT_TRUE(telemetry.model_usages.empty()); +} + +TEST(WebServiceTelemetryTest, StatusEndpointIsSampledHourly) { + test::MockCatalog catalog; + StderrLogger logger; + test::CpuOnlyEpDetector ep_detector; + ModelLoadManager model_load_manager(ep_detector, logger); + SessionManager session_manager(logger); + CapturingTelemetry telemetry; + + WebService service(catalog, logger, "/tmp/test", model_load_manager, session_manager, telemetry, []() {}); + auto urls = service.Start({"http://127.0.0.1:0"}); + + // Three rapid heartbeats — only the first inside the hourly window is recorded. + TestHttpGet(urls[0] + "/status"); + TestHttpGet(urls[0] + "/status"); + TestHttpGet(urls[0] + "/status"); + + service.Stop(); + + EXPECT_EQ(telemetry.Count(Action::kServiceStatus), 1); +} + // ======================================================================== // Streaming validation tests — same errors apply with stream=true // ======================================================================== From 9989d31c194514879cfab1088dc73f4b638d7ccc Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 20 Jun 2026 11:51:08 -0500 Subject: [PATCH 21/77] sdk_v2/cpp/telemetry: track Azure Catalog accesses (CatalogFetch event) Every access to a model catalog source is now recorded, with success/failure: - New CatalogFetch typed event carrying operation ("FetchAll" or the cached-id "FetchByIds" lookup), endpoint/region/format parsed from the catalog URL, status, duration, model count, error message, and a correlation id shared across the accesses of one refresh. - FetchAllModelInfosWithCachedModels gains optional telemetry params (defaulted, so the snapshot tool and existing tests are unchanged) and emits an event for the primary fetch and, when it runs, the secondary cached-id lookup. - AzureModelCatalog parses the catalog URL into endpoint/region/format (https://ai.azure.com/api/eastus/ux/v1.0 -> {ai.azure.com, eastus, ux/v1.0}; "static" for the embedded snapshot) and threads an ITelemetry from Manager. - Tests assert the primary and secondary accesses are each tracked with the right operation, status, endpoint, correlation id and model count. Builds clean (/W4 /WX); catalog + telemetry + webservice tests pass. --- sdk_v2/cpp/src/catalog/azure_model_catalog.cc | 88 ++++++++++++++++++- sdk_v2/cpp/src/catalog/azure_model_catalog.h | 6 +- sdk_v2/cpp/src/catalog/catalog_client.cc | 46 +++++++++- sdk_v2/cpp/src/catalog/catalog_client.h | 12 ++- sdk_v2/cpp/src/manager.cc | 5 +- sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc | 19 ++++ sdk_v2/cpp/src/telemetry/one_ds_telemetry.h | 1 + sdk_v2/cpp/src/telemetry/telemetry.h | 18 ++++ sdk_v2/cpp/src/telemetry/telemetry_logger.cc | 9 ++ sdk_v2/cpp/src/telemetry/telemetry_logger.h | 1 + sdk_v2/cpp/test/internal_api/null_telemetry.h | 2 + .../cpp/test/internal_api/telemetry_test.cc | 5 ++ .../cpp/test/internal_api/web_service_test.cc | 5 ++ 13 files changed, 206 insertions(+), 11 deletions(-) diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index 39afcae37..484b17353 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -6,6 +6,8 @@ #include "catalog/local_model_scanner.h" #include "model.h" #include "model_info.h" +#include "telemetry/invocation_context.h" +#include "telemetry/telemetry.h" #include "utils.h" #include @@ -16,6 +18,72 @@ namespace fl { +namespace { + +/// Split a catalog URL into structured telemetry dimensions. The Azure Foundry +/// catalog URL looks like "https://ai.azure.com/api//", e.g. +/// "https://ai.azure.com/api/eastus/ux/v1.0" -> {ai.azure.com, eastus, ux/v1.0}. +/// Custom URLs that don't follow the "/api//" convention keep an empty +/// region and put the whole path in `format`. The embedded snapshot is "static". +struct ParsedCatalogUrl { + std::string endpoint; + std::string region; + std::string format; +}; + +ParsedCatalogUrl ParseCatalogUrl(const std::string& url) { + if (url == "static") { + return {"static", "", ""}; + } + + std::string rest = url; + if (auto scheme = rest.find("://"); scheme != std::string::npos) { + rest = rest.substr(scheme + 3); + } + + ParsedCatalogUrl out; + std::string path; + if (auto slash = rest.find('/'); slash == std::string::npos) { + out.endpoint = rest; + } else { + out.endpoint = rest.substr(0, slash); + path = rest.substr(slash + 1); + } + + if (auto q = path.find_first_of("?#"); q != std::string::npos) { + path = path.substr(0, q); + } + + std::vector segments; + size_t pos = 0; + while (pos < path.size()) { + auto next = path.find('/', pos); + if (next == std::string::npos) { + next = path.size(); + } + if (next > pos) { + segments.push_back(path.substr(pos, next - pos)); + } + pos = next + 1; + } + + size_t format_start = 0; + if (segments.size() >= 2 && segments[0] == "api") { + out.region = segments[1]; + format_start = 2; + } + for (size_t i = format_start; i < segments.size(); ++i) { + if (!out.format.empty()) { + out.format += '/'; + } + out.format += segments[i]; + } + + return out; +} + +} // namespace + AzureModelCatalog::AzureModelCatalog(std::vector>> catalog_urls, std::string cache_dir, ModelFactory model_factory, @@ -23,7 +91,8 @@ AzureModelCatalog::AzureModelCatalog(std::vector(kDefaultCatalogFilter)); } @@ -77,6 +147,9 @@ std::vector AzureModelCatalog::FetchModels() const { std::vector fetched_infos; const std::string& cache_dir = cache_dir_; + // One correlation id groups every catalog access made by this refresh. + const std::string correlation_id = MakeGuidV4Hex(); + logger_.Log(LogLevel::Information, "Getting latest info from the Azure catalog and for locally cached models."); @@ -96,7 +169,16 @@ std::vector AzureModelCatalog::FetchModels() const { // while letting callers explicitly request "" as a real filter override. auto client = MakeCatalogClient(url, filter.value_or(""), ep_detector_, logger_, cache_dir, catalog_region_, disable_region_fallback_); - auto model_infos = FetchAllModelInfosWithCachedModels(*client, cached_model_ids, logger_); + + auto parsed = ParseCatalogUrl(url); + CatalogFetchInfo base_info; + base_info.endpoint = parsed.endpoint; + base_info.region = parsed.region; + base_info.format = parsed.format; + base_info.correlation_id = correlation_id; + + auto model_infos = FetchAllModelInfosWithCachedModels(*client, cached_model_ids, logger_, + telemetry_, &base_info); for (const auto& info : model_infos) { // Check if the model is locally cached and pass the path if so. diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.h b/sdk_v2/cpp/src/catalog/azure_model_catalog.h index 5769a3ef7..38a91ffc2 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.h +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.h @@ -14,6 +14,8 @@ namespace fl { +class ITelemetry; // forward declaration + /// Azure-specific catalog. Fetches from Azure Foundry catalog API, /// scans local cache, merges results. /// Maps to C# AzureModelCatalog. @@ -28,7 +30,8 @@ class AzureModelCatalog : public BaseModelCatalog { ILogger& logger, bool cache_only = false, std::string catalog_region = "", - bool disable_region_fallback = false); + bool disable_region_fallback = false, + ITelemetry* telemetry = nullptr); ~AzureModelCatalog() override; protected: @@ -50,6 +53,7 @@ class AzureModelCatalog : public BaseModelCatalog { // Configured Azure region: empty/"auto" → auto-detect, explicit → hard override. std::string catalog_region_; bool disable_region_fallback_; + ITelemetry* telemetry_; }; } // namespace fl diff --git a/sdk_v2/cpp/src/catalog/catalog_client.cc b/sdk_v2/cpp/src/catalog/catalog_client.cc index 6f0a51e38..c81070568 100644 --- a/sdk_v2/cpp/src/catalog/catalog_client.cc +++ b/sdk_v2/cpp/src/catalog/catalog_client.cc @@ -1,12 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #include "catalog/catalog_client.h" +#include "ep_detection/ep_detector.h" +#include "telemetry/telemetry.h" #include "utils.h" #include #include +#include +#include #include namespace fl { @@ -14,9 +18,40 @@ namespace fl { std::vector FetchAllModelInfosWithCachedModels( ICatalogClient& client, const std::vector& cached_model_ids, - ILogger& logger) { - // Step 1: Fetch latest catalog models (existing flow). - auto result = client.FetchAllModelInfos(); + ILogger& logger, + ITelemetry* telemetry, + const CatalogFetchInfo* base_info) { + auto now = [] { return std::chrono::steady_clock::now(); }; + auto elapsed_ms = [](std::chrono::steady_clock::time_point start) { + return std::chrono::duration_cast(std::chrono::steady_clock::now() - start) + .count(); + }; + auto emit = [&](const std::string& operation, ActionStatus status, int64_t duration_ms, + int32_t model_count, const std::string& error) { + if (telemetry == nullptr || base_info == nullptr) { + return; + } + CatalogFetchInfo info = *base_info; + info.operation = operation; + info.status = status; + info.duration_ms = duration_ms; + info.model_count = model_count; + info.error_message = error; + telemetry->RecordCatalogFetch(info); + }; + + // Step 1: Fetch latest catalog models (the primary catalog access). + std::vector result; + { + const auto start = now(); + try { + result = client.FetchAllModelInfos(); + } catch (const std::exception& ex) { + emit("FetchAll", ActionStatus::kFailure, elapsed_ms(start), 0, ex.what()); + throw; + } + emit("FetchAll", ActionStatus::kSuccess, elapsed_ms(start), static_cast(result.size()), ""); + } if (cached_model_ids.empty()) { return result; @@ -37,17 +72,22 @@ std::vector FetchAllModelInfosWithCachedModels( // Step 3: Look up unresolved IDs from the catalog (older versions, etc.) if (!unresolved_ids.empty()) { + const auto start = now(); try { auto additional = client.FetchModelsByIds(unresolved_ids); + const auto additional_count = static_cast(additional.size()); for (auto& info : additional) { resolved_ids.insert(info.model_id); result.push_back(std::move(info)); } + emit("FetchByIds", ActionStatus::kSuccess, elapsed_ms(start), additional_count, ""); } catch (const std::exception& ex) { logger.Log(LogLevel::Warning, fmt::format("catalog: failed to fetch cached model IDs — {}", ex.what())); + emit("FetchByIds", ActionStatus::kFailure, elapsed_ms(start), 0, ex.what()); } catch (...) { logger.Log(LogLevel::Warning, "catalog: failed to fetch cached model IDs — unknown error"); + emit("FetchByIds", ActionStatus::kFailure, elapsed_ms(start), 0, "unknown error"); } // Step 4: Create basic entries for any IDs still unresolved (BYO models). diff --git a/sdk_v2/cpp/src/catalog/catalog_client.h b/sdk_v2/cpp/src/catalog/catalog_client.h index e3afcfe78..af84e8453 100644 --- a/sdk_v2/cpp/src/catalog/catalog_client.h +++ b/sdk_v2/cpp/src/catalog/catalog_client.h @@ -12,6 +12,9 @@ namespace fl { +class ITelemetry; // forward declaration +struct CatalogFetchInfo; // forward declaration + /// Abstract catalog client. Implemented by the live Azure catalog client, /// which queries the Azure Foundry catalog REST API. class ICatalogClient { @@ -50,11 +53,16 @@ class ICatalogClient { }; /// Production helper that combines a catalog fetch with locally cached model -/// resolution and BYO synthesis. +/// resolution and BYO synthesis. When `telemetry` and `base_info` are provided, +/// emits a CatalogFetch event for the primary fetch and (if it runs) the +/// cached-id lookup, copying the endpoint/region/format/correlation fields from +/// `base_info` and filling in operation/status/duration/model_count/error. std::vector FetchAllModelInfosWithCachedModels( ICatalogClient& client, const std::vector& cached_model_ids, - ILogger& logger); + ILogger& logger, + ITelemetry* telemetry = nullptr, + const CatalogFetchInfo* base_info = nullptr); /// Construct a client for the live Azure Foundry catalog. /// - `ep_detector` limits results to models supported by this machine. diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index 872f4ce1b..df72c5554 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -24,11 +24,11 @@ #include "spdlog_logger.h" #include "telemetry/telemetry_action_tracker.h" #include "telemetry/telemetry_logger.h" -#include "util/string_utils.h" #if FOUNDRY_LOCAL_HAS_1DS #include "one_ds_tenant_token.h" // auto-generated header, in ${CMAKE_BINARY_DIR}/generated #include "telemetry/one_ds_telemetry.h" #endif +#include "util/string_utils.h" #include "utils.h" #if FOUNDRY_LOCAL_HAS_EP_CATALOG @@ -350,7 +350,8 @@ Manager::Manager(const Configuration& config) *ep_detector_, *logger_, config_.external_service_url.has_value(), config_.catalog_region.value_or("auto"), - disable_region_fallback); + disable_region_fallback, + telemetry_.get()); } Manager::~Manager() { diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc index 324566be7..84b6d6a7b 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc @@ -252,4 +252,23 @@ void OneDsTelemetry::RecordDownload(const DownloadInfo& info) { SafeLog(GetMatLogger(), ev); } +void OneDsTelemetry::RecordCatalogFetch(const CatalogFetchInfo& info) { + local_log_.RecordCatalogFetch(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("CatalogFetch", metadata_.test_mode); + ev.SetProperty("Operation", info.operation); + ev.SetProperty("Endpoint", info.endpoint); + ev.SetProperty("Region", info.region); + ev.SetProperty("Format", info.format); + ev.SetProperty("Status", std::string(ActionStatusToString(info.status))); + ev.SetProperty("TimeMs", info.duration_ms); + ev.SetProperty("ModelCount", static_cast(info.model_count)); + ev.SetProperty("ErrorMessage", info.error_message); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); + SafeLog(GetMatLogger(), ev); +} + } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h index d7f0fe303..13339f6b2 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h @@ -64,6 +64,7 @@ class OneDsTelemetry : public ITelemetry { void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; void RecordDownload(const DownloadInfo& info) override; + void RecordCatalogFetch(const CatalogFetchInfo& info) override; /// True if 1DS Initialize succeeded (i.e. events are actually uploaded). /// False in CI or when the tenant token was empty. diff --git a/sdk_v2/cpp/src/telemetry/telemetry.h b/sdk_v2/cpp/src/telemetry/telemetry.h index 4c93282d8..c99d863fd 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.h +++ b/sdk_v2/cpp/src/telemetry/telemetry.h @@ -137,6 +137,21 @@ struct DownloadInfo { int32_t max_concurrency = 0; }; +/// Payload for the CatalogFetch event — emitted once per access to a model +/// catalog source (the live Azure catalog or the embedded static snapshot). +struct CatalogFetchInfo { + std::string operation; // "FetchAll" (full catalog) or "FetchByIds" (cached-id lookup) + std::string endpoint; // catalog host (e.g. "ai.azure.com"), or "static" for the embedded snapshot + std::string region; // region parsed from the catalog URL (e.g. "eastus"); empty if not present + std::string format; // catalog API path/version after the region (e.g. "ux/v1.0") + ActionStatus status = ActionStatus::kInvalid; + int64_t duration_ms = 0; + int32_t model_count = 0; // models returned by this access + std::string error_message; // populated on failure + std::string user_agent; + std::string correlation_id; // shared across the accesses of one catalog refresh +}; + /// Abstract telemetry interface. /// Implementations may send events to a telemetry backend (1DS, ETW, OpenTelemetry, …) /// or simply log them. The OneDsTelemetry implementation sends to 1DS; the @@ -170,6 +185,9 @@ class ITelemetry { /// Record one model file download (Download event). virtual void RecordDownload(const DownloadInfo& info) = 0; + + /// Record one access to a model catalog source (CatalogFetch event). + virtual void RecordCatalogFetch(const CatalogFetchInfo& info) = 0; }; } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc index 5a5354b93..a48885b27 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc @@ -83,4 +83,13 @@ void TelemetryLogger::RecordDownload(const DownloadInfo& info) { info.download_wait_result, info.max_concurrency)); } +void TelemetryLogger::RecordCatalogFetch(const CatalogFetchInfo& info) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] CatalogFetch AppName={} Operation={} Endpoint={} Region={} Format={} " + "Status={} TimeMs={} ModelCount={} Error={} UserAgent={} CorrelationId={}", + app_name_, info.operation, info.endpoint, info.region, info.format, + ActionStatusToString(info.status), info.duration_ms, info.model_count, + info.error_message, info.user_agent, info.correlation_id)); +} + } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.h b/sdk_v2/cpp/src/telemetry/telemetry_logger.h index 8679ef537..da3928f86 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.h @@ -35,6 +35,7 @@ class TelemetryLogger : public ITelemetry { void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; void RecordDownload(const DownloadInfo& info) override; + void RecordCatalogFetch(const CatalogFetchInfo& info) override; private: std::string app_name_; diff --git a/sdk_v2/cpp/test/internal_api/null_telemetry.h b/sdk_v2/cpp/test/internal_api/null_telemetry.h index c7c5966e8..c0a122fe6 100644 --- a/sdk_v2/cpp/test/internal_api/null_telemetry.h +++ b/sdk_v2/cpp/test/internal_api/null_telemetry.h @@ -25,6 +25,8 @@ class NullTelemetry : public ITelemetry { void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& /*info*/) override {} void RecordDownload(const DownloadInfo& /*info*/) override {} + + void RecordCatalogFetch(const CatalogFetchInfo& /*info*/) override {} }; } // namespace fl::test diff --git a/sdk_v2/cpp/test/internal_api/telemetry_test.cc b/sdk_v2/cpp/test/internal_api/telemetry_test.cc index 217fe22aa..cd0d3f57c 100644 --- a/sdk_v2/cpp/test/internal_api/telemetry_test.cc +++ b/sdk_v2/cpp/test/internal_api/telemetry_test.cc @@ -73,6 +73,10 @@ class RecordingTelemetry : public ITelemetry { download_calls.push_back(info); } + void RecordCatalogFetch(const CatalogFetchInfo& info) override { + catalog_fetch_calls.push_back(info); + } + std::vector action_calls; std::vector> exception_calls; std::vector model_usage_calls; @@ -80,6 +84,7 @@ class RecordingTelemetry : public ITelemetry { std::vector ep_attempt_calls; std::vector ep_register_calls; std::vector download_calls; + std::vector catalog_fetch_calls; }; } // namespace diff --git a/sdk_v2/cpp/test/internal_api/web_service_test.cc b/sdk_v2/cpp/test/internal_api/web_service_test.cc index 68bb270ad..6f76601bc 100644 --- a/sdk_v2/cpp/test/internal_api/web_service_test.cc +++ b/sdk_v2/cpp/test/internal_api/web_service_test.cc @@ -84,6 +84,10 @@ class CapturingTelemetry : public ITelemetry { void RecordEpDownloadAttempt(const EpDownloadAttemptInfo&) override {} void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo&) override {} void RecordDownload(const DownloadInfo&) override {} + void RecordCatalogFetch(const CatalogFetchInfo& info) override { + std::lock_guard lock(mutex_); + catalog_fetches.push_back(info); + } int Count(Action action) { std::lock_guard lock(mutex_); @@ -108,6 +112,7 @@ class CapturingTelemetry : public ITelemetry { std::vector actions; std::vector model_usages; + std::vector catalog_fetches; std::mutex mutex_; }; From 5433c9e3d7c681b4d906215052a24901ef97fc49 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 21 Jun 2026 19:07:33 -0500 Subject: [PATCH 22/77] sdk_v2/cpp/telemetry: plain AppSessionGuid + ext.app.sesId usage sessions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Drop the UTCReplace_ magic from the per-process correlation GUID: it only fires on the Windows UTC transmission path (not our direct upload, and never off-Windows), so it was dead weight. The field is now plainly "AppSessionGuid" — a stable per-process correlation id on every platform. - Add ITelemetry::StartSession()/EndSession() (default no-op; OneDsTelemetry maps them to 1DS LogSession(Started/Ended), TelemetryLogger logs them). Manager opens a session when the web service starts and closes it on stop, so events carry the standard, cross-platform usage-session id (ext.app.sesId) and the backend gets session duration. This is additive — it complements the per-run AppSessionGuid and the per-operation CorrelationId. Builds clean (/W4 /WX); telemetry/webservice/catalog tests pass. --- sdk_v2/cpp/src/manager.cc | 4 +++ sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc | 35 ++++++++++++++++--- sdk_v2/cpp/src/telemetry/one_ds_telemetry.h | 2 ++ sdk_v2/cpp/src/telemetry/telemetry.h | 7 ++++ sdk_v2/cpp/src/telemetry/telemetry_logger.cc | 8 +++++ sdk_v2/cpp/src/telemetry/telemetry_logger.h | 2 ++ sdk_v2/cpp/src/telemetry/telemetry_metadata.h | 10 +++--- 7 files changed, 59 insertions(+), 9 deletions(-) diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index df72c5554..5f81eed3c 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -485,6 +485,9 @@ void Manager::StartWebService() { bound_urls_ = web_service_->Start(endpoints); web_service_running_ = true; + // Open an app-usage session for the lifetime of the running service so events + // carry ext.app.sesId and the backend gets session duration. + telemetry_->StartSession(); tracker.SetStatus(ActionStatus::kSuccess); #else FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -514,6 +517,7 @@ void Manager::StopWebService() { web_service_.reset(); web_service_running_ = false; bound_urls_.clear(); + telemetry_->EndSession(); tracker.SetStatus(ActionStatus::kSuccess); #else FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc index 84b6d6a7b..d46840744 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc @@ -29,6 +29,7 @@ using MatILogger = ::Microsoft::Applications::Events::ILogger; using ::Microsoft::Applications::Events::EventProperties; using ::Microsoft::Applications::Events::EventPriority; using ::Microsoft::Applications::Events::PiiKind_None; +using ::Microsoft::Applications::Events::SessionState; constexpr uint64_t kCriticalData = MICROSOFT_KEYWORD_CRITICAL_DATA; @@ -36,10 +37,10 @@ void SetCommonContext(MatILogger* mat_logger, const TelemetryMetadata& m) { // Process-wide context — stamped on every event uploaded through this ILogger. mat_logger->SetContext("AppName", m.app_name); mat_logger->SetContext("Version", m.version); - // 1DS recognizes UTCReplace_AppSessionGuid as a magic field name on Windows UTC - // and substitutes the OS session GUID. On other platforms we just send the - // GUID we computed at startup — same correlation semantics either way. - mat_logger->SetContext("UTCReplace_AppSessionGuid", m.app_session_guid); + // Per-process correlation GUID, generated at startup. Lets the backend group + // every event from one FL run. (This is distinct from ext.app.sesId, the SDK's + // rotating usage-session id set via LogSession.) + mat_logger->SetContext("AppSessionGuid", m.app_session_guid); mat_logger->SetContext("OsName", m.os_name); mat_logger->SetContext("OsVersion", m.os_version); mat_logger->SetContext("CpuArch", m.cpu_arch); @@ -271,4 +272,30 @@ void OneDsTelemetry::RecordCatalogFetch(const CatalogFetchInfo& info) { SafeLog(GetMatLogger(), ev); } +void OneDsTelemetry::StartSession() { + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto* mat_logger = GetMatLogger(); + if (mat_logger == nullptr) { + return; + } + // LogSession(Started) opens an app-usage session; the SDK stamps ext.app.sesId + // on subsequent events and records session duration on End. + auto ev = MakeEvent("Session", metadata_.test_mode); + mat_logger->LogSession(SessionState::Session_Started, ev); +} + +void OneDsTelemetry::EndSession() { + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto* mat_logger = GetMatLogger(); + if (mat_logger == nullptr) { + return; + } + auto ev = MakeEvent("Session", metadata_.test_mode); + mat_logger->LogSession(SessionState::Session_Ended, ev); +} + } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h index 13339f6b2..c1af838bd 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h @@ -65,6 +65,8 @@ class OneDsTelemetry : public ITelemetry { void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; void RecordDownload(const DownloadInfo& info) override; void RecordCatalogFetch(const CatalogFetchInfo& info) override; + void StartSession() override; + void EndSession() override; /// True if 1DS Initialize succeeded (i.e. events are actually uploaded). /// False in CI or when the tenant token was empty. diff --git a/sdk_v2/cpp/src/telemetry/telemetry.h b/sdk_v2/cpp/src/telemetry/telemetry.h index c99d863fd..91ee2e4b2 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.h +++ b/sdk_v2/cpp/src/telemetry/telemetry.h @@ -188,6 +188,13 @@ class ITelemetry { /// Record one access to a model catalog source (CatalogFetch event). virtual void RecordCatalogFetch(const CatalogFetchInfo& info) = 0; + + /// Mark the start / end of an app-usage session via 1DS LogSession. Between + /// these the SDK stamps a session id (ext.app.sesId) on events and emits a + /// session start/end with duration — standard, cross-platform engagement + /// sessions. Default no-op for the non-1DS implementations. + virtual void StartSession() {} + virtual void EndSession() {} }; } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc index a48885b27..9b594770d 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc @@ -92,4 +92,12 @@ void TelemetryLogger::RecordCatalogFetch(const CatalogFetchInfo& info) { info.error_message, info.user_agent, info.correlation_id)); } +void TelemetryLogger::StartSession() { + logger_.Log(LogLevel::Debug, fmt::format("[Telemetry] SessionStart AppName={}", app_name_)); +} + +void TelemetryLogger::EndSession() { + logger_.Log(LogLevel::Debug, fmt::format("[Telemetry] SessionEnd AppName={}", app_name_)); +} + } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.h b/sdk_v2/cpp/src/telemetry/telemetry_logger.h index da3928f86..b07273987 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.h @@ -36,6 +36,8 @@ class TelemetryLogger : public ITelemetry { void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; void RecordDownload(const DownloadInfo& info) override; void RecordCatalogFetch(const CatalogFetchInfo& info) override; + void StartSession() override; + void EndSession() override; private: std::string app_name_; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_metadata.h b/sdk_v2/cpp/src/telemetry/telemetry_metadata.h index 58f880c1f..8d47fc6e2 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_metadata.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_metadata.h @@ -9,11 +9,11 @@ namespace fl { /// Process-wide metadata stamped onto every 1DS event as common context. /// Computed once at startup and cached. Cheap to copy. struct TelemetryMetadata { - /// Hex-encoded random 128-bit GUID; correlates events from the same process. - /// 1DS interprets the magic field name "UTCReplace_AppSessionGuid" by replacing - /// the value with the OS-supplied app session GUID on Windows, and accepts our - /// random GUID on other platforms. We always provide a value so the event is - /// well-formed even if UTC's magic isn't honored. + /// Hex-encoded random 128-bit GUID, generated once at startup. Stamped on + /// every event as `AppSessionGuid` so the backend can group all events from a + /// single FL process run. This is a stable per-process correlation id and is + /// distinct from the SDK's rotating usage-session id (ext.app.sesId), which is + /// driven separately via LogSession(Started/Ended). std::string app_session_guid; /// Foundry Local SDK version (FoundryLocalGetVersionString). From bb372c7a8c40f22de79e9285eb6fe94413086013 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 9 Jul 2026 17:14:09 -0500 Subject: [PATCH 23/77] Fix telemetry merge validation issues Address post-merge review findings before opening the PR: keep telemetry tokens out of logged commands, make the WinML fetch-url path compatible with CMake 3.20, preserve newly discovered catalog variants on refresh, propagate file close failures before deleting download state, and mark cross-process cache hits as skipped downloads. Files changed: - sdk_v2/cpp/build.py and cmake/FindWinMLEpCatalog.cmake - sdk_v2/cpp/src/catalog/base_model_catalog.* - sdk_v2/cpp/src/download/download_manager.cc and file_writer.cc - sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/build.py | 9 +++- sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake | 2 +- sdk_v2/cpp/src/catalog/base_model_catalog.cc | 51 ++++++------------- sdk_v2/cpp/src/catalog/base_model_catalog.h | 1 + sdk_v2/cpp/src/download/download_manager.cc | 4 ++ sdk_v2/cpp/src/download/file_writer.cc | 33 +++++++++--- .../internal_api/base_model_catalog_test.cc | 18 +++++++ 7 files changed, 73 insertions(+), 45 deletions(-) diff --git a/sdk_v2/cpp/build.py b/sdk_v2/cpp/build.py index be56d826d..a947e24df 100644 --- a/sdk_v2/cpp/build.py +++ b/sdk_v2/cpp/build.py @@ -57,9 +57,16 @@ def _path_from_env_var(var: str) -> Path | None: # Subprocess runner # --------------------------------------------------------------------------- +def _redact_command_arg(arg: str) -> str: + secret_define = "-DFOUNDRY_LOCAL_TELEMETRY_TOKEN=" + if arg.startswith(secret_define): + return f"{secret_define}" + return arg + + def run(cmd: list[str], cwd: Path | str | None = None, env: dict[str, str] | None = None) -> None: """Run a command, logging and streaming output. Raises on non-zero exit.""" - log.info("Running: %s", " ".join(str(c) for c in cmd)) + log.info("Running: %s", " ".join(_redact_command_arg(str(c)) for c in cmd)) subprocess.run(cmd, cwd=cwd, env=env, check=True) diff --git a/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake b/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake index dc732a4ff..debfee8f2 100644 --- a/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake +++ b/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake @@ -73,7 +73,7 @@ if(WINML_EP_CATALOG_FETCH_URL) set(_WINML_ZIP_PATH "${CMAKE_BINARY_DIR}/_deps/winml_ep_catalog-download/winml_ep_catalog.zip") get_filename_component(_WINML_ZIP_DIR "${_WINML_ZIP_PATH}" DIRECTORY) file(MAKE_DIRECTORY "${_WINML_ZIP_DIR}") - file(COPY_FILE "${WINML_EP_CATALOG_FETCH_URL}" "${_WINML_ZIP_PATH}") + configure_file("${WINML_EP_CATALOG_FETCH_URL}" "${_WINML_ZIP_PATH}" COPYONLY) set(WINML_EP_CATALOG_FETCH_URL "${_WINML_ZIP_PATH}") endif() FetchContent_Declare(winml_ep_catalog URL ${WINML_EP_CATALOG_FETCH_URL} DOWNLOAD_EXTRACT_TIMESTAMP TRUE DOWNLOAD_NAME winml_ep_catalog.zip) diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.cc b/sdk_v2/cpp/src/catalog/base_model_catalog.cc index 3c96eb51c..d5ee9844a 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.cc @@ -20,6 +20,11 @@ BaseModelCatalog::BaseModelCatalog(std::string name, ILogger& logger) BaseModelCatalog::~BaseModelCatalog() = default; void BaseModelCatalog::PopulateModels(std::vector variants) const { + if (populated_) { + IntegrateVariantsLocked(std::move(variants)); + return; + } + // Group variants by alias into Model containers. // Matches C# Catalog.UpdateModels() pattern: // foreach (modelInfo) { find or create Model by alias, add variant } @@ -47,50 +52,25 @@ void BaseModelCatalog::PopulateModels(std::vector variants) const { model.SelectDefaultVariant(); } - // On refresh: merge new models into stable storage. Existing models keep their addresses. - // New aliases are appended. Existing aliases are left unchanged (their Model* stays valid). - if (populated_) { - // Build a set of existing aliases for fast lookup. - std::unordered_map existing_aliases; - for (auto& m : models_) { - existing_aliases[m->Alias()] = m.get(); - } - - size_t new_count = 0; - for (auto& [alias, model] : alias_to_model) { - if (!existing_aliases.contains(alias)) { - models_.push_back(std::make_unique(std::move(model))); - ++new_count; - } - } - - if (new_count > 0) { - logger_.Log(LogLevel::Information, - fmt::format("Catalog '{}' refresh: {} new model(s) added, {} total.", - name_, new_count, models_.size())); - } else { - logger_.Log(LogLevel::Debug, - fmt::format("Catalog '{}' refresh: no new models. {} total.", - name_, models_.size())); - } - } else { - // Initial population: move all models into stable storage. - models_.reserve(alias_to_model.size()); - for (auto& [alias, model] : alias_to_model) { - models_.push_back(std::make_unique(std::move(model))); - } - - logger_.Log(LogLevel::Debug, - fmt::format("Catalog '{}' populated with {} model(s).", name_, models_.size())); + // Initial population: move all models into stable storage. + models_.reserve(alias_to_model.size()); + for (auto& [alias, model] : alias_to_model) { + models_.push_back(std::make_unique(std::move(model))); } + logger_.Log(LogLevel::Debug, + fmt::format("Catalog '{}' populated with {} model(s).", name_, models_.size())); + RebuildIndex(); populated_ = true; } void BaseModelCatalog::IntegrateVariants(std::vector variants) const { std::lock_guard lock(mutex_); + IntegrateVariantsLocked(std::move(variants)); +} +void BaseModelCatalog::IntegrateVariantsLocked(std::vector variants) const { if (variants.empty()) { return; } @@ -141,6 +121,7 @@ void BaseModelCatalog::IntegrateVariants(std::vector variants) const { it->second->AddVariant(std::move(v)); ++added_variants; } + it->second->SelectDefaultVariant(); } else { // New alias: build a container and choose default after all variants are added. auto first = std::move(alias_variants.front()); diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.h b/sdk_v2/cpp/src/catalog/base_model_catalog.h index 97411e395..8e91a9c6b 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.h +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.h @@ -111,6 +111,7 @@ class BaseModelCatalog : public ICatalog { /// present. For new aliases, creates a new container. Rebuilds the lookup /// index when the model set actually changed. void IntegrateVariants(std::vector variants) const; + void IntegrateVariantsLocked(std::vector variants) const; /// Build lookup indices from the current models_ collection. /// Builds a complete new ModelIndex locally, then atomically swaps it into index_. diff --git a/sdk_v2/cpp/src/download/download_manager.cc b/sdk_v2/cpp/src/download/download_manager.cc index 82d304d6b..09d923743 100644 --- a/sdk_v2/cpp/src/download/download_manager.cc +++ b/sdk_v2/cpp/src/download/download_manager.cc @@ -342,6 +342,10 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, if (progress_cb) { progress_cb(100.0f); } + if (tracker != nullptr) { + tracker->SetStatus(ActionStatus::kSkipped); + tracker->SetDownloadWaitResult("CompletedByOtherProcess"); + } return ResolveEffectiveModelPath(model_path); } diff --git a/sdk_v2/cpp/src/download/file_writer.cc b/sdk_v2/cpp/src/download/file_writer.cc index 6ffb6dd5e..5c9b4f5eb 100644 --- a/sdk_v2/cpp/src/download/file_writer.cc +++ b/sdk_v2/cpp/src/download/file_writer.cc @@ -71,7 +71,15 @@ FileWriter::FileWriter(ILogger& logger) : logger_(logger) {} #ifdef _WIN32 -FileWriter::~FileWriter() { Close(); } +FileWriter::~FileWriter() { + try { + Close(); + } catch (const std::exception& ex) { + logger_.Log(LogLevel::Warning, std::string("FileWriter: close failed in destructor: ") + ex.what()); + } catch (...) { + logger_.Log(LogLevel::Warning, "FileWriter: close failed in destructor with unknown error"); + } +} void FileWriter::Open(const fs::path& path, int64_t expected_size) { EnsureFileExistsAtSize(path, expected_size); @@ -119,8 +127,9 @@ void FileWriter::Close() { if (handle_ != nullptr) { if (!::CloseHandle(handle_)) { const DWORD err = ::GetLastError(); - logger_.Log(LogLevel::Warning, - "FileWriter: CloseHandle failed (Win32 err " + std::to_string(err) + ")"); + handle_ = nullptr; + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "FileWriter CloseHandle failed (Win32 err " + std::to_string(err) + ")"); } handle_ = nullptr; } @@ -128,7 +137,15 @@ void FileWriter::Close() { #else // POSIX -FileWriter::~FileWriter() { Close(); } +FileWriter::~FileWriter() { + try { + Close(); + } catch (const std::exception& ex) { + logger_.Log(LogLevel::Warning, std::string("FileWriter: close failed in destructor: ") + ex.what()); + } catch (...) { + logger_.Log(LogLevel::Warning, "FileWriter: close failed in destructor with unknown error"); + } +} void FileWriter::Open(const fs::path& path, int64_t expected_size) { EnsureFileExistsAtSize(path, expected_size); @@ -163,13 +180,13 @@ void FileWriter::Close() { if (fd_ >= 0) { // A failing close() can surface a deferred write error (e.g. EIO, or ENOSPC // on delayed allocation / a networked filesystem), so the file may be - // incomplete even though every pwrite returned success. Log it for - // diagnosis. Don't retry: on Linux the descriptor is freed even when close() + // incomplete even though every pwrite returned success. Don't retry: on Linux the descriptor is freed even when close() // returns EINTR, so a retry could close an unrelated, since-reused fd. if (::close(fd_) != 0) { const int err = errno; - logger_.Log(LogLevel::Warning, - "FileWriter: close failed (errno " + std::to_string(err) + ")"); + fd_ = -1; + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "FileWriter close failed (errno " + std::to_string(err) + ")"); } fd_ = -1; } diff --git a/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc b/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc index 4e331d038..30aeef4ce 100644 --- a/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc +++ b/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc @@ -208,6 +208,24 @@ TEST_F(BaseModelCatalogTest, GetModel_VariantsAccessible) { EXPECT_EQ(container->Variants().size(), 2u); } +TEST_F(BaseModelCatalogTest, RefreshMergesNewVariantsForExistingAlias) { + TestCatalog catalog(logger_); + catalog.AddModel(MakeModel("phi-3-mini-cpu:1", "phi-3-mini-cpu", 1, "phi-3")); + + Model* container = catalog.GetModel("phi-3"); + ASSERT_NE(container, nullptr); + EXPECT_EQ(container->Variants().size(), 1u); + EXPECT_EQ(catalog.GetModelVariant("phi-3-mini-gpu:1"), nullptr); + + catalog.AddModel(MakeModel("phi-3-mini-gpu:1", "phi-3-mini-gpu", 1, "phi-3")); + catalog.InvalidateCache(); + + auto refreshed = catalog.ListModels(); + ASSERT_EQ(refreshed.size(), 1u); + EXPECT_EQ(refreshed[0]->Variants().size(), 2u); + EXPECT_NE(catalog.GetModelVariant("phi-3-mini-gpu:1"), nullptr); +} + TEST_F(BaseModelCatalogTest, GetModel_NotFound_ReturnsNullptr) { TestCatalog catalog(logger_); catalog.AddModel(MakeModel("phi-3-mini:1", "phi-3-mini", 1, "phi-3")); From de660d5d23bfc76d1ebfff98f4b9aa9452c8b1f4 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 9 Jul 2026 17:53:07 -0500 Subject: [PATCH 24/77] Harden WinML fetch and model selection refresh Prevent supported CMake 3.20 builds from seeing newer FetchContent options and remove a catalog-refresh data race by making selected model variants atomic. Files changed: - sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake - sdk_v2/cpp/src/model.cc - sdk_v2/cpp/src/model.h Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake | 6 ++- sdk_v2/cpp/src/model.cc | 60 +++++++++++------------ sdk_v2/cpp/src/model.h | 6 ++- 3 files changed, 39 insertions(+), 33 deletions(-) diff --git a/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake b/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake index debfee8f2..21fc9313e 100644 --- a/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake +++ b/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake @@ -76,7 +76,11 @@ if(WINML_EP_CATALOG_FETCH_URL) configure_file("${WINML_EP_CATALOG_FETCH_URL}" "${_WINML_ZIP_PATH}" COPYONLY) set(WINML_EP_CATALOG_FETCH_URL "${_WINML_ZIP_PATH}") endif() - FetchContent_Declare(winml_ep_catalog URL ${WINML_EP_CATALOG_FETCH_URL} DOWNLOAD_EXTRACT_TIMESTAMP TRUE DOWNLOAD_NAME winml_ep_catalog.zip) + set(_WINML_FETCH_ARGS URL ${WINML_EP_CATALOG_FETCH_URL} DOWNLOAD_NAME winml_ep_catalog.zip) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24") + list(APPEND _WINML_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP TRUE) + endif() + FetchContent_Declare(winml_ep_catalog ${_WINML_FETCH_ARGS}) FetchContent_MakeAvailable(winml_ep_catalog) set(_WINML_EP_ROOT "${winml_ep_catalog_SOURCE_DIR}") message(STATUS "WinML EP Catalog via FetchContent: ${_WINML_EP_ROOT}") diff --git a/sdk_v2/cpp/src/model.cc b/sdk_v2/cpp/src/model.cc index 1f06201ce..5aa5d64fa 100644 --- a/sdk_v2/cpp/src/model.cc +++ b/sdk_v2/cpp/src/model.cc @@ -111,11 +111,11 @@ Model::Model(Model&& other) noexcept download_manager_(other.download_manager_), model_load_manager_(other.model_load_manager_), variants_(std::move(other.variants_)), - selected_variant_(other.selected_variant_) { + selected_variant_(other.selected_variant_.load(std::memory_order_acquire)) { // After vector move, selected_variant_ still points into the transferred buffer. other.download_manager_ = nullptr; other.model_load_manager_ = nullptr; - other.selected_variant_ = nullptr; + other.selected_variant_.store(nullptr, std::memory_order_release); } Model& Model::operator=(Model&& other) noexcept { @@ -126,10 +126,10 @@ Model& Model::operator=(Model&& other) noexcept { download_manager_ = other.download_manager_; model_load_manager_ = other.model_load_manager_; variants_ = std::move(other.variants_); - selected_variant_ = other.selected_variant_; + selected_variant_.store(other.selected_variant_.load(std::memory_order_acquire), std::memory_order_release); other.download_manager_ = nullptr; other.model_load_manager_ = nullptr; - other.selected_variant_ = nullptr; + other.selected_variant_.store(nullptr, std::memory_order_release); } return *this; @@ -163,7 +163,7 @@ Model Model::FromModelInfo(ModelInfo info, Model Model::MakeContainer(Model first_variant) { Model container; container.variants_.push_back(std::make_unique(std::move(first_variant))); - container.selected_variant_ = container.variants_.back().get(); + container.selected_variant_.store(container.variants_.back().get(), std::memory_order_release); return container; } @@ -196,12 +196,12 @@ void Model::SelectDefaultVariant() { for (auto& v : variants_) { if (v->IsCached()) { - selected_variant_ = v.get(); + selected_variant_.store(v.get(), std::memory_order_release); return; } } - selected_variant_ = variants_.front().get(); + selected_variant_.store(variants_.front().get(), std::memory_order_release); } // --------------------------------------------------------------------------- @@ -209,24 +209,24 @@ void Model::SelectDefaultVariant() { // --------------------------------------------------------------------------- const std::string& Model::Id() const { - if (selected_variant_) { - return selected_variant_->Id(); + if (auto* selected = SelectedVariant()) { + return selected->Id(); } return info_.model_id; } const std::string& Model::Alias() const { - if (selected_variant_) { - return selected_variant_->Alias(); + if (auto* selected = SelectedVariant()) { + return selected->Alias(); } return info_.alias; } const ModelInfo& Model::Info() const { - if (selected_variant_) { - return selected_variant_->Info(); + if (auto* selected = SelectedVariant()) { + return selected->Info(); } return info_; @@ -251,16 +251,16 @@ std::vector Model::Variants() const { } bool Model::IsCached() const { - if (selected_variant_) { - return selected_variant_->IsCached(); + if (auto* selected = SelectedVariant()) { + return selected->IsCached(); } return cached_; } bool Model::IsLoaded() const { - if (selected_variant_) { - return selected_variant_->IsLoaded(); + if (auto* selected = SelectedVariant()) { + return selected->IsLoaded(); } // ModelLoadManager owns the authoritative loaded-instance map. The pointer is set at @@ -274,8 +274,8 @@ bool Model::IsLoaded() const { // --------------------------------------------------------------------------- void Model::Download(std::function progress_cb) { - if (selected_variant_) { - selected_variant_->Download(std::move(progress_cb)); + if (auto* selected = SelectedVariant()) { + selected->Download(std::move(progress_cb)); return; } @@ -296,16 +296,16 @@ void Model::Download(std::function progress_cb) { } const std::string& Model::GetPath() const { - if (selected_variant_) { - return selected_variant_->GetPath(); + if (auto* selected = SelectedVariant()) { + return selected->GetPath(); } return local_path_; } void Model::Load(ExecutionProvider ep) { - if (selected_variant_) { - selected_variant_->Load(ep); + if (auto* selected = SelectedVariant()) { + selected->Load(ep); return; } @@ -319,8 +319,8 @@ void Model::Load(ExecutionProvider ep) { } void Model::Unload() { - if (selected_variant_) { - selected_variant_->Unload(); + if (auto* selected = SelectedVariant()) { + selected->Unload(); return; } @@ -329,8 +329,8 @@ void Model::Unload() { } void Model::RemoveFromCache() { - if (selected_variant_) { - selected_variant_->RemoveFromCache(); + if (auto* selected = SelectedVariant()) { + selected->RemoveFromCache(); return; } @@ -359,7 +359,7 @@ void Model::SelectVariant(const Model& variant) { for (auto& v : variants_) { if (v.get() == &variant) { - selected_variant_ = v.get(); + selected_variant_.store(v.get(), std::memory_order_release); return; } } @@ -434,8 +434,8 @@ Model::IOInfo IOInfoFromCache(const StaticIOCache& cache) { } // namespace Model::IOInfo Model::GetInputOutputInfo() const { - if (selected_variant_) { - return selected_variant_->GetInputOutputInfo(); + if (auto* selected = SelectedVariant()) { + return selected->GetInputOutputInfo(); } const auto& task = Info().task; diff --git a/sdk_v2/cpp/src/model.h b/sdk_v2/cpp/src/model.h index 90710768a..930beaef2 100644 --- a/sdk_v2/cpp/src/model.h +++ b/sdk_v2/cpp/src/model.h @@ -116,7 +116,7 @@ class Model { IOInfo GetInputOutputInfo() const; /// True if this is a multi-variant container. - bool IsContainer() const { return selected_variant_ != nullptr; } + bool IsContainer() const { return SelectedVariant() != nullptr; } // --- Mutation methods --- @@ -169,11 +169,13 @@ class Model { // Container data (empty/null for leaves). unique_ptr keeps Model addresses // stable across vector growth/reordering. std::vector> variants_; - Model* selected_variant_ = nullptr; // non-null = this is a container + std::atomic selected_variant_{nullptr}; // non-null = this is a container // Guards variants_ across reader/writer threads (catalog refresh adding variants // while another thread enumerates via Variants()). mutable std::mutex state_mutex_; + + Model* SelectedVariant() const { return selected_variant_.load(std::memory_order_acquire); } }; } // namespace fl From e4a518a23e9f3b91b049967dbf779f7ad7419603 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 9 Jul 2026 18:07:23 -0500 Subject: [PATCH 25/77] Close final telemetry review gaps Redact typed telemetry-token CMake defines, serialize manual variant selection against catalog refresh, and bucket custom catalog URLs before uploading CatalogFetch telemetry. Files changed: - sdk_v2/cpp/build.py - sdk_v2/cpp/src/catalog/azure_model_catalog.cc - sdk_v2/cpp/src/model.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/build.py | 11 +++++++++-- sdk_v2/cpp/src/catalog/azure_model_catalog.cc | 13 ++++++------- sdk_v2/cpp/src/model.cc | 2 ++ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/sdk_v2/cpp/build.py b/sdk_v2/cpp/build.py index a947e24df..f919a1cfc 100644 --- a/sdk_v2/cpp/build.py +++ b/sdk_v2/cpp/build.py @@ -58,9 +58,16 @@ def _path_from_env_var(var: str) -> Path | None: # --------------------------------------------------------------------------- def _redact_command_arg(arg: str) -> str: - secret_define = "-DFOUNDRY_LOCAL_TELEMETRY_TOKEN=" + secret_define = "-DFOUNDRY_LOCAL_TELEMETRY_TOKEN" if arg.startswith(secret_define): - return f"{secret_define}" + suffix = arg[len(secret_define):] + if suffix.startswith("="): + return f"{secret_define}=" + if suffix.startswith(":"): + separator = suffix.find("=") + if separator >= 0: + return f"{secret_define}{suffix[:separator]}=" + return f"{secret_define}:" return arg diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index 484b17353..f89a9bbac 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -23,8 +23,7 @@ namespace { /// Split a catalog URL into structured telemetry dimensions. The Azure Foundry /// catalog URL looks like "https://ai.azure.com/api//", e.g. /// "https://ai.azure.com/api/eastus/ux/v1.0" -> {ai.azure.com, eastus, ux/v1.0}. -/// Custom URLs that don't follow the "/api//" convention keep an empty -/// region and put the whole path in `format`. The embedded snapshot is "static". +/// Custom/private URLs are bucketed so telemetry does not disclose hostnames or paths. struct ParsedCatalogUrl { std::string endpoint; std::string region; @@ -67,12 +66,12 @@ ParsedCatalogUrl ParseCatalogUrl(const std::string& url) { pos = next + 1; } - size_t format_start = 0; - if (segments.size() >= 2 && segments[0] == "api") { - out.region = segments[1]; - format_start = 2; + if (out.endpoint != "ai.azure.com" || segments.size() < 2 || segments[0] != "api") { + return {"custom", "", ""}; } - for (size_t i = format_start; i < segments.size(); ++i) { + + out.region = segments[1]; + for (size_t i = 2; i < segments.size(); ++i) { if (!out.format.empty()) { out.format += '/'; } diff --git a/sdk_v2/cpp/src/model.cc b/sdk_v2/cpp/src/model.cc index 5aa5d64fa..09bb61e65 100644 --- a/sdk_v2/cpp/src/model.cc +++ b/sdk_v2/cpp/src/model.cc @@ -357,6 +357,8 @@ void Model::SelectVariant(const Model& variant) { "with all variants available."); } + std::lock_guard lock(state_mutex_); + for (auto& v : variants_) { if (v.get() == &variant) { selected_variant_.store(v.get(), std::memory_order_release); From 5353ec9efca3e9e062d4a13711070b964e2375bb Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 9 Jul 2026 18:38:29 -0500 Subject: [PATCH 26/77] Bump C ABI and scrub URL telemetry Request the new API table version after adding Catalog.GetModelVersions, keep C#/Python mirrors aligned, and prevent custom catalog failure telemetry from retaining URL query or fragment data. Files changed: - sdk_v2/cpp/include/foundry_local/foundry_local_c.h and src/c_api.cc - sdk_v2/cs/src/Detail/NativeMethods.cs - sdk_v2/python/src/foundry_local_sdk API-version mirrors - sdk_v2/cpp/src/telemetry/telemetry_redaction.h and telemetry tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/include/foundry_local/foundry_local_c.h | 6 +++--- sdk_v2/cpp/src/c_api.cc | 6 +++--- sdk_v2/cpp/src/telemetry/telemetry_redaction.h | 7 +++++++ sdk_v2/cpp/test/internal_api/telemetry_test.cc | 10 ++++++++++ sdk_v2/cs/src/Detail/NativeMethods.cs | 2 +- sdk_v2/python/src/foundry_local_sdk/_native/api.py | 4 ++-- sdk_v2/python/src/foundry_local_sdk/items.py | 2 +- sdk_v2/python/src/foundry_local_sdk/request.py | 2 +- sdk_v2/python/src/foundry_local_sdk/response.py | 2 +- sdk_v2/python/src/foundry_local_sdk/session.py | 2 +- 10 files changed, 30 insertions(+), 13 deletions(-) diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_c.h b/sdk_v2/cpp/include/foundry_local/foundry_local_c.h index 034348e99..3974d75d4 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_c.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_c.h @@ -60,7 +60,7 @@ * Incremented with each release. * Used to request the API function table via FoundryLocalGetApi. * ----------------------------------------------------------------------- */ -#define FOUNDRY_LOCAL_API_VERSION 1 +#define FOUNDRY_LOCAL_API_VERSION 2 /* ----------------------------------------------------------------------- * Platform export macros (C version) @@ -816,7 +816,7 @@ struct flItemApi { void FL_API_T(ItemQueue_MarkFinished, _In_ flItemQueue* queue); ///< Producer is done bool FL_API_T(ItemQueue_IsFinished, _In_ const flItemQueue* queue); - // End V1 + // End V2 }; struct flInferenceApi { @@ -889,7 +889,7 @@ struct flInferenceApi { /// If all turns are undone, the cached generator is destroyed. FL_API_STATUS(Session_UndoTurns, _In_ flSession* session, size_t count); - // End V1 + // End V2 }; /* --- Configuration API ------------------------------------------------- */ diff --git a/sdk_v2/cpp/src/c_api.cc b/sdk_v2/cpp/src/c_api.cc index dbf49cc03..5cc689cdd 100644 --- a/sdk_v2/cpp/src/c_api.cc +++ b/sdk_v2/cpp/src/c_api.cc @@ -1844,10 +1844,10 @@ static const flModelApi* FL_API_CALL GetModelApiImpl() FL_NO_EXCEPTION { } // ======================================================================== -// Root API function table (version 1) +// Root API function table (version 2) // ======================================================================== -static const flApi g_api_v1 = { +static const flApi g_api_v2 = { /* Status */ Status_CreateImpl, Status_ReleaseImpl, @@ -1898,7 +1898,7 @@ extern "C" { FL_EXPORT const flApi* FL_API_CALL FoundryLocalGetApi(uint32_t version) FL_NO_EXCEPTION { if (version == 0 || version <= FOUNDRY_LOCAL_API_VERSION) { - return &g_api_v1; + return &g_api_v2; } return nullptr; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_redaction.h b/sdk_v2/cpp/src/telemetry/telemetry_redaction.h index 2866ecca2..199f5fe4b 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_redaction.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_redaction.h @@ -10,6 +10,9 @@ namespace fl { namespace telemetry_detail { inline bool LooksLikePath(std::string_view token) { + if (token.find("://") != std::string_view::npos) { + return true; + } if (token.find('\\') != std::string_view::npos) { return true; } @@ -32,6 +35,10 @@ inline bool LooksLikePath(std::string_view token) { inline std::string RedactPathToken(std::string_view token) { const auto is_sep = [](char c) { return c == '/' || c == '\\'; }; + const auto query_start = token.find_first_of("?#"); + if (query_start != std::string_view::npos) { + token = token.substr(0, query_start); + } std::string normalized(token); for (char& c : normalized) { diff --git a/sdk_v2/cpp/test/internal_api/telemetry_test.cc b/sdk_v2/cpp/test/internal_api/telemetry_test.cc index 6c44be977..17952dd9b 100644 --- a/sdk_v2/cpp/test/internal_api/telemetry_test.cc +++ b/sdk_v2/cpp/test/internal_api/telemetry_test.cc @@ -152,6 +152,16 @@ TEST(TelemetryRedactionTest, ScrubsHomePathsAndKeepsUsefulTail) { EXPECT_NE(scrubbed.find("[path]\\AppData\\file.bin"), std::string::npos); } +TEST(TelemetryRedactionTest, ScrubsUrlQueryAndFragment) { + const auto scrubbed = ScrubTelemetryErrorMessage( + "GET https://custom.example.com/private/catalog?token=secret#fragment failed"); + + EXPECT_EQ(scrubbed.find("custom.example.com"), std::string::npos); + EXPECT_EQ(scrubbed.find("token=secret"), std::string::npos); + EXPECT_EQ(scrubbed.find("#fragment"), std::string::npos); + EXPECT_NE(scrubbed.find("[path]/private/catalog"), std::string::npos); +} + TEST(TelemetryRedactionTest, TruncatesLongErrorMessages) { const auto scrubbed = ScrubTelemetryErrorMessage(std::string(300, 'x')); diff --git a/sdk_v2/cs/src/Detail/NativeMethods.cs b/sdk_v2/cs/src/Detail/NativeMethods.cs index fa8e920d7..f5f9541bc 100644 --- a/sdk_v2/cs/src/Detail/NativeMethods.cs +++ b/sdk_v2/cs/src/Detail/NativeMethods.cs @@ -33,7 +33,7 @@ namespace Microsoft.AI.Foundry.Local.Detail.Interop; // ----------------------------------------------------------------------- public static partial class NativeMethods { - public const uint ApiVersion = 1; + public const uint ApiVersion = 2; public const string LibraryName = "foundry_local"; // The first P/Invoke through this class can come from any of several entry diff --git a/sdk_v2/python/src/foundry_local_sdk/_native/api.py b/sdk_v2/python/src/foundry_local_sdk/_native/api.py index e2a392d3d..1e56572a4 100644 --- a/sdk_v2/python/src/foundry_local_sdk/_native/api.py +++ b/sdk_v2/python/src/foundry_local_sdk/_native/api.py @@ -16,8 +16,8 @@ from foundry_local_sdk._native.lib_loader import find_library, prepare_native_dependencies from foundry_local_sdk.exception import FoundryLocalException -# FOUNDRY_LOCAL_API_VERSION = 1 (from foundry_local_c.h) -_FOUNDRY_LOCAL_API_VERSION: int = 1 +# FOUNDRY_LOCAL_API_VERSION = 2 (from foundry_local_c.h) +_FOUNDRY_LOCAL_API_VERSION: int = 2 _lib_path = find_library() diff --git a/sdk_v2/python/src/foundry_local_sdk/items.py b/sdk_v2/python/src/foundry_local_sdk/items.py index 0f1f3c14f..809fe2b62 100644 --- a/sdk_v2/python/src/foundry_local_sdk/items.py +++ b/sdk_v2/python/src/foundry_local_sdk/items.py @@ -11,7 +11,7 @@ from enum import IntEnum -_API_VERSION = 1 # FOUNDRY_LOCAL_API_VERSION +_API_VERSION = 2 # FOUNDRY_LOCAL_API_VERSION # Sentinel for absent time fields in flSpeech* structs. INT64_MIN, matches # C ABI FOUNDRY_LOCAL_DURATION_UNSET. cffi cdef does not process #define. _DURATION_UNSET: int = -(2**63) diff --git a/sdk_v2/python/src/foundry_local_sdk/request.py b/sdk_v2/python/src/foundry_local_sdk/request.py index fe6879e81..5c3a90ca5 100644 --- a/sdk_v2/python/src/foundry_local_sdk/request.py +++ b/sdk_v2/python/src/foundry_local_sdk/request.py @@ -12,7 +12,7 @@ from foundry_local_sdk.items import Item from foundry_local_sdk.session_types import RequestOptions -_API_VERSION = 1 # FOUNDRY_LOCAL_API_VERSION +_API_VERSION = 2 # FOUNDRY_LOCAL_API_VERSION class Request: diff --git a/sdk_v2/python/src/foundry_local_sdk/response.py b/sdk_v2/python/src/foundry_local_sdk/response.py index df6748bc3..8df2e2f24 100644 --- a/sdk_v2/python/src/foundry_local_sdk/response.py +++ b/sdk_v2/python/src/foundry_local_sdk/response.py @@ -12,7 +12,7 @@ from foundry_local_sdk.items import Item from foundry_local_sdk.session_types import FinishReason, TokenUsage -_API_VERSION = 1 # FOUNDRY_LOCAL_API_VERSION +_API_VERSION = 2 # FOUNDRY_LOCAL_API_VERSION class Response: diff --git a/sdk_v2/python/src/foundry_local_sdk/session.py b/sdk_v2/python/src/foundry_local_sdk/session.py index 136609085..e913c0138 100644 --- a/sdk_v2/python/src/foundry_local_sdk/session.py +++ b/sdk_v2/python/src/foundry_local_sdk/session.py @@ -17,7 +17,7 @@ from foundry_local_sdk.response import Response from foundry_local_sdk.session_types import RequestOptions -_API_VERSION = 1 # FOUNDRY_LOCAL_API_VERSION +_API_VERSION = 2 # FOUNDRY_LOCAL_API_VERSION # Sentinel placed on the stream queue by the background thread when inference finishes. _DONE = object() From b44dff942d86ffde6e6837078019f0ef9d98f302 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 9 Jul 2026 19:26:02 -0500 Subject: [PATCH 27/77] Redact catalog logs and package WinML runtime Scrub catalog failure details in local logs, make C++ wrapper API-version mismatches fail clearly, and keep required C++ SDK archive dependencies including GSL headers and the WinML runtime DLL. Files changed: - .pipelines/v2/templates/steps-pack-cpp-sdk.yml - sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h - sdk_v2/cpp/src/catalog/azure_model_catalog.cc - sdk_v2/cpp/src/telemetry/telemetry_logger.cc and telemetry_redaction.h - sdk_v2/cpp/test/internal_api/telemetry_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .../v2/templates/steps-pack-cpp-sdk.yml | 9 +-- .../include/foundry_local/foundry_local_cpp.h | 55 ++++++++++++++++--- sdk_v2/cpp/src/catalog/azure_model_catalog.cc | 5 +- sdk_v2/cpp/src/telemetry/telemetry_logger.cc | 6 +- .../cpp/src/telemetry/telemetry_redaction.h | 4 ++ .../cpp/test/internal_api/telemetry_test.cc | 2 +- 6 files changed, 66 insertions(+), 15 deletions(-) diff --git a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml index d18bfa8b7..448abef0b 100644 --- a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml +++ b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml @@ -57,9 +57,8 @@ steps: Copy-Item -Path (Join-Path $includeSrc '*') -Destination $includeDst -Recurse -Force - # The SDK tgz should include only public wrapper headers. + # The SDK tgz should include public wrapper headers and their header-only dependencies. $pathsToPrune = @( - (Join-Path $includeDst 'gsl'), (Join-Path $includeDst '_manifest') ) foreach ($pathToPrune in $pathsToPrune) { @@ -96,13 +95,15 @@ steps: New-SdkArchive -Rid 'win-x64' -NativeArtifact 'cpp-native-win-x64' -Files @( 'foundry_local.dll', 'foundry_local.pdb', - 'foundry_local.lib' + 'foundry_local.lib', + 'Microsoft.Windows.AI.MachineLearning.dll' ) New-SdkArchive -Rid 'win-arm64' -NativeArtifact 'cpp-native-win-arm64' -Files @( 'foundry_local.dll', 'foundry_local.pdb', - 'foundry_local.lib' + 'foundry_local.lib', + 'Microsoft.Windows.AI.MachineLearning.dll' ) New-SdkArchive -Rid 'linux-x64' -NativeArtifact 'cpp-native-linux-x64' -Files @( diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h index fe6281302..941665cfc 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h @@ -61,39 +61,80 @@ inline const char* Version() noexcept; namespace detail { /// Get the API function table (cached on first call). -/// Returns nullptr if the library does not support the requested API version. inline const flApi* api() { - static const flApi* p = FoundryLocalGetApi(FOUNDRY_LOCAL_API_VERSION); + static const flApi* p = [] { + const flApi* root = FoundryLocalGetApi(FOUNDRY_LOCAL_API_VERSION); + if (!root) { + throw Error("Foundry Local native library does not support the requested API version", + FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } + return root; + }(); return p; } /// Get the Catalog sub-API (cached on first call). inline const flCatalogApi* catalog_api() { - static const flCatalogApi* p = api()->GetCatalogApi(); + static const flCatalogApi* p = [] { + const flCatalogApi* catalog = api()->GetCatalogApi(); + if (!catalog) { + throw Error("Foundry Local native library returned a null Catalog API table", + FOUNDRY_LOCAL_ERROR_INTERNAL); + } + return catalog; + }(); return p; } /// Get the Configuration sub-API (cached on first call). inline const flConfigurationApi* config_api() { - static const flConfigurationApi* p = api()->GetConfigurationApi(); + static const flConfigurationApi* p = [] { + const flConfigurationApi* config = api()->GetConfigurationApi(); + if (!config) { + throw Error("Foundry Local native library returned a null Configuration API table", + FOUNDRY_LOCAL_ERROR_INTERNAL); + } + return config; + }(); return p; } /// Get the Inference sub-API (cached on first call). inline const flInferenceApi* inference_api() { - static const flInferenceApi* p = api()->GetInferenceApi(); + static const flInferenceApi* p = [] { + const flInferenceApi* inference = api()->GetInferenceApi(); + if (!inference) { + throw Error("Foundry Local native library returned a null Inference API table", + FOUNDRY_LOCAL_ERROR_INTERNAL); + } + return inference; + }(); return p; } /// Get the Item sub-API (cached on first call). inline const flItemApi* item_api() { - static const flItemApi* p = api()->GetItemApi(); + static const flItemApi* p = [] { + const flItemApi* item = api()->GetItemApi(); + if (!item) { + throw Error("Foundry Local native library returned a null Item API table", + FOUNDRY_LOCAL_ERROR_INTERNAL); + } + return item; + }(); return p; } /// Get the Model sub-API (cached on first call). inline const flModelApi* model_api() { - static const flModelApi* p = api()->GetModelApi(); + static const flModelApi* p = [] { + const flModelApi* model = api()->GetModelApi(); + if (!model) { + throw Error("Foundry Local native library returned a null Model API table", + FOUNDRY_LOCAL_ERROR_INTERNAL); + } + return model; + }(); return p; } diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index f89a9bbac..453ae39a7 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -8,6 +8,7 @@ #include "model_info.h" #include "telemetry/invocation_context.h" #include "telemetry/telemetry.h" +#include "telemetry/telemetry_redaction.h" #include "utils.h" #include @@ -197,8 +198,10 @@ std::vector AzureModelCatalog::FetchModels() const { fetch_from(url, filter); } catch (const std::exception& ex) { // One failing URL shouldn't block others — skip and continue. + auto parsed = ParseCatalogUrl(url); logger_.Log(LogLevel::Error, - fmt::format("failed to fetch catalog from {}: {}", url, ex.what())); + fmt::format("failed to fetch catalog from {}: {}", parsed.endpoint, + ScrubTelemetryErrorMessage(ex.what()))); } } diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc index 9b594770d..60e44faba 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc @@ -2,6 +2,8 @@ // Licensed under the MIT License. #include "telemetry/telemetry_logger.h" +#include "telemetry/telemetry_redaction.h" + #include namespace fl { @@ -24,7 +26,7 @@ void TelemetryLogger::RecordException(Action action, const std::exception& excep logger_.Log(LogLevel::Debug, fmt::format("[Telemetry] Error AppName={} UserAgent={} CorrelationId={} Action={} Exception={}", app_name_, context.user_agent, context.correlation_id, ActionToString(action), - exception.what())); + ScrubTelemetryErrorMessage(exception.what()))); } void TelemetryLogger::RecordModelUsage(const ModelUsageInfo& info) { @@ -89,7 +91,7 @@ void TelemetryLogger::RecordCatalogFetch(const CatalogFetchInfo& info) { "Status={} TimeMs={} ModelCount={} Error={} UserAgent={} CorrelationId={}", app_name_, info.operation, info.endpoint, info.region, info.format, ActionStatusToString(info.status), info.duration_ms, info.model_count, - info.error_message, info.user_agent, info.correlation_id)); + ScrubTelemetryErrorMessage(info.error_message), info.user_agent, info.correlation_id)); } void TelemetryLogger::StartSession() { diff --git a/sdk_v2/cpp/src/telemetry/telemetry_redaction.h b/sdk_v2/cpp/src/telemetry/telemetry_redaction.h index 199f5fe4b..cd429e11f 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_redaction.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_redaction.h @@ -34,6 +34,10 @@ inline bool LooksLikePath(std::string_view token) { } inline std::string RedactPathToken(std::string_view token) { + if (token.find("://") != std::string_view::npos) { + return "[url]"; + } + const auto is_sep = [](char c) { return c == '/' || c == '\\'; }; const auto query_start = token.find_first_of("?#"); if (query_start != std::string_view::npos) { diff --git a/sdk_v2/cpp/test/internal_api/telemetry_test.cc b/sdk_v2/cpp/test/internal_api/telemetry_test.cc index 17952dd9b..128d3722b 100644 --- a/sdk_v2/cpp/test/internal_api/telemetry_test.cc +++ b/sdk_v2/cpp/test/internal_api/telemetry_test.cc @@ -159,7 +159,7 @@ TEST(TelemetryRedactionTest, ScrubsUrlQueryAndFragment) { EXPECT_EQ(scrubbed.find("custom.example.com"), std::string::npos); EXPECT_EQ(scrubbed.find("token=secret"), std::string::npos); EXPECT_EQ(scrubbed.find("#fragment"), std::string::npos); - EXPECT_NE(scrubbed.find("[path]/private/catalog"), std::string::npos); + EXPECT_NE(scrubbed.find("[url]"), std::string::npos); } TEST(TelemetryRedactionTest, TruncatesLongErrorMessages) { From 16ec313e468dd91699c2fb8e8bc955ef378f0905 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 9 Jul 2026 19:55:25 -0500 Subject: [PATCH 28/77] Harden catalog logging and WinML acquisition Avoid leaking telemetry tokens from failed build commands, acquire WinML through the flat-container FetchContent path by default, sanitize catalog failure logs consistently, and preserve explicit variant selections across catalog refresh. Files changed: - sdk_v2/cpp/build.py - sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake - sdk_v2/cpp/src/catalog/azure_model_catalog.cc - sdk_v2/cpp/src/catalog/base_model_catalog.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/build.py | 7 ++- sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake | 46 +++++++++---------- sdk_v2/cpp/src/catalog/azure_model_catalog.cc | 8 +++- sdk_v2/cpp/src/catalog/base_model_catalog.cc | 1 - 4 files changed, 33 insertions(+), 29 deletions(-) diff --git a/sdk_v2/cpp/build.py b/sdk_v2/cpp/build.py index f919a1cfc..ee5930919 100644 --- a/sdk_v2/cpp/build.py +++ b/sdk_v2/cpp/build.py @@ -73,8 +73,11 @@ def _redact_command_arg(arg: str) -> str: def run(cmd: list[str], cwd: Path | str | None = None, env: dict[str, str] | None = None) -> None: """Run a command, logging and streaming output. Raises on non-zero exit.""" - log.info("Running: %s", " ".join(_redact_command_arg(str(c)) for c in cmd)) - subprocess.run(cmd, cwd=cwd, env=env, check=True) + redacted = " ".join(_redact_command_arg(str(c)) for c in cmd) + log.info("Running: %s", redacted) + result = subprocess.run(cmd, cwd=cwd, env=env, check=False) + if result.returncode != 0: + raise RuntimeError(f"Command failed with exit code {result.returncode}: {redacted}") # --------------------------------------------------------------------------- diff --git a/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake b/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake index 21fc9313e..4bf4eb0f7 100644 --- a/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake +++ b/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake @@ -60,34 +60,32 @@ if(NOT _WINML_PLATFORM_UPPER MATCHES "^(AMD64|X64|X86_64|ARM64|AARCH64)$") return() endif() -include(cmake/nuget.cmake) - # WINML_EP_CATALOG_FETCH_URL can be set externally (e.g. for CI where nuget.org is blocked). set(WINML_EP_CATALOG_FETCH_URL "" CACHE STRING "Override URL or local path for the WinML EP Catalog NuGet package") -if(WINML_EP_CATALOG_FETCH_URL) - # Use FetchContent to download/extract the pre-downloaded package - include(FetchContent) - string(REPLACE "\\" "/" WINML_EP_CATALOG_FETCH_URL "${WINML_EP_CATALOG_FETCH_URL}") - if(WINML_EP_CATALOG_FETCH_URL MATCHES "\\.nupkg$" AND NOT WINML_EP_CATALOG_FETCH_URL MATCHES "^https?://") - set(_WINML_ZIP_PATH "${CMAKE_BINARY_DIR}/_deps/winml_ep_catalog-download/winml_ep_catalog.zip") - get_filename_component(_WINML_ZIP_DIR "${_WINML_ZIP_PATH}" DIRECTORY) - file(MAKE_DIRECTORY "${_WINML_ZIP_DIR}") - configure_file("${WINML_EP_CATALOG_FETCH_URL}" "${_WINML_ZIP_PATH}" COPYONLY) - set(WINML_EP_CATALOG_FETCH_URL "${_WINML_ZIP_PATH}") - endif() - set(_WINML_FETCH_ARGS URL ${WINML_EP_CATALOG_FETCH_URL} DOWNLOAD_NAME winml_ep_catalog.zip) - if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24") - list(APPEND _WINML_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP TRUE) - endif() - FetchContent_Declare(winml_ep_catalog ${_WINML_FETCH_ARGS}) - FetchContent_MakeAvailable(winml_ep_catalog) - set(_WINML_EP_ROOT "${winml_ep_catalog_SOURCE_DIR}") - message(STATUS "WinML EP Catalog via FetchContent: ${_WINML_EP_ROOT}") -else() - install_nuget_package(Microsoft.Windows.AI.MachineLearning ${WINML_EP_CATALOG_VERSION} _WINML_EP_ROOT - SOURCE https://api.nuget.org/v3/index.json) +include(FetchContent) +set(_WINML_EP_CATALOG_URL "${WINML_EP_CATALOG_FETCH_URL}") +if(NOT _WINML_EP_CATALOG_URL) + set(_WINML_EP_CATALOG_URL + "https://api.nuget.org/v3-flatcontainer/microsoft.windows.ai.machinelearning/${WINML_EP_CATALOG_VERSION}/microsoft.windows.ai.machinelearning.${WINML_EP_CATALOG_VERSION}.nupkg") +endif() + +string(REPLACE "\\" "/" _WINML_EP_CATALOG_URL "${_WINML_EP_CATALOG_URL}") +if(_WINML_EP_CATALOG_URL MATCHES "\\.nupkg$" AND NOT _WINML_EP_CATALOG_URL MATCHES "^https?://") + set(_WINML_ZIP_PATH "${CMAKE_BINARY_DIR}/_deps/winml_ep_catalog-download/winml_ep_catalog.zip") + get_filename_component(_WINML_ZIP_DIR "${_WINML_ZIP_PATH}" DIRECTORY) + file(MAKE_DIRECTORY "${_WINML_ZIP_DIR}") + configure_file("${_WINML_EP_CATALOG_URL}" "${_WINML_ZIP_PATH}" COPYONLY) + set(_WINML_EP_CATALOG_URL "${_WINML_ZIP_PATH}") +endif() +set(_WINML_FETCH_ARGS URL ${_WINML_EP_CATALOG_URL} DOWNLOAD_NAME winml_ep_catalog.zip) +if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24") + list(APPEND _WINML_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP TRUE) endif() +FetchContent_Declare(winml_ep_catalog ${_WINML_FETCH_ARGS}) +FetchContent_MakeAvailable(winml_ep_catalog) +set(_WINML_EP_ROOT "${winml_ep_catalog_SOURCE_DIR}") +message(STATUS "WinML EP Catalog via FetchContent: ${_WINML_EP_ROOT}") # Load the package's first-party CMake config for target discovery and layout # resolution. The config lives at build/cmake/-config.cmake diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index 453ae39a7..0799cea92 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -241,8 +241,10 @@ std::vector AzureModelCatalog::FetchModelVersions( out.push_back(model_factory_(std::move(info), /*local_path=*/"")); } } catch (const std::exception& ex) { + auto parsed = ParseCatalogUrl(url); logger_.Log(LogLevel::Error, - fmt::format("FetchModelVersions: failed to query {} — {}", url, ex.what())); + fmt::format("FetchModelVersions: failed to query {} — {}", parsed.endpoint, + ScrubTelemetryErrorMessage(ex.what()))); } } @@ -297,8 +299,10 @@ std::vector AzureModelCatalog::FetchModelsByIds(const std::vector variants) cons it->second->AddVariant(std::move(v)); ++added_variants; } - it->second->SelectDefaultVariant(); } else { // New alias: build a container and choose default after all variants are added. auto first = std::move(alias_variants.front()); From 47a18d80c7834fa2cb14094c4aa3253bc1517b1f Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 9 Jul 2026 20:25:44 -0500 Subject: [PATCH 29/77] Sanitize catalog errors and unsafe resume state Scrub remaining catalog failure logs, bucket nonstandard ai.azure.com catalog paths, and invalidate blob data/resume state when final close surfaces deferred write errors. Files changed: - sdk_v2/cpp/src/catalog/azure_model_catalog.cc - sdk_v2/cpp/src/catalog/catalog_client.cc - sdk_v2/cpp/src/download/blob_downloader.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/catalog/azure_model_catalog.cc | 3 ++- sdk_v2/cpp/src/catalog/catalog_client.cc | 6 ++++-- sdk_v2/cpp/src/download/blob_downloader.cc | 14 +++++++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index 0799cea92..1f687c596 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -67,7 +67,8 @@ ParsedCatalogUrl ParseCatalogUrl(const std::string& url) { pos = next + 1; } - if (out.endpoint != "ai.azure.com" || segments.size() < 2 || segments[0] != "api") { + if (out.endpoint != "ai.azure.com" || segments.size() < 4 || segments[0] != "api" || + segments[2] != "ux" || segments[3].empty() || segments[3][0] != 'v') { return {"custom", "", ""}; } diff --git a/sdk_v2/cpp/src/catalog/catalog_client.cc b/sdk_v2/cpp/src/catalog/catalog_client.cc index c81070568..f19d699bc 100644 --- a/sdk_v2/cpp/src/catalog/catalog_client.cc +++ b/sdk_v2/cpp/src/catalog/catalog_client.cc @@ -3,6 +3,7 @@ #include "catalog/catalog_client.h" #include "ep_detection/ep_detector.h" #include "telemetry/telemetry.h" +#include "telemetry/telemetry_redaction.h" #include "utils.h" #include @@ -82,9 +83,10 @@ std::vector FetchAllModelInfosWithCachedModels( } emit("FetchByIds", ActionStatus::kSuccess, elapsed_ms(start), additional_count, ""); } catch (const std::exception& ex) { + auto error_message = ScrubTelemetryErrorMessage(ex.what()); logger.Log(LogLevel::Warning, - fmt::format("catalog: failed to fetch cached model IDs — {}", ex.what())); - emit("FetchByIds", ActionStatus::kFailure, elapsed_ms(start), 0, ex.what()); + fmt::format("catalog: failed to fetch cached model IDs — {}", error_message)); + emit("FetchByIds", ActionStatus::kFailure, elapsed_ms(start), 0, error_message); } catch (...) { logger.Log(LogLevel::Warning, "catalog: failed to fetch cached model IDs — unknown error"); emit("FetchByIds", ActionStatus::kFailure, elapsed_ms(start), 0, "unknown error"); diff --git a/sdk_v2/cpp/src/download/blob_downloader.cc b/sdk_v2/cpp/src/download/blob_downloader.cc index e5411d4db..9374e1b6a 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.cc +++ b/sdk_v2/cpp/src/download/blob_downloader.cc @@ -360,7 +360,19 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, // Release the OS handle before persisting / deleting the sidecar so any // observer that watches the data file sees a fully-closed handle. - writer.Close(); + try { + writer.Close(); + } catch (...) { + std::error_code remove_ec; + std::filesystem::remove(local_path, remove_ec); + if (remove_ec) { + logger_.Log(LogLevel::Warning, + "failed to remove blob file after close failure: " + local_path + + " (" + remove_ec.message() + ")"); + } + BlobDownloadState::DeleteState(local_path, logger_); + throw; + } const bool was_cancelled = cancelled && cancelled->load(std::memory_order_relaxed); if (first_error || was_cancelled) { From e82b4b5fe7407bc85a71636a0be078a796246990 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 9 Jul 2026 20:55:17 -0500 Subject: [PATCH 30/77] Secure telemetry token and EP snapshots Pass telemetry tokens through the environment instead of argv/cache, publish immutable C API EP snapshots, track skipped download bytes/files, and force a safe redownload when final close reports deferred write failures. Files changed: - sdk_v2/cpp/CMakeLists.txt and build.py - sdk_v2/cpp/src/ep_detection/ep_detector.* - sdk_v2/cpp/src/download/blob_downloader.* and download_manager.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/CMakeLists.txt | 13 +++---- sdk_v2/cpp/build.py | 4 +- sdk_v2/cpp/src/download/blob_downloader.cc | 42 ++++++++++++++++----- sdk_v2/cpp/src/download/blob_downloader.h | 2 + sdk_v2/cpp/src/download/download_manager.cc | 2 + sdk_v2/cpp/src/ep_detection/ep_detector.cc | 30 ++++++++++----- sdk_v2/cpp/src/ep_detection/ep_detector.h | 12 ++++-- 7 files changed, 73 insertions(+), 32 deletions(-) diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index cbc668d1a..729b77090 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -47,14 +47,11 @@ option(FOUNDRY_LOCAL_USE_TELEMETRY "Build with 1DS telemetry uploads (requires cpp-client-telemetry vcpkg port)" ON) option(FOUNDRY_LOCAL_ENABLE_ASAN "Enable AddressSanitizer + UndefinedBehaviorSanitizer (Linux only)" OFF) -# 1DS ingestion token. Treat as a secret: pass via CI as -# cmake -DFOUNDRY_LOCAL_TELEMETRY_TOKEN="" -# Defaults to empty so developer builds and forks don't accidentally bind to a -# real tenant. When empty (or when in CI at runtime), OneDsTelemetry initializes -# but does not transmit events. -set(FOUNDRY_LOCAL_TELEMETRY_TOKEN "" - CACHE STRING "1DS / Aria ingestion token baked into the binary. Leave empty in dev builds.") -mark_as_advanced(FOUNDRY_LOCAL_TELEMETRY_TOKEN) +# 1DS ingestion token. Treat as a secret: pass via the +# FOUNDRY_LOCAL_TELEMETRY_TOKEN environment variable so it does not land in +# process argv or CMakeCache.txt. Defaults to empty so developer builds and forks +# don't accidentally bind to a real tenant. +set(FOUNDRY_LOCAL_TELEMETRY_TOKEN "$ENV{FOUNDRY_LOCAL_TELEMETRY_TOKEN}") # Android: interactive examples and host tools don't run on device if(ANDROID) diff --git a/sdk_v2/cpp/build.py b/sdk_v2/cpp/build.py index ee5930919..aad2071ef 100644 --- a/sdk_v2/cpp/build.py +++ b/sdk_v2/cpp/build.py @@ -495,8 +495,10 @@ def configure(args: argparse.Namespace) -> None: command += ["-DFOUNDRY_LOCAL_USE_TELEMETRY=OFF"] else: command += ["-DFOUNDRY_LOCAL_USE_TELEMETRY=ON"] + env = None if args.telemetry_token is not None: - command += [f"-DFOUNDRY_LOCAL_TELEMETRY_TOKEN={args.telemetry_token}"] + env = os.environ.copy() + env["FOUNDRY_LOCAL_TELEMETRY_TOKEN"] = args.telemetry_token # WinML EP catalog is enabled automatically on Windows by CMake. Allow an # optional version override for the Microsoft.Windows.AI.MachineLearning NuGet. diff --git a/sdk_v2/cpp/src/download/blob_downloader.cc b/sdk_v2/cpp/src/download/blob_downloader.cc index 9374e1b6a..68c0a6534 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.cc +++ b/sdk_v2/cpp/src/download/blob_downloader.cc @@ -364,13 +364,30 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, writer.Close(); } catch (...) { std::error_code remove_ec; - std::filesystem::remove(local_path, remove_ec); - if (remove_ec) { + const bool removed = std::filesystem::remove(local_path, remove_ec) || + !std::filesystem::exists(local_path); + if (removed) { + BlobDownloadState::DeleteState(local_path, logger_); + } else { logger_.Log(LogLevel::Warning, "failed to remove blob file after close failure: " + local_path + " (" + remove_ec.message() + ")"); + std::error_code resize_ec; + std::filesystem::resize_file(local_path, 0, resize_ec); + if (resize_ec) { + logger_.Log(LogLevel::Warning, + "failed to truncate blob file after close failure: " + local_path + + " (" + resize_ec.message() + ")"); + auto fresh_state = BlobDownloadState::CreateNew(blob_name, local_path, blob_size, + static_cast(kChunkSize), num_chunks); + if (!fresh_state->SaveState(logger_)) { + logger_.Log(LogLevel::Error, + "failed to reset download state after close failure for '" + local_path + "'"); + } + } else { + BlobDownloadState::DeleteState(local_path, logger_); + } } - BlobDownloadState::DeleteState(local_path, logger_); throw; } @@ -536,17 +553,24 @@ void DownloadBlobsToDirectory(IBlobDownloader& downloader, // count toward "downloaded" so the percentage stays accurate when this is a // resume of a partially-completed download. int64_t skipped_bytes = 0; + int32_t skipped_file_count = 0; blobs_to_download.erase( std::remove_if(blobs_to_download.begin(), blobs_to_download.end(), - [&skipped_bytes](const auto& pair) { - if (IsDownloadNeeded(pair.first, pair.second)) { - return false; - } - skipped_bytes += pair.first.content_length; - return true; + [&skipped_bytes, &skipped_file_count](const auto& pair) { + if (IsDownloadNeeded(pair.first, pair.second)) { + return false; + } + skipped_bytes += pair.first.content_length; + ++skipped_file_count; + return true; }), blobs_to_download.end()); + if (stats != nullptr) { + stats->already_cached_bytes = skipped_bytes; + stats->skipped_file_count = skipped_file_count; + } + // Step 6: Emit initial progress reflecting any already-on-disk bytes. // If everything was skipped, emit 100% directly and return. if (blobs_to_download.empty()) { diff --git a/sdk_v2/cpp/src/download/blob_downloader.h b/sdk_v2/cpp/src/download/blob_downloader.h index 66aadab89..1ecb53a83 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.h +++ b/sdk_v2/cpp/src/download/blob_downloader.h @@ -134,7 +134,9 @@ void DownloadBlobsToDirectory(IBlobDownloader& downloader, /// stamp telemetry events with download-shape data (file count, byte volume, timing). struct BlobDownloadStats { int64_t total_size_bytes = 0; + int64_t already_cached_bytes = 0; int32_t file_count = 0; + int32_t skipped_file_count = 0; int64_t enumeration_ms = 0; // Time spent listing blobs from the SAS URI. int64_t download_ms = 0; // Time spent transferring blob bytes (excludes enumeration). }; diff --git a/sdk_v2/cpp/src/download/download_manager.cc b/sdk_v2/cpp/src/download/download_manager.cc index 09d923743..cf25d1751 100644 --- a/sdk_v2/cpp/src/download/download_manager.cc +++ b/sdk_v2/cpp/src/download/download_manager.cc @@ -396,6 +396,8 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, tracker->SetEnumerationMs(stats.enumeration_ms); tracker->SetFileCount(stats.file_count); tracker->SetTotalSizeBytes(stats.total_size_bytes); + tracker->SetAlreadyCachedBytes(stats.already_cached_bytes); + tracker->SetSkippedFileCount(stats.skipped_file_count); tracker->SetDownloadWaitResult("Completed"); } diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.cc b/sdk_v2/cpp/src/ep_detection/ep_detector.cc index c454d1406..f6d9da4cb 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.cc @@ -24,21 +24,28 @@ EpDetector::EpDetector(const OrtApi& ort_api, OrtEnv& ort_env, bootstrappers_(std::move(bootstrappers)), logger_(logger), telemetry_(telemetry) { - // Populate both cache vectors exact-sized from bootstrappers_. After this point - // size and element addresses (including the EpInfo::name string storage backing - // flEpInfo::name) are immutable for the detector's lifetime — only is_registered - // is ever updated, in place, under cache_mutex_. + // Populate the C++ cache once from bootstrappers_. Name string storage is stable + // because cached_eps_ size is fixed for the detector's lifetime. cached_eps_.reserve(bootstrappers_.size()); - cached_eps_c_.reserve(bootstrappers_.size()); for (const auto& bs : bootstrappers_) { cached_eps_.push_back(EpInfo{bs->Name(), bs->IsRegistered()}); - cached_eps_c_.push_back(flEpInfo{ + } + + PublishCApiSnapshotLocked(); +} + +void EpDetector::PublishCApiSnapshotLocked() { + auto& snapshot = cached_eps_c_snapshots_.emplace_back(); + snapshot.reserve(cached_eps_.size()); + for (const auto& ep : cached_eps_) { + snapshot.push_back(flEpInfo{ FOUNDRY_LOCAL_API_VERSION, - cached_eps_.back().name.c_str(), - bs->IsRegistered(), + ep.name.c_str(), + ep.is_registered, }); } + current_cached_eps_c_ = &snapshot; } std::map> EpDetector::GetAvailableDevicesToEPs() const { @@ -122,7 +129,10 @@ const std::vector& EpDetector::GetDiscoverableEps() const { std::span EpDetector::GetDiscoverableEpsCApi() const { std::lock_guard lock(cache_mutex_); - return cached_eps_c_; + if (current_cached_eps_c_ == nullptr) { + return {}; + } + return *current_cached_eps_c_; } EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector* names, @@ -226,7 +236,7 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector cache_lock(cache_mutex_); cached_eps_[i].is_registered = true; - cached_eps_c_[i].is_registered = true; + PublishCApiSnapshotLocked(); if (tracker) { tracker->RecordDownloadComplete(ActionStatus::kSuccess, "Installed"); diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.h b/sdk_v2/cpp/src/ep_detection/ep_detector.h index e4c9aceaa..eb0ac5239 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.h +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include #include #include @@ -103,11 +104,14 @@ class EpDetector : public IEpDetector { std::mutex download_mutex_; std::atomic download_in_progress_{false}; mutable std::mutex cache_mutex_; - // Populated once in the constructor; size and element addresses (including name strings) - // are stable for the detector's lifetime. Only is_registered fields are mutated, under - // cache_mutex_. cached_eps_c_ mirrors cached_eps_ for the C ABI. + // cached_eps_ is updated under cache_mutex_. Each C ABI view is an immutable + // snapshot retained for the detector lifetime so previously returned spans are + // never mutated or freed while callers may still read them. std::vector cached_eps_; - std::vector cached_eps_c_; + std::deque> cached_eps_c_snapshots_; + const std::vector* current_cached_eps_c_ = nullptr; + + void PublishCApiSnapshotLocked(); }; } // namespace fl From d530ae449118e31bdcaf90eb49aecc1c9124d9b1 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 9 Jul 2026 21:38:28 -0500 Subject: [PATCH 31/77] Finish telemetry and download hardening Remove the telemetry-token CLI path, require environment-based token injection, raise the CMake floor for WinML package config, make catalog failure telemetry generic, and force safe redownload after unfinalized sidecars or close failures. Files changed: - sdk_v2/cpp/CMakeLists.txt and build.py - sdk_v2/cpp/src/catalog/catalog_client.cc - sdk_v2/cpp/src/download/blob_downloader.cc - sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/CMakeLists.txt | 2 +- sdk_v2/cpp/build.py | 18 ++--- sdk_v2/cpp/src/catalog/catalog_client.cc | 12 +++- sdk_v2/cpp/src/download/blob_downloader.cc | 66 +++++++++++++------ .../src/telemetry/one_ds_tenant_token.h.in | 9 ++- 5 files changed, 67 insertions(+), 40 deletions(-) diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index 729b77090..eb8362255 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright (c) Microsoft. All rights reserved. -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.21) # Android: web service not applicable on device. # Detect Android early via the vcpkg triplet name because the ANDROID variable diff --git a/sdk_v2/cpp/build.py b/sdk_v2/cpp/build.py index aad2071ef..ef6e59822 100644 --- a/sdk_v2/cpp/build.py +++ b/sdk_v2/cpp/build.py @@ -189,13 +189,6 @@ class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescript "not link any telemetry transport. Local diagnostic logging via TelemetryLogger " "still works.", ) - parser.add_argument( - "--telemetry_token", default=None, type=str, - help="1DS / Aria ingestion token. Baked into the binary at configure time as a " - "secret. Treat with care — do not commit to source. Defaults to empty, which " - "means OneDsTelemetry initializes but skips upload.", - ) - # Cross-compilation (mutually exclusive targets) cross_group = parser.add_mutually_exclusive_group() cross_group.add_argument( @@ -360,6 +353,12 @@ def _validate_args(args: argparse.Namespace) -> None: args.cmake_extra_defines = ( [f"-D{d}" for j in args.cmake_extra_defines for d in j] if args.cmake_extra_defines else [] ) + for define in args.cmake_extra_defines: + if define.startswith("-DFOUNDRY_LOCAL_TELEMETRY_TOKEN"): + raise ValueError( + "FOUNDRY_LOCAL_TELEMETRY_TOKEN must be supplied as an environment variable, " + "not via --cmake_extra_defines or command-line arguments." + ) # Cross-compilation if args.arm64: @@ -495,11 +494,6 @@ def configure(args: argparse.Namespace) -> None: command += ["-DFOUNDRY_LOCAL_USE_TELEMETRY=OFF"] else: command += ["-DFOUNDRY_LOCAL_USE_TELEMETRY=ON"] - env = None - if args.telemetry_token is not None: - env = os.environ.copy() - env["FOUNDRY_LOCAL_TELEMETRY_TOKEN"] = args.telemetry_token - # WinML EP catalog is enabled automatically on Windows by CMake. Allow an # optional version override for the Microsoft.Windows.AI.MachineLearning NuGet. if args.winml_sdk_version: diff --git a/sdk_v2/cpp/src/catalog/catalog_client.cc b/sdk_v2/cpp/src/catalog/catalog_client.cc index f19d699bc..65fc2e05c 100644 --- a/sdk_v2/cpp/src/catalog/catalog_client.cc +++ b/sdk_v2/cpp/src/catalog/catalog_client.cc @@ -16,6 +16,12 @@ namespace fl { +namespace { + +constexpr const char* kCatalogFetchFailure = "catalog request failed"; + +} // namespace + std::vector FetchAllModelInfosWithCachedModels( ICatalogClient& client, const std::vector& cached_model_ids, @@ -48,7 +54,9 @@ std::vector FetchAllModelInfosWithCachedModels( try { result = client.FetchAllModelInfos(); } catch (const std::exception& ex) { - emit("FetchAll", ActionStatus::kFailure, elapsed_ms(start), 0, ex.what()); + logger.Log(LogLevel::Warning, + fmt::format("catalog: failed to fetch models — {}", ScrubTelemetryErrorMessage(ex.what()))); + emit("FetchAll", ActionStatus::kFailure, elapsed_ms(start), 0, kCatalogFetchFailure); throw; } emit("FetchAll", ActionStatus::kSuccess, elapsed_ms(start), static_cast(result.size()), ""); @@ -86,7 +94,7 @@ std::vector FetchAllModelInfosWithCachedModels( auto error_message = ScrubTelemetryErrorMessage(ex.what()); logger.Log(LogLevel::Warning, fmt::format("catalog: failed to fetch cached model IDs — {}", error_message)); - emit("FetchByIds", ActionStatus::kFailure, elapsed_ms(start), 0, error_message); + emit("FetchByIds", ActionStatus::kFailure, elapsed_ms(start), 0, kCatalogFetchFailure); } catch (...) { logger.Log(LogLevel::Warning, "catalog: failed to fetch cached model IDs — unknown error"); emit("FetchByIds", ActionStatus::kFailure, elapsed_ms(start), 0, "unknown error"); diff --git a/sdk_v2/cpp/src/download/blob_downloader.cc b/sdk_v2/cpp/src/download/blob_downloader.cc index 68c0a6534..0df2a8bc5 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.cc +++ b/sdk_v2/cpp/src/download/blob_downloader.cc @@ -218,12 +218,21 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, auto pending = state->GetPendingChunks(); if (pending.empty()) { - // Already complete on disk — drop the sidecar. + // A complete sidecar means a previous attempt finished all chunks but did not reach finalization. Do not trust + // that state: a late close error may have left a full-size corrupt file. Start a fresh all-chunks pass. + logger_.Log(LogLevel::Information, + "Resume sidecar for '" + local_path + "' is complete but unfinalized; starting fresh"); + std::error_code remove_ec; + std::filesystem::remove(local_path, remove_ec); BlobDownloadState::DeleteState(local_path, logger_); - if (bytes_written_cb) { - bytes_written_cb(blob_size); + state = BlobDownloadState::CreateNew(blob_name, local_path, blob_size, + static_cast(kChunkSize), num_chunks); + if (!state->SaveState(logger_)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "failed to persist reset download state for '" + local_path + "'"); } - return; + bytes_completed.store(0); + pending = state->GetPendingChunks(); } // Open the file writer once for the whole download. Open() pre-allocates @@ -363,31 +372,48 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, try { writer.Close(); } catch (...) { + bool safe_to_delete_state = false; std::error_code remove_ec; - const bool removed = std::filesystem::remove(local_path, remove_ec) || - !std::filesystem::exists(local_path); - if (removed) { - BlobDownloadState::DeleteState(local_path, logger_); + if (std::filesystem::remove(local_path, remove_ec) || !std::filesystem::exists(local_path)) { + safe_to_delete_state = true; } else { logger_.Log(LogLevel::Warning, "failed to remove blob file after close failure: " + local_path + " (" + remove_ec.message() + ")"); - std::error_code resize_ec; - std::filesystem::resize_file(local_path, 0, resize_ec); - if (resize_ec) { + + const std::filesystem::path invalid_path = local_path + ".invalid"; + std::error_code cleanup_ec; + std::filesystem::remove(invalid_path, cleanup_ec); + std::error_code rename_ec; + std::filesystem::rename(local_path, invalid_path, rename_ec); + if (!rename_ec) { + safe_to_delete_state = true; + } else { logger_.Log(LogLevel::Warning, - "failed to truncate blob file after close failure: " + local_path + - " (" + resize_ec.message() + ")"); - auto fresh_state = BlobDownloadState::CreateNew(blob_name, local_path, blob_size, - static_cast(kChunkSize), num_chunks); - if (!fresh_state->SaveState(logger_)) { - logger_.Log(LogLevel::Error, - "failed to reset download state after close failure for '" + local_path + "'"); + "failed to quarantine blob file after close failure: " + local_path + + " (" + rename_ec.message() + ")"); + + std::error_code resize_ec; + std::filesystem::resize_file(local_path, 0, resize_ec); + if (!resize_ec) { + safe_to_delete_state = true; + } else { + logger_.Log(LogLevel::Warning, + "failed to truncate blob file after close failure: " + local_path + + " (" + resize_ec.message() + ")"); + auto fresh_state = BlobDownloadState::CreateNew(blob_name, local_path, blob_size, + static_cast(kChunkSize), num_chunks); + if (!fresh_state->SaveState(logger_)) { + logger_.Log(LogLevel::Error, + "failed to reset download state after close failure for '" + local_path + "'"); + } } - } else { - BlobDownloadState::DeleteState(local_path, logger_); } } + + if (safe_to_delete_state) { + BlobDownloadState::DeleteState(local_path, logger_); + } throw; } diff --git a/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in b/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in index 1417e9281..2257ad0be 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in +++ b/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in @@ -1,11 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. // Auto-generated by CMake from one_ds_tenant_token.h.in — do not edit manually. // -// The 1DS tenant token is passed at configure time via -// cmake -DFOUNDRY_LOCAL_TELEMETRY_TOKEN="" -// and is intended to be supplied by CI as a secret, similar to how the build -// system supplies ORT paths via -DORT_HOME. The default is an empty string so -// developer builds and forks don't accidentally ship a tenant binding. +// The 1DS tenant token is read by CMake from the FOUNDRY_LOCAL_TELEMETRY_TOKEN +// environment variable. Do not pass it with -D: that would place the secret in +// process argv and CMakeCache.txt. The default is an empty string so developer +// builds and forks don't accidentally ship a tenant binding. // // When the constant is empty, OneDsTelemetry::Initialize is skipped and the // Manager falls back to TelemetryLogger (which only writes to the local From ba3435169cef13d8e6fd09ca18c341e1fdb52350 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 9 Jul 2026 22:12:11 -0500 Subject: [PATCH 32/77] Preserve variant intent and reject cached tokens Reject telemetry tokens supplied through the CMake cache, broaden build-command secret redaction, and reselect default variants after refresh only when the caller has not explicitly selected a variant. Files changed: - sdk_v2/cpp/CMakeLists.txt and build.py - sdk_v2/cpp/src/model.* and src/catalog/base_model_catalog.cc - sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/CMakeLists.txt | 8 ++++++++ sdk_v2/cpp/build.py | 16 ++++++--------- sdk_v2/cpp/src/catalog/base_model_catalog.cc | 3 +++ sdk_v2/cpp/src/model.cc | 10 +++++++++- sdk_v2/cpp/src/model.h | 4 ++++ .../internal_api/base_model_catalog_test.cc | 20 +++++++++++++++++++ 6 files changed, 50 insertions(+), 11 deletions(-) diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index eb8362255..a98b734a9 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -51,6 +51,14 @@ option(FOUNDRY_LOCAL_ENABLE_ASAN "Enable AddressSanitizer + UndefinedBehaviorSan # FOUNDRY_LOCAL_TELEMETRY_TOKEN environment variable so it does not land in # process argv or CMakeCache.txt. Defaults to empty so developer builds and forks # don't accidentally bind to a real tenant. +get_property(_FL_TELEMETRY_TOKEN_IN_CACHE CACHE FOUNDRY_LOCAL_TELEMETRY_TOKEN PROPERTY TYPE SET) +if(_FL_TELEMETRY_TOKEN_IN_CACHE) + get_property(_FL_TELEMETRY_TOKEN_CACHE_VALUE CACHE FOUNDRY_LOCAL_TELEMETRY_TOKEN PROPERTY VALUE) + unset(FOUNDRY_LOCAL_TELEMETRY_TOKEN CACHE) + if(_FL_TELEMETRY_TOKEN_CACHE_VALUE) + message(FATAL_ERROR "FOUNDRY_LOCAL_TELEMETRY_TOKEN must be supplied as an environment variable, not via -D.") + endif() +endif() set(FOUNDRY_LOCAL_TELEMETRY_TOKEN "$ENV{FOUNDRY_LOCAL_TELEMETRY_TOKEN}") # Android: interactive examples and host tools don't run on device diff --git a/sdk_v2/cpp/build.py b/sdk_v2/cpp/build.py index ef6e59822..34bc484ea 100644 --- a/sdk_v2/cpp/build.py +++ b/sdk_v2/cpp/build.py @@ -58,16 +58,12 @@ def _path_from_env_var(var: str) -> Path | None: # --------------------------------------------------------------------------- def _redact_command_arg(arg: str) -> str: - secret_define = "-DFOUNDRY_LOCAL_TELEMETRY_TOKEN" - if arg.startswith(secret_define): - suffix = arg[len(secret_define):] - if suffix.startswith("="): - return f"{secret_define}=" - if suffix.startswith(":"): - separator = suffix.find("=") - if separator >= 0: - return f"{secret_define}{suffix[:separator]}=" - return f"{secret_define}:" + sensitive_words = ("token", "secret", "password", "passwd", "apikey", "api_key", "accesskey", "sig") + if "=" in arg: + key, _ = arg.split("=", 1) + normalized_key = key.lstrip("-").lower() + if any(word in normalized_key for word in sensitive_words): + return f"{key}=" return arg diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.cc b/sdk_v2/cpp/src/catalog/base_model_catalog.cc index 02d6f959b..fc541e6cb 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.cc @@ -121,6 +121,9 @@ void BaseModelCatalog::IntegrateVariantsLocked(std::vector variants) cons it->second->AddVariant(std::move(v)); ++added_variants; } + if (!it->second->HasExplicitVariantSelection()) { + it->second->SelectDefaultVariant(); + } } else { // New alias: build a container and choose default after all variants are added. auto first = std::move(alias_variants.front()); diff --git a/sdk_v2/cpp/src/model.cc b/sdk_v2/cpp/src/model.cc index 09bb61e65..e036ed468 100644 --- a/sdk_v2/cpp/src/model.cc +++ b/sdk_v2/cpp/src/model.cc @@ -111,11 +111,13 @@ Model::Model(Model&& other) noexcept download_manager_(other.download_manager_), model_load_manager_(other.model_load_manager_), variants_(std::move(other.variants_)), - selected_variant_(other.selected_variant_.load(std::memory_order_acquire)) { + selected_variant_(other.selected_variant_.load(std::memory_order_acquire)), + explicit_variant_selected_(other.explicit_variant_selected_.load(std::memory_order_acquire)) { // After vector move, selected_variant_ still points into the transferred buffer. other.download_manager_ = nullptr; other.model_load_manager_ = nullptr; other.selected_variant_.store(nullptr, std::memory_order_release); + other.explicit_variant_selected_.store(false, std::memory_order_release); } Model& Model::operator=(Model&& other) noexcept { @@ -127,9 +129,12 @@ Model& Model::operator=(Model&& other) noexcept { model_load_manager_ = other.model_load_manager_; variants_ = std::move(other.variants_); selected_variant_.store(other.selected_variant_.load(std::memory_order_acquire), std::memory_order_release); + explicit_variant_selected_.store(other.explicit_variant_selected_.load(std::memory_order_acquire), + std::memory_order_release); other.download_manager_ = nullptr; other.model_load_manager_ = nullptr; other.selected_variant_.store(nullptr, std::memory_order_release); + other.explicit_variant_selected_.store(false, std::memory_order_release); } return *this; @@ -197,11 +202,13 @@ void Model::SelectDefaultVariant() { for (auto& v : variants_) { if (v->IsCached()) { selected_variant_.store(v.get(), std::memory_order_release); + explicit_variant_selected_.store(false, std::memory_order_release); return; } } selected_variant_.store(variants_.front().get(), std::memory_order_release); + explicit_variant_selected_.store(false, std::memory_order_release); } // --------------------------------------------------------------------------- @@ -362,6 +369,7 @@ void Model::SelectVariant(const Model& variant) { for (auto& v : variants_) { if (v.get() == &variant) { selected_variant_.store(v.get(), std::memory_order_release); + explicit_variant_selected_.store(true, std::memory_order_release); return; } } diff --git a/sdk_v2/cpp/src/model.h b/sdk_v2/cpp/src/model.h index 930beaef2..0dcb329c1 100644 --- a/sdk_v2/cpp/src/model.h +++ b/sdk_v2/cpp/src/model.h @@ -70,6 +70,9 @@ class Model { /// first cached variant if any, else the best variant. /// Requires IsContainer() to be true. void SelectDefaultVariant(); + bool HasExplicitVariantSelection() const { + return explicit_variant_selected_.load(std::memory_order_acquire); + } /// Best-first comparator: device priority asc, version desc, created-at desc, model_id asc. /// Exposed so callers that return Model* lists can produce consistent ordering with Variants(). @@ -170,6 +173,7 @@ class Model { // stable across vector growth/reordering. std::vector> variants_; std::atomic selected_variant_{nullptr}; // non-null = this is a container + std::atomic explicit_variant_selected_{false}; // Guards variants_ across reader/writer threads (catalog refresh adding variants // while another thread enumerates via Variants()). diff --git a/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc b/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc index 30aeef4ce..8e0c9b434 100644 --- a/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc +++ b/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc @@ -224,6 +224,26 @@ TEST_F(BaseModelCatalogTest, RefreshMergesNewVariantsForExistingAlias) { ASSERT_EQ(refreshed.size(), 1u); EXPECT_EQ(refreshed[0]->Variants().size(), 2u); EXPECT_NE(catalog.GetModelVariant("phi-3-mini-gpu:1"), nullptr); + EXPECT_EQ(catalog.GetModel("phi-3")->Id(), "phi-3-mini-gpu:1"); +} + +TEST_F(BaseModelCatalogTest, RefreshPreservesExplicitVariantSelection) { + TestCatalog catalog(logger_); + catalog.AddModel(MakeModel("phi-3-mini-cpu:1", "phi-3-mini-cpu", 1, "phi-3")); + + Model* container = catalog.GetModel("phi-3"); + ASSERT_NE(container, nullptr); + Model* cpu_variant = catalog.GetModelVariant("phi-3-mini-cpu:1"); + ASSERT_NE(cpu_variant, nullptr); + container->SelectVariant(*cpu_variant); + + catalog.AddModel(MakeModel("phi-3-mini-gpu:1", "phi-3-mini-gpu", 1, "phi-3")); + catalog.InvalidateCache(); + + auto refreshed = catalog.ListModels(); + ASSERT_EQ(refreshed.size(), 1u); + EXPECT_EQ(refreshed[0]->Variants().size(), 2u); + EXPECT_EQ(container->Id(), "phi-3-mini-cpu:1"); } TEST_F(BaseModelCatalogTest, GetModel_NotFound_ReturnsNullptr) { From 02f8323d10cc75c49e402c28f094313a6335cc6e Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 9 Jul 2026 22:47:39 -0500 Subject: [PATCH 33/77] Guard fetch content and streaming cleanup Keep CMake 3.21-compatible FetchContent calls, redact URL-valued build defines, report unmeasured TTFT as unset, and let WebService join streaming threads after service-dependent cleanup. Files changed: - sdk_v2/cpp/build.py - sdk_v2/cpp/cmake/FindOnnxRuntime*.cmake - sdk_v2/cpp/src/service streaming handlers and web_service.h - sdk_v2/cpp/src/telemetry/telemetry.h Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/build.py | 4 +++- sdk_v2/cpp/cmake/FindOnnxRuntime.cmake | 20 +++++++++++++------ sdk_v2/cpp/cmake/FindOnnxRuntimeGenAI.cmake | 16 +++++++++------ .../service/audio_transcriptions_handler.cc | 1 - .../src/service/chat_completions_handler.cc | 1 - sdk_v2/cpp/src/service/responses_handler.cc | 1 - sdk_v2/cpp/src/service/web_service.h | 16 +-------------- sdk_v2/cpp/src/telemetry/telemetry.h | 2 +- 8 files changed, 29 insertions(+), 32 deletions(-) diff --git a/sdk_v2/cpp/build.py b/sdk_v2/cpp/build.py index 34bc484ea..8817c3b70 100644 --- a/sdk_v2/cpp/build.py +++ b/sdk_v2/cpp/build.py @@ -60,8 +60,10 @@ def _path_from_env_var(var: str) -> Path | None: def _redact_command_arg(arg: str) -> str: sensitive_words = ("token", "secret", "password", "passwd", "apikey", "api_key", "accesskey", "sig") if "=" in arg: - key, _ = arg.split("=", 1) + key, value = arg.split("=", 1) normalized_key = key.lstrip("-").lower() + if "://" in value: + return f"{key}=" if any(word in normalized_key for word in sensitive_words): return f"{key}=" return arg diff --git a/sdk_v2/cpp/cmake/FindOnnxRuntime.cmake b/sdk_v2/cpp/cmake/FindOnnxRuntime.cmake index f11385873..5ed855cf1 100644 --- a/sdk_v2/cpp/cmake/FindOnnxRuntime.cmake +++ b/sdk_v2/cpp/cmake/FindOnnxRuntime.cmake @@ -97,7 +97,7 @@ else() message(STATUS "Downloading ${ORT_PACKAGE_NAME} ${ORT_VERSION} from nuget.org") endif() else() - message(STATUS "Using pre-configured ORT_FETCH_URL: ${ORT_FETCH_URL}") + message(STATUS "Using pre-configured ORT_FETCH_URL") endif() # Normalize backslashes (Windows paths) and handle .nupkg extension @@ -106,11 +106,15 @@ else() set(_ORT_ZIP_PATH "${CMAKE_BINARY_DIR}/_deps/ortlib-download/ort.zip") get_filename_component(_ORT_ZIP_DIR "${_ORT_ZIP_PATH}" DIRECTORY) file(MAKE_DIRECTORY "${_ORT_ZIP_DIR}") - file(COPY_FILE "${ORT_FETCH_URL}" "${_ORT_ZIP_PATH}") + configure_file("${ORT_FETCH_URL}" "${_ORT_ZIP_PATH}" COPYONLY) set(ORT_FETCH_URL "${_ORT_ZIP_PATH}") endif() - FetchContent_Declare(ortlib URL ${ORT_FETCH_URL} DOWNLOAD_EXTRACT_TIMESTAMP TRUE DOWNLOAD_NAME ort.zip) + set(_ORT_FETCH_ARGS URL ${ORT_FETCH_URL} DOWNLOAD_NAME ort.zip) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24") + list(APPEND _ORT_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP TRUE) + endif() + FetchContent_Declare(ortlib ${_ORT_FETCH_ARGS}) FetchContent_MakeAvailable(ortlib) set(_ORT_HEADER_DIR "${ortlib_SOURCE_DIR}/build/native/include") @@ -149,7 +153,7 @@ else() message(STATUS "Downloading ${_ORT_GPU_LINUX_PACKAGE} ${ORT_VERSION} from nuget.org") endif() else() - message(STATUS "Using pre-configured ORT_GPU_LINUX_FETCH_URL: ${ORT_GPU_LINUX_FETCH_URL}") + message(STATUS "Using pre-configured ORT_GPU_LINUX_FETCH_URL") endif() # Normalize backslashes and handle .nupkg extension @@ -158,11 +162,15 @@ else() set(_ORT_GPU_ZIP_PATH "${CMAKE_BINARY_DIR}/_deps/ort_gpu_linux-download/ort_gpu_linux.zip") get_filename_component(_ORT_GPU_ZIP_DIR "${_ORT_GPU_ZIP_PATH}" DIRECTORY) file(MAKE_DIRECTORY "${_ORT_GPU_ZIP_DIR}") - file(COPY_FILE "${ORT_GPU_LINUX_FETCH_URL}" "${_ORT_GPU_ZIP_PATH}") + configure_file("${ORT_GPU_LINUX_FETCH_URL}" "${_ORT_GPU_ZIP_PATH}" COPYONLY) set(ORT_GPU_LINUX_FETCH_URL "${_ORT_GPU_ZIP_PATH}") endif() - FetchContent_Declare(ort_gpu_linux URL ${ORT_GPU_LINUX_FETCH_URL} DOWNLOAD_EXTRACT_TIMESTAMP TRUE DOWNLOAD_NAME ort_gpu_linux.zip) + set(_ORT_GPU_FETCH_ARGS URL ${ORT_GPU_LINUX_FETCH_URL} DOWNLOAD_NAME ort_gpu_linux.zip) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24") + list(APPEND _ORT_GPU_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP TRUE) + endif() + FetchContent_Declare(ort_gpu_linux ${_ORT_GPU_FETCH_ARGS}) FetchContent_MakeAvailable(ort_gpu_linux) set(_ORT_LIB_DIR "${ort_gpu_linux_SOURCE_DIR}/runtimes/${_ORT_PLATFORM}/native") diff --git a/sdk_v2/cpp/cmake/FindOnnxRuntimeGenAI.cmake b/sdk_v2/cpp/cmake/FindOnnxRuntimeGenAI.cmake index f83002d64..f8bdc4b10 100644 --- a/sdk_v2/cpp/cmake/FindOnnxRuntimeGenAI.cmake +++ b/sdk_v2/cpp/cmake/FindOnnxRuntimeGenAI.cmake @@ -160,7 +160,7 @@ else() message(STATUS "Downloading ${_GENAI_PACKAGE_NAME} ${ORT_GENAI_VERSION} from nuget.org") endif() else() - message(STATUS "Using caller-provided GENAI_FETCH_URL: ${GENAI_FETCH_URL}") + message(STATUS "Using caller-provided GENAI_FETCH_URL") endif() # Normalize to forward slashes — backslashes from Windows paths cause CMake @@ -173,15 +173,19 @@ else() set(_GENAI_ZIP_PATH "${CMAKE_BINARY_DIR}/_deps/genai-download/genai.zip") get_filename_component(_GENAI_ZIP_DIR "${_GENAI_ZIP_PATH}" DIRECTORY) file(MAKE_DIRECTORY "${_GENAI_ZIP_DIR}") - file(COPY_FILE "${GENAI_FETCH_URL}" "${_GENAI_ZIP_PATH}") + configure_file("${GENAI_FETCH_URL}" "${_GENAI_ZIP_PATH}" COPYONLY) set(GENAI_FETCH_URL "${_GENAI_ZIP_PATH}") - message(STATUS "Copied .nupkg to .zip for CMake extraction: ${GENAI_FETCH_URL}") + message(STATUS "Copied local GenAI .nupkg to .zip for CMake extraction") endif() - FetchContent_Declare(genailib + set(_GENAI_FETCH_ARGS URL ${GENAI_FETCH_URL} - DOWNLOAD_EXTRACT_TIMESTAMP TRUE - DOWNLOAD_NAME genai.zip # .nupkg is a ZIP; force CMake to recognize the format + DOWNLOAD_NAME genai.zip) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24") + list(APPEND _GENAI_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP TRUE) + endif() + FetchContent_Declare(genailib + ${_GENAI_FETCH_ARGS} ) FetchContent_MakeAvailable(genailib) diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc index c62468500..ef6a94340 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc @@ -249,7 +249,6 @@ std::shared_ptr AudioTranscriptionsHandler } body_ptr->Finish(); - thread_tracker.Remove(std::this_thread::get_id()); }); thread_tracker.Track(std::move(streaming_thread)); diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.cc b/sdk_v2/cpp/src/service/chat_completions_handler.cc index 3475c82ac..957d69d87 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.cc +++ b/sdk_v2/cpp/src/service/chat_completions_handler.cc @@ -287,7 +287,6 @@ std::shared_ptr ChatCompletionsHandler::Ha body_ptr->Finish(); // route_tracker is destroyed with this closure once the thread completes, // recording the route action with the full streaming duration and final status. - thread_tracker.Remove(std::this_thread::get_id()); }); thread_tracker.Track(std::move(streaming_thread)); diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index b343e443b..9a0ad75be 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -602,7 +602,6 @@ std::shared_ptr ResponsesHandler::HandleSt body_ptr->Push("data: [DONE]\n\n"); body_ptr->Finish(); - tracker.Remove(std::this_thread::get_id()); }); tracker.Track(std::move(streaming_thread)); diff --git a/sdk_v2/cpp/src/service/web_service.h b/sdk_v2/cpp/src/service/web_service.h index b7d72bd17..f915e276e 100644 --- a/sdk_v2/cpp/src/service/web_service.h +++ b/sdk_v2/cpp/src/service/web_service.h @@ -21,7 +21,6 @@ class ResponseStore; /// Tracks streaming threads so they can be joined on shutdown. /// Handlers call Track() instead of std::thread::detach(). -/// Threads call Remove() when done to clean up immediately. class StreamingThreadTracker { public: /// Take ownership of a streaming thread. @@ -30,21 +29,8 @@ class StreamingThreadTracker { threads_.push_back(std::move(t)); } - /// Called from within a thread to untrack itself after work is done. - /// Detaches the thread (can't join itself) and removes the entry. - void Remove(std::thread::id id) { - std::lock_guard lock(mutex_); - for (auto it = threads_.begin(); it != threads_.end(); ++it) { - if (it->get_id() == id) { - it->detach(); - threads_.erase(it); - return; - } - } - } - /// Join all remaining threads. Called by WebService::Stop(). - /// Moves entries out before joining to avoid deadlock with Remove(). + /// Moves entries out before joining so completed streaming threads are cleaned up safely. void JoinAll() { std::vector local; { diff --git a/sdk_v2/cpp/src/telemetry/telemetry.h b/sdk_v2/cpp/src/telemetry/telemetry.h index 91ee2e4b2..c5075e42c 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.h +++ b/sdk_v2/cpp/src/telemetry/telemetry.h @@ -110,7 +110,7 @@ struct ModelUsageInfo { std::string correlation_id; bool stream = false; // True if the inference was streamed (SSE) vs a single response bool indirect = false; // True if the inference was driven by another action (e.g. an HTTP route) - int64_t time_to_first_token_ms = 0; + int64_t time_to_first_token_ms = -1; int64_t total_time_ms = 0; int32_t total_tokens = 0; int32_t input_token_count = 0; From a59accf5696990b459ce83e0396e0f68399d7bfe Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 9 Jul 2026 23:34:54 -0500 Subject: [PATCH 34/77] Package C++ runtime dependencies Include ORT/GenAI runtime dependencies in C++ SDK bundles, keep no-telemetry builds from materializing tenant tokens, and join streaming threads after connection shutdown prevents new producers. Files changed: - .pipelines/v2/templates/steps-build-{windows,linux,macos}.yml - .pipelines/v2/templates/steps-pack-cpp-sdk.yml - sdk_v2/cpp/CMakeLists.txt - sdk_v2/cpp/src/service/web_service.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .pipelines/v2/templates/steps-build-linux.yml | 34 +++++++++++-------- .pipelines/v2/templates/steps-build-macos.yml | 33 +++++++++++------- .../v2/templates/steps-build-windows.yml | 4 ++- .../v2/templates/steps-pack-cpp-sdk.yml | 30 ++++++++++------ sdk_v2/cpp/CMakeLists.txt | 3 ++ sdk_v2/cpp/src/service/web_service.cc | 7 ++-- 6 files changed, 70 insertions(+), 41 deletions(-) diff --git a/.pipelines/v2/templates/steps-build-linux.yml b/.pipelines/v2/templates/steps-build-linux.yml index f092abc41..3c36c2950 100644 --- a/.pipelines/v2/templates/steps-build-linux.yml +++ b/.pipelines/v2/templates/steps-build-linux.yml @@ -75,24 +75,30 @@ steps: displayName: 'Dump vcpkg error logs' condition: failed() -# Stage the redistributable native artifact. Vcpkg uses static linkage on -# Linux (see sdk_v2/cpp/triplets/x64-linux.cmake), so libfoundry_local.so -# carries its transitive deps (azure-*, spdlog, fmt, libcurl, libssl/libcrypto, -# zlib, brotli*) inside itself — the only file we need to forward downstream -# is libfoundry_local.so. ORT/GenAI .so files are also present in bin/ but are -# supplied to consumers separately (pip on the Python side, NuGet on the C# -# side); test/example binaries are not part of the redistributable surface. +# Stage the redistributable native artifacts for standalone C++ consumers. - bash: | set -euo pipefail src='$(Build.SourcesDirectory)/sdk_v2/cpp/build/Linux/${{ parameters.buildConfig }}/bin' dst='$(Build.ArtifactStagingDirectory)/native' mkdir -p "$dst" - primary="$src/libfoundry_local.so" - if [ ! -f "$primary" ]; then - echo "ERROR: libfoundry_local.so not found at $primary" >&2 - exit 1 - fi - cp -P "$primary" "$dst/" - echo " staged libfoundry_local.so" + required=( + "$src/libfoundry_local.so" + "$src/libonnxruntime-genai.so" + "$src/libonnxruntime.so" + ) + for f in "${required[@]}"; do + if [ ! -e "$f" ]; then + echo "ERROR: required native artifact not found: $f" >&2 + exit 1 + fi + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + done + for f in "$src"/libonnxruntime.so.*; do + if [ -e "$f" ]; then + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + fi + done displayName: 'Stage native artifacts' diff --git a/.pipelines/v2/templates/steps-build-macos.yml b/.pipelines/v2/templates/steps-build-macos.yml index 92f9825e9..8bd0ecb95 100644 --- a/.pipelines/v2/templates/steps-build-macos.yml +++ b/.pipelines/v2/templates/steps-build-macos.yml @@ -81,23 +81,30 @@ steps: displayName: 'Dump vcpkg error logs' condition: failed() -# Stage the redistributable native artifact. macOS has no triplet overlay so -# vcpkg uses its default arm64-osx triplet, which is static — libfoundry_local.dylib -# carries its transitive deps (azure-*, spdlog, fmt, libcurl, libssl/libcrypto, -# zlib, brotli*) inside itself. ORT/GenAI .dylib files are supplied to consumers -# separately (pip on the Python side, NuGet on the C# side); test/example binaries -# are not part of the redistributable surface. +# Stage the redistributable native artifacts for standalone C++ consumers. - bash: | set -euo pipefail src='$(Build.SourcesDirectory)/sdk_v2/cpp/build/macOS/${{ parameters.buildConfig }}/bin' dst='$(Build.ArtifactStagingDirectory)/native' mkdir -p "$dst" - primary="$src/libfoundry_local.dylib" - if [ ! -f "$primary" ]; then - echo "ERROR: libfoundry_local.dylib not found at $primary" >&2 - exit 1 - fi - cp -P "$primary" "$dst/" - echo " staged libfoundry_local.dylib" + required=( + "$src/libfoundry_local.dylib" + "$src/libonnxruntime-genai.dylib" + "$src/libonnxruntime.dylib" + ) + for f in "${required[@]}"; do + if [ ! -e "$f" ]; then + echo "ERROR: required native artifact not found: $f" >&2 + exit 1 + fi + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + done + for f in "$src"/libonnxruntime.*.dylib; do + if [ -e "$f" ]; then + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + fi + done displayName: 'Stage native artifacts' diff --git a/.pipelines/v2/templates/steps-build-windows.yml b/.pipelines/v2/templates/steps-build-windows.yml index 5f1d0f761..afc39a909 100644 --- a/.pipelines/v2/templates/steps-build-windows.yml +++ b/.pipelines/v2/templates/steps-build-windows.yml @@ -138,7 +138,9 @@ steps: (Join-Path $binDir 'foundry_local.dll'), (Join-Path $binDir 'foundry_local.pdb'), (Join-Path $linkDir 'foundry_local.lib'), - (Join-Path $binDir 'Microsoft.Windows.AI.MachineLearning.dll') + (Join-Path $binDir 'Microsoft.Windows.AI.MachineLearning.dll'), + (Join-Path $binDir 'onnxruntime.dll'), + (Join-Path $binDir 'onnxruntime-genai.dll') ) foreach ($s in $sources) { diff --git a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml index 448abef0b..3a7322564 100644 --- a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml +++ b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml @@ -67,13 +67,15 @@ steps: } } - foreach ($f in $Files) { - $src = Join-Path $nativeSrc $f - if (-not (Test-Path $src)) { - throw "Required native file not found: $src" + foreach ($pattern in $Files) { + $matches = @(Get-ChildItem -Path $nativeSrc -Filter $pattern -File -ErrorAction SilentlyContinue) + if ($matches.Count -eq 0) { + throw "Required native file not found: $(Join-Path $nativeSrc $pattern)" + } + foreach ($match in $matches) { + $dst = Join-Path $libDst $match.Name + Copy-Item -Path $match.FullName -Destination $dst -Force } - $dst = Join-Path $libDst (Split-Path -Leaf $src) - Copy-Item -Path $src -Destination $dst -Force } $archive = Join-Path $outDir ("cpp-sdk-v2-{0}.tgz" -f $Rid) @@ -96,22 +98,30 @@ steps: 'foundry_local.dll', 'foundry_local.pdb', 'foundry_local.lib', - 'Microsoft.Windows.AI.MachineLearning.dll' + 'Microsoft.Windows.AI.MachineLearning.dll', + 'onnxruntime.dll', + 'onnxruntime-genai.dll' ) New-SdkArchive -Rid 'win-arm64' -NativeArtifact 'cpp-native-win-arm64' -Files @( 'foundry_local.dll', 'foundry_local.pdb', 'foundry_local.lib', - 'Microsoft.Windows.AI.MachineLearning.dll' + 'Microsoft.Windows.AI.MachineLearning.dll', + 'onnxruntime.dll', + 'onnxruntime-genai.dll' ) New-SdkArchive -Rid 'linux-x64' -NativeArtifact 'cpp-native-linux-x64' -Files @( - 'libfoundry_local.so' + 'libfoundry_local.so', + 'libonnxruntime-genai.so', + 'libonnxruntime.so*' ) New-SdkArchive -Rid 'osx-arm64' -NativeArtifact 'cpp-native-osx-arm64' -Files @( - 'libfoundry_local.dylib' + 'libfoundry_local.dylib', + 'libonnxruntime-genai.dylib', + 'libonnxruntime*.dylib' ) Write-Host "Staged archives:" diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index a98b734a9..6b336c6b7 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -60,6 +60,9 @@ if(_FL_TELEMETRY_TOKEN_IN_CACHE) endif() endif() set(FOUNDRY_LOCAL_TELEMETRY_TOKEN "$ENV{FOUNDRY_LOCAL_TELEMETRY_TOKEN}") +if(NOT FOUNDRY_LOCAL_USE_TELEMETRY) + set(FOUNDRY_LOCAL_TELEMETRY_TOKEN "") +endif() # Android: interactive examples and host tools don't run on device if(ANDROID) diff --git a/sdk_v2/cpp/src/service/web_service.cc b/sdk_v2/cpp/src/service/web_service.cc index e49fbad8f..97eb4f362 100644 --- a/sdk_v2/cpp/src/service/web_service.cc +++ b/sdk_v2/cpp/src/service/web_service.cc @@ -435,9 +435,6 @@ void WebService::Stop() { return; } - // Join streaming threads first — they may still be pushing to SSE bodies. - impl_->thread_tracker.JoinAll(); - // Stop accepting new connections first, then stop server loops. for (auto& provider : impl_->providers) { provider->stop(); @@ -458,6 +455,10 @@ void WebService::Stop() { impl_->connection_handler->stop(); } + // Join streaming producers after connection shutdown so no new streaming + // thread can be tracked while Stop() is taking its join snapshot. + impl_->thread_tracker.JoinAll(); + for (auto& thread : impl_->listener_threads) { if (thread.joinable()) { thread.join(); From f861d9c24d9e7155f391e6361e07cf1999045b6b Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 00:26:48 -0500 Subject: [PATCH 35/77] Package ORT companions and harden resume skips Include provider companion patterns in Windows C++ artifacts, ensure Linux outputs include the ORT soname symlink, and avoid trusting size-only blob cache hits while resuming incomplete model downloads. Files changed: - .pipelines/v2/templates/steps-build-windows.yml - .pipelines/v2/templates/steps-pack-cpp-sdk.yml - sdk_v2/cpp/CMakeLists.txt - sdk_v2/cpp/src/download/blob_downloader.* and download_manager.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .pipelines/v2/templates/steps-build-windows.yml | 9 +++++++++ .pipelines/v2/templates/steps-pack-cpp-sdk.yml | 8 ++++++-- sdk_v2/cpp/CMakeLists.txt | 3 +++ sdk_v2/cpp/src/download/blob_downloader.cc | 4 ++-- sdk_v2/cpp/src/download/blob_downloader.h | 4 ++++ sdk_v2/cpp/src/download/download_manager.cc | 1 + 6 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.pipelines/v2/templates/steps-build-windows.yml b/.pipelines/v2/templates/steps-build-windows.yml index afc39a909..7d89f4de1 100644 --- a/.pipelines/v2/templates/steps-build-windows.yml +++ b/.pipelines/v2/templates/steps-build-windows.yml @@ -142,6 +142,9 @@ steps: (Join-Path $binDir 'onnxruntime.dll'), (Join-Path $binDir 'onnxruntime-genai.dll') ) + $optionalPatterns = @( + 'onnxruntime_providers_*.dll' + ) foreach ($s in $sources) { if (-not (Test-Path $s)) { @@ -151,6 +154,12 @@ steps: Copy-Item -Path $s -Destination $dst -Force Write-Host " staged $(Split-Path -Leaf $s)" } + foreach ($pattern in $optionalPatterns) { + Get-ChildItem -Path $binDir -Filter $pattern -File -ErrorAction SilentlyContinue | ForEach-Object { + Copy-Item -Path $_.FullName -Destination $dst -Force + Write-Host " staged $($_.Name)" + } + } - ${{ if eq(parameters.stageHeaders, true) }}: - task: CopyFiles@2 diff --git a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml index 3a7322564..ff0709ecf 100644 --- a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml +++ b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml @@ -100,7 +100,8 @@ steps: 'foundry_local.lib', 'Microsoft.Windows.AI.MachineLearning.dll', 'onnxruntime.dll', - 'onnxruntime-genai.dll' + 'onnxruntime-genai.dll', + 'onnxruntime_providers_*.dll' ) New-SdkArchive -Rid 'win-arm64' -NativeArtifact 'cpp-native-win-arm64' -Files @( @@ -109,12 +110,15 @@ steps: 'foundry_local.lib', 'Microsoft.Windows.AI.MachineLearning.dll', 'onnxruntime.dll', - 'onnxruntime-genai.dll' + 'onnxruntime-genai.dll', + 'onnxruntime_providers_*.dll' ) New-SdkArchive -Rid 'linux-x64' -NativeArtifact 'cpp-native-linux-x64' -Files @( 'libfoundry_local.so', 'libonnxruntime-genai.so', + 'libonnxruntime-genai*.so', + 'libonnxruntime_providers_*.so', 'libonnxruntime.so*' ) diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index 6b336c6b7..d5cce782b 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -510,6 +510,9 @@ if(TARGET OnnxRuntime::OnnxRuntime) COMMAND ${CMAKE_COMMAND} -E copy_if_different "${ORT_LIB_DIR}/libonnxruntime.so" $ + COMMAND ${CMAKE_COMMAND} -E create_symlink + libonnxruntime.so + $/libonnxruntime.so.1 ) if(EXISTS "${ORT_LIB_DIR}/libonnxruntime_providers_shared.so") diff --git a/sdk_v2/cpp/src/download/blob_downloader.cc b/sdk_v2/cpp/src/download/blob_downloader.cc index 0df2a8bc5..976db3316 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.cc +++ b/sdk_v2/cpp/src/download/blob_downloader.cc @@ -582,8 +582,8 @@ void DownloadBlobsToDirectory(IBlobDownloader& downloader, int32_t skipped_file_count = 0; blobs_to_download.erase( std::remove_if(blobs_to_download.begin(), blobs_to_download.end(), - [&skipped_bytes, &skipped_file_count](const auto& pair) { - if (IsDownloadNeeded(pair.first, pair.second)) { + [&skipped_bytes, &skipped_file_count, &options](const auto& pair) { + if (!options.skip_completed_files || IsDownloadNeeded(pair.first, pair.second)) { return false; } skipped_bytes += pair.first.content_length; diff --git a/sdk_v2/cpp/src/download/blob_downloader.h b/sdk_v2/cpp/src/download/blob_downloader.h index 1ecb53a83..279fc5d26 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.h +++ b/sdk_v2/cpp/src/download/blob_downloader.h @@ -28,6 +28,10 @@ struct BlobDownloadOptions { /// Maximum concurrent chunk downloads per blob. Default matches C# desktop. int max_concurrency = 64; + /// When false, full-size files without a sidecar are redownloaded instead of + /// skipped. Use this while recovering an incomplete model directory. + bool skip_completed_files = true; + /// Progress callback (optional). Return non-zero to cancel the download. DownloadProgressFn progress; }; diff --git a/sdk_v2/cpp/src/download/download_manager.cc b/sdk_v2/cpp/src/download/download_manager.cc index cf25d1751..688faaa49 100644 --- a/sdk_v2/cpp/src/download/download_manager.cc +++ b/sdk_v2/cpp/src/download/download_manager.cc @@ -376,6 +376,7 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, // and the blob container has all files at the root or in variant subdirectories. download_opts.path_prefix = ""; download_opts.max_concurrency = max_concurrency_; + download_opts.skip_completed_files = !std::filesystem::exists(signal_path); if (progress_cb) { download_opts.progress = [&progress_cb](float percent) { From d8e825cad38ebc502f26cb93da5d074a9bb6e53d Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 01:06:49 -0500 Subject: [PATCH 36/77] Reap completed streams and optional providers Reap completed streaming producer threads during normal operation, stage optional provider companion libraries, and make optional provider patterns non-fatal in C++ SDK archive packaging. Files changed: - .pipelines/v2/templates/steps-build-{linux,macos}.yml - .pipelines/v2/templates/steps-pack-cpp-sdk.yml - sdk_v2/cpp/src/service streaming handlers and web_service.h Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .pipelines/v2/templates/steps-build-linux.yml | 6 ++++ .pipelines/v2/templates/steps-build-macos.yml | 6 ++++ .../v2/templates/steps-pack-cpp-sdk.yml | 17 ++++++---- .../service/audio_transcriptions_handler.cc | 7 ++-- .../src/service/chat_completions_handler.cc | 7 ++-- sdk_v2/cpp/src/service/responses_handler.cc | 7 ++-- sdk_v2/cpp/src/service/web_service.h | 34 +++++++++++++++---- 7 files changed, 64 insertions(+), 20 deletions(-) diff --git a/.pipelines/v2/templates/steps-build-linux.yml b/.pipelines/v2/templates/steps-build-linux.yml index 3c36c2950..20d246d08 100644 --- a/.pipelines/v2/templates/steps-build-linux.yml +++ b/.pipelines/v2/templates/steps-build-linux.yml @@ -101,4 +101,10 @@ steps: echo " staged $(basename "$f")" fi done + for f in "$src"/libonnxruntime_providers_*.so; do + if [ -e "$f" ]; then + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + fi + done displayName: 'Stage native artifacts' diff --git a/.pipelines/v2/templates/steps-build-macos.yml b/.pipelines/v2/templates/steps-build-macos.yml index 8bd0ecb95..a3decb433 100644 --- a/.pipelines/v2/templates/steps-build-macos.yml +++ b/.pipelines/v2/templates/steps-build-macos.yml @@ -107,4 +107,10 @@ steps: echo " staged $(basename "$f")" fi done + for f in "$src"/libonnxruntime_providers_*.dylib; do + if [ -e "$f" ]; then + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + fi + done displayName: 'Stage native artifacts' diff --git a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml index ff0709ecf..7a7145fd1 100644 --- a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml +++ b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml @@ -67,9 +67,11 @@ steps: } } - foreach ($pattern in $Files) { + foreach ($entry in $Files) { + $optional = $entry.StartsWith('?') + $pattern = if ($optional) { $entry.Substring(1) } else { $entry } $matches = @(Get-ChildItem -Path $nativeSrc -Filter $pattern -File -ErrorAction SilentlyContinue) - if ($matches.Count -eq 0) { + if ($matches.Count -eq 0 -and -not $optional) { throw "Required native file not found: $(Join-Path $nativeSrc $pattern)" } foreach ($match in $matches) { @@ -101,7 +103,7 @@ steps: 'Microsoft.Windows.AI.MachineLearning.dll', 'onnxruntime.dll', 'onnxruntime-genai.dll', - 'onnxruntime_providers_*.dll' + '?onnxruntime_providers_*.dll' ) New-SdkArchive -Rid 'win-arm64' -NativeArtifact 'cpp-native-win-arm64' -Files @( @@ -111,21 +113,22 @@ steps: 'Microsoft.Windows.AI.MachineLearning.dll', 'onnxruntime.dll', 'onnxruntime-genai.dll', - 'onnxruntime_providers_*.dll' + '?onnxruntime_providers_*.dll' ) New-SdkArchive -Rid 'linux-x64' -NativeArtifact 'cpp-native-linux-x64' -Files @( 'libfoundry_local.so', 'libonnxruntime-genai.so', - 'libonnxruntime-genai*.so', - 'libonnxruntime_providers_*.so', + '?libonnxruntime-genai*.so', + '?libonnxruntime_providers_*.so', 'libonnxruntime.so*' ) New-SdkArchive -Rid 'osx-arm64' -NativeArtifact 'cpp-native-osx-arm64' -Files @( 'libfoundry_local.dylib', 'libonnxruntime-genai.dylib', - 'libonnxruntime*.dylib' + 'libonnxruntime*.dylib', + '?libonnxruntime_providers_*.dylib' ) Write-Host "Staged archives:" diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc index ef6a94340..0b514a9c9 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc @@ -197,11 +197,13 @@ std::shared_ptr AudioTranscriptionsHandler auto body_ptr = body; auto& logger = ctx_.logger; auto& thread_tracker = ctx_.thread_tracker; + auto stream_done = std::make_shared>(false); std::thread streaming_thread([bg_session = std::move(session), body_ptr, &logger, req = std::move(session_request), &thread_tracker, route_tracker = std::move(route_tracker), - &session_manager = ctx_.session_manager]() mutable { + &session_manager = ctx_.session_manager, + stream_done]() mutable { SessionRegistration reg(session_manager, bg_session); try { @@ -249,9 +251,10 @@ std::shared_ptr AudioTranscriptionsHandler } body_ptr->Finish(); + stream_done->store(true, std::memory_order_release); }); - thread_tracker.Track(std::move(streaming_thread)); + thread_tracker.Track(std::move(streaming_thread), stream_done); auto response = oatpp::web::protocol::http::outgoing::Response::createShared(Status::CODE_200, body); response->putHeader("Content-Type", "text/event-stream"); diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.cc b/sdk_v2/cpp/src/service/chat_completions_handler.cc index 957d69d87..f7ca7ff1e 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.cc +++ b/sdk_v2/cpp/src/service/chat_completions_handler.cc @@ -211,12 +211,14 @@ std::shared_ptr ChatCompletionsHandler::Ha auto body_ptr = body; auto& logger = ctx_.logger; auto& thread_tracker = ctx_.thread_tracker; + auto stream_done = std::make_shared>(false); std::thread streaming_thread([bg_session = std::move(session), body_ptr, &logger, req = std::move(session_request), include_usage, &thread_tracker, route_tracker = std::move(route_tracker), - &session_manager = ctx_.session_manager]() mutable { + &session_manager = ctx_.session_manager, + stream_done]() mutable { SessionRegistration reg(session_manager, bg_session); try { @@ -287,9 +289,10 @@ std::shared_ptr ChatCompletionsHandler::Ha body_ptr->Finish(); // route_tracker is destroyed with this closure once the thread completes, // recording the route action with the full streaming duration and final status. + stream_done->store(true, std::memory_order_release); }); - thread_tracker.Track(std::move(streaming_thread)); + thread_tracker.Track(std::move(streaming_thread), stream_done); auto response = oatpp::web::protocol::http::outgoing::Response::createShared( Status::CODE_200, body); diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index 9a0ad75be..22cdeb44f 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -329,6 +329,7 @@ std::shared_ptr ResponsesHandler::HandleSt auto& logger = ctx_.logger; auto& session_manager = ctx_.session_manager; auto& tracker = ctx_.thread_tracker; + auto stream_done = std::make_shared>(false); // Background thread is required: oatpp needs the Response returned immediately so it can // start writing SSE events. ProcessRequest blocks until generation completes. @@ -340,7 +341,8 @@ std::shared_ptr ResponsesHandler::HandleSt req_copy = std::move(req_copy), params_copy = std::move(params_copy), route_tracker = std::move(route_tracker), - &tracker]() mutable { + &tracker, + stream_done]() mutable { SessionRegistration reg(session_manager, *session); int seq = 2; @@ -601,10 +603,11 @@ std::shared_ptr ResponsesHandler::HandleSt // Terminal event per spec body_ptr->Push("data: [DONE]\n\n"); body_ptr->Finish(); + stream_done->store(true, std::memory_order_release); }); - tracker.Track(std::move(streaming_thread)); + tracker.Track(std::move(streaming_thread), stream_done); auto response = oatpp::web::protocol::http::outgoing::Response::createShared( Status::CODE_200, body); diff --git a/sdk_v2/cpp/src/service/web_service.h b/sdk_v2/cpp/src/service/web_service.h index f915e276e..37d78f938 100644 --- a/sdk_v2/cpp/src/service/web_service.h +++ b/sdk_v2/cpp/src/service/web_service.h @@ -4,6 +4,7 @@ #include "logger.h" +#include #include #include #include @@ -22,32 +23,51 @@ class ResponseStore; /// Tracks streaming threads so they can be joined on shutdown. /// Handlers call Track() instead of std::thread::detach(). class StreamingThreadTracker { + struct TrackedThread { + std::thread thread; + std::shared_ptr> done; + }; + public: /// Take ownership of a streaming thread. - void Track(std::thread t) { + void Track(std::thread t, std::shared_ptr> done) { std::lock_guard lock(mutex_); - threads_.push_back(std::move(t)); + ReapCompletedLocked(); + threads_.push_back(TrackedThread{std::move(t), std::move(done)}); } /// Join all remaining threads. Called by WebService::Stop(). /// Moves entries out before joining so completed streaming threads are cleaned up safely. void JoinAll() { - std::vector local; + std::vector local; { std::lock_guard lock(mutex_); local = std::move(threads_); } - for (auto& t : local) { - if (t.joinable()) { - t.join(); + for (auto& tracked : local) { + if (tracked.thread.joinable()) { + tracked.thread.join(); } } } private: + void ReapCompletedLocked() { + for (auto it = threads_.begin(); it != threads_.end();) { + if (it->done && it->done->load(std::memory_order_acquire)) { + if (it->thread.joinable()) { + it->thread.join(); + } + it = threads_.erase(it); + } else { + ++it; + } + } + } + std::mutex mutex_; - std::vector threads_; + std::vector threads_; }; /// Context shared with all HTTP controllers. From b56a815a749ed8ed5a7c43b23037fed978797e4d Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 01:41:11 -0500 Subject: [PATCH 37/77] Complete runtime packaging and stream aborts Forward DirectML with WinML, keep Python wheels limited to intended native payloads, and add SSE abort/reap handling so shutdown can wake streams and cancel producers. Files changed: - .pipelines/v2/templates/steps-build-{windows,python}.yml and steps-pack-cpp-sdk.yml - sdk_v2/cpp/CMakeLists.txt and nuget/pack.py - sdk_v2/cpp/src/service streaming helpers/handlers - sdk_v2/js/script/copy-native.mjs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .../v2/templates/steps-build-python.yml | 14 ++++++++++- .../v2/templates/steps-build-windows.yml | 1 + .../v2/templates/steps-pack-cpp-sdk.yml | 2 ++ sdk_v2/cpp/CMakeLists.txt | 7 ++++++ sdk_v2/cpp/nuget/pack.py | 1 + .../service/audio_transcriptions_handler.cc | 6 +++-- .../src/service/chat_completions_handler.cc | 6 +++-- sdk_v2/cpp/src/service/handler_utils.h | 23 ++++++++++++++++--- sdk_v2/cpp/src/service/responses_handler.cc | 12 ++++++---- sdk_v2/cpp/src/service/web_service.cc | 2 ++ sdk_v2/cpp/src/service/web_service.h | 14 +++++++++-- sdk_v2/js/script/copy-native.mjs | 1 + 12 files changed, 75 insertions(+), 14 deletions(-) diff --git a/.pipelines/v2/templates/steps-build-python.yml b/.pipelines/v2/templates/steps-build-python.yml index 50f86c9da..177e44b66 100644 --- a/.pipelines/v2/templates/steps-build-python.yml +++ b/.pipelines/v2/templates/steps-build-python.yml @@ -106,7 +106,19 @@ steps: # deps (onnxruntime-{core,gpu} / onnxruntime-genai-{core,cuda}). Test # and example binaries are also filtered out upstream. We just copy # the curated artifact contents straight in. - $files = Get-ChildItem -Path "${{ parameters.nativeArtifactDir }}" -Recurse -File + $rid = '${{ parameters.rid }}' + if ($rid -like 'win-*') { + $allowed = @('foundry_local.dll', 'Microsoft.Windows.AI.MachineLearning.dll', 'DirectML.dll') + } elseif ($rid -like 'linux-*') { + $allowed = @('libfoundry_local.so') + } elseif ($rid -like 'osx-*') { + $allowed = @('libfoundry_local.dylib') + } else { + throw "Unsupported Python wheel RID: $rid" + } + $files = foreach ($name in $allowed) { + Get-ChildItem -Path "${{ parameters.nativeArtifactDir }}" -Filter $name -File -ErrorAction SilentlyContinue + } if ($files.Count -eq 0) { throw "No native artifacts found under ${{ parameters.nativeArtifactDir }}" } diff --git a/.pipelines/v2/templates/steps-build-windows.yml b/.pipelines/v2/templates/steps-build-windows.yml index 7d89f4de1..e72fde515 100644 --- a/.pipelines/v2/templates/steps-build-windows.yml +++ b/.pipelines/v2/templates/steps-build-windows.yml @@ -139,6 +139,7 @@ steps: (Join-Path $binDir 'foundry_local.pdb'), (Join-Path $linkDir 'foundry_local.lib'), (Join-Path $binDir 'Microsoft.Windows.AI.MachineLearning.dll'), + (Join-Path $binDir 'DirectML.dll'), (Join-Path $binDir 'onnxruntime.dll'), (Join-Path $binDir 'onnxruntime-genai.dll') ) diff --git a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml index 7a7145fd1..ab5a94415 100644 --- a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml +++ b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml @@ -101,6 +101,7 @@ steps: 'foundry_local.pdb', 'foundry_local.lib', 'Microsoft.Windows.AI.MachineLearning.dll', + 'DirectML.dll', 'onnxruntime.dll', 'onnxruntime-genai.dll', '?onnxruntime_providers_*.dll' @@ -111,6 +112,7 @@ steps: 'foundry_local.pdb', 'foundry_local.lib', 'Microsoft.Windows.AI.MachineLearning.dll', + 'DirectML.dll', 'onnxruntime.dll', 'onnxruntime-genai.dll', '?onnxruntime_providers_*.dll' diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index d5cce782b..eb0dfd5ea 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -481,6 +481,13 @@ if(TARGET OnnxRuntime::OnnxRuntime) $ ) endif() + if(WinMLEpCatalog_FOUND AND EXISTS "${WINML_EP_CATALOG_DLL_DIR}/DirectML.dll") + add_custom_command(TARGET foundry_local POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${WINML_EP_CATALOG_DLL_DIR}/DirectML.dll" + $ + ) + endif() elseif(APPLE) # macOS: copy dylibs so consumers that only link libfoundry_local.dylib (e.g. cache_only_tests) find the correct # ORT version instead of any system-installed ORT, which would cause an Ort::InitApi() version mismatch. diff --git a/sdk_v2/cpp/nuget/pack.py b/sdk_v2/cpp/nuget/pack.py index 902ba288a..0ef856e1a 100644 --- a/sdk_v2/cpp/nuget/pack.py +++ b/sdk_v2/cpp/nuget/pack.py @@ -60,6 +60,7 @@ # foundry_local.dll; other platforms don't, so presence alone drives inclusion. OPTIONAL_SIBLINGS: tuple[str, ...] = ( "Microsoft.Windows.AI.MachineLearning.dll", + "DirectML.dll", ) log = logging.getLogger("pack") diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc index 0b514a9c9..2ca3a14d0 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc @@ -219,7 +219,9 @@ std::shared_ptr AudioTranscriptionsHandler if (item->type == FOUNDRY_LOCAL_ITEM_TEXT) { auto& text_item = static_cast(*item); - body_ptr->Push("data: " + text_item.text + "\n\n"); + if (!body_ptr->Push("data: " + text_item.text + "\n\n")) { + return 1; + } } else { logger.Log(LogLevel::Error, fmt::format("Unexpected item type {} in audio streaming callback", @@ -254,7 +256,7 @@ std::shared_ptr AudioTranscriptionsHandler stream_done->store(true, std::memory_order_release); }); - thread_tracker.Track(std::move(streaming_thread), stream_done); + thread_tracker.Track(std::move(streaming_thread), stream_done, [body] { body->Abort(); }); auto response = oatpp::web::protocol::http::outgoing::Response::createShared(Status::CODE_200, body); response->putHeader("Content-Type", "text/event-stream"); diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.cc b/sdk_v2/cpp/src/service/chat_completions_handler.cc index f7ca7ff1e..b309597fa 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.cc +++ b/sdk_v2/cpp/src/service/chat_completions_handler.cc @@ -234,7 +234,9 @@ std::shared_ptr ChatCompletionsHandler::Ha if (item->type == FOUNDRY_LOCAL_ITEM_TEXT) { auto& text_item = static_cast(*item); - body_ptr->Push("data: " + text_item.text + "\n\n"); + if (!body_ptr->Push("data: " + text_item.text + "\n\n")) { + return 1; + } } else { logger.Log(LogLevel::Error, fmt::format("Unexpected item type {} in chat streaming callback", static_cast(item->type))); @@ -292,7 +294,7 @@ std::shared_ptr ChatCompletionsHandler::Ha stream_done->store(true, std::memory_order_release); }); - thread_tracker.Track(std::move(streaming_thread), stream_done); + thread_tracker.Track(std::move(streaming_thread), stream_done, [body] { body->Abort(); }); auto response = oatpp::web::protocol::http::outgoing::Response::createShared( Status::CODE_200, body); diff --git a/sdk_v2/cpp/src/service/handler_utils.h b/sdk_v2/cpp/src/service/handler_utils.h index 7216892ce..c54fd7e91 100644 --- a/sdk_v2/cpp/src/service/handler_utils.h +++ b/sdk_v2/cpp/src/service/handler_utils.h @@ -80,13 +80,17 @@ inline std::string GetUserAgent(const std::shared_ptr lock(mutex_); + if (aborted_) { + return false; + } queue_.push(std::move(chunk)); cv_.notify_one(); + return true; } /// Signal that no more data will be pushed. @@ -96,6 +100,18 @@ class SseStreamBody : public oatpp::web::protocol::http::outgoing::Body { cv_.notify_one(); } + void Abort() { + std::lock_guard lock(mutex_); + aborted_ = true; + done_ = true; + cv_.notify_all(); + } + + bool IsAborted() const { + std::lock_guard lock(mutex_); + return aborted_; + } + // -- Body interface -- oatpp::v_io_size read(void* buffer, v_buff_size count, oatpp::async::Action& /*action*/) override { @@ -143,10 +159,11 @@ class SseStreamBody : public oatpp::web::protocol::http::outgoing::Body { v_int64 getKnownSize() override { return -1; } // unknown → chunked transfer private: - std::mutex mutex_; + mutable std::mutex mutex_; std::condition_variable cv_; std::queue queue_; bool done_; + bool aborted_; }; } // namespace fl diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index 22cdeb44f..86e25f678 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -365,7 +365,7 @@ std::shared_ptr ResponsesHandler::HandleSt std::vector closed_items; auto push_event = [&](const std::string& event_name, const StreamEvent& ev) { - body_ptr->Push("event: " + event_name + "\ndata: " + nlohmann::json(ev).dump() + "\n\n"); + return body_ptr->Push("event: " + event_name + "\ndata: " + nlohmann::json(ev).dump() + "\n\n"); }; auto close_current = [&]() { @@ -527,7 +527,9 @@ std::shared_ptr ResponsesHandler::HandleSt delta.output_index = current_output_index; delta.item_id = current_id; delta.delta = text_item->text; - push_event("response.reasoning.delta", delta); + if (!push_event("response.reasoning.delta", delta)) { + return 1; + } } else { full_text += text_item->text; @@ -538,7 +540,9 @@ std::shared_ptr ResponsesHandler::HandleSt text_delta.content_index = 0; text_delta.item_id = current_id; text_delta.delta = text_item->text; - push_event("response.output_text.delta", text_delta); + if (!push_event("response.output_text.delta", text_delta)) { + return 1; + } } return 0; @@ -607,7 +611,7 @@ std::shared_ptr ResponsesHandler::HandleSt }); - tracker.Track(std::move(streaming_thread), stream_done); + tracker.Track(std::move(streaming_thread), stream_done, [body] { body->Abort(); }); auto response = oatpp::web::protocol::http::outgoing::Response::createShared( Status::CODE_200, body); diff --git a/sdk_v2/cpp/src/service/web_service.cc b/sdk_v2/cpp/src/service/web_service.cc index 97eb4f362..2e8450042 100644 --- a/sdk_v2/cpp/src/service/web_service.cc +++ b/sdk_v2/cpp/src/service/web_service.cc @@ -444,6 +444,8 @@ void WebService::Stop() { server->stop(); } + impl_->thread_tracker.AbortAll(); + // Stop per-connection worker tasks before releasing router/handlers. // // HttpConnectionHandler::stop() invalidates every open connection (our ForceCloseConnectionProvider's invalidator diff --git a/sdk_v2/cpp/src/service/web_service.h b/sdk_v2/cpp/src/service/web_service.h index 37d78f938..93e5828bb 100644 --- a/sdk_v2/cpp/src/service/web_service.h +++ b/sdk_v2/cpp/src/service/web_service.h @@ -26,14 +26,24 @@ class StreamingThreadTracker { struct TrackedThread { std::thread thread; std::shared_ptr> done; + std::function abort; }; public: /// Take ownership of a streaming thread. - void Track(std::thread t, std::shared_ptr> done) { + void Track(std::thread t, std::shared_ptr> done, std::function abort) { std::lock_guard lock(mutex_); ReapCompletedLocked(); - threads_.push_back(TrackedThread{std::move(t), std::move(done)}); + threads_.push_back(TrackedThread{std::move(t), std::move(done), std::move(abort)}); + } + + void AbortAll() { + std::lock_guard lock(mutex_); + for (auto& tracked : threads_) { + if (tracked.abort) { + tracked.abort(); + } + } } /// Join all remaining threads. Called by WebService::Stop(). diff --git a/sdk_v2/js/script/copy-native.mjs b/sdk_v2/js/script/copy-native.mjs index 494d26671..34682f7dc 100644 --- a/sdk_v2/js/script/copy-native.mjs +++ b/sdk_v2/js/script/copy-native.mjs @@ -55,6 +55,7 @@ const wanted = (() => { "onnxruntime-genai.dll", "onnxruntime_providers_shared.dll", "Microsoft.Windows.AI.MachineLearning.dll", + "DirectML.dll", ]; } if (process.platform === "darwin") { From 2dafa794e1e9cf055e87a95df7e73087e13e5a70 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 03:00:08 -0500 Subject: [PATCH 38/77] Complete WinML and CUDA packaging hardening Forward DirectML into JS/Python Windows payloads, verify the CUDA EP archive before extraction, and keep size-only blob skips disabled only for true incomplete-download resumes. Files changed: - .pipelines/v2/templates/steps-build-{js,python}.yml - sdk_v2/js/script/pack-prebuilds.mjs - sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc - sdk_v2/cpp/src/download/download_manager.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .pipelines/v2/templates/steps-build-js.yml | 8 ++++++++ .pipelines/v2/templates/steps-build-python.yml | 2 +- sdk_v2/cpp/src/download/download_manager.cc | 4 +++- sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc | 8 ++++++++ sdk_v2/js/script/pack-prebuilds.mjs | 2 +- 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.pipelines/v2/templates/steps-build-js.yml b/.pipelines/v2/templates/steps-build-js.yml index 657704bd9..cf18c80c4 100644 --- a/.pipelines/v2/templates/steps-build-js.yml +++ b/.pipelines/v2/templates/steps-build-js.yml @@ -156,6 +156,14 @@ steps: throw "WinML runtime DLL not found in native artifact: $winmlDll" } + $directMlDll = Join-Path "${{ parameters.nativeArtifactDir }}" 'DirectML.dll' + if (Test-Path $directMlDll) { + Copy-Item $directMlDll -Destination $dst -Force + Write-Host " staged DirectML.dll" + } else { + throw "DirectML.dll not found in native artifact: $directMlDll" + } + Get-ChildItem $dst | ForEach-Object { Write-Host " $($_.Name) $($_.Length) bytes" } displayName: 'Stage prebuild directory' diff --git a/.pipelines/v2/templates/steps-build-python.yml b/.pipelines/v2/templates/steps-build-python.yml index 177e44b66..c65c72b3d 100644 --- a/.pipelines/v2/templates/steps-build-python.yml +++ b/.pipelines/v2/templates/steps-build-python.yml @@ -108,7 +108,7 @@ steps: # the curated artifact contents straight in. $rid = '${{ parameters.rid }}' if ($rid -like 'win-*') { - $allowed = @('foundry_local.dll', 'Microsoft.Windows.AI.MachineLearning.dll', 'DirectML.dll') + $allowed = @('foundry_local.dll', 'foundry_local.lib', 'Microsoft.Windows.AI.MachineLearning.dll', 'DirectML.dll') } elseif ($rid -like 'linux-*') { $allowed = @('libfoundry_local.so') } elseif ($rid -like 'osx-*') { diff --git a/sdk_v2/cpp/src/download/download_manager.cc b/sdk_v2/cpp/src/download/download_manager.cc index 688faaa49..05ee2392f 100644 --- a/sdk_v2/cpp/src/download/download_manager.cc +++ b/sdk_v2/cpp/src/download/download_manager.cc @@ -349,6 +349,8 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, return ResolveEffectiveModelPath(model_path); } + const bool was_incomplete_download = std::filesystem::exists(signal_path); + // Create download signal file { std::ofstream signal(signal_path); @@ -376,7 +378,7 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, // and the blob container has all files at the root or in variant subdirectories. download_opts.path_prefix = ""; download_opts.max_concurrency = max_concurrency_; - download_opts.skip_completed_files = !std::filesystem::exists(signal_path); + download_opts.skip_completed_files = !was_incomplete_download; if (progress_cb) { download_opts.progress = [&progress_cb](float percent) { diff --git a/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc b/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc index 7857655c0..837eea163 100644 --- a/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc +++ b/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc @@ -8,6 +8,7 @@ #include "utils.h" #include "http/http_download.h" #include "util/zip_extract.h" +#include "util/sha256.h" #include @@ -28,6 +29,8 @@ constexpr int kMaxInstallAttempts = 5; // CUDA EP package is built against the ONNX Runtime version we link against. constexpr const char* kDownloadUrl = "https://foundrypackages-ffhrdhbxb7gpdreh.b02.azurefd.net/cuda-ep-20260501-062935.zip"; +constexpr const char* kPackageSha256 = + "D0DBCE52D121954F2D86E01E6D3418690A5BBA787028CCEE448E4BCDDD4F01E1"; struct ExpectedBinary { const char* filename; @@ -156,6 +159,11 @@ bool CudaEpBootstrapper::DownloadAndRegister(bool force, return false; } + if (!VerifyEpArchive(zip_path, kPackageSha256, "CUDA EP", logger)) { + logger.Log(LogLevel::Warning, "CUDA EP: downloaded archive verification failed"); + return false; + } + // Extract logger.Log(LogLevel::Information, "CUDA EP: extracting..."); diff --git a/sdk_v2/js/script/pack-prebuilds.mjs b/sdk_v2/js/script/pack-prebuilds.mjs index 36e1d0913..8dad4a509 100644 --- a/sdk_v2/js/script/pack-prebuilds.mjs +++ b/sdk_v2/js/script/pack-prebuilds.mjs @@ -54,7 +54,7 @@ const wanted = (() => { // not. The reg-free WinML 2.x runtime ships next to foundry_local.dll on Windows // so WinML hardware EPs work out of the box without an install-time download. const optional = - process.platform === "win32" ? ["Microsoft.Windows.AI.MachineLearning.dll"] : []; + process.platform === "win32" ? ["Microsoft.Windows.AI.MachineLearning.dll", "DirectML.dll"] : []; let copied = 0; const available = new Set(readdirSync(sourceDir)); From 2dc4ebd5b4173b1c5393efa7f3f967fabb8e4979 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 09:20:10 -0500 Subject: [PATCH 39/77] Fix shutdown and platform loader edge cases Keep Windows Python DLL-directory handles alive, run /shutdown asynchronously, abort late-tracked streams during shutdown, and expose the CDN CUDA bootstrapper only on Windows. Files changed: - sdk_v2/python/src/foundry_local_sdk/_native/{api.py,lib_loader.py} - sdk_v2/cpp/src/service/web_service.{cc,h} - sdk_v2/cpp/src/manager.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/manager.cc | 9 +++++++- sdk_v2/cpp/src/service/web_service.cc | 6 ++++-- sdk_v2/cpp/src/service/web_service.h | 21 +++++++++++++++++++ .../src/foundry_local_sdk/_native/api.py | 3 ++- .../foundry_local_sdk/_native/lib_loader.py | 2 +- 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index 5f81eed3c..5b223f338 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -252,7 +252,11 @@ Manager::Manager(const Configuration& config) // Detected once and reused below for both the WinML-catalog skip-list and the // Foundry CUDA bootstrapper. HasNvidiaGpu() shells out to nvidia-smi, so caching // the result here avoids a second subprocess spawn. +#ifdef _WIN32 const bool has_nvidia_gpu = CudaEpBootstrapper::HasNvidiaGpu(); +#else + const bool has_nvidia_gpu = false; +#endif #if FOUNDRY_LOCAL_HAS_EP_CATALOG // WinML EPs — enumerate from the OS EP catalog (Windows 10 19H1+ reg-free runtime). @@ -289,11 +293,14 @@ Manager::Manager(const Configuration& config) const auto cache_dir = std::filesystem::path(*config_.model_cache_dir).parent_path(); - // CUDA EP — only if an NVIDIA GPU is detected +#ifdef _WIN32 + // CUDA EP — only if an NVIDIA GPU is detected. The current Foundry CUDA + // package contains Windows DLLs. if (has_nvidia_gpu) { const auto cuda_ep_dir = cache_dir / "cuda-ep"; bootstrappers.push_back(std::make_unique(cuda_ep_dir.string(), register_ep)); } +#endif // WebGPU EP — always available (no hardware detection needed). const auto webgpu_ep_dir = cache_dir / "webgpu-ep"; diff --git a/sdk_v2/cpp/src/service/web_service.cc b/sdk_v2/cpp/src/service/web_service.cc index 2e8450042..c6d87779c 100644 --- a/sdk_v2/cpp/src/service/web_service.cc +++ b/sdk_v2/cpp/src/service/web_service.cc @@ -192,9 +192,8 @@ class ShutdownHandler : public HttpRequestHandler { : shutdown_fn_(std::move(shutdown_fn)) {} std::shared_ptr handle(const std::shared_ptr&) override { - shutdown_fn_(); - nlohmann::json body = {{"status", "shutting_down"}}; + std::thread([fn = shutdown_fn_] { fn(); }).detach(); return JsonResponse(Status::CODE_200, body); } @@ -426,6 +425,7 @@ std::vector WebService::Start(const std::vector& endpo ctx.bound_urls = bound_urls; impl_->running.store(true); + impl_->thread_tracker.Reset(); return bound_urls; } @@ -435,6 +435,8 @@ void WebService::Stop() { return; } + impl_->thread_tracker.BeginStopping(); + // Stop accepting new connections first, then stop server loops. for (auto& provider : impl_->providers) { provider->stop(); diff --git a/sdk_v2/cpp/src/service/web_service.h b/sdk_v2/cpp/src/service/web_service.h index 93e5828bb..5f2fb89b8 100644 --- a/sdk_v2/cpp/src/service/web_service.h +++ b/sdk_v2/cpp/src/service/web_service.h @@ -34,11 +34,31 @@ class StreamingThreadTracker { void Track(std::thread t, std::shared_ptr> done, std::function abort) { std::lock_guard lock(mutex_); ReapCompletedLocked(); + if (stopping_) { + if (abort) { + abort(); + } + } threads_.push_back(TrackedThread{std::move(t), std::move(done), std::move(abort)}); } + void BeginStopping() { + std::lock_guard lock(mutex_); + stopping_ = true; + AbortAllLocked(); + } + void AbortAll() { std::lock_guard lock(mutex_); + AbortAllLocked(); + } + + void Reset() { + std::lock_guard lock(mutex_); + stopping_ = false; + } + + void AbortAllLocked() { for (auto& tracked : threads_) { if (tracked.abort) { tracked.abort(); @@ -78,6 +98,7 @@ class StreamingThreadTracker { std::mutex mutex_; std::vector threads_; + bool stopping_ = false; }; /// Context shared with all HTTP controllers. diff --git a/sdk_v2/python/src/foundry_local_sdk/_native/api.py b/sdk_v2/python/src/foundry_local_sdk/_native/api.py index 1e56572a4..0f19e5188 100644 --- a/sdk_v2/python/src/foundry_local_sdk/_native/api.py +++ b/sdk_v2/python/src/foundry_local_sdk/_native/api.py @@ -32,6 +32,7 @@ # module-level name so the GC never collects them (which would unload the DLLs # mid-process and crash any in-flight ORT call). _preloaded_native_deps: list = [] +_dll_directory_handles: list = [] if _lib_path is not None: _preloaded_native_deps = prepare_native_dependencies(_lib_path.parent) @@ -46,7 +47,7 @@ # foundry_local.dll resolves. ORT/GenAI were already preloaded by absolute path above. _dll_parent = _lib_path.parent.resolve() if _dll_parent.is_dir(): - os.add_dll_directory(str(_dll_parent)) + _dll_directory_handles.append(os.add_dll_directory(str(_dll_parent))) else: # Linux/macOS: preload libfoundry_local with RTLD_GLOBAL so its exported symbols satisfy the cffi # extension's NEEDED entry by symbol resolution rather than by file lookup. ORT/GenAI are already diff --git a/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py b/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py index 1369759c8..0b67d466a 100644 --- a/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py +++ b/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py @@ -248,7 +248,7 @@ def prepare_native_dependencies(foundry_local_dir: pathlib.Path) -> list: if add_dll_directory is not None: for d in {ort_path.parent, genai_path.parent, foundry_local_dir}: try: - add_dll_directory(str(d)) + handles.append(add_dll_directory(str(d))) except OSError as exc: logger.warning("os.add_dll_directory(%s) failed: %s", d, exc) From 5e12c10d340dbb5a664632838296cdf4332408ec Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 15:14:36 -0500 Subject: [PATCH 40/77] Harden platform and constructor edges Auto-enable telemetry vcpkg feature for direct CMake, fix Windows ARM64 Python native probing, fully validate cached CUDA DLL payloads, gate unsupported WebGPU/CUDA bootstrapper platforms, clean up ORT/GenAI callbacks on constructor failure, and clean up partial web-service startup. Files changed: - sdk_v2/cpp/CMakeLists.txt - sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py - sdk_v2/cpp/src/ep_detection/{cuda_ep_bootstrapper,webgpu_ep_bootstrapper,ep_utils}.* - sdk_v2/cpp/src/manager.cc and src/service/web_service.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/CMakeLists.txt | 3 ++ .../src/ep_detection/cuda_ep_bootstrapper.cc | 33 ++++++++++++++----- sdk_v2/cpp/src/ep_detection/ep_utils.cc | 25 ++++++++++++++ sdk_v2/cpp/src/ep_detection/ep_utils.h | 8 +++++ .../ep_detection/webgpu_ep_bootstrapper.cc | 16 ++++++--- .../src/ep_detection/webgpu_ep_bootstrapper.h | 1 + sdk_v2/cpp/src/manager.cc | 29 +++++++++++++--- sdk_v2/cpp/src/service/web_service.cc | 6 ++++ .../foundry_local_sdk/_native/lib_loader.py | 5 ++- 9 files changed, 108 insertions(+), 18 deletions(-) diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index eb0dfd5ea..ad63ef343 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -13,6 +13,9 @@ endif() if(NOT DEFINED FOUNDRY_LOCAL_BUILD_SERVICE OR FOUNDRY_LOCAL_BUILD_SERVICE) list(APPEND VCPKG_MANIFEST_FEATURES "service") endif() +if(NOT DEFINED FOUNDRY_LOCAL_USE_TELEMETRY OR FOUNDRY_LOCAL_USE_TELEMETRY) + list(APPEND VCPKG_MANIFEST_FEATURES "telemetry") +endif() project(foundry_local VERSION 0.1.0 LANGUAGES CXX C) diff --git a/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc b/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc index 837eea163..b7ea216be 100644 --- a/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc +++ b/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc @@ -17,7 +17,10 @@ #include #include #include +#include #include +#include +#include namespace { @@ -38,6 +41,17 @@ struct ExpectedBinary { }; constexpr ExpectedBinary kExpectedBinaries[] = { + {"cublas64_12.dll", "9513540E4EC4C51EE9E7304138C2CC255C29A8C181F9E80C38EFA25738BECD99"}, + {"cublasLt64_12.dll", "B199D1FF892A81B7FD3D57BA1781549609B41500B36008FEF326038393AD46C7"}, + {"cudart64_12.dll", "C2C9A9C22A9BCBA90E261825968836787B331038047A26770CFFB7A583C28344"}, + {"cudnn64_9.dll", "0D1D71325EB5E91570AB8BA8E399E07BF717FFD76511B2407229A8F45E0B1305"}, + {"cudnn_adv64_9.dll", "6D66BCE22502C2582A9C0E5398EE8CC38ADDCE2C837EB6DB8786ABC650E48DD8"}, + {"cudnn_engines_precompiled64_9.dll", "B410C3B42921AFC6E668FF994FCE1BF12C5A8A9B1A9445EBEE61958BF49B1E0A"}, + {"cudnn_engines_runtime_compiled64_9.dll", "8E62214495C96B93C6333C084FEC49B43F272B7E1977A12FE62275E9070647EB"}, + {"cudnn_graph64_9.dll", "82F710B01D15D20C311009721C771B76360A4954EBF7B5F4A407B0F96587F568"}, + {"cudnn_heuristic64_9.dll", "50719EEFB6692074096BF83C87E9CD186F7CE5B953201DA33669C1277A61949B"}, + {"cudnn_ops64_9.dll", "49487537744256A3D4365C4792B03BF31130AD1FAEA0A13EAFA219620941D837"}, + {"cufft64_11.dll", "F4FEA9227B14843894AD5436725F9638B172171142C95291FC6AE7A493248221"}, {"onnxruntime_providers_cuda.dll", "DD540FCFECFBC68B4675C9ADF09C2858CF6B054563859D79598AA2524406A76F"}, {"onnxruntime-genai-cuda.dll", "BC953F8E2AAFC6219B2D723B65AB8F1A9426A6B7724D6A01ED756FAE8C3DE6AE"}, }; @@ -46,6 +60,15 @@ constexpr const char* kRegistrationName = "Foundry.CUDA"; constexpr const char* kCudaProviderDll = "onnxruntime_providers_cuda.dll"; constexpr const char* kCudaProviderOverrideEnv = "FOUNDRY_LOCAL_CUDA_EP_LIBRARY"; +std::vector> ExpectedBinaryHashes() { + std::vector> expected; + expected.reserve(std::size(kExpectedBinaries)); + for (const auto& binary : kExpectedBinaries) { + expected.emplace_back(binary.filename, binary.sha256); + } + return expected; +} + } // anonymous namespace namespace fl { @@ -125,10 +148,7 @@ bool CudaEpBootstrapper::DownloadAndRegister(bool force, FileLock lock(lock_path); // Check if package already exists and is valid - if (fl::VerifyEpBinaries(ep_dir, - {{kExpectedBinaries[0].filename, kExpectedBinaries[0].sha256}, - {kExpectedBinaries[1].filename, kExpectedBinaries[1].sha256}}, - "CUDA EP", logger)) { + if (fl::VerifyEpBinaries(ep_dir, ExpectedBinaryHashes(), "CUDA EP", logger)) { logger.Log(LogLevel::Information, "CUDA EP: package already valid, skipping download"); } else { // Clean up any partial install @@ -176,10 +196,7 @@ bool CudaEpBootstrapper::DownloadAndRegister(bool force, std::filesystem::remove(zip_path); // Verify - if (!fl::VerifyEpBinaries(ep_dir, - {{kExpectedBinaries[0].filename, kExpectedBinaries[0].sha256}, - {kExpectedBinaries[1].filename, kExpectedBinaries[1].sha256}}, - "CUDA EP", logger)) { + if (!fl::VerifyEpBinaries(ep_dir, ExpectedBinaryHashes(), "CUDA EP", logger)) { logger.Log(LogLevel::Warning, "CUDA EP: verification failed after download"); return false; } diff --git a/sdk_v2/cpp/src/ep_detection/ep_utils.cc b/sdk_v2/cpp/src/ep_detection/ep_utils.cc index 5cd025b1b..2f8c53fc3 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_utils.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_utils.cc @@ -9,6 +9,7 @@ #include #include +#include #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN @@ -75,6 +76,30 @@ bool VerifyEpBinaries( return true; } +bool VerifyEpBinaries( + const std::filesystem::path& dir, + const std::vector>& expected, + std::string_view ep_name, + ILogger& logger) { + for (const auto& [filename, expected_hash] : expected) { + auto file_path = dir / filename; + + if (!std::filesystem::exists(file_path)) { + return false; + } + + auto hash = Sha256File(file_path); + if (CompareCaseInsensitive(hash, expected_hash) != 0) { + logger.Log(LogLevel::Warning, + fmt::format("{}: hash mismatch for {}: got {}, expected {}", + ep_name, filename, hash, expected_hash)); + return false; + } + } + + return true; +} + void PrependDirToProcessPath([[maybe_unused]] const std::filesystem::path& dir) { #ifdef _WIN32 DWORD len = GetEnvironmentVariableW(L"PATH", nullptr, 0); diff --git a/sdk_v2/cpp/src/ep_detection/ep_utils.h b/sdk_v2/cpp/src/ep_detection/ep_utils.h index ba4690e65..985248ab3 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_utils.h +++ b/sdk_v2/cpp/src/ep_detection/ep_utils.h @@ -4,8 +4,10 @@ #include #include +#include #include #include +#include namespace fl { @@ -37,6 +39,12 @@ bool VerifyEpBinaries( std::string_view ep_name, ILogger& logger); +bool VerifyEpBinaries( + const std::filesystem::path& dir, + const std::vector>& expected, + std::string_view ep_name, + ILogger& logger); + /// Prepend @p dir to the process `PATH` environment variable for the lifetime of the process. /// /// EP provider libraries (CUDA, WebGPU) delay-load sibling dependency DLLs from their own directory, diff --git a/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.cc b/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.cc index d8b63db52..2f5d38aab 100644 --- a/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.cc +++ b/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.cc @@ -56,7 +56,7 @@ const std::unordered_map kPackageMetada "0767448ABEADE590821E88E56C471E0F2DFE89EC9A7642D08ADC3DC94F14AB92"} }, }; -#elif defined(__APPLE__) +#elif defined(__APPLE__) && defined(__aarch64__) const std::unordered_map kPackageMetadata = { {"macos-arm64", {"https://foundrypackages-ffhrdhbxb7gpdreh.b02.azurefd.net/webgpu_ep_0.1.0_macos-arm64.zip", @@ -64,7 +64,7 @@ const std::unordered_map kPackageMetada "A08BCEBE097B555E23938FCC71A5FAAD461F586CAB0B63DC9D21E970F6CA4C87"} }, }; -#else +#elif defined(__linux__) && defined(__x86_64__) && !defined(__ANDROID__) const std::unordered_map kPackageMetadata = { {"linux-x64", {"https://foundrypackages-ffhrdhbxb7gpdreh.b02.azurefd.net/webgpu_ep_0.1.0_linux-x64.zip", @@ -72,6 +72,8 @@ const std::unordered_map kPackageMetada "CBDFF74E6569E3CF66B46F0D194D87CD3CF49E83B7AA46552C39B0218D58B215"} }, }; +#else +const std::unordered_map kPackageMetadata = {}; #endif // Platform-specific package metadata is baked into the binary to keep @@ -80,10 +82,12 @@ const std::unordered_map kPackageMetada constexpr const char* kPlatformKey = "win-arm64"; #elif defined(_WIN32) constexpr const char* kPlatformKey = "win-x64"; -#elif defined(__APPLE__) +#elif defined(__APPLE__) && defined(__aarch64__) constexpr const char* kPlatformKey = "macos-arm64"; -#else +#elif defined(__linux__) && defined(__x86_64__) && !defined(__ANDROID__) constexpr const char* kPlatformKey = "linux-x64"; +#else +constexpr const char* kPlatformKey = ""; #endif const WebGpuPackageMetadata& GetPackageMetadata() { @@ -124,6 +128,10 @@ bool WebGpuEpBootstrapper::IsRegistered() const { return registered_; } +bool WebGpuEpBootstrapper::IsSupported() { + return kPackageMetadata.find(kPlatformKey) != kPackageMetadata.end(); +} + bool WebGpuEpBootstrapper::DownloadAndRegister(bool force, const ProgressCallback& progress_cb, ILogger& logger) { diff --git a/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.h b/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.h index e80d4651d..21a147d32 100644 --- a/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.h +++ b/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.h @@ -30,6 +30,7 @@ class WebGpuEpBootstrapper : public IEpBootstrapper { WebGpuEpBootstrapper& operator=(const WebGpuEpBootstrapper&) = delete; const std::string& Name() const override; + static bool IsSupported(); bool IsRegistered() const override; bool DownloadAndRegister(bool force, const ProgressCallback& progress_cb, diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index 5b223f338..42b8f96f1 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -182,6 +182,7 @@ std::unique_ptr Manager::s_instance_; Manager::Manager(const Configuration& config) : config_(config) { + try { config_.Validate(); const bool genai_verbose_logging = IsGenAIVerboseLoggingEnabled(); @@ -302,9 +303,10 @@ Manager::Manager(const Configuration& config) } #endif - // WebGPU EP — always available (no hardware detection needed). - const auto webgpu_ep_dir = cache_dir / "webgpu-ep"; - bootstrappers.push_back(std::make_unique(webgpu_ep_dir.string(), register_ep)); + if (WebGpuEpBootstrapper::IsSupported()) { + const auto webgpu_ep_dir = cache_dir / "webgpu-ep"; + bootstrappers.push_back(std::make_unique(webgpu_ep_dir.string(), register_ep)); + } // Telemetry must be constructed before subsystems that emit events so we can // pass it to them at construction time (e.g. EpDetector emits EPDownloadAttempt @@ -359,6 +361,22 @@ Manager::Manager(const Configuration& config) config_.catalog_region.value_or("auto"), disable_region_fallback, telemetry_.get()); + } catch (...) { + if (ort_api_ != nullptr && ort_env_ != nullptr) { + for (const auto& name : registered_ep_libraries_) { + OrtStatus* status = ort_api_->UnregisterExecutionProviderLibrary(ort_env_, name.c_str()); + if (status != nullptr) { + ort_api_->ReleaseStatus(status); + } + } + registered_ep_libraries_.clear(); + ort_api_->ReleaseEnv(ort_env_); + ort_env_ = nullptr; + } + SetOgaLogCallback(nullptr); + s_ort_logger.store(nullptr, std::memory_order_release); + throw; + } } Manager::~Manager() { @@ -540,6 +558,9 @@ void Manager::Shutdown() { logger_->Log(LogLevel::Information, "Shutdown requested"); + model_load_manager_->RejectNewLoads(); + session_manager_->CancelAll(); + if (web_service_running_) { StopWebService(); } @@ -550,8 +571,6 @@ void Manager::Shutdown() { // 3. Unload all models, polling per-model session refcount for direct-API users // who haven't dropped their flSession* yet. Bounded by timeout so a stuck // caller can't block process shutdown indefinitely. - model_load_manager_->RejectNewLoads(); - session_manager_->CancelAll(); session_manager_->WaitForDrain(); model_load_manager_->UnloadAll(); } diff --git a/sdk_v2/cpp/src/service/web_service.cc b/sdk_v2/cpp/src/service/web_service.cc index c6d87779c..49e8f03df 100644 --- a/sdk_v2/cpp/src/service/web_service.cc +++ b/sdk_v2/cpp/src/service/web_service.cc @@ -351,6 +351,7 @@ std::vector WebService::Start(const std::vector& endpo std::vector bound_urls; + try { for (const auto& endpoint : endpoints) { // Parse "http://host:port" — strip scheme, extract host:port std::string host = "127.0.0.1"; @@ -422,6 +423,11 @@ std::vector WebService::Start(const std::vector& endpo ctx.logger.Log(LogLevel::Information, fmt::format("Web service listening on {}", bound_url)); } + } catch (...) { + impl_->running.store(true); + Stop(); + throw; + } ctx.bound_urls = bound_urls; impl_->running.store(true); diff --git a/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py b/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py index 0b67d466a..97582b65a 100644 --- a/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py +++ b/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py @@ -29,7 +29,10 @@ def _lib_name() -> str: def _platform_subdir() -> str: if sys.platform == "win32": - return "win-x64" + machine = platform.machine().lower() + arch = os.environ.get("PROCESSOR_ARCHITECTURE", "").lower() + wow64_arch = os.environ.get("PROCESSOR_ARCHITEW6432", "").lower() + return "win-arm64" if "arm64" in {machine, arch, wow64_arch} or machine == "aarch64" else "win-x64" if sys.platform == "darwin": return "osx-arm64" if platform.machine() == "arm64" else "osx-x64" return "linux-x64" From dc5683727d1be586a431e76eb673ee250c88f796 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 16:08:32 -0500 Subject: [PATCH 41/77] Fix Python RID and session telemetry edges Select Windows Python native RID from the interpreter platform tag and make web-service telemetry session start best-effort so listener startup does not remain half-failed. Files changed: - sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py - sdk_v2/cpp/src/manager.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/manager.cc | 8 +++++++- sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py | 6 ++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index 42b8f96f1..f6d4fcf2a 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -512,7 +512,13 @@ void Manager::StartWebService() { web_service_running_ = true; // Open an app-usage session for the lifetime of the running service so events // carry ext.app.sesId and the backend gets session duration. - telemetry_->StartSession(); + try { + telemetry_->StartSession(); + } catch (const std::exception& ex) { + logger_->Log(LogLevel::Warning, std::string("telemetry StartSession failed: ") + ex.what()); + } catch (...) { + logger_->Log(LogLevel::Warning, "telemetry StartSession failed with unknown error"); + } tracker.SetStatus(ActionStatus::kSuccess); #else FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, diff --git a/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py b/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py index 97582b65a..ab23a8642 100644 --- a/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py +++ b/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py @@ -15,6 +15,7 @@ import pathlib import platform import sys +import sysconfig logger = logging.getLogger(__name__) @@ -29,10 +30,7 @@ def _lib_name() -> str: def _platform_subdir() -> str: if sys.platform == "win32": - machine = platform.machine().lower() - arch = os.environ.get("PROCESSOR_ARCHITECTURE", "").lower() - wow64_arch = os.environ.get("PROCESSOR_ARCHITEW6432", "").lower() - return "win-arm64" if "arm64" in {machine, arch, wow64_arch} or machine == "aarch64" else "win-x64" + return "win-arm64" if sysconfig.get_platform().lower() == "win-arm64" else "win-x64" if sys.platform == "darwin": return "osx-arm64" if platform.machine() == "arm64" else "osx-x64" return "linux-x64" From 3c5e74e565ca748da75be5ffb91de77e1373f22e Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 17:29:06 -0500 Subject: [PATCH 42/77] Force catalog refresh and stream registration safety Force invalidated catalogs to refresh on the next lookup, and make streaming worker registration/cancellation safe during shutdown by catching registration failures and wiring abort callbacks to request cancellation. Files changed: - sdk_v2/cpp/src/catalog/base_model_catalog.* - sdk_v2/cpp/src/service/{chat_completions,audio_transcriptions,responses}_handler.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/catalog/base_model_catalog.cc | 6 ++++-- sdk_v2/cpp/src/catalog/base_model_catalog.h | 1 + .../src/service/audio_transcriptions_handler.cc | 13 ++++++++----- .../cpp/src/service/chat_completions_handler.cc | 13 ++++++++----- sdk_v2/cpp/src/service/responses_handler.cc | 16 ++++++++++------ 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.cc b/sdk_v2/cpp/src/catalog/base_model_catalog.cc index fc541e6cb..fb32005c3 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.cc @@ -197,6 +197,7 @@ void BaseModelCatalog::InvalidateCache() { // This is called after EP registration changes — the catalog needs to // re-query with updated device/EP filters. std::lock_guard lock(mutex_); + force_refresh_ = true; next_refresh_at_ = std::chrono::steady_clock::time_point{}; } @@ -207,8 +208,8 @@ void BaseModelCatalog::EnsurePopulated(bool allow_refresh) const { // not worth the complexity to optimise.) std::lock_guard lock(mutex_); - bool needs_refresh = allow_refresh && - std::chrono::steady_clock::now() >= next_refresh_at_; + bool needs_refresh = force_refresh_ || + (allow_refresh && std::chrono::steady_clock::now() >= next_refresh_at_); if (populated_ && !needs_refresh) { return; @@ -221,6 +222,7 @@ void BaseModelCatalog::EnsurePopulated(bool allow_refresh) const { auto variants = FetchModels(); PopulateModels(std::move(variants)); + force_refresh_ = false; next_refresh_at_ = std::chrono::steady_clock::now() + kCacheDuration; } diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.h b/sdk_v2/cpp/src/catalog/base_model_catalog.h index 8e91a9c6b..d7adb5d80 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.h +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.h @@ -98,6 +98,7 @@ class BaseModelCatalog : public ICatalog { std::shared_ptr GetIndex() const; mutable bool populated_ = false; + mutable bool force_refresh_ = false; mutable std::mutex mutex_; mutable std::chrono::steady_clock::time_point next_refresh_at_{}; diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc index 2ca3a14d0..ab21c5bd1 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc @@ -198,15 +198,15 @@ std::shared_ptr AudioTranscriptionsHandler auto& logger = ctx_.logger; auto& thread_tracker = ctx_.thread_tracker; auto stream_done = std::make_shared>(false); + auto stream_request = std::make_shared(std::move(session_request)); std::thread streaming_thread([bg_session = std::move(session), body_ptr, &logger, - req = std::move(session_request), &thread_tracker, + req = stream_request, &thread_tracker, route_tracker = std::move(route_tracker), &session_manager = ctx_.session_manager, stream_done]() mutable { - SessionRegistration reg(session_manager, bg_session); - try { + SessionRegistration reg(session_manager, bg_session); fl::Response bg_response; // Callback receives OPENAI_JSON-tagged TextItem chunks from AudioSession — just wrap in SSE framing. @@ -232,7 +232,7 @@ std::shared_ptr AudioTranscriptionsHandler }; bg_session.SetStreamingCallback(callback_fn); - bg_session.ProcessRequest(req, bg_response); + bg_session.ProcessRequest(*req, bg_response); // Send terminal event body_ptr->Push("data: [DONE]\n\n"); @@ -256,7 +256,10 @@ std::shared_ptr AudioTranscriptionsHandler stream_done->store(true, std::memory_order_release); }); - thread_tracker.Track(std::move(streaming_thread), stream_done, [body] { body->Abort(); }); + thread_tracker.Track(std::move(streaming_thread), stream_done, [body, stream_request] { + body->Abort(); + stream_request->canceled.store(true, std::memory_order_relaxed); + }); auto response = oatpp::web::protocol::http::outgoing::Response::createShared(Status::CODE_200, body); response->putHeader("Content-Type", "text/event-stream"); diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.cc b/sdk_v2/cpp/src/service/chat_completions_handler.cc index b309597fa..605c55f2a 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.cc +++ b/sdk_v2/cpp/src/service/chat_completions_handler.cc @@ -212,16 +212,16 @@ std::shared_ptr ChatCompletionsHandler::Ha auto& logger = ctx_.logger; auto& thread_tracker = ctx_.thread_tracker; auto stream_done = std::make_shared>(false); + auto stream_request = std::make_shared(std::move(session_request)); std::thread streaming_thread([bg_session = std::move(session), body_ptr, &logger, - req = std::move(session_request), + req = stream_request, include_usage, &thread_tracker, route_tracker = std::move(route_tracker), &session_manager = ctx_.session_manager, stream_done]() mutable { - SessionRegistration reg(session_manager, bg_session); - try { + SessionRegistration reg(session_manager, bg_session); fl::Response bg_response; // Callback receives OPENAI_JSON-tagged TextItem chunks from ChatSession — just wrap in SSE framing. @@ -246,7 +246,7 @@ std::shared_ptr ChatCompletionsHandler::Ha }; bg_session.SetStreamingCallback(callback_fn); - bg_session.ProcessRequest(req, bg_response); + bg_session.ProcessRequest(*req, bg_response); // Usage chunk — only if stream_options.include_usage was true if (include_usage) { @@ -294,7 +294,10 @@ std::shared_ptr ChatCompletionsHandler::Ha stream_done->store(true, std::memory_order_release); }); - thread_tracker.Track(std::move(streaming_thread), stream_done, [body] { body->Abort(); }); + thread_tracker.Track(std::move(streaming_thread), stream_done, [body, stream_request] { + body->Abort(); + stream_request->canceled.store(true, std::memory_order_relaxed); + }); auto response = oatpp::web::protocol::http::outgoing::Response::createShared( Status::CODE_200, body); diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index 86e25f678..988f9739c 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -330,12 +330,13 @@ std::shared_ptr ResponsesHandler::HandleSt auto& session_manager = ctx_.session_manager; auto& tracker = ctx_.thread_tracker; auto stream_done = std::make_shared>(false); + auto stream_request = std::make_shared(std::move(session_request)); // Background thread is required: oatpp needs the Response returned immediately so it can // start writing SSE events. ProcessRequest blocks until generation completes. std::thread streaming_thread([body_ptr, &logger, &session_manager, session = std::move(session), - req = std::move(session_request), + req = stream_request, model_name, response_id, created_at, should_store, &store, req_copy = std::move(req_copy), @@ -343,8 +344,6 @@ std::shared_ptr ResponsesHandler::HandleSt route_tracker = std::move(route_tracker), &tracker, stream_done]() mutable { - SessionRegistration reg(session_manager, *session); - int seq = 2; std::string full_text; // concatenation of all visible runs, used for output_text in completed_response @@ -363,6 +362,7 @@ std::shared_ptr ResponsesHandler::HandleSt // Items that have been *closed* (or, for the currently-open item at end-of-stream, finalized in place). // Used to construct the final `output[]` array for the response.completed event. std::vector closed_items; + std::optional reg; auto push_event = [&](const std::string& event_name, const StreamEvent& ev) { return body_ptr->Push("event: " + event_name + "\ndata: " + nlohmann::json(ev).dump() + "\n\n"); @@ -490,6 +490,7 @@ std::shared_ptr ResponsesHandler::HandleSt }; try { + reg.emplace(session_manager, *session); fl::Response bg_response; fl::Session::StreamingCallbackFn callback_fn = [&](flStreamingCallbackData event, void* /*user_data*/) -> int { fl::ItemQueue* queue = reinterpret_cast(event.item_queue); @@ -550,7 +551,7 @@ std::shared_ptr ResponsesHandler::HandleSt session->SetStreamingCallback(callback_fn); - session->ProcessRequest(req, bg_response); + session->ProcessRequest(*req, bg_response); // Close whatever item is still open at end-of-generation so the SSE stream is well-formed. close_current(); @@ -571,7 +572,7 @@ std::shared_ptr ResponsesHandler::HandleSt store.Store(response_id, response_json, std::move(input_items)); // Deregister before caching — see non-streaming path comment. - reg.Release(); + reg->Release(); // Clear per-request streaming callback before caching — see non-streaming path comment. session->SetStreamingCallback(nullptr); @@ -611,7 +612,10 @@ std::shared_ptr ResponsesHandler::HandleSt }); - tracker.Track(std::move(streaming_thread), stream_done, [body] { body->Abort(); }); + tracker.Track(std::move(streaming_thread), stream_done, [body, stream_request] { + body->Abort(); + stream_request->canceled.store(true, std::memory_order_relaxed); + }); auto response = oatpp::web::protocol::http::outgoing::Response::createShared( Status::CODE_200, body); From e3ad607fcb2a3009be192ab8dbe6d8b68cc95dc4 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 18:17:51 -0500 Subject: [PATCH 43/77] Let C++ wrapper version errors propagate Remove noexcept from ModelInfo wrapper accessors that call API-table helpers so version-mismatch/null-table errors throw normally instead of terminating. Files changed: - sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h - sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .../include/foundry_local/foundry_local_cpp.h | 62 +++++++++---------- .../foundry_local/foundry_local_cpp.inline.h | 62 +++++++++---------- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h index 941665cfc..70224e219 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h @@ -346,71 +346,71 @@ class ModelInfo { explicit ModelInfo(const flModelInfo& info) noexcept : info_(&info) {} // Core identity. - std::string_view Id() const noexcept; - std::string_view Name() const noexcept; - int Version() const noexcept; - std::string_view Alias() const noexcept; - std::string_view Uri() const noexcept; - flDeviceType DeviceType() const noexcept; - std::optional ExecutionProvider() const noexcept; + std::string_view Id() const; + std::string_view Name() const; + int Version() const; + std::string_view Alias() const; + std::string_view Uri() const; + flDeviceType DeviceType() const; + std::optional ExecutionProvider() const; /// Returns the device + execution provider pair, or nullopt if device_type is unknown/invalid. - std::optional GetRuntime() const noexcept; + std::optional GetRuntime() const; // Key-value lookups - std::optional GetPromptTemplate(const char* key) const noexcept; - std::optional GetModelSetting(const char* key) const noexcept; + std::optional GetPromptTemplate(const char* key) const; + std::optional GetModelSetting(const char* key) const; /// Default/recommended inference settings declared by the model author. /// Returns a non-owning read-only view; nullopt if no settings are declared. /// Useful for discovering which parameters a model supports overriding. - std::optional GetModelSettings() const noexcept; + std::optional GetModelSettings() const; /// Get a string property by key. Known keys are defined by the FOUNDRY_LOCAL_MODEL_PROP_*_STR constants. - std::optional GetStringProperty(const char* key) const noexcept; + std::optional GetStringProperty(const char* key) const; /// Get an int property by key. Known keys are defined by the FOUNDRY_LOCAL_MODEL_PROP_*_INT constants. /// Returns default_value if key is not set. - int64_t GetIntProperty(const char* key, int64_t default_value = -1) const noexcept; + int64_t GetIntProperty(const char* key, int64_t default_value = -1) const; // --- Typed property accessors (convenience wrappers over Get{String,Int}Property) --- /// Display name shown in UIs. May differ from name(). - std::optional DisplayName() const noexcept; + std::optional DisplayName() const; /// Model format, e.g. "onnx". - std::optional ModelType() const noexcept; + std::optional ModelType() const; /// Publisher / organization. - std::optional Publisher() const noexcept; + std::optional Publisher() const; /// SPDX license identifier. - std::optional License() const noexcept; + std::optional License() const; /// Human-readable license description. - std::optional LicenseDescription() const noexcept; + std::optional LicenseDescription() const; /// Task the model is designed for, e.g. "chat", "text-generation". - std::optional Task() const noexcept; + std::optional Task() const; /// Source of the model, e.g. "AzureCatalog". - std::optional ModelProvider() const noexcept; + std::optional ModelProvider() const; /// Minimum Foundry Local version required to run this model. - std::optional MinFlVersion() const noexcept; + std::optional MinFlVersion() const; /// URI of the parent model (for derived/quantized variants). - std::optional ParentUri() const noexcept; + std::optional ParentUri() const; /// Whether the model supports tool/function calling. nullopt if unspecified. - std::optional SupportsToolCalling() const noexcept; + std::optional SupportsToolCalling() const; /// Download size in megabytes. nullopt if unspecified. - std::optional FilesizeMb() const noexcept; + std::optional FilesizeMb() const; /// Maximum output tokens the model can generate. nullopt if unspecified. - std::optional MaxOutputTokens() const noexcept; + std::optional MaxOutputTokens() const; /// Unix timestamp when the model entry was created. 0 if unset. - int64_t CreatedAtUnix() const noexcept; + int64_t CreatedAtUnix() const; /// Whether this is a test/synthetic model (not for production). - bool IsTestModel() const noexcept; + bool IsTestModel() const; /// Maximum context length in tokens. nullopt if unspecified. - std::optional ContextLength() const noexcept; + std::optional ContextLength() const; /// Comma-separated list of supported input modalities (e.g. "text,image"). nullopt if unspecified. - std::optional InputModalities() const noexcept; + std::optional InputModalities() const; /// Comma-separated list of supported output modalities. nullopt if unspecified. - std::optional OutputModalities() const noexcept; + std::optional OutputModalities() const; /// Comma-separated list of model capabilities. nullopt if unspecified. - std::optional Capabilities() const noexcept; + std::optional Capabilities() const; private: static std::string_view safe(const char* s) noexcept { return s ? s : ""; } diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h index 08c8f594f..159247937 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h @@ -289,36 +289,36 @@ inline flManager* detail::CreateManager(const Configuration& config) { // ModelInfo // =========================================================================== -inline std::string_view ModelInfo::Id() const noexcept { +inline std::string_view ModelInfo::Id() const { return safe(detail::model_api()->Info_GetId(info_)); } -inline std::string_view ModelInfo::Name() const noexcept { +inline std::string_view ModelInfo::Name() const { return safe(detail::model_api()->Info_GetName(info_)); } -inline int ModelInfo::Version() const noexcept { +inline int ModelInfo::Version() const { return detail::model_api()->Info_GetVersion(info_); } -inline std::string_view ModelInfo::Alias() const noexcept { +inline std::string_view ModelInfo::Alias() const { return safe(detail::model_api()->Info_GetAlias(info_)); } -inline std::string_view ModelInfo::Uri() const noexcept { +inline std::string_view ModelInfo::Uri() const { return safe(detail::model_api()->Info_GetUri(info_)); } -inline flDeviceType ModelInfo::DeviceType() const noexcept { +inline flDeviceType ModelInfo::DeviceType() const { return detail::model_api()->Info_GetDeviceType(info_); } -inline std::optional ModelInfo::ExecutionProvider() const noexcept { +inline std::optional ModelInfo::ExecutionProvider() const { const char* v = detail::model_api()->Info_GetExecutionProvider(info_); return v ? std::optional{v} : std::nullopt; } -inline std::optional ModelInfo::GetRuntime() const noexcept { +inline std::optional ModelInfo::GetRuntime() const { flDeviceType dt = DeviceType(); if (dt == FOUNDRY_LOCAL_DEVICE_NOTSET) { return std::nullopt; @@ -326,7 +326,7 @@ inline std::optional ModelInfo::GetRuntime() const noexcept { return Runtime{dt, ExecutionProvider()}; } -inline std::optional ModelInfo::GetPromptTemplate(const char* key) const noexcept { +inline std::optional ModelInfo::GetPromptTemplate(const char* key) const { const flKeyValuePairs* kvps = detail::model_api()->Info_GetPromptTemplates(info_); if (!kvps) { return std::nullopt; @@ -335,7 +335,7 @@ inline std::optional ModelInfo::GetPromptTemplate(const char* return v ? std::optional{v} : std::nullopt; } -inline std::optional ModelInfo::GetModelSetting(const char* key) const noexcept { +inline std::optional ModelInfo::GetModelSetting(const char* key) const { const flKeyValuePairs* kvps = detail::model_api()->Info_GetModelSettings(info_); if (!kvps) { return std::nullopt; @@ -344,7 +344,7 @@ inline std::optional ModelInfo::GetModelSetting(const char* ke return v ? std::optional{v} : std::nullopt; } -inline std::optional ModelInfo::GetModelSettings() const noexcept { +inline std::optional ModelInfo::GetModelSettings() const { const flKeyValuePairs* kvps = detail::model_api()->Info_GetModelSettings(info_); if (!kvps) { return std::nullopt; @@ -352,54 +352,54 @@ inline std::optional ModelInfo::GetModelSettings() const noexcept return KeyValuePairs(*kvps); } -inline std::optional ModelInfo::GetStringProperty(const char* key) const noexcept { +inline std::optional ModelInfo::GetStringProperty(const char* key) const { const char* v = detail::model_api()->Info_GetStringProperty(info_, key); return v ? std::optional{v} : std::nullopt; } -inline int64_t ModelInfo::GetIntProperty(const char* key, int64_t default_value) const noexcept { +inline int64_t ModelInfo::GetIntProperty(const char* key, int64_t default_value) const { return detail::model_api()->Info_GetIntProperty(info_, key, default_value); } // --- Typed property accessors --- -inline std::optional ModelInfo::DisplayName() const noexcept { +inline std::optional ModelInfo::DisplayName() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_DISPLAY_NAME_STR); } -inline std::optional ModelInfo::ModelType() const noexcept { +inline std::optional ModelInfo::ModelType() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR); } -inline std::optional ModelInfo::Publisher() const noexcept { +inline std::optional ModelInfo::Publisher() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR); } -inline std::optional ModelInfo::License() const noexcept { +inline std::optional ModelInfo::License() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_LICENSE_STR); } -inline std::optional ModelInfo::LicenseDescription() const noexcept { +inline std::optional ModelInfo::LicenseDescription() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_LICENSE_DESCRIPTION_STR); } -inline std::optional ModelInfo::Task() const noexcept { +inline std::optional ModelInfo::Task() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_TASK_STR); } -inline std::optional ModelInfo::ModelProvider() const noexcept { +inline std::optional ModelInfo::ModelProvider() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR); } -inline std::optional ModelInfo::MinFlVersion() const noexcept { +inline std::optional ModelInfo::MinFlVersion() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_MIN_FL_VERSION_STR); } -inline std::optional ModelInfo::ParentUri() const noexcept { +inline std::optional ModelInfo::ParentUri() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_PARENT_URI_STR); } -inline std::optional ModelInfo::SupportsToolCalling() const noexcept { +inline std::optional ModelInfo::SupportsToolCalling() const { int64_t v = GetIntProperty(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT); if (v < 0) { return std::nullopt; @@ -407,7 +407,7 @@ inline std::optional ModelInfo::SupportsToolCalling() const noexcept { return v != 0; } -inline std::optional ModelInfo::FilesizeMb() const noexcept { +inline std::optional ModelInfo::FilesizeMb() const { int64_t v = GetIntProperty(FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT); if (v < 0) { return std::nullopt; @@ -415,7 +415,7 @@ inline std::optional ModelInfo::FilesizeMb() const noexcept { return v; } -inline std::optional ModelInfo::MaxOutputTokens() const noexcept { +inline std::optional ModelInfo::MaxOutputTokens() const { int64_t v = GetIntProperty(FOUNDRY_LOCAL_MODEL_PROP_MAX_OUTPUT_TOKENS_INT); if (v < 0) { return std::nullopt; @@ -423,15 +423,15 @@ inline std::optional ModelInfo::MaxOutputTokens() const noexcept { return v; } -inline int64_t ModelInfo::CreatedAtUnix() const noexcept { +inline int64_t ModelInfo::CreatedAtUnix() const { return GetIntProperty(FOUNDRY_LOCAL_MODEL_PROP_CREATED_AT_UNIX_INT, 0); } -inline bool ModelInfo::IsTestModel() const noexcept { +inline bool ModelInfo::IsTestModel() const { return GetIntProperty(FOUNDRY_LOCAL_MODEL_PROP_IS_TEST_MODEL_INT, 0) != 0; } -inline std::optional ModelInfo::ContextLength() const noexcept { +inline std::optional ModelInfo::ContextLength() const { int64_t v = GetIntProperty(FOUNDRY_LOCAL_MODEL_PROP_CONTEXT_LENGTH_INT, -1); if (v < 0) { return std::nullopt; @@ -439,15 +439,15 @@ inline std::optional ModelInfo::ContextLength() const noexcept { return v; } -inline std::optional ModelInfo::InputModalities() const noexcept { +inline std::optional ModelInfo::InputModalities() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR); } -inline std::optional ModelInfo::OutputModalities() const noexcept { +inline std::optional ModelInfo::OutputModalities() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR); } -inline std::optional ModelInfo::Capabilities() const noexcept { +inline std::optional ModelInfo::Capabilities() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_CAPABILITIES_STR); } From aead4277633ffe0c3fd5f8b0190b00a7af190fe9 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 19:04:43 -0500 Subject: [PATCH 44/77] Fix JS metadata and safe EP/download snapshots Stage JS dependency metadata, remove unsupported Linux ARM64 install path, publish immutable discoverable-EP snapshots, and make low-level blob size-only skipping opt-in while preserving explicit tests. Files changed: - .pipelines/v2/templates/steps-pack-js.yml - sdk_v2/js/script/install-native.cjs - sdk_v2/cpp/src/ep_detection/ep_detector.* - sdk_v2/cpp/src/download/blob_downloader.h, download_manager.cc, and download tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .pipelines/v2/templates/steps-pack-js.yml | 1 + sdk_v2/cpp/src/download/blob_downloader.h | 2 +- sdk_v2/cpp/src/download/download_manager.cc | 4 +-- sdk_v2/cpp/src/ep_detection/ep_detector.cc | 26 ++++++++++++------- sdk_v2/cpp/src/ep_detection/ep_detector.h | 3 +++ sdk_v2/cpp/test/internal_api/download_test.cc | 4 +++ sdk_v2/js/script/install-native.cjs | 1 - 7 files changed, 27 insertions(+), 14 deletions(-) diff --git a/.pipelines/v2/templates/steps-pack-js.yml b/.pipelines/v2/templates/steps-pack-js.yml index 439ab41fd..9d7a4aff1 100644 --- a/.pipelines/v2/templates/steps-pack-js.yml +++ b/.pipelines/v2/templates/steps-pack-js.yml @@ -74,6 +74,7 @@ steps: $dst = '${{ parameters.outputDir }}' New-Item -ItemType Directory -Force -Path $dst | Out-Null Set-Location "$(Build.SourcesDirectory)/sdk_v2/js" + Copy-Item "$(Build.SourcesDirectory)/sdk_v2/deps_versions.json" -Destination "deps_versions.json" -Force npm pack --pack-destination $dst if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } Get-ChildItem $dst | ForEach-Object { Write-Host " $($_.Name) $($_.Length) bytes" } diff --git a/sdk_v2/cpp/src/download/blob_downloader.h b/sdk_v2/cpp/src/download/blob_downloader.h index 279fc5d26..a38097d08 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.h +++ b/sdk_v2/cpp/src/download/blob_downloader.h @@ -30,7 +30,7 @@ struct BlobDownloadOptions { /// When false, full-size files without a sidecar are redownloaded instead of /// skipped. Use this while recovering an incomplete model directory. - bool skip_completed_files = true; + bool skip_completed_files = false; /// Progress callback (optional). Return non-zero to cancel the download. DownloadProgressFn progress; diff --git a/sdk_v2/cpp/src/download/download_manager.cc b/sdk_v2/cpp/src/download/download_manager.cc index 05ee2392f..670b42ff8 100644 --- a/sdk_v2/cpp/src/download/download_manager.cc +++ b/sdk_v2/cpp/src/download/download_manager.cc @@ -349,8 +349,6 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, return ResolveEffectiveModelPath(model_path); } - const bool was_incomplete_download = std::filesystem::exists(signal_path); - // Create download signal file { std::ofstream signal(signal_path); @@ -378,7 +376,7 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, // and the blob container has all files at the root or in variant subdirectories. download_opts.path_prefix = ""; download_opts.max_concurrency = max_concurrency_; - download_opts.skip_completed_files = !was_incomplete_download; + download_opts.skip_completed_files = false; if (progress_cb) { download_opts.progress = [&progress_cb](float percent) { diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.cc b/sdk_v2/cpp/src/ep_detection/ep_detector.cc index f6d9da4cb..7fbd72ebd 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.cc @@ -32,13 +32,22 @@ EpDetector::EpDetector(const OrtApi& ort_api, OrtEnv& ort_env, cached_eps_.push_back(EpInfo{bs->Name(), bs->IsRegistered()}); } + PublishEpSnapshotLocked(); PublishCApiSnapshotLocked(); } +void EpDetector::PublishEpSnapshotLocked() { + auto& snapshot = cached_eps_snapshots_.emplace_back(cached_eps_); + current_cached_eps_ = &snapshot; +} + void EpDetector::PublishCApiSnapshotLocked() { auto& snapshot = cached_eps_c_snapshots_.emplace_back(); - snapshot.reserve(cached_eps_.size()); - for (const auto& ep : cached_eps_) { + if (current_cached_eps_ == nullptr) { + current_cached_eps_ = &cached_eps_; + } + snapshot.reserve(current_cached_eps_->size()); + for (const auto& ep : *current_cached_eps_) { snapshot.push_back(flEpInfo{ FOUNDRY_LOCAL_API_VERSION, ep.name.c_str(), @@ -117,14 +126,12 @@ std::map> EpDetector::GetAvailableDevicesT } const std::vector& EpDetector::GetDiscoverableEps() const { - // Take the cache lock for strict correctness of the is_registered field reads. - // Vector size and element addresses are immutable after construction; only - // is_registered fields can be mutated (by DownloadAndRegisterEps under the same - // mutex). The lock is released when this function returns, so the snapshot may - // be stale by the time the caller reads individual fields — that is documented - // and acceptable. std::lock_guard lock(cache_mutex_); - return cached_eps_; + if (current_cached_eps_ == nullptr) { + static const std::vector empty; + return empty; + } + return *current_cached_eps_; } std::span EpDetector::GetDiscoverableEpsCApi() const { @@ -236,6 +243,7 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector cache_lock(cache_mutex_); cached_eps_[i].is_registered = true; + PublishEpSnapshotLocked(); PublishCApiSnapshotLocked(); if (tracker) { diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.h b/sdk_v2/cpp/src/ep_detection/ep_detector.h index eb0ac5239..673ac4812 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.h +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.h @@ -108,9 +108,12 @@ class EpDetector : public IEpDetector { // snapshot retained for the detector lifetime so previously returned spans are // never mutated or freed while callers may still read them. std::vector cached_eps_; + std::deque> cached_eps_snapshots_; + const std::vector* current_cached_eps_ = nullptr; std::deque> cached_eps_c_snapshots_; const std::vector* current_cached_eps_c_ = nullptr; + void PublishEpSnapshotLocked(); void PublishCApiSnapshotLocked(); }; diff --git a/sdk_v2/cpp/test/internal_api/download_test.cc b/sdk_v2/cpp/test/internal_api/download_test.cc index 96ce4f543..21102ebe4 100644 --- a/sdk_v2/cpp/test/internal_api/download_test.cc +++ b/sdk_v2/cpp/test/internal_api/download_test.cc @@ -490,6 +490,7 @@ TEST(BlobDownloadTest, FiltersOutInferenceModelJson) { }; BlobDownloadOptions opts; + opts.skip_completed_files = true; DownloadBlobsToDirectory(mock, "https://test.blob/c?sig=x", tmpdir.string(), opts); ASSERT_EQ(mock.downloaded_blobs.size(), 2u); @@ -553,6 +554,7 @@ TEST(BlobDownloadTest, SkipsExistingFilesWithCorrectSize) { }; BlobDownloadOptions opts; + opts.skip_completed_files = true; DownloadBlobsToDirectory(mock, "https://test.blob/c?sig=x", tmpdir.string(), opts); // Only the missing blob should be downloaded. @@ -591,6 +593,7 @@ TEST(BlobDownloadTest, ReportsSkippedBytesInInitialProgress) { std::vector progress_values; BlobDownloadOptions opts; + opts.skip_completed_files = true; opts.progress = [&](float pct) { progress_values.push_back(pct); return 0; @@ -618,6 +621,7 @@ TEST(BlobDownloadTest, EmitsHundredPercentWhenEverythingIsCached) { std::vector progress_values; BlobDownloadOptions opts; + opts.skip_completed_files = true; opts.progress = [&](float pct) { progress_values.push_back(pct); return 0; diff --git a/sdk_v2/js/script/install-native.cjs b/sdk_v2/js/script/install-native.cjs index bbb372091..ae32ce0b4 100644 --- a/sdk_v2/js/script/install-native.cjs +++ b/sdk_v2/js/script/install-native.cjs @@ -30,7 +30,6 @@ const PLATFORM_MAP = { 'win32-x64': 'win-x64', 'win32-arm64': 'win-arm64', 'linux-x64': 'linux-x64', - 'linux-arm64': 'linux-arm64', 'darwin-arm64': 'osx-arm64', }; const platformKey = `${os.platform()}-${os.arch()}`; From e447b9c0f67f4b26fb6f6a816407fd9711b8a060 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 19:44:37 -0500 Subject: [PATCH 45/77] Close shutdown and wrapper review gaps Make singleton shutdown callback safe, preserve ICatalog source compatibility, propagate wrapper API-version errors through formerly noexcept accessors, and preserve forced catalog refresh after all-source failures. Files changed: - sdk_v2/cpp/include/foundry_local/foundry_local_cpp{,.inline}.h - sdk_v2/cpp/src/catalog/azure_model_catalog.cc - sdk_v2/cpp/src/manager.{cc,h} - sdk_v2/cpp/src/service/web_service.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .../include/foundry_local/foundry_local_cpp.h | 16 +++++++++++----- .../foundry_local/foundry_local_cpp.inline.h | 8 ++++---- sdk_v2/cpp/src/catalog/azure_model_catalog.cc | 6 ++++++ sdk_v2/cpp/src/manager.cc | 9 ++++++++- sdk_v2/cpp/src/manager.h | 3 +++ sdk_v2/cpp/src/service/web_service.cc | 7 ++++++- 6 files changed, 38 insertions(+), 11 deletions(-) diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h index 70224e219..19c4cef31 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h @@ -246,7 +246,7 @@ class KeyValuePairs { KeyValuePairs& operator=(KeyValuePairs&&) noexcept = default; /// Get a value by key. Returns nullopt if not found. - std::optional Get(const char* key) const noexcept; + std::optional Get(const char* key) const; /// Get all key/value pairs. std::vector GetAll() const; @@ -551,7 +551,7 @@ class Item { // --- Read accessors (always work) --- - flItemType GetType() const noexcept; + flItemType GetType() const; BytesContent GetBytes() const; TextContent GetText() const; TensorContent GetTensor() const; @@ -817,7 +817,13 @@ class ICatalog { /// Queries for different aliases do not invalidate each other's results. virtual ModelList GetModelVersions(const std::string& model_alias, const std::string& variant_name = {}, - int max_versions = 50) = 0; + int max_versions = 50) { + (void)model_alias; + (void)variant_name; + (void)max_versions; + throw Error("GetModelVersions is not implemented by this catalog", + FOUNDRY_LOCAL_ERROR_NOT_IMPLEMENTED); + } }; // =========================================================================== @@ -983,7 +989,7 @@ class Request { Request& AddItem(Item&& item) { return AddItem(item, true); } - size_t GetItemCount() const noexcept; + size_t GetItemCount() const; Item GetItem(size_t idx) const; /// Options for this request. Overrides session options for the duration of this request. @@ -1005,7 +1011,7 @@ class Response { /// Lifetime is tied to this Response object. const std::vector& GetItems() const noexcept { return items_; } - flFinishReason GetFinishReason() const noexcept; + flFinishReason GetFinishReason() const; flUsage GetUsage() const; private: diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h index 159247937..3fc4d9a66 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h @@ -91,7 +91,7 @@ inline KeyValuePairs::KeyValuePairs( } } -inline std::optional KeyValuePairs::Get(const char* key) const noexcept { +inline std::optional KeyValuePairs::Get(const char* key) const { if (!handle_.get()) { return std::nullopt; } @@ -634,7 +634,7 @@ inline Item::Item(flItem& raw) inline Item::Item(flItemType type) : handle_(detail::CreateItem(type), detail::item_api()->Item_Release) {} -inline flItemType Item::GetType() const noexcept { +inline flItemType Item::GetType() const { return detail::item_api()->GetType(handle_.get()); } @@ -1076,7 +1076,7 @@ inline Request& Request::AddItem(Item& item, bool take_ownership) { return *this; } -inline size_t Request::GetItemCount() const noexcept { +inline size_t Request::GetItemCount() const { return detail::inference_api()->Request_GetItemCount(handle_.get()); } @@ -1118,7 +1118,7 @@ inline Response::Response(flResponse* response) } } -inline flFinishReason Response::GetFinishReason() const noexcept { +inline flFinishReason Response::GetFinishReason() const { return detail::inference_api()->Response_GetFinishReason(handle_.get()); } diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index 1f687c596..2135a6fe7 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -194,9 +194,11 @@ std::vector AzureModelCatalog::FetchModels() const { } }; + bool fetched_any_catalog = false; for (const auto& [url, filter] : catalog_urls_) { try { fetch_from(url, filter); + fetched_any_catalog = true; } catch (const std::exception& ex) { // One failing URL shouldn't block others — skip and continue. auto parsed = ParseCatalogUrl(url); @@ -206,6 +208,10 @@ std::vector AzureModelCatalog::FetchModels() const { } } + if (!fetched_any_catalog) { + FL_THROW(FOUNDRY_LOCAL_ERROR_NETWORK, "failed to fetch any configured catalog source"); + } + logger_.Log(LogLevel::Information, fmt::format("Populated model info for {} models.", models.size())); diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index f6d4fcf2a..97761fe91 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -482,6 +482,13 @@ void Manager::Destroy() { s_instance_.reset(); } +void Manager::RequestShutdown() { + std::lock_guard lock(s_mutex_); + if (s_instance_ != nullptr) { + s_instance_->Shutdown(); + } +} + ICatalog& Manager::GetCatalog() { return *catalog_; } @@ -501,7 +508,7 @@ void Manager::StartWebService() { #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE web_service_ = std::make_unique(*catalog_, *logger_, *config_.model_cache_dir, *model_load_manager_, *session_manager_, *telemetry_, - [this]() { Shutdown(); }); + []() { Manager::RequestShutdown(); }); auto endpoints = config_.web_service_endpoints; if (endpoints.empty()) { diff --git a/sdk_v2/cpp/src/manager.h b/sdk_v2/cpp/src/manager.h index 187b5b3ff..e42120e2f 100644 --- a/sdk_v2/cpp/src/manager.h +++ b/sdk_v2/cpp/src/manager.h @@ -45,6 +45,9 @@ class Manager { /// Destroy the singleton and release all resources. static void Destroy(); + /// Request shutdown on the singleton if it still exists. + static void RequestShutdown(); + /// Get the shared catalog interface for querying models. /// The catalog is owned by the manager and shared across all consumers /// (web service, C API, etc.) so model state (e.g. IsLoaded) is consistent. diff --git a/sdk_v2/cpp/src/service/web_service.cc b/sdk_v2/cpp/src/service/web_service.cc index 49e8f03df..0698a8a22 100644 --- a/sdk_v2/cpp/src/service/web_service.cc +++ b/sdk_v2/cpp/src/service/web_service.cc @@ -193,7 +193,12 @@ class ShutdownHandler : public HttpRequestHandler { std::shared_ptr handle(const std::shared_ptr&) override { nlohmann::json body = {{"status", "shutting_down"}}; - std::thread([fn = shutdown_fn_] { fn(); }).detach(); + std::thread([fn = shutdown_fn_] { + try { + fn(); + } catch (...) { + } + }).detach(); return JsonResponse(Status::CODE_200, body); } From 630e475228a2e6aa871d7396a6d8552a1aa7c673 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 20:52:26 -0500 Subject: [PATCH 46/77] Drop response session check-ins during shutdown Prevent completed streaming Responses requests from re-caching sessions after shutdown cancellation has begun, so model unload can drain cleanly. Files changed: - sdk_v2/cpp/src/inferencing/session/session_manager.cc - sdk_v2/cpp/test/internal_api/session_manager_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/inferencing/session/session_manager.cc | 6 ++++++ sdk_v2/cpp/test/internal_api/session_manager_test.cc | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/sdk_v2/cpp/src/inferencing/session/session_manager.cc b/sdk_v2/cpp/src/inferencing/session/session_manager.cc index a48e81bb6..ae0df19e3 100644 --- a/sdk_v2/cpp/src/inferencing/session/session_manager.cc +++ b/sdk_v2/cpp/src/inferencing/session/session_manager.cc @@ -108,6 +108,12 @@ void SessionManager::CheckIn(const std::string& key, std::unique_ptr lock(mutex_); + if (shutting_down_.load()) { + logger_.Log(LogLevel::Debug, + fmt::format("SessionManager: dropping check-in for '{}' during shutdown", key)); + return; + } + // Replace existing entry for this key (if any) auto existing = cache_.find(key); if (existing != cache_.end()) { diff --git a/sdk_v2/cpp/test/internal_api/session_manager_test.cc b/sdk_v2/cpp/test/internal_api/session_manager_test.cc index 9e485781a..cbd3e988b 100644 --- a/sdk_v2/cpp/test/internal_api/session_manager_test.cc +++ b/sdk_v2/cpp/test/internal_api/session_manager_test.cc @@ -145,6 +145,17 @@ TEST_F(SessionManagerTest, CheckInAndCheckOutRoundTrip) { EXPECT_EQ(mgr.CacheSize(), 0u); } +TEST(SessionManagerStandaloneTest, CheckInAfterCancelAllDropsSession) { + StderrLogger logger; + SessionManager mgr(logger); + mgr.CancelAll(); + + mgr.CheckIn("resp-1", nullptr); + + EXPECT_EQ(mgr.CacheSize(), 0u); + EXPECT_EQ(mgr.CheckOut("resp-1"), nullptr); +} + TEST_F(SessionManagerTest, CheckOutRemovesFromCache) { SessionManager mgr(GetLogger()); auto session = MakeSession(); From 8fd7c3e2801a241e6e5f10dfec5fde90607883a7 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 22:23:17 -0500 Subject: [PATCH 47/77] Harden telemetry redaction and refresh fallback Avoid re-emitting redacted home-user segments in retained path tails and keep serving the populated catalog when a forced refresh fails transiently. Files changed: - sdk_v2/cpp/src/telemetry/telemetry_redaction.h - sdk_v2/cpp/test/internal_api/telemetry_test.cc - sdk_v2/cpp/src/catalog/base_model_catalog.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/catalog/base_model_catalog.cc | 13 +++++++++- .../cpp/src/telemetry/telemetry_redaction.h | 26 ++++++++++++++++++- .../cpp/test/internal_api/telemetry_test.cc | 8 ++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.cc b/sdk_v2/cpp/src/catalog/base_model_catalog.cc index fb32005c3..fc2d776fe 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.cc @@ -220,7 +220,18 @@ void BaseModelCatalog::EnsurePopulated(bool allow_refresh) const { fmt::format("Catalog '{}' refreshing (cache expired).", name_)); } - auto variants = FetchModels(); + std::vector variants; + try { + variants = FetchModels(); + } catch (const std::exception& ex) { + if (populated_) { + logger_.Log(LogLevel::Warning, + fmt::format("Catalog '{}' refresh failed; keeping existing catalog: {}", name_, ex.what())); + next_refresh_at_ = std::chrono::steady_clock::now() + std::chrono::minutes(1); + return; + } + throw; + } PopulateModels(std::move(variants)); force_refresh_ = false; next_refresh_at_ = std::chrono::steady_clock::now() + kCacheDuration; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_redaction.h b/sdk_v2/cpp/src/telemetry/telemetry_redaction.h index cd429e11f..72f2fdf8e 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_redaction.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_redaction.h @@ -50,6 +50,7 @@ inline std::string RedactPathToken(std::string_view token) { } size_t safe_start = 0; + std::string redacted_user; const auto guard = [&](std::string_view marker, bool has_user) { for (size_t pos = normalized.find(marker); pos != std::string::npos; pos = normalized.find(marker, pos + 1)) { @@ -60,9 +61,16 @@ inline std::string RedactPathToken(std::string_view token) { (token[end] == '.' && (end + 1 == token.size() || is_sep(token[end + 1]))))) { ++end; } + const size_t user_start = end; while (end < token.size() && !is_sep(token[end])) { ++end; } + if (end > user_start && redacted_user.empty()) { + redacted_user.assign(token.data() + user_start, end - user_start); + for (char& c : redacted_user) { + c = static_cast(std::tolower(static_cast(c))); + } + } } else { --end; } @@ -94,7 +102,23 @@ inline std::string RedactPathToken(std::string_view token) { const size_t keep_from = (tail_start > safe_start) ? tail_start : safe_start; std::string out = "[path]"; if (keep_from < token.size()) { - out.append(token.data() + keep_from, token.size() - keep_from); + size_t pos = keep_from; + while (pos < token.size()) { + if (is_sep(token[pos])) { + out.push_back(token[pos++]); + continue; + } + const size_t segment_start = pos; + while (pos < token.size() && !is_sep(token[pos])) { + ++pos; + } + std::string segment(token.data() + segment_start, pos - segment_start); + std::string lowered = segment; + for (char& c : lowered) { + c = static_cast(std::tolower(static_cast(c))); + } + out += (!redacted_user.empty() && lowered == redacted_user) ? "[user]" : segment; + } } return out; } diff --git a/sdk_v2/cpp/test/internal_api/telemetry_test.cc b/sdk_v2/cpp/test/internal_api/telemetry_test.cc index 128d3722b..82fadcc1e 100644 --- a/sdk_v2/cpp/test/internal_api/telemetry_test.cc +++ b/sdk_v2/cpp/test/internal_api/telemetry_test.cc @@ -152,6 +152,14 @@ TEST(TelemetryRedactionTest, ScrubsHomePathsAndKeepsUsefulTail) { EXPECT_NE(scrubbed.find("[path]\\AppData\\file.bin"), std::string::npos); } +TEST(TelemetryRedactionTest, ScrubsRepeatedUserNameInRetainedTail) { + const auto scrubbed = ScrubTelemetryErrorMessage(R"(failed C:\Users\Alice\src\Alice\model.onnx)"); + + EXPECT_EQ(scrubbed.find("Alice"), std::string::npos); + EXPECT_EQ(scrubbed.find("alice"), std::string::npos); + EXPECT_NE(scrubbed.find("[path]\\[user]\\model.onnx"), std::string::npos); +} + TEST(TelemetryRedactionTest, ScrubsUrlQueryAndFragment) { const auto scrubbed = ScrubTelemetryErrorMessage( "GET https://custom.example.com/private/catalog?token=secret#fragment failed"); From d64200a5786086ec2e11b36ddf82fb13ea7733b0 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Fri, 10 Jul 2026 23:49:43 -0500 Subject: [PATCH 48/77] Make telemetry emission best-effort Guard telemetry tracker destructor emissions, exception telemetry, and 1DS event construction/logging so telemetry failures cannot terminate host operations. Files changed: - sdk_v2/cpp/src/telemetry/download_tracker.cc - sdk_v2/cpp/src/telemetry/ep_download_tracker.cc - sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc - sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/telemetry/download_tracker.cc | 12 ++++++-- .../cpp/src/telemetry/ep_download_tracker.cc | 12 ++++++-- sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc | 30 +++++++++++++++++++ .../src/telemetry/telemetry_action_tracker.cc | 14 ++++++--- 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/sdk_v2/cpp/src/telemetry/download_tracker.cc b/sdk_v2/cpp/src/telemetry/download_tracker.cc index 549bf0187..67322f23b 100644 --- a/sdk_v2/cpp/src/telemetry/download_tracker.cc +++ b/sdk_v2/cpp/src/telemetry/download_tracker.cc @@ -20,12 +20,18 @@ DownloadTracker::DownloadTracker(std::string model_id, DownloadTracker::~DownloadTracker() { // Emit the Download event regardless of outcome. The default status is // kFailure so abrupt exits (exceptions) are recorded as failures. - telemetry_.RecordDownload(info_); + try { + telemetry_.RecordDownload(info_); + } catch (...) { + } } void DownloadTracker::RecordException(const std::exception& exception) { - telemetry_.RecordException(Action::kModelFileDownload, exception, - InvocationContext{info_.user_agent, info_.correlation_id, /*indirect=*/false}); + try { + telemetry_.RecordException(Action::kModelFileDownload, exception, + InvocationContext{info_.user_agent, info_.correlation_id, /*indirect=*/false}); + } catch (...) { + } } } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc b/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc index f7a29ea81..ae82ce96f 100644 --- a/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc +++ b/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc @@ -31,7 +31,10 @@ EpDownloadTracker::~EpDownloadTracker() { // Mirror neutron-server: if the caller didn't reach Done() or // RecordRegisterComplete, assume the abrupt exit was an exception path and // record any unfinished stage as kFailure. - RecordEvent(ActionStatus::kFailure); + try { + RecordEvent(ActionStatus::kFailure); + } catch (...) { + } } void EpDownloadTracker::RecordInitialState(std::string ready_state) { @@ -62,8 +65,11 @@ void EpDownloadTracker::Done() { void EpDownloadTracker::RecordException(const std::exception& ex) { // The per-provider attempt happens as a consequence of the overall // DownloadAndRegisterEps call, so it is indirect and shares its correlation id. - telemetry_.RecordException(Action::kEpDownloadAndRegister, ex, - InvocationContext{user_agent_, correlation_id_, /*indirect=*/true}); + try { + telemetry_.RecordException(Action::kEpDownloadAndRegister, ex, + InvocationContext{user_agent_, correlation_id_, /*indirect=*/true}); + } catch (...) { + } } void EpDownloadTracker::RecordEvent(ActionStatus incomplete_stage_status) { diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc index 3a3993737..d380d8e56 100644 --- a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc @@ -124,6 +124,7 @@ OneDsTelemetry::~OneDsTelemetry() { void OneDsTelemetry::RecordAction(Action action, ActionStatus status, const InvocationContext& context, int64_t duration_ms) { + try { local_log_.RecordAction(action, status, context, duration_ms); if (!initialized_.load(std::memory_order_acquire)) { return; @@ -136,10 +137,13 @@ void OneDsTelemetry::RecordAction(Action action, ActionStatus status, const Invo ev.SetProperty("Direct", !context.indirect); ev.SetProperty("TimeMs", duration_ms); SafeLog(GetMatLogger(), ev); + } catch (...) { + } } void OneDsTelemetry::RecordException(Action action, const std::exception& exception, const InvocationContext& context) { + try { local_log_.RecordException(action, exception, context); if (!initialized_.load(std::memory_order_acquire)) { return; @@ -155,9 +159,12 @@ void OneDsTelemetry::RecordException(Action action, const std::exception& except ev.SetProperty("StackTrace", ""); ev.SetProperty("InnerStackTrace", ""); SafeLog(GetMatLogger(), ev); + } catch (...) { + } } void OneDsTelemetry::RecordModelUsage(const ModelUsageInfo& info) { + try { local_log_.RecordModelUsage(info); if (!initialized_.load(std::memory_order_acquire)) { return; @@ -178,10 +185,13 @@ void OneDsTelemetry::RecordModelUsage(const ModelUsageInfo& info) { ev.SetProperty("CpuTimeMs", info.cpu_time_ms); ev.SetProperty("GpuMemoryUsedMB", info.gpu_memory_used_mb); SafeLog(GetMatLogger(), ev); + } catch (...) { + } } void OneDsTelemetry::RecordModelId(Action action, const std::string& model_id, ActionStatus status, const InvocationContext& context) { + try { local_log_.RecordModelId(action, model_id, status, context); if (!initialized_.load(std::memory_order_acquire) || model_id.empty()) { return; @@ -193,9 +203,12 @@ void OneDsTelemetry::RecordModelId(Action action, const std::string& model_id, ev.SetProperty("UserAgent", context.user_agent); ev.SetProperty("CorrelationId", context.correlation_id); SafeLog(GetMatLogger(), ev); + } catch (...) { + } } void OneDsTelemetry::RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) { + try { local_log_.RecordEpDownloadAttempt(info); if (!initialized_.load(std::memory_order_acquire)) { return; @@ -211,9 +224,12 @@ void OneDsTelemetry::RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) ev.SetProperty("Status", std::string(ActionStatusToString(info.status))); ev.SetProperty("TimeMs", info.duration_ms); SafeLog(GetMatLogger(), ev); + } catch (...) { + } } void OneDsTelemetry::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) { + try { local_log_.RecordEpDownloadAndRegister(info); if (!initialized_.load(std::memory_order_acquire)) { return; @@ -230,9 +246,12 @@ void OneDsTelemetry::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo ev.SetProperty("RegisterStatus", std::string(ActionStatusToString(info.register_status))); ev.SetProperty("RegisterTimeMs", info.register_duration_ms); SafeLog(GetMatLogger(), ev); + } catch (...) { + } } void OneDsTelemetry::RecordDownload(const DownloadInfo& info) { + try { local_log_.RecordDownload(info); if (!initialized_.load(std::memory_order_acquire)) { return; @@ -252,9 +271,12 @@ void OneDsTelemetry::RecordDownload(const DownloadInfo& info) { ev.SetProperty("DownloadWaitResult", info.download_wait_result); ev.SetProperty("MaxConcurrency", static_cast(info.max_concurrency)); SafeLog(GetMatLogger(), ev); + } catch (...) { + } } void OneDsTelemetry::RecordCatalogFetch(const CatalogFetchInfo& info) { + try { local_log_.RecordCatalogFetch(info); if (!initialized_.load(std::memory_order_acquire)) { return; @@ -271,9 +293,12 @@ void OneDsTelemetry::RecordCatalogFetch(const CatalogFetchInfo& info) { ev.SetProperty("UserAgent", info.user_agent); ev.SetProperty("CorrelationId", info.correlation_id); SafeLog(GetMatLogger(), ev); + } catch (...) { + } } void OneDsTelemetry::StartSession() { + try { if (!initialized_.load(std::memory_order_acquire)) { return; } @@ -285,9 +310,12 @@ void OneDsTelemetry::StartSession() { // on subsequent events and records session duration on End. auto ev = MakeEvent("Session", metadata_.test_mode); mat_logger->LogSession(SessionState::Session_Started, ev); + } catch (...) { + } } void OneDsTelemetry::EndSession() { + try { if (!initialized_.load(std::memory_order_acquire)) { return; } @@ -297,6 +325,8 @@ void OneDsTelemetry::EndSession() { } auto ev = MakeEvent("Session", metadata_.test_mode); mat_logger->LogSession(SessionState::Session_Ended, ev); + } catch (...) { + } } } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc index 54bcd938a..50a960e35 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc @@ -20,10 +20,13 @@ ActionTracker::~ActionTracker() { auto end = std::chrono::steady_clock::now(); auto duration_ms = std::chrono::duration_cast(end - start_).count(); - telemetry_.RecordAction(action_, status_, context_, duration_ms); + try { + telemetry_.RecordAction(action_, status_, context_, duration_ms); - if (!model_id_.empty()) { - telemetry_.RecordModelId(action_, model_id_, status_, context_); + if (!model_id_.empty()) { + telemetry_.RecordModelId(action_, model_id_, status_, context_); + } + } catch (...) { } } @@ -32,7 +35,10 @@ void ActionTracker::SetStatus(ActionStatus status) { } void ActionTracker::RecordException(const std::exception& exception) { - telemetry_.RecordException(action_, exception, context_); + try { + telemetry_.RecordException(action_, exception, context_); + } catch (...) { + } } void ActionTracker::SetModelId(const std::string& model_id) { From ff1dc5f122aa4b91332640a58f5368055d1eac5f Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 11 Jul 2026 00:50:43 -0500 Subject: [PATCH 49/77] Validate blob identity during resume Persist blob identity in download sidecars and apply If-Match to ranged blob downloads so resumable downloads cannot mix chunks across same-sized blob replacements. Files changed: - sdk_v2/cpp/src/download/blob_download_state.* - sdk_v2/cpp/src/download/blob_downloader.* Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .../cpp/src/download/blob_download_state.cc | 53 ++++++++++++++++--- sdk_v2/cpp/src/download/blob_download_state.h | 12 ++++- sdk_v2/cpp/src/download/blob_downloader.cc | 28 ++++++++-- sdk_v2/cpp/src/download/blob_downloader.h | 2 + 4 files changed, 82 insertions(+), 13 deletions(-) diff --git a/sdk_v2/cpp/src/download/blob_download_state.cc b/sdk_v2/cpp/src/download/blob_download_state.cc index 07680bd73..9384e0781 100644 --- a/sdk_v2/cpp/src/download/blob_download_state.cc +++ b/sdk_v2/cpp/src/download/blob_download_state.cc @@ -21,7 +21,7 @@ constexpr const char* kStateFileExtension = ".dlstate"; // bytes | field // -------|-------------------------------------------------------- // 0..3 | magic "FLDS" -// 4 | version (currently 1) +// 4 | version (currently 2) // 5..12 | blob_size (int64) // 13..16 | chunk_size (int32) // 17..20 | total_chunks (int32) @@ -29,12 +29,14 @@ constexpr const char* kStateFileExtension = ".dlstate"; // 25..28 | highest_completed_chunk (int32) // 29..32 | completed_count (int32) // 33..40 | last_modified_unix_ms (int64) -// 41..44 | trunc_bitmap_byte_len (uint32) -// 45.. | trunc_bitmap_byte_len bytes of bitmap data, copied directly out of +// 41..44 | blob_identity_len (uint32) +// 45.. | blob_identity bytes +// ... | trunc_bitmap_byte_len (uint32) +// ... | trunc_bitmap_byte_len bytes of bitmap data, copied directly out of // full_completion_bitmap starting at the byte offset implied by // bitmap_byte_aligned_start. constexpr char kMagic[4] = {'F', 'L', 'D', 'S'}; -constexpr uint8_t kVersion = 1; +constexpr uint8_t kVersion = 2; constexpr int32_t kBitsPerWord = 64; @@ -54,6 +56,25 @@ bool ReadNative(std::istream& in, T& out_value) { return static_cast(in); } +void WriteString(std::ostream& out, const std::string& value) { + WriteNative(out, static_cast(value.size())); + if (!value.empty()) { + out.write(value.data(), static_cast(value.size())); + } +} + +bool ReadString(std::istream& in, std::string& value) { + uint32_t size = 0; + if (!ReadNative(in, size)) { + return false; + } + value.resize(size); + if (size > 0) { + in.read(value.data(), size); + } + return static_cast(in); +} + int64_t NowUnixMs() { return std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) @@ -72,13 +93,15 @@ std::unique_ptr BlobDownloadState::CreateNew(std::string blob const std::filesystem::path& local_file_path, int64_t blob_size, int32_t chunk_size, - int32_t total_chunks) { + int32_t total_chunks, + std::string blob_identity) { auto state = std::make_unique(); state->blob_name = std::move(blob_name); state->local_file_path = local_file_path.string(); state->blob_size = blob_size; state->chunk_size = chunk_size; state->total_chunks = total_chunks; + state->blob_identity = std::move(blob_identity); state->bitmap_byte_aligned_start = 0; state->highest_completed_chunk = -1; state->completed_count = 0; @@ -93,6 +116,7 @@ std::unique_ptr BlobDownloadState::LoadState(std::string blob int64_t expected_blob_size, int32_t expected_chunk_size, int32_t expected_total_chunks, + std::string_view expected_blob_identity, ILogger& logger) { auto state_path = GetStateFilePath(local_file_path); std::error_code ec; @@ -122,17 +146,20 @@ std::unique_ptr BlobDownloadState::LoadState(std::string blob int32_t highest_completed_chunk = 0; int32_t completed_count = 0; int64_t last_modified_unix_ms = 0; + std::string blob_identity; uint32_t trunc_len = 0; if (!ReadNative(in, blob_size) || !ReadNative(in, chunk_size) || !ReadNative(in, total_chunks) || !ReadNative(in, bitmap_byte_aligned_start) || !ReadNative(in, highest_completed_chunk) || - !ReadNative(in, completed_count) || !ReadNative(in, last_modified_unix_ms) || !ReadNative(in, trunc_len)) { + !ReadNative(in, completed_count) || !ReadNative(in, last_modified_unix_ms) || + !ReadString(in, blob_identity) || !ReadNative(in, trunc_len)) { logger.Log(LogLevel::Warning, "Download state header truncated: " + state_path.string()); return nullptr; } // Sanity / compatibility checks. if (blob_size != expected_blob_size || chunk_size != expected_chunk_size || - total_chunks != expected_total_chunks) { + total_chunks != expected_total_chunks || + (!expected_blob_identity.empty() && blob_identity != expected_blob_identity)) { logger.Log(LogLevel::Information, "Download state for " + state_path.string() + " is incompatible with current blob layout; starting fresh"); @@ -191,6 +218,7 @@ std::unique_ptr BlobDownloadState::LoadState(std::string blob state->highest_completed_chunk = highest_completed_chunk; state->completed_count = completed_count; state->last_modified_unix_ms = last_modified_unix_ms; + state->blob_identity = std::move(blob_identity); state->full_completion_bitmap = std::move(bitmap); logger.Log(LogLevel::Information, @@ -200,6 +228,16 @@ std::unique_ptr BlobDownloadState::LoadState(std::string blob return state; } +std::unique_ptr BlobDownloadState::LoadState(std::string blob_name, + const std::filesystem::path& local_file_path, + int64_t expected_blob_size, + int32_t expected_chunk_size, + int32_t expected_total_chunks, + ILogger& logger) { + return LoadState(std::move(blob_name), local_file_path, expected_blob_size, expected_chunk_size, + expected_total_chunks, {}, logger); +} + int64_t BlobDownloadState::CalculateDownloadedSize() const noexcept { int64_t bytes = static_cast(completed_count) * chunk_size; // If the final chunk is partial and was completed, adjust the overcount. @@ -321,6 +359,7 @@ bool BlobDownloadState::SaveState(ILogger& logger) { WriteNative(out, highest_completed_chunk); WriteNative(out, completed_count); WriteNative(out, last_modified_unix_ms); + WriteString(out, blob_identity); WriteNative(out, trunc_len); if (trunc_len > 0) { auto* src = reinterpret_cast(full_completion_bitmap.data()) + byte_offset; diff --git a/sdk_v2/cpp/src/download/blob_download_state.h b/sdk_v2/cpp/src/download/blob_download_state.h index 54f114340..acd43be99 100644 --- a/sdk_v2/cpp/src/download/blob_download_state.h +++ b/sdk_v2/cpp/src/download/blob_download_state.h @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace fl { @@ -35,6 +36,7 @@ class BlobDownloadState { int64_t blob_size = 0; int32_t chunk_size = 0; int32_t total_chunks = 0; + std::string blob_identity; /// Serialization marker (always a multiple of 8): chunks below this index are /// complete and dropped from the sidecar's truncated bitmap. The in-memory @@ -64,7 +66,8 @@ class BlobDownloadState { const std::filesystem::path& local_file_path, int64_t blob_size, int32_t chunk_size, - int32_t total_chunks); + int32_t total_chunks, + std::string blob_identity = {}); /// Load existing state from `.dlstate`. Returns nullptr if /// the file does not exist, is corrupted, or has incompatible @@ -73,6 +76,13 @@ class BlobDownloadState { /// and the partial download is no longer valid). /// `logger` receives diagnostics for corrupt/incompatible state files. Required: the /// downloader always has a logger, so there is no optional/null case to handle. + static std::unique_ptr LoadState(std::string blob_name, + const std::filesystem::path& local_file_path, + int64_t expected_blob_size, + int32_t expected_chunk_size, + int32_t expected_total_chunks, + std::string_view expected_blob_identity, + ILogger& logger); static std::unique_ptr LoadState(std::string blob_name, const std::filesystem::path& local_file_path, int64_t expected_blob_size, diff --git a/sdk_v2/cpp/src/download/blob_downloader.cc b/sdk_v2/cpp/src/download/blob_downloader.cc index 976db3316..14b3ab58a 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.cc +++ b/sdk_v2/cpp/src/download/blob_downloader.cc @@ -49,6 +49,7 @@ constexpr size_t kStreamingBufferBytes = 64 * 1024; struct AzureBlobDownloader::ChunkContext { const Azure::Storage::Blobs::BlobClient& blob_client; const Azure::Core::Context& azure_ctx; + std::string blob_identity; }; AzureBlobDownloader::AzureBlobDownloader(ILogger& logger) : logger_(logger) {} @@ -63,6 +64,11 @@ std::vector AzureBlobDownloader::ListBlobs(const std::string& sas_ BlobItemInfo info; info.name = blob.Name; info.content_length = blob.BlobSize; + if (blob.Details.ETag.HasValue()) { + info.blob_identity = blob.Details.ETag.ToString(); + } else if (blob.VersionId.HasValue()) { + info.blob_identity = blob.VersionId.Value(); + } items.push_back(std::move(info)); } } @@ -79,6 +85,11 @@ int64_t AzureBlobDownloader::GetBlobSize(ChunkContext& ctx) { return props.BlobSize; } +std::string AzureBlobDownloader::GetBlobIdentity(ChunkContext& ctx) { + auto props = ctx.blob_client.GetProperties({}, ctx.azure_ctx).Value; + return props.ETag.HasValue() ? props.ETag.ToString() : std::string{}; +} + bool AzureBlobDownloader::IsCancellationRequested(const ChunkContext& ctx) const { return ctx.azure_ctx.IsCancelled(); } @@ -88,6 +99,9 @@ void AzureBlobDownloader::DownloadChunkStreaming( const std::function& sink) { Azure::Storage::Blobs::DownloadBlobOptions range_opts; range_opts.Range = Azure::Core::Http::HttpRange{offset, size}; + if (!ctx.blob_identity.empty()) { + range_opts.AccessConditions.IfMatch = Azure::ETag(ctx.blob_identity); + } auto result = ctx.blob_client.Download(range_opts, ctx.azure_ctx); auto& body_stream = *result.Value.BodyStream; @@ -155,9 +169,10 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, // or by external cancellation; checked by workers between iterations. std::atomic internal_cancel{false}; - ChunkContext chunk_ctx{blob_client, azure_ctx}; + ChunkContext chunk_ctx{blob_client, azure_ctx, ""}; int64_t blob_size = GetBlobSize(chunk_ctx); + chunk_ctx.blob_identity = GetBlobIdentity(chunk_ctx); if (blob_size == 0) { EnsureEmptyBlobFile(local_path); @@ -172,7 +187,7 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, // Resume from existing sidecar if it matches the current blob layout. auto state = BlobDownloadState::LoadState(blob_name, local_path, blob_size, static_cast(kChunkSize), - num_chunks, logger_); + num_chunks, chunk_ctx.blob_identity, logger_); if (state) { // Only trust the sidecar if the data file it describes is actually on disk // at full size. If the data file was truncated or removed (e.g. an external @@ -192,7 +207,8 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, if (!state) { state = BlobDownloadState::CreateNew(blob_name, local_path, blob_size, - static_cast(kChunkSize), num_chunks); + static_cast(kChunkSize), num_chunks, + chunk_ctx.blob_identity); // Persist the sidecar now, before Open() pre-allocates the data file. // IsDownloadNeeded treats "data file at full size + no sidecar" as a // completed download and skips it. The periodic save below does not run @@ -226,7 +242,8 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, std::filesystem::remove(local_path, remove_ec); BlobDownloadState::DeleteState(local_path, logger_); state = BlobDownloadState::CreateNew(blob_name, local_path, blob_size, - static_cast(kChunkSize), num_chunks); + static_cast(kChunkSize), num_chunks, + chunk_ctx.blob_identity); if (!state->SaveState(logger_)) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "failed to persist reset download state for '" + local_path + "'"); @@ -402,7 +419,8 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, "failed to truncate blob file after close failure: " + local_path + " (" + resize_ec.message() + ")"); auto fresh_state = BlobDownloadState::CreateNew(blob_name, local_path, blob_size, - static_cast(kChunkSize), num_chunks); + static_cast(kChunkSize), num_chunks, + chunk_ctx.blob_identity); if (!fresh_state->SaveState(logger_)) { logger_.Log(LogLevel::Error, "failed to reset download state after close failure for '" + local_path + "'"); diff --git a/sdk_v2/cpp/src/download/blob_downloader.h b/sdk_v2/cpp/src/download/blob_downloader.h index a38097d08..28a5a8cc4 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.h +++ b/sdk_v2/cpp/src/download/blob_downloader.h @@ -40,6 +40,7 @@ struct BlobDownloadOptions { struct BlobItemInfo { std::string name; int64_t content_length = 0; + std::string blob_identity; }; /// Interface for blob download operations. Enables testing via mock implementations. @@ -96,6 +97,7 @@ class AzureBlobDownloader : public IBlobDownloader { /// Return the blob size in bytes. Production calls `BlobClient::GetProperties`. virtual int64_t GetBlobSize(ChunkContext& ctx); + virtual std::string GetBlobIdentity(ChunkContext& ctx); /// Read `size` bytes starting at `offset` from the blob and forward them /// piecewise to `sink`. Pulls from the blob client referenced by `ctx`. From 63a78156916c85efef8422bcbc78894299eee7f6 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 11 Jul 2026 02:27:34 -0500 Subject: [PATCH 50/77] Treat cancelled response streams as failures Abort cancelled streaming Responses before emitting/storing completed results or caching the session. Files changed: - sdk_v2/cpp/src/service/responses_handler.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/service/responses_handler.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index 988f9739c..c20ae9c88 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -6,6 +6,7 @@ #include "c_api_types.h" #include "catalog.h" +#include "exception.h" #include "inferencing/generative/chat/chat_session.h" #include "inferencing/generative/openresponses/response_converter.h" #include "inferencing/generative/openresponses/response_store.h" @@ -552,6 +553,9 @@ std::shared_ptr ResponsesHandler::HandleSt session->SetStreamingCallback(callback_fn); session->ProcessRequest(*req, bg_response); + if (req->canceled.load(std::memory_order_relaxed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "response stream cancelled"); + } // Close whatever item is still open at end-of-generation so the SSE stream is well-formed. close_current(); From 224f2f45ee7a7a62a3fd0e8d70a79bc6540cc29f Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 11 Jul 2026 04:01:38 -0500 Subject: [PATCH 51/77] Treat cancelled chat audio streams as failures Mirror Responses streaming cancellation handling for chat/audio streams and cap blob identity allocations when reading corrupt resume sidecars. Files changed: - sdk_v2/cpp/src/service/chat_completions_handler.cc - sdk_v2/cpp/src/service/audio_transcriptions_handler.cc - sdk_v2/cpp/src/download/blob_download_state.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/download/blob_download_state.cc | 4 ++++ sdk_v2/cpp/src/service/audio_transcriptions_handler.cc | 4 ++++ sdk_v2/cpp/src/service/chat_completions_handler.cc | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/sdk_v2/cpp/src/download/blob_download_state.cc b/sdk_v2/cpp/src/download/blob_download_state.cc index 9384e0781..b76ff7180 100644 --- a/sdk_v2/cpp/src/download/blob_download_state.cc +++ b/sdk_v2/cpp/src/download/blob_download_state.cc @@ -39,6 +39,7 @@ constexpr char kMagic[4] = {'F', 'L', 'D', 'S'}; constexpr uint8_t kVersion = 2; constexpr int32_t kBitsPerWord = 64; +constexpr uint32_t kMaxBlobIdentityLength = 4096; // Serialize a scalar field in host byte order. Every target we build for // (x64 / arm64) is little-endian, so the on-disk layout is little-endian in @@ -68,6 +69,9 @@ bool ReadString(std::istream& in, std::string& value) { if (!ReadNative(in, size)) { return false; } + if (size > kMaxBlobIdentityLength) { + return false; + } value.resize(size); if (size > 0) { in.read(value.data(), size); diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc index ab21c5bd1..c5d845c42 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc @@ -7,6 +7,7 @@ #include "c_api_types.h" #include "catalog.h" #include "contracts/audio_transcriptions.h" +#include "exception.h" #include "inferencing/generative/audio/audio_session.h" #include "inferencing/model_load_manager.h" #include "inferencing/session/session_manager.h" @@ -233,6 +234,9 @@ std::shared_ptr AudioTranscriptionsHandler bg_session.SetStreamingCallback(callback_fn); bg_session.ProcessRequest(*req, bg_response); + if (req->canceled.load(std::memory_order_relaxed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "audio transcription stream cancelled"); + } // Send terminal event body_ptr->Push("data: [DONE]\n\n"); diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.cc b/sdk_v2/cpp/src/service/chat_completions_handler.cc index 605c55f2a..cbeedcd35 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.cc +++ b/sdk_v2/cpp/src/service/chat_completions_handler.cc @@ -7,6 +7,7 @@ #include "c_api_types.h" #include "catalog.h" #include "contracts/chat_completions.h" +#include "exception.h" #include "inferencing/generative/chat/chat_session.h" #include "inferencing/model_load_manager.h" #include "inferencing/session/session.h" @@ -247,6 +248,9 @@ std::shared_ptr ChatCompletionsHandler::Ha bg_session.SetStreamingCallback(callback_fn); bg_session.ProcessRequest(*req, bg_response); + if (req->canceled.load(std::memory_order_relaxed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "chat completion stream cancelled"); + } // Usage chunk — only if stream_options.include_usage was true if (include_usage) { From 69093b530ec1ee783328ef5a98d4c2898a77a3c0 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sat, 11 Jul 2026 05:04:02 -0500 Subject: [PATCH 52/77] Sanitize catalog names and JS exit cleanup Use sanitized catalog display names in refresh logs and avoid disposing the native manager from Node's synchronous exit event where native workers may still be unwinding. Files changed: - sdk_v2/cpp/src/catalog/azure_model_catalog.cc - sdk_v2/cpp/src/catalog/base_model_catalog.cc - sdk_v2/js/src/foundryLocalManager.ts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/catalog/azure_model_catalog.cc | 3 ++- sdk_v2/cpp/src/catalog/base_model_catalog.cc | 4 +++- sdk_v2/js/src/foundryLocalManager.ts | 12 ++++-------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index 2135a6fe7..cc4342dac 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -94,7 +94,8 @@ AzureModelCatalog::AzureModelCatalog(std::vector #include "exception.h" +#include "telemetry/telemetry_redaction.h" #include #include @@ -226,7 +227,8 @@ void BaseModelCatalog::EnsurePopulated(bool allow_refresh) const { } catch (const std::exception& ex) { if (populated_) { logger_.Log(LogLevel::Warning, - fmt::format("Catalog '{}' refresh failed; keeping existing catalog: {}", name_, ex.what())); + fmt::format("Catalog '{}' refresh failed; keeping existing catalog: {}", + name_, ScrubTelemetryErrorMessage(ex.what()))); next_refresh_at_ = std::chrono::steady_clock::now() + std::chrono::minutes(1); return; } diff --git a/sdk_v2/js/src/foundryLocalManager.ts b/sdk_v2/js/src/foundryLocalManager.ts index 9cf9946cf..2d2772a47 100644 --- a/sdk_v2/js/src/foundryLocalManager.ts +++ b/sdk_v2/js/src/foundryLocalManager.ts @@ -14,11 +14,10 @@ import { import type { EpDownloadResult, EpInfo } from "./types.js"; // A native Manager holds process-global native resources. Dispose the live -// Manager on process exit so native teardown happens at a deterministic point -// rather than being left entirely to environment finalizers. `beforeExit` -// covers a natural event-loop drain; `exit` covers callers who invoke -// `process.exit()` themselves. Both are idempotent. The native layer permits -// only one live Manager at a time, so a single reference is enough. +// Manager when the event loop naturally drains so native teardown happens at a +// deterministic point rather than being left entirely to environment finalizers. +// Do not dispose during the synchronous `exit` event: native worker threads may +// still be unwinding, and freeing the process-global manager there can race them. let liveManager: FoundryLocalManager | undefined; let exitHandlersInstalled = false; @@ -42,9 +41,6 @@ function installExitHandlersOnce(): void { process.on("beforeExit", () => { disposeLiveManager(); }); - process.on("exit", () => { - disposeLiveManager(); - }); } export class FoundryLocalManager { From 10744b3ad6b5d83928c1117796569b0720661cdb Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 01:36:45 -0500 Subject: [PATCH 53/77] Close pre-PR telemetry review findings Fetch blob size and identity atomically so resume state and ranged reads agree on the same remote version, validate final model cache path segments, and make web-service shutdown stop safely around startup. Files changed: - sdk_v2/cpp/src/download/blob_downloader.cc - sdk_v2/cpp/src/download/blob_downloader.h - sdk_v2/cpp/src/download/download_manager.cc - sdk_v2/cpp/src/manager.cc - sdk_v2/cpp/src/manager.h - sdk_v2/cpp/src/service/web_service.cc - sdk_v2/cpp/test/internal_api/download_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/download/blob_downloader.cc | 19 +-- sdk_v2/cpp/src/download/blob_downloader.h | 13 +- sdk_v2/cpp/src/download/download_manager.cc | 2 +- sdk_v2/cpp/src/manager.cc | 96 +++++++++----- sdk_v2/cpp/src/manager.h | 1 + sdk_v2/cpp/src/service/web_service.cc | 124 +++++++++--------- sdk_v2/cpp/test/internal_api/download_test.cc | 18 ++- 7 files changed, 162 insertions(+), 111 deletions(-) diff --git a/sdk_v2/cpp/src/download/blob_downloader.cc b/sdk_v2/cpp/src/download/blob_downloader.cc index 14b3ab58a..efd09200f 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.cc +++ b/sdk_v2/cpp/src/download/blob_downloader.cc @@ -80,14 +80,14 @@ std::vector AzureBlobDownloader::ListBlobs(const std::string& sas_ } } -int64_t AzureBlobDownloader::GetBlobSize(ChunkContext& ctx) { +AzureBlobDownloader::BlobProperties AzureBlobDownloader::GetBlobProperties(ChunkContext& ctx) { auto props = ctx.blob_client.GetProperties({}, ctx.azure_ctx).Value; - return props.BlobSize; -} - -std::string AzureBlobDownloader::GetBlobIdentity(ChunkContext& ctx) { - auto props = ctx.blob_client.GetProperties({}, ctx.azure_ctx).Value; - return props.ETag.HasValue() ? props.ETag.ToString() : std::string{}; + BlobProperties result; + result.content_length = props.BlobSize; + if (props.ETag.HasValue()) { + result.blob_identity = props.ETag.ToString(); + } + return result; } bool AzureBlobDownloader::IsCancellationRequested(const ChunkContext& ctx) const { @@ -171,8 +171,9 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, ChunkContext chunk_ctx{blob_client, azure_ctx, ""}; - int64_t blob_size = GetBlobSize(chunk_ctx); - chunk_ctx.blob_identity = GetBlobIdentity(chunk_ctx); + auto blob_properties = GetBlobProperties(chunk_ctx); + int64_t blob_size = blob_properties.content_length; + chunk_ctx.blob_identity = std::move(blob_properties.blob_identity); if (blob_size == 0) { EnsureEmptyBlobFile(local_path); diff --git a/sdk_v2/cpp/src/download/blob_downloader.h b/sdk_v2/cpp/src/download/blob_downloader.h index 28a5a8cc4..3e3c671a7 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.h +++ b/sdk_v2/cpp/src/download/blob_downloader.h @@ -40,7 +40,7 @@ struct BlobDownloadOptions { struct BlobItemInfo { std::string name; int64_t content_length = 0; - std::string blob_identity; + std::string blob_identity{}; }; /// Interface for blob download operations. Enables testing via mock implementations. @@ -95,9 +95,14 @@ class AzureBlobDownloader : public IBlobDownloader { /// SDK BlobClient + Context pointers used by the production virtuals. struct ChunkContext; - /// Return the blob size in bytes. Production calls `BlobClient::GetProperties`. - virtual int64_t GetBlobSize(ChunkContext& ctx); - virtual std::string GetBlobIdentity(ChunkContext& ctx); + struct BlobProperties { + int64_t content_length = 0; + std::string blob_identity{}; + }; + + /// Return the blob size and identity from one properties read so chunk planning + /// and conditional range reads refer to the same remote blob version. + virtual BlobProperties GetBlobProperties(ChunkContext& ctx); /// Read `size` bytes starting at `offset` from the blob and forward them /// piecewise to `sink`. Pulls from the blob client referenced by `ctx`. diff --git a/sdk_v2/cpp/src/download/download_manager.cc b/sdk_v2/cpp/src/download/download_manager.cc index 670b42ff8..862c00c81 100644 --- a/sdk_v2/cpp/src/download/download_manager.cc +++ b/sdk_v2/cpp/src/download/download_manager.cc @@ -222,7 +222,7 @@ std::string DownloadManager::ComputeModelPath(const ModelInfo& info) const { auto version = std::string_view(info.model_id).substr(last_colon + 1); SanitizeForPathSegment(bare_id); SanitizeForPathSegment(version); - sanitized_model_dir = FixVersionSuffix(info.model_id); + sanitized_model_dir = SanitizeForPathSegment(FixVersionSuffix(info.model_id)); } std::filesystem::path full_path(cache_directory_); diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index 97761fe91..d61031b26 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -494,38 +494,63 @@ ICatalog& Manager::GetCatalog() { } void Manager::StartWebService() { - if (web_service_running_) { - FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "web service is already running"); - } - - if (config_.external_service_url.has_value()) { - FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, - "cannot start local web service when external_service_url is configured"); - } - ActionTracker tracker(Action::kCoreServiceStart, *telemetry_); #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE - web_service_ = std::make_unique(*catalog_, *logger_, *config_.model_cache_dir, *model_load_manager_, - *session_manager_, *telemetry_, - []() { Manager::RequestShutdown(); }); + bool stop_after_start = false; + + { + std::lock_guard lock(web_service_mutex_); + + if (web_service_ != nullptr) { + FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "web service is already running"); + } + + if (config_.external_service_url.has_value()) { + FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "cannot start local web service when external_service_url is configured"); + } + + web_service_ = std::make_unique(*catalog_, *logger_, *config_.model_cache_dir, *model_load_manager_, + *session_manager_, *telemetry_, + []() { Manager::RequestShutdown(); }); - auto endpoints = config_.web_service_endpoints; - if (endpoints.empty()) { - endpoints.push_back("http://127.0.0.1:0"); + auto endpoints = config_.web_service_endpoints; + if (endpoints.empty()) { + endpoints.push_back("http://127.0.0.1:0"); + } + + try { + bound_urls_ = web_service_->Start(endpoints); + web_service_running_ = true; + stop_after_start = shutdown_requested_.load(std::memory_order_acquire); + + if (!stop_after_start) { + // Open an app-usage session for the lifetime of the running service so events + // carry ext.app.sesId and the backend gets session duration. + try { + telemetry_->StartSession(); + } catch (const std::exception& ex) { + logger_->Log(LogLevel::Warning, std::string("telemetry StartSession failed: ") + ex.what()); + } catch (...) { + logger_->Log(LogLevel::Warning, "telemetry StartSession failed with unknown error"); + } + } + } catch (...) { + if (web_service_) { + web_service_->Stop(); + } + web_service_.reset(); + web_service_running_ = false; + bound_urls_.clear(); + throw; + } } - bound_urls_ = web_service_->Start(endpoints); - web_service_running_ = true; - // Open an app-usage session for the lifetime of the running service so events - // carry ext.app.sesId and the backend gets session duration. - try { - telemetry_->StartSession(); - } catch (const std::exception& ex) { - logger_->Log(LogLevel::Warning, std::string("telemetry StartSession failed: ") + ex.what()); - } catch (...) { - logger_->Log(LogLevel::Warning, "telemetry StartSession failed with unknown error"); + if (stop_after_start) { + StopWebService(); } + tracker.SetStatus(ActionStatus::kSuccess); #else FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -541,21 +566,28 @@ const std::vector& Manager::GetWebServiceUrls() const { } void Manager::StopWebService() { - if (!web_service_running_) { +#ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE + std::lock_guard lock(web_service_mutex_); + + if (web_service_ == nullptr) { // No-op rather than throw: the public-API contract treats StopWebService() as idempotent so // callers can shut down unconditionally without first probing service state. logger_->Log(LogLevel::Information, "StopWebService called but web service is not running; ignoring"); return; } - ActionTracker tracker(Action::kCoreServiceStop, *telemetry_); -#ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE web_service_->Stop(); web_service_.reset(); web_service_running_ = false; bound_urls_.clear(); - telemetry_->EndSession(); + try { + telemetry_->EndSession(); + } catch (const std::exception& ex) { + logger_->Log(LogLevel::Warning, std::string("telemetry EndSession failed: ") + ex.what()); + } catch (...) { + logger_->Log(LogLevel::Warning, "telemetry EndSession failed with unknown error"); + } tracker.SetStatus(ActionStatus::kSuccess); #else FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -574,9 +606,9 @@ void Manager::Shutdown() { model_load_manager_->RejectNewLoads(); session_manager_->CancelAll(); - if (web_service_running_) { - StopWebService(); - } +#ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE + StopWebService(); +#endif // Order matters: // 1. Reject new loads so callers gated on IsShutdownRequested can stop early. diff --git a/sdk_v2/cpp/src/manager.h b/sdk_v2/cpp/src/manager.h index e42120e2f..9eccfdd58 100644 --- a/sdk_v2/cpp/src/manager.h +++ b/sdk_v2/cpp/src/manager.h @@ -149,6 +149,7 @@ class Manager { std::unique_ptr session_manager_; std::atomic shutdown_requested_{false}; std::atomic web_service_running_{false}; + std::mutex web_service_mutex_; std::vector bound_urls_; #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE diff --git a/sdk_v2/cpp/src/service/web_service.cc b/sdk_v2/cpp/src/service/web_service.cc index 0698a8a22..e743ee411 100644 --- a/sdk_v2/cpp/src/service/web_service.cc +++ b/sdk_v2/cpp/src/service/web_service.cc @@ -314,11 +314,13 @@ WebService::~WebService() { std::vector WebService::Start(const std::vector& endpoints) { auto& ctx = *impl_->context; - if (impl_->running.load()) { + if (impl_->running.exchange(true)) { ctx.logger.Log(LogLevel::Information, "Web service is already running."); FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Web service is already running"); } + impl_->thread_tracker.Reset(); + // Create shared router and register all routes impl_->router = oatpp::web::server::HttpRouter::createShared(); @@ -357,92 +359,88 @@ std::vector WebService::Start(const std::vector& endpo std::vector bound_urls; try { - for (const auto& endpoint : endpoints) { - // Parse "http://host:port" — strip scheme, extract host:port - std::string host = "127.0.0.1"; - uint16_t port = 0; - - std::string addr = endpoint; - auto scheme_end = addr.find("://"); - if (scheme_end != std::string::npos) { - addr = addr.substr(scheme_end + 3); - } + for (const auto& endpoint : endpoints) { + // Parse "http://host:port" — strip scheme, extract host:port + std::string host = "127.0.0.1"; + uint16_t port = 0; + + std::string addr = endpoint; + auto scheme_end = addr.find("://"); + if (scheme_end != std::string::npos) { + addr = addr.substr(scheme_end + 3); + } - if (!addr.empty() && addr.back() == '/') { - addr.pop_back(); - } + if (!addr.empty() && addr.back() == '/') { + addr.pop_back(); + } - auto colon = addr.rfind(':'); - if (colon != std::string::npos) { - host = addr.substr(0, colon); - port = static_cast(std::stoi(addr.substr(colon + 1))); - } else { - host = addr; - } + auto colon = addr.rfind(':'); + if (colon != std::string::npos) { + host = addr.substr(0, colon); + port = static_cast(std::stoi(addr.substr(colon + 1))); + } else { + host = addr; + } - auto tcp_provider = oatpp::network::tcp::server::ConnectionProvider::createShared({host.c_str(), port}); + auto tcp_provider = oatpp::network::tcp::server::ConnectionProvider::createShared({host.c_str(), port}); - // oatpp resolves ephemeral port 0 during construction via getsockname() - // and stores it in PROPERTY_PORT. getAddress().port is stale — use the property. - auto resolved_port = tcp_provider->getProperty(oatpp::network::ConnectionProvider::PROPERTY_PORT); + // oatpp resolves ephemeral port 0 during construction via getsockname() + // and stores it in PROPERTY_PORT. getAddress().port is stale — use the property. + auto resolved_port = tcp_provider->getProperty(oatpp::network::ConnectionProvider::PROPERTY_PORT); - if (resolved_port) { - port = static_cast(std::stoi(resolved_port.std_str())); - } + if (resolved_port) { + port = static_cast(std::stoi(resolved_port.std_str())); + } - // Wrap with our force-close provider so that connection invalidation also unblocks any pending recv() in worker - // threads (via CancelIoEx on Windows; shutdown() suffices on POSIX). Without this, HttpConnectionHandler::stop() - // can wait ~120s for a single keep-alive client to drop its connection. - auto provider = std::static_pointer_cast( - std::make_shared(tcp_provider)); + // Wrap with our force-close provider so that connection invalidation also unblocks any pending recv() in worker + // threads (via CancelIoEx on Windows; shutdown() suffices on POSIX). Without this, HttpConnectionHandler::stop() + // can wait ~120s for a single keep-alive client to drop its connection. + auto provider = std::static_pointer_cast( + std::make_shared(tcp_provider)); - auto server = std::make_shared(provider, impl_->connection_handler); + auto server = std::make_shared(provider, impl_->connection_handler); - impl_->providers.push_back(provider); - impl_->servers.push_back(server); + impl_->providers.push_back(provider); + impl_->servers.push_back(server); - // Start server on a background thread - impl_->listener_threads.emplace_back([server]() { - server->run(); - }); + // Start server on a background thread + impl_->listener_threads.emplace_back([server]() { + server->run(); + }); - // Wait until oatpp's server reports STATUS_RUNNING (i.e. the listener loop - // has started and is accepting connections). Bounded poll with a 5s timeout. - const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - while (server->getStatus() != oatpp::network::Server::STATUS_RUNNING && - std::chrono::steady_clock::now() < deadline) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } + // Wait until oatpp's server reports STATUS_RUNNING (i.e. the listener loop + // has started and is accepting connections). Bounded poll with a 5s timeout. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (server->getStatus() != oatpp::network::Server::STATUS_RUNNING && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } - if (server->getStatus() != oatpp::network::Server::STATUS_RUNNING) { - // Tear down anything we already started in this call so we don't leak - // listener threads. `running` is still false, so the destructor would skip Stop(). - impl_->running.store(true); - Stop(); - FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, - "Web service failed to reach RUNNING state for endpoint ", endpoint, " within 5s"); - } + if (server->getStatus() != oatpp::network::Server::STATUS_RUNNING) { + // Tear down anything we already started in this call so we don't leak listener threads. + Stop(); + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "Web service failed to reach RUNNING state for endpoint ", endpoint, " within 5s"); + } - std::string bound_url = "http://" + host + ":" + std::to_string(port); - bound_urls.push_back(bound_url); + std::string bound_url = "http://" + host + ":" + std::to_string(port); + bound_urls.push_back(bound_url); - ctx.logger.Log(LogLevel::Information, fmt::format("Web service listening on {}", bound_url)); - } + ctx.logger.Log(LogLevel::Information, fmt::format("Web service listening on {}", bound_url)); + } } catch (...) { - impl_->running.store(true); Stop(); throw; } ctx.bound_urls = bound_urls; - impl_->running.store(true); - impl_->thread_tracker.Reset(); return bound_urls; } void WebService::Stop() { - if (!impl_->running.load()) { + if (!impl_->running.load() && impl_->servers.empty() && impl_->providers.empty() && + impl_->listener_threads.empty() && !impl_->connection_handler) { return; } diff --git a/sdk_v2/cpp/test/internal_api/download_test.cc b/sdk_v2/cpp/test/internal_api/download_test.cc index 21102ebe4..fe4dc17c2 100644 --- a/sdk_v2/cpp/test/internal_api/download_test.cc +++ b/sdk_v2/cpp/test/internal_api/download_test.cc @@ -1528,6 +1528,17 @@ TEST(DownloadManagerTest, RejectsColonInBareModelId) { EXPECT_THROW(manager.GetModelCachePath(info), fl::Exception); } +TEST(DownloadManagerTest, RejectsNonNumericVersionSuffix) { + TempDir tmpdir; + DownloadManager manager(tmpdir.string(), "eastus", 64, fl::test::NullLog()); + + ModelInfo info; + info.model_id = "test:preview"; + info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR] = "Publisher"; + + EXPECT_THROW(manager.GetModelCachePath(info), fl::Exception); +} + TEST(DownloadManagerTest, RejectsTrailingDotInPublisher) { TempDir tmpdir; DownloadManager manager(tmpdir.string(), "eastus", 64, fl::test::NullLog()); @@ -1566,7 +1577,7 @@ TEST(DownloadManagerTest, AcceptsNormalModelIdAndPublisher) { // ======================================================================== // AzureBlobDownloader resume + cancel-cascade tests -// Use a subclass that overrides the protected GetBlobSize / DownloadChunkStreaming +// Use a subclass that overrides the protected GetBlobProperties / DownloadChunkStreaming // virtuals to bypass the real Azure SDK and simulate per-chunk behavior. // ======================================================================== @@ -1577,6 +1588,7 @@ namespace { class FakeChunkAzureDownloader : public AzureBlobDownloader { public: int64_t blob_size = 0; + std::string blob_identity; /// Per-call hook. Receives the chunk offset and size plus a `sink` callback /// that forwards bytes to the file writer. Allowed to: @@ -1601,7 +1613,9 @@ class FakeChunkAzureDownloader : public AzureBlobDownloader { FakeChunkAzureDownloader() : AzureBlobDownloader(fl::test::NullLog()) {} protected: - int64_t GetBlobSize(ChunkContext& /*ctx*/) override { return blob_size; } + BlobProperties GetBlobProperties(ChunkContext& /*ctx*/) override { + return BlobProperties{blob_size, blob_identity}; + } void DownloadChunkStreaming(ChunkContext& ctx, int64_t offset, int64_t size, std::vector& scratch, From 97839adcf9a684ab82d75aaa2d88056cb183563c Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 02:48:33 -0500 Subject: [PATCH 54/77] Harden endpoint snapshots and binding lifetimes Return web-service endpoint snapshots safely, keep JS native manager state alive across in-flight async workers and derived handles, and use ORT_LIB_PATH instead of mutating macOS Python dependency packages. Files changed: - sdk_v2/cpp/src/c_api.cc - sdk_v2/cpp/src/manager.cc - sdk_v2/cpp/src/manager.h - sdk_v2/js/native/src/catalog.cc - sdk_v2/js/native/src/catalog.h - sdk_v2/js/native/src/manager.cc - sdk_v2/js/native/src/manager.h - sdk_v2/js/native/src/model.cc - sdk_v2/js/native/src/model.h - sdk_v2/js/native/src/session.cc - sdk_v2/js/native/src/session.h - sdk_v2/js/test/manager-dispose.test.ts - sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/c_api.cc | 34 ++++++++++++++----- sdk_v2/cpp/src/manager.cc | 3 +- sdk_v2/cpp/src/manager.h | 4 +-- sdk_v2/js/native/src/catalog.cc | 24 ++++++++----- sdk_v2/js/native/src/catalog.h | 3 ++ sdk_v2/js/native/src/manager.cc | 12 ++++--- sdk_v2/js/native/src/manager.h | 4 +-- sdk_v2/js/native/src/model.cc | 33 ++++++++++++------ sdk_v2/js/native/src/model.h | 3 ++ sdk_v2/js/native/src/session.cc | 3 ++ sdk_v2/js/native/src/session.h | 3 ++ sdk_v2/js/test/manager-dispose.test.ts | 8 +++++ .../foundry_local_sdk/_native/lib_loader.py | 21 +++--------- 13 files changed, 101 insertions(+), 54 deletions(-) diff --git a/sdk_v2/cpp/src/c_api.cc b/sdk_v2/cpp/src/c_api.cc index 5cc689cdd..750a5d3b3 100644 --- a/sdk_v2/cpp/src/c_api.cc +++ b/sdk_v2/cpp/src/c_api.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -70,9 +71,17 @@ struct flCatalog { // --- Manager --- struct flManager { + explicit flManager(fl::Manager& manager) : impl(manager) {} + + struct UrlSnapshot { + std::vector strings; + std::vector pointers; + }; + fl::Manager& impl; std::unique_ptr catalog; // stores the flCatalog wrapper around impl.GetCatalog() - mutable std::vector urls_cache; + mutable std::mutex urls_cache_mutex; + mutable std::vector> urls_snapshots; }; // ======================================================================== @@ -327,7 +336,7 @@ FL_API_STATUS_IMPL(Manager_CreateImpl, const flConfiguration* config, flManager* } auto& mgr = fl::Manager::Create(*cfg); - auto wrapper = std::make_unique(flManager{mgr, nullptr, {}}); + auto wrapper = std::make_unique(mgr); wrapper->catalog = std::make_unique(flCatalog{mgr.GetCatalog()}); *out_manager = wrapper.release(); return nullptr; @@ -372,15 +381,22 @@ FL_API_STATUS_IMPL(Manager_WebServiceUrlsImpl, const flManager* manager, return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null argument"); } - const auto& urls = manager->impl.GetWebServiceUrls(); - manager->urls_cache.clear(); - manager->urls_cache.reserve(urls.size()); - for (const auto& u : urls) { - manager->urls_cache.push_back(u.c_str()); + auto urls = manager->impl.GetWebServiceUrls(); + auto snapshot = std::make_unique(); + snapshot->strings = std::move(urls); + snapshot->pointers.reserve(snapshot->strings.size()); + for (const auto& u : snapshot->strings) { + snapshot->pointers.push_back(u.c_str()); + } + + auto* snapshot_ptr = snapshot.get(); + { + std::lock_guard lock(manager->urls_cache_mutex); + manager->urls_snapshots.push_back(std::move(snapshot)); } - *out_urls = manager->urls_cache.data(); - *out_num_urls = manager->urls_cache.size(); + *out_urls = snapshot_ptr->pointers.empty() ? nullptr : snapshot_ptr->pointers.data(); + *out_num_urls = snapshot_ptr->pointers.size(); return nullptr; API_IMPL_END } diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index d61031b26..7f22ca451 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -558,10 +558,11 @@ void Manager::StartWebService() { #endif } -const std::vector& Manager::GetWebServiceUrls() const { +std::vector Manager::GetWebServiceUrls() const { // No "not running" check: bound_urls_ is cleared in StopWebService() and is empty before // StartWebService(), so the empty vector is the documented "service is not running" signal // (see GetWebServiceEndpoints() docstring in foundry_local_cpp.h). + std::lock_guard lock(web_service_mutex_); return bound_urls_; } diff --git a/sdk_v2/cpp/src/manager.h b/sdk_v2/cpp/src/manager.h index 9eccfdd58..9b71a9410 100644 --- a/sdk_v2/cpp/src/manager.h +++ b/sdk_v2/cpp/src/manager.h @@ -84,7 +84,7 @@ class Manager { /// Get the bound service URLs. The returned reference is valid as long as /// the web service is running. Throws if web service is not running. - const std::vector& GetWebServiceUrls() const; + std::vector GetWebServiceUrls() const; /// Stop the embedded web service. void StopWebService(); @@ -149,7 +149,7 @@ class Manager { std::unique_ptr session_manager_; std::atomic shutdown_requested_{false}; std::atomic web_service_running_{false}; - std::mutex web_service_mutex_; + mutable std::mutex web_service_mutex_; std::vector bound_urls_; #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE diff --git a/sdk_v2/js/native/src/catalog.cc b/sdk_v2/js/native/src/catalog.cc index 8372187a2..53c37ef90 100644 --- a/sdk_v2/js/native/src/catalog.cc +++ b/sdk_v2/js/native/src/catalog.cc @@ -18,8 +18,8 @@ namespace { // Wrap a ModelList (rvalue) into a JS array of Model handles, each pinning the // passed-in manager reference. -Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, - Napi::ObjectReference manager) { +Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, Napi::ObjectReference manager, + std::shared_ptr manager_keepalive) { auto list = std::make_shared(std::move(ml)); auto models = list->Models(); Napi::Array arr = Napi::Array::New(env, models.size()); @@ -28,6 +28,7 @@ Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, token.impl = models[i].get(); token.keepalive = list; token.manager = Napi::Reference::New(manager.Value(), 1); + token.manager_keepalive = manager_keepalive; arr.Set(static_cast(i), Model::NewInstance(env, std::move(token))); } return arr; @@ -35,7 +36,8 @@ Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, // Wrap an owning unique_ptr into a JS Model (or undefined when null). Napi::Value WrapOwnedModelOrUndefined(Napi::Env env, std::unique_ptr owned, - Napi::ObjectReference manager) { + Napi::ObjectReference manager, + std::shared_ptr manager_keepalive) { if (!owned) { return env.Undefined(); } @@ -46,6 +48,7 @@ Napi::Value WrapOwnedModelOrUndefined(Napi::Env env, std::unique_ptr>(std::move(owned)); token.keepalive = holder; token.manager = std::move(manager); + token.manager_keepalive = std::move(manager_keepalive); return Model::NewInstance(env, std::move(token)); } @@ -109,6 +112,7 @@ Catalog::Catalog(const Napi::CallbackInfo& info) : Napi::ObjectWrap(inf } impl_ = token->impl; manager_ = std::move(token->manager); + manager_keepalive_ = std::move(token->manager_keepalive); } Napi::Value Catalog::GetName(const Napi::CallbackInfo& info) { @@ -125,14 +129,16 @@ Napi::Value Catalog::GetModels(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked( - env, [&]() -> Napi::Value { return WrapModelList(env, impl_->GetModels(), std::move(mgr)); }); + env, [&]() -> Napi::Value { + return WrapModelList(env, impl_->GetModels(), std::move(mgr), manager_keepalive_); + }); } Napi::Value Catalog::GetCachedModels(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { - return WrapModelList(env, impl_->GetCachedModels(), std::move(mgr)); + return WrapModelList(env, impl_->GetCachedModels(), std::move(mgr), manager_keepalive_); }); } @@ -140,7 +146,7 @@ Napi::Value Catalog::GetLoadedModels(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { - return WrapModelList(env, impl_->GetLoadedModels(), std::move(mgr)); + return WrapModelList(env, impl_->GetLoadedModels(), std::move(mgr), manager_keepalive_); }); } @@ -156,7 +162,7 @@ Napi::Value Catalog::GetModel(const Napi::CallbackInfo& info) { Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { auto owned = impl_->GetModel(alias); - return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr)); + return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_); }); } @@ -170,7 +176,7 @@ Napi::Value Catalog::GetModelVariant(const Napi::CallbackInfo& info) { Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { auto owned = impl_->GetModelVariant(model_id); - return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr)); + return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_); }); } @@ -188,7 +194,7 @@ Napi::Value Catalog::GetLatestVersion(const Napi::CallbackInfo& info) { Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { auto owned = impl_->GetLatestVersion(*arg); - return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr)); + return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_); }); } diff --git a/sdk_v2/js/native/src/catalog.h b/sdk_v2/js/native/src/catalog.h index 29ff89cd9..fd24c7c39 100644 --- a/sdk_v2/js/native/src/catalog.h +++ b/sdk_v2/js/native/src/catalog.h @@ -16,6 +16,7 @@ #include +#include #include namespace foundry_local_node { @@ -23,6 +24,7 @@ namespace foundry_local_node { struct CatalogCtorToken { foundry_local::ICatalog* impl = nullptr; Napi::ObjectReference manager; // pins the owning Manager + std::shared_ptr manager_keepalive; }; class Catalog : public Napi::ObjectWrap { @@ -43,6 +45,7 @@ class Catalog : public Napi::ObjectWrap { foundry_local::ICatalog* impl_ = nullptr; Napi::ObjectReference manager_; + std::shared_ptr manager_keepalive_; }; } // namespace foundry_local_node diff --git a/sdk_v2/js/native/src/manager.cc b/sdk_v2/js/native/src/manager.cc index 6d359b588..3b11e6d25 100644 --- a/sdk_v2/js/native/src/manager.cc +++ b/sdk_v2/js/native/src/manager.cc @@ -193,7 +193,7 @@ Manager::Manager(const Napi::CallbackInfo& info) : Napi::ObjectWrap(inf } config.SetAdditionalOptions(kvp); } - impl_ = std::make_unique(std::move(config)); + impl_ = std::make_shared(std::move(config)); }); } @@ -227,6 +227,7 @@ Napi::Value Manager::GetCatalog(const Napi::CallbackInfo& info) { CatalogCtorToken token; token.impl = &cat; token.manager = std::move(owner); + token.manager_keepalive = impl_; return Catalog::NewInstance(env, std::move(token)); }); } @@ -288,8 +289,9 @@ namespace { // progress callback. Mirrors the pattern in model.cc's DownloadWorker. class EpDownloadWorker : public Napi::AsyncWorker { public: - EpDownloadWorker(Napi::Env env, foundry_local::Manager* impl, std::vector ep_names, - Napi::ObjectReference owner, Napi::ThreadSafeFunction tsfn) + EpDownloadWorker(Napi::Env env, std::shared_ptr impl, + std::vector ep_names, Napi::ObjectReference owner, + Napi::ThreadSafeFunction tsfn) : Napi::AsyncWorker(env), deferred_(Napi::Promise::Deferred::New(env)), impl_(impl), @@ -356,7 +358,7 @@ class EpDownloadWorker : public Napi::AsyncWorker { } Napi::Promise::Deferred deferred_; - foundry_local::Manager* impl_; + std::shared_ptr impl_; std::vector ep_names_; Napi::ObjectReference owner_; Napi::ThreadSafeFunction tsfn_; @@ -413,7 +415,7 @@ Napi::Value Manager::DownloadAndRegisterEps(const Napi::CallbackInfo& info) { } Napi::ObjectReference owner = Napi::Reference::New(info.This().As(), 1); - auto* w = new EpDownloadWorker(env, impl_.get(), std::move(ep_names), std::move(owner), std::move(tsfn)); + auto* w = new EpDownloadWorker(env, impl_, std::move(ep_names), std::move(owner), std::move(tsfn)); Napi::Promise p = w->Promise(); w->Queue(); return p; diff --git a/sdk_v2/js/native/src/manager.h b/sdk_v2/js/native/src/manager.h index 0cdd35207..8251b6568 100644 --- a/sdk_v2/js/native/src/manager.h +++ b/sdk_v2/js/native/src/manager.h @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // -// Napi::ObjectWrap over std::unique_ptr. +// Napi::ObjectWrap over std::shared_ptr. // // Surface: // - ctor accepts { appName, modelCacheDir?, serviceEndpoint? } @@ -55,7 +55,7 @@ class Manager : public Napi::ObjectWrap { // on env and returns true. Callers should return env.Undefined() when true. bool ThrowIfDisposed(Napi::Env env); - std::unique_ptr impl_; + std::shared_ptr impl_; }; } // namespace foundry_local_node diff --git a/sdk_v2/js/native/src/model.cc b/sdk_v2/js/native/src/model.cc index c7164778f..185a0e72c 100644 --- a/sdk_v2/js/native/src/model.cc +++ b/sdk_v2/js/native/src/model.cc @@ -144,7 +144,8 @@ Napi::Object SnapshotModelInfo(Napi::Env env, const foundry_local::ModelInfo& in // Drain a ModelList into a JS array, with each entry wrapped as a JS Model // whose keepalive holds the shared ModelList. Napi::Array WrapModelList(Napi::Env env, std::shared_ptr list, - Napi::ObjectReference manager) { + Napi::ObjectReference manager, + std::shared_ptr manager_keepalive) { auto models = list->Models(); Napi::Array arr = Napi::Array::New(env, models.size()); for (size_t i = 0; i < models.size(); ++i) { @@ -153,6 +154,7 @@ Napi::Array WrapModelList(Napi::Env env, std::shared_ptr::New(manager.Value(), 1); + token.manager_keepalive = manager_keepalive; arr.Set(static_cast(i), Model::NewInstance(env, std::move(token))); } return arr; @@ -198,6 +200,7 @@ Model::Model(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) { } impl_ = token->impl; keepalive_ = std::move(token->keepalive); + manager_keepalive_ = std::move(token->manager_keepalive); manager_ = std::move(token->manager); } @@ -238,17 +241,16 @@ Napi::Value Model::GetVariants(const Napi::CallbackInfo& info) { Napi::ObjectReference owner_clone = Napi::Reference::New(manager_.Value(), 1); return CallChecked(env, [&]() -> Napi::Value { auto list = std::make_shared(impl_->GetVariants()); - return WrapModelList(env, std::move(list), std::move(owner_clone)); + return WrapModelList(env, std::move(list), std::move(owner_clone), manager_keepalive_); }); } // ── Async lifecycle ───────────────────────────────────────────────────────── // // Load/Unload/Download dispatch the underlying virtual call onto a libuv -// worker so the event loop stays responsive. The Model itself is pinned -// against GC for the duration of the worker via an ObjectReference to the -// parent Manager (the Manager owns the catalog whose ModelList views the -// IModel*). +// worker so the event loop stays responsive. Each worker captures the model's +// native keepalives so explicit Manager.dispose() or JS GC cannot release the +// underlying Manager/ModelList/owned IModel before the worker finishes. Napi::Value Model::Load(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); @@ -258,8 +260,11 @@ Napi::Value Model::Load(const Napi::CallbackInfo& info) { } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); foundry_local::IModel* m = impl_; + auto keepalive = keepalive_; + auto manager_keepalive = manager_keepalive_; return PromiseWorkerVoid::Run( - env, [m]() { m->Load(); }, std::move(owner)); + env, [m, keepalive = std::move(keepalive), manager_keepalive = std::move(manager_keepalive)]() { m->Load(); }, + std::move(owner)); } Napi::Value Model::Unload(const Napi::CallbackInfo& info) { @@ -270,8 +275,11 @@ Napi::Value Model::Unload(const Napi::CallbackInfo& info) { } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); foundry_local::IModel* m = impl_; + auto keepalive = keepalive_; + auto manager_keepalive = manager_keepalive_; return PromiseWorkerVoid::Run( - env, [m]() { m->Unload(); }, std::move(owner)); + env, [m, keepalive = std::move(keepalive), manager_keepalive = std::move(manager_keepalive)]() { m->Unload(); }, + std::move(owner)); } namespace { @@ -282,11 +290,14 @@ namespace { // worker queues and released in OnOK/OnError. class DownloadWorker : public Napi::AsyncWorker { public: - DownloadWorker(Napi::Env env, foundry_local::IModel* impl, Napi::ObjectReference owner, + DownloadWorker(Napi::Env env, foundry_local::IModel* impl, std::shared_ptr keepalive, + std::shared_ptr manager_keepalive, Napi::ObjectReference owner, Napi::ThreadSafeFunction tsfn) : Napi::AsyncWorker(env), deferred_(Napi::Promise::Deferred::New(env)), impl_(impl), + keepalive_(std::move(keepalive)), + manager_keepalive_(std::move(manager_keepalive)), owner_(std::move(owner)), tsfn_(std::move(tsfn)) {} @@ -350,6 +361,8 @@ class DownloadWorker : public Napi::AsyncWorker { Napi::Promise::Deferred deferred_; foundry_local::IModel* impl_; + std::shared_ptr keepalive_; + std::shared_ptr manager_keepalive_; Napi::ObjectReference owner_; Napi::ThreadSafeFunction tsfn_; std::string err_msg_; @@ -379,7 +392,7 @@ Napi::Value Model::Download(const Napi::CallbackInfo& info) { } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - auto* w = new DownloadWorker(env, impl_, std::move(owner), std::move(tsfn)); + auto* w = new DownloadWorker(env, impl_, keepalive_, manager_keepalive_, std::move(owner), std::move(tsfn)); Napi::Promise p = w->Promise(); w->Queue(); return p; diff --git a/sdk_v2/js/native/src/model.h b/sdk_v2/js/native/src/model.h index 3ae4b6e63..42107a07a 100644 --- a/sdk_v2/js/native/src/model.h +++ b/sdk_v2/js/native/src/model.h @@ -39,6 +39,7 @@ struct ModelCtorToken { // or a std::shared_ptr) alive for the JS Model's // lifetime. std::shared_ptr keepalive; + std::shared_ptr manager_keepalive; // Pins the parent Manager so its native handle (and the Catalog's flCatalog* // which the IModel views into) cannot be released first. Napi::ObjectReference manager; @@ -62,6 +63,7 @@ class Model : public Napi::ObjectWrap { // Internal accessor used by Session / ChatSession ctors so they can clone // the parent Manager ObjectReference and pin it for the session lifetime. const Napi::ObjectReference& manager() const noexcept { return manager_; } + const std::shared_ptr& manager_keepalive() const noexcept { return manager_keepalive_; } private: Napi::Value GetInfo(const Napi::CallbackInfo& info); @@ -78,6 +80,7 @@ class Model : public Napi::ObjectWrap { foundry_local::IModel* impl_ = nullptr; std::shared_ptr keepalive_; + std::shared_ptr manager_keepalive_; Napi::ObjectReference manager_; }; diff --git a/sdk_v2/js/native/src/session.cc b/sdk_v2/js/native/src/session.cc index 0d1247463..b7aabd2f0 100644 --- a/sdk_v2/js/native/src/session.cc +++ b/sdk_v2/js/native/src/session.cc @@ -331,6 +331,7 @@ ChatSession::ChatSession(const Napi::CallbackInfo& info) : Napi::ObjectWrap::New(model->manager().Value(), 1); + manager_keepalive_ = model->manager_keepalive(); } bool ChatSession::ThrowIfDisposed(Napi::Env env) { @@ -488,6 +489,7 @@ EmbeddingsSession::EmbeddingsSession(const Napi::CallbackInfo& info) return; } manager_ = Napi::Reference::New(model->manager().Value(), 1); + manager_keepalive_ = model->manager_keepalive(); } bool EmbeddingsSession::ThrowIfDisposed(Napi::Env env) { @@ -582,6 +584,7 @@ AudioSession::AudioSession(const Napi::CallbackInfo& info) return; } manager_ = Napi::Reference::New(model->manager().Value(), 1); + manager_keepalive_ = model->manager_keepalive(); } bool AudioSession::ThrowIfDisposed(Napi::Env env) { diff --git a/sdk_v2/js/native/src/session.h b/sdk_v2/js/native/src/session.h index 2b1db7b86..e361ee197 100644 --- a/sdk_v2/js/native/src/session.h +++ b/sdk_v2/js/native/src/session.h @@ -53,6 +53,7 @@ class ChatSession : public Napi::ObjectWrap { std::unique_ptr impl_; Napi::ObjectReference manager_; + std::shared_ptr manager_keepalive_; }; // Napi::ObjectWrap over foundry_local::EmbeddingsSession. @@ -84,6 +85,7 @@ class EmbeddingsSession : public Napi::ObjectWrap { std::unique_ptr impl_; Napi::ObjectReference manager_; + std::shared_ptr manager_keepalive_; }; // Napi::ObjectWrap over foundry_local::AudioSession. @@ -113,6 +115,7 @@ class AudioSession : public Napi::ObjectWrap { std::unique_ptr impl_; Napi::ObjectReference manager_; + std::shared_ptr manager_keepalive_; }; } // namespace foundry_local_node diff --git a/sdk_v2/js/test/manager-dispose.test.ts b/sdk_v2/js/test/manager-dispose.test.ts index fe7c7ba19..d6c9a22af 100644 --- a/sdk_v2/js/test/manager-dispose.test.ts +++ b/sdk_v2/js/test/manager-dispose.test.ts @@ -70,6 +70,14 @@ describeIfBuilt("FoundryLocalManager.dispose", () => { expect(mgr.disposed).toBe(true); }); + it("dispose() does not invalidate an in-flight native EP worker", async () => { + const mgr = freshManager("async-worker"); + const pending = mgr.downloadAndRegisterEps(["__not-a-provider__"]); + mgr.dispose(); + await expect(pending).resolves.toMatchObject({ success: true }); + expect(mgr.disposed).toBe(true); + }); + it("`using` declaration disposes at scope exit", () => { let captured: FoundryLocalManager | undefined; { diff --git a/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py b/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py index ab23a8642..57d8b8868 100644 --- a/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py +++ b/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py @@ -279,22 +279,11 @@ def prepare_native_dependencies(foundry_local_dir: pathlib.Path) -> list: logger.warning("Failed to preload ORT (%s): %s", ort_path, exc) return handles - # macOS only: GenAI's static initializer does its own dlopen("libonnxruntime.dylib"), - # which on Darwin only matches by leafname against dyld search paths or images - # whose install_name leaf is exactly "libonnxruntime.dylib". ORT's dylib has a - # versioned install_name instead (the soversion, e.g. "@rpath/libonnxruntime.1.dylib"), - # so neither match path succeeds and GenAI aborts in dyld init before any of our code - # runs. The second name GenAI tries is "/libonnxruntime.dylib", so a symlink - # there fixes it. Linux dlopen consults the loaded-soname table by leafname and finds - # our already-RTLD_GLOBAL'd image without help. - if sys.platform == "darwin": - symlink_path = genai_path.parent / "libonnxruntime.dylib" - try: - if not symlink_path.exists(): - symlink_path.symlink_to(ort_path) - logger.info("Created macOS GenAI->ORT symlink: %s -> %s", symlink_path, ort_path) - except OSError as exc: - logger.warning("Failed to create macOS ORT symlink at %s: %s", symlink_path, exc) + # GenAI's lazy InitApi() first checks ORT_LIB_PATH before falling back to a + # platform leafname. Point it at the exact ORT dylib/so we resolved instead + # of mutating the installed GenAI package to add an unversioned alias. + if "ORT_LIB_PATH" not in os.environ: + os.environ["ORT_LIB_PATH"] = str(ort_path) try: handles.append(ctypes.CDLL(str(genai_path), mode=ctypes.RTLD_GLOBAL)) From 055cba32db543a53cff7a79031888344f8095c97 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 03:51:47 -0500 Subject: [PATCH 55/77] Fix native lifetime races Lease loaded models before session construction, keep JS session workers alive while async operations run, invalidate stale catalog/model handles after manager disposal, clear C API URL snapshots on stop, and report cancelled requests as cancelled telemetry. Files changed: - sdk_v2/cpp/src/c_api.cc - sdk_v2/cpp/src/inferencing/generative/audio/audio_session.* - sdk_v2/cpp/src/inferencing/generative/chat/chat_session.* - sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.* - sdk_v2/cpp/src/inferencing/model_load_manager.* - sdk_v2/cpp/src/inferencing/session/session.cc - sdk_v2/cpp/src/service/*handler.* - sdk_v2/cpp/test/internal_api/model_load_manager_test.cc - sdk_v2/js/native/src/{catalog,model,session}.* - sdk_v2/js/src/catalog.ts - sdk_v2/js/test/manager-dispose.test.ts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/c_api.cc | 8 ++- .../generative/audio/audio_session.cc | 6 +- .../generative/audio/audio_session.h | 3 +- .../generative/chat/chat_session.cc | 7 +- .../generative/chat/chat_session.h | 3 +- .../embeddings/embeddings_session.cc | 6 +- .../embeddings/embeddings_session.h | 2 +- .../cpp/src/inferencing/model_load_manager.cc | 38 +++++++++++ .../cpp/src/inferencing/model_load_manager.h | 31 +++++++++ sdk_v2/cpp/src/inferencing/session/session.cc | 14 ++-- .../service/audio_transcriptions_handler.cc | 9 +-- .../service/audio_transcriptions_handler.h | 4 +- .../src/service/chat_completions_handler.cc | 9 +-- .../src/service/chat_completions_handler.h | 4 +- sdk_v2/cpp/src/service/embeddings_handler.cc | 5 +- sdk_v2/cpp/src/service/responses_handler.cc | 9 +-- sdk_v2/cpp/src/service/responses_handler.h | 4 +- .../internal_api/model_load_manager_test.cc | 13 ++++ sdk_v2/js/native/src/catalog.cc | 27 +++++++- sdk_v2/js/native/src/catalog.h | 4 +- sdk_v2/js/native/src/model.cc | 66 +++++++++++++++++-- sdk_v2/js/native/src/model.h | 6 +- sdk_v2/js/native/src/session.cc | 55 ++++++++++------ sdk_v2/js/native/src/session.h | 6 +- sdk_v2/js/src/catalog.ts | 1 + sdk_v2/js/test/manager-dispose.test.ts | 15 +++++ 26 files changed, 286 insertions(+), 69 deletions(-) diff --git a/sdk_v2/cpp/src/c_api.cc b/sdk_v2/cpp/src/c_api.cc index 750a5d3b3..2987bdc17 100644 --- a/sdk_v2/cpp/src/c_api.cc +++ b/sdk_v2/cpp/src/c_api.cc @@ -393,10 +393,10 @@ FL_API_STATUS_IMPL(Manager_WebServiceUrlsImpl, const flManager* manager, { std::lock_guard lock(manager->urls_cache_mutex); manager->urls_snapshots.push_back(std::move(snapshot)); + *out_urls = snapshot_ptr->pointers.empty() ? nullptr : snapshot_ptr->pointers.data(); + *out_num_urls = snapshot_ptr->pointers.size(); } - *out_urls = snapshot_ptr->pointers.empty() ? nullptr : snapshot_ptr->pointers.data(); - *out_num_urls = snapshot_ptr->pointers.size(); return nullptr; API_IMPL_END } @@ -408,6 +408,10 @@ FL_API_STATUS_IMPL(Manager_WebServiceStopImpl, flManager* manager) { } manager->impl.StopWebService(); + { + std::lock_guard lock(manager->urls_cache_mutex); + manager->urls_snapshots.clear(); + } return nullptr; API_IMPL_END } diff --git a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc index 06a597ea9..94206864d 100644 --- a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc @@ -74,11 +74,13 @@ std::string JoinTokens(const std::vector& token_texts) { } // namespace AudioSession::AudioSession(const fl::Model& catalog_model, GenAIModelInstance& model, - ILogger& logger, ITelemetry& telemetry) + ILogger& logger, ITelemetry& telemetry, bool session_ref_acquired) : Session(catalog_model, logger, telemetry), logger_(logger), model_(model) { logger_.Log(LogLevel::Debug, fmt::format("Creating AudioSession for model: {}", model.ModelId())); // Last so a throw above does not leak a refcount; nothing below can throw. - model_.AcquireSession(); + if (!session_ref_acquired) { + model_.AcquireSession(); + } } AudioSession::~AudioSession() { diff --git a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h index 1a35d096a..28b9dd886 100644 --- a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h @@ -35,7 +35,8 @@ struct SpeechSegmentItem; /// AudioTranscriptionResponse payload. class AudioSession : public Session { public: - AudioSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry); + AudioSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry, + bool session_ref_acquired = false); ~AudioSession(); // Movable: transfers session refcount ownership to the moved-to instance. diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index eb76ef94d..7515f26b9 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -48,11 +48,14 @@ void ApplyToolChoiceToContext(std::optional tool_choice, ToolCallC } // namespace -ChatSession::ChatSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry) +ChatSession::ChatSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, + ITelemetry& telemetry, bool session_ref_acquired) : Session(catalog_model, logger, telemetry), logger_(logger), model_(model) { logger_.Log(LogLevel::Debug, fmt::format("Creating ChatSession for model: {}", model.ModelId())); // Last so a throw above does not leak a refcount; nothing below can throw. - model_.AcquireSession(); + if (!session_ref_acquired) { + model_.AcquireSession(); + } } ChatSession::~ChatSession() { diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h index 4bd761bc3..f3b9ef29b 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h @@ -39,7 +39,8 @@ class ChatSession : public Session { // The assistant reply is at history_[history_start + input_count] }; - ChatSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry); + ChatSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry, + bool session_ref_acquired = false); ~ChatSession(); // Movable: transfers session refcount ownership to the moved-to instance. diff --git a/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.cc b/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.cc index ae571dbf4..08a9a4425 100644 --- a/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.cc @@ -22,13 +22,15 @@ namespace fl { EmbeddingsSession::EmbeddingsSession(const fl::Model& catalog_model, GenAIModelInstance& model, - ILogger& logger, ITelemetry& telemetry) + ILogger& logger, ITelemetry& telemetry, bool session_ref_acquired) : Session(catalog_model, logger, telemetry, /*allow_concurrent_requests=*/true), logger_(logger), model_(model) { logger_.Log(LogLevel::Debug, fmt::format("Creating EmbeddingsSession for model: {}", model.ModelId())); // Last so a throw above does not leak a refcount; nothing below can throw. - model_.AcquireSession(); + if (!session_ref_acquired) { + model_.AcquireSession(); + } } EmbeddingsSession::~EmbeddingsSession() { diff --git a/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.h b/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.h index 26f9a317d..99f751fea 100644 --- a/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.h @@ -22,7 +22,7 @@ class GenAIModelInstance; class EmbeddingsSession : public Session { public: EmbeddingsSession(const fl::Model& catalog_model, GenAIModelInstance& model, - ILogger& logger, ITelemetry& telemetry); + ILogger& logger, ITelemetry& telemetry, bool session_ref_acquired = false); ~EmbeddingsSession(); EmbeddingsSession(EmbeddingsSession&&) = delete; diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.cc b/sdk_v2/cpp/src/inferencing/model_load_manager.cc index 0bc321b9b..f44527c07 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.cc +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.cc @@ -62,6 +62,31 @@ ModelLoadManager::~ModelLoadManager() { loaded_models_.clear(); } +ModelLoadManager::LoadedModelLease::~LoadedModelLease() { + Reset(); +} + +ModelLoadManager::LoadedModelLease::LoadedModelLease(LoadedModelLease&& other) noexcept + : model_(other.model_) { + other.model_ = nullptr; +} + +ModelLoadManager::LoadedModelLease& ModelLoadManager::LoadedModelLease::operator=(LoadedModelLease&& other) noexcept { + if (this != &other) { + Reset(); + model_ = other.model_; + other.model_ = nullptr; + } + return *this; +} + +void ModelLoadManager::LoadedModelLease::Reset() noexcept { + if (model_ != nullptr) { + model_->ReleaseSession(); + model_ = nullptr; + } +} + bool ModelLoadManager::HasEP(const std::string& ep_name) const { const auto& device_map = ep_detector_.GetAvailableDevicesToEPs(); for (const auto& [device, eps] : device_map) { @@ -270,4 +295,17 @@ GenAIModelInstance* ModelLoadManager::GetLoadedModel(std::string_view model_id) return nullptr; } +ModelLoadManager::LoadedModelLease ModelLoadManager::AcquireLoadedModel(std::string_view model_id) { + std::lock_guard lock(mutex_); + + std::string id_str(model_id); + auto it = loaded_models_.find(id_str); + if (it == loaded_models_.end()) { + return {}; + } + + it->second->AcquireSession(); + return LoadedModelLease(it->second.get()); +} + } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.h b/sdk_v2/cpp/src/inferencing/model_load_manager.h index f582217ad..9082c3bfa 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.h +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.h @@ -39,6 +39,33 @@ class ModelLoadManager { GenAIModelInstance* model = nullptr; // non-owning pointer; lifetime managed by this class }; + class LoadedModelLease { + public: + LoadedModelLease() = default; + ~LoadedModelLease(); + + LoadedModelLease(const LoadedModelLease&) = delete; + LoadedModelLease& operator=(const LoadedModelLease&) = delete; + + LoadedModelLease(LoadedModelLease&& other) noexcept; + LoadedModelLease& operator=(LoadedModelLease&& other) noexcept; + + explicit operator bool() const { return model_ != nullptr; } + GenAIModelInstance* get() const { return model_; } + GenAIModelInstance& operator*() const { return *model_; } + GenAIModelInstance* operator->() const { return model_; } + + void Reset() noexcept; + void Release() noexcept { model_ = nullptr; } + + private: + explicit LoadedModelLease(GenAIModelInstance* model) : model_(model) {} + + GenAIModelInstance* model_ = nullptr; + + friend class ModelLoadManager; + }; + ModelLoadManager(IEpDetector& ep_detector, ILogger& logger); ~ModelLoadManager(); @@ -67,6 +94,10 @@ class ModelLoadManager { /// The returned pointer is valid until UnloadModel is called for this model_id. GenAIModelInstance* GetLoadedModel(std::string_view model_id); + /// Get a loaded model by ID and increment its live-session refcount while + /// holding the load-manager lock. Returns an empty lease if not loaded. + LoadedModelLease AcquireLoadedModel(std::string_view model_id); + /// Reject all future LoadModel calls. Called by Manager::Shutdown(). /// Idempotent and thread-safe. void RejectNewLoads(); diff --git a/sdk_v2/cpp/src/inferencing/session/session.cc b/sdk_v2/cpp/src/inferencing/session/session.cc index b9afd6e83..137a9eb5a 100644 --- a/sdk_v2/cpp/src/inferencing/session/session.cc +++ b/sdk_v2/cpp/src/inferencing/session/session.cc @@ -49,7 +49,7 @@ std::unique_ptr Session::Create(const fl::Model& model) { } auto& lm = mgr.GetModelLoadManager(); - auto* loaded = lm.GetLoadedModel(model.Id()); + auto loaded = lm.AcquireLoadedModel(model.Id()); if (!loaded) { FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL, "loaded model not found in load manager"); } @@ -58,19 +58,22 @@ std::unique_ptr Session::Create(const fl::Model& model) { const auto& info = model.Info(); if (info.task == "chat-completion" || info.task == "vision-language-chat") { - auto session = std::make_unique(model, *loaded, logger, telemetry); + auto session = std::make_unique(model, *loaded, logger, telemetry, true); + loaded.Release(); tracker.SetStatus(ActionStatus::kSuccess); return session; } if (info.task == "automatic-speech-recognition") { - auto session = std::make_unique(model, *loaded, logger, telemetry); + auto session = std::make_unique(model, *loaded, logger, telemetry, true); + loaded.Release(); tracker.SetStatus(ActionStatus::kSuccess); return session; } if (info.task == "embeddings") { - auto session = std::make_unique(model, *loaded, logger, telemetry); + auto session = std::make_unique(model, *loaded, logger, telemetry, true); + loaded.Release(); tracker.SetStatus(ActionStatus::kSuccess); return session; } @@ -115,6 +118,9 @@ void Session::ProcessRequest(const Request& request, Response& response) { const auto start = std::chrono::steady_clock::now(); try { ProcessRequestImpl(request, response); + if (request.canceled.load(std::memory_order_relaxed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "request cancelled"); + } tracker.SetStatus(ActionStatus::kSuccess); } catch (const std::exception& ex) { diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc index c5d845c42..870f0a557 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc @@ -58,13 +58,13 @@ std::shared_ptr AudioTranscriptionsHandler } std::shared_ptr AudioTranscriptionsHandler::ResolveModel( - const std::string& model_name, Model*& model, GenAIModelInstance*& loaded) { + const std::string& model_name, Model*& model, ModelLoadManager::LoadedModelLease& loaded) { model = ctx_.catalog.GetModelVariant(model_name); if (!model) { return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + model_name + "'"); } - loaded = ctx_.model_load_manager.GetLoadedModel(model->Id()); + loaded = ctx_.model_load_manager.AcquireLoadedModel(model->Id()); if (!loaded) { return ErrorResponse(Status::CODE_400, "Model not loaded", "Model '" + model_name + "' must be loaded before inference"); @@ -123,7 +123,7 @@ std::shared_ptr AudioTranscriptionsHandler // 3. Resolve model std::string model_name = req.model; Model* model = nullptr; - GenAIModelInstance* loaded = nullptr; + ModelLoadManager::LoadedModelLease loaded; if (auto err = ResolveModel(model_name, model, loaded)) { tracker->SetStatus(ActionStatus::kClientError); return err; @@ -140,7 +140,8 @@ std::shared_ptr AudioTranscriptionsHandler // 5. Dispatch to streaming or non-streaming try { - AudioSession session(*model, *loaded, ctx_.logger, ctx_.telemetry); + AudioSession session(*model, *loaded, ctx_.logger, ctx_.telemetry, true); + loaded.Release(); { ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); create_tracker.SetModelId(model_name); diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.h b/sdk_v2/cpp/src/service/audio_transcriptions_handler.h index 1f68c058c..9f25345fa 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.h +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.h @@ -4,6 +4,7 @@ #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE +#include "inferencing/model_load_manager.h" #include "service/handler_utils.h" #include @@ -15,7 +16,6 @@ struct ServiceContext; struct AudioTranscriptionRequest; class AudioSession; class Model; -class GenAIModelInstance; class ActionTracker; struct Request; @@ -36,7 +36,7 @@ class AudioTranscriptionsHandler : public HttpRequestHandler { /// Look up model in catalog and verify it's loaded and supports audio. std::shared_ptr ResolveModel(const std::string& model_name, - Model*& model, GenAIModelInstance*& loaded); + Model*& model, ModelLoadManager::LoadedModelLease& loaded); /// Build a Request with an OPENAI_JSON-tagged TEXT item from the original body string. void BuildOpenAIJsonRequest(const std::string& body, Request& session_request); diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.cc b/sdk_v2/cpp/src/service/chat_completions_handler.cc index cbeedcd35..289309d9d 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.cc +++ b/sdk_v2/cpp/src/service/chat_completions_handler.cc @@ -61,13 +61,13 @@ std::shared_ptr ChatCompletionsHandler::Pa } std::shared_ptr ChatCompletionsHandler::ResolveModel( - const std::string& model_name, Model*& model, GenAIModelInstance*& loaded) { + const std::string& model_name, Model*& model, ModelLoadManager::LoadedModelLease& loaded) { model = ctx_.catalog.GetModelVariant(model_name); if (!model) { return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + model_name + "'"); } - loaded = ctx_.model_load_manager.GetLoadedModel(model->Id()); + loaded = ctx_.model_load_manager.AcquireLoadedModel(model->Id()); if (!loaded) { return ErrorResponse(Status::CODE_400, "Model not loaded", "Model '" + model_name + "' must be loaded before inference"); @@ -132,7 +132,7 @@ std::shared_ptr ChatCompletionsHandler::ha // 2. Resolve model std::string model_name = req.model; Model* model = nullptr; - GenAIModelInstance* loaded = nullptr; + ModelLoadManager::LoadedModelLease loaded; if (auto err = ResolveModel(model_name, model, loaded)) { tracker->SetStatus(ActionStatus::kClientError); return err; @@ -156,7 +156,8 @@ std::shared_ptr ChatCompletionsHandler::ha // 6. Run inference via ChatSession try { - ChatSession session(*model, *loaded, ctx_.logger, ctx_.telemetry); + ChatSession session(*model, *loaded, ctx_.logger, ctx_.telemetry, true); + loaded.Release(); { ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); create_tracker.SetModelId(model_name); diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.h b/sdk_v2/cpp/src/service/chat_completions_handler.h index c7225bf77..bb6a010af 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.h +++ b/sdk_v2/cpp/src/service/chat_completions_handler.h @@ -4,6 +4,7 @@ #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE +#include "inferencing/model_load_manager.h" #include "service/handler_utils.h" #include @@ -15,7 +16,6 @@ struct ServiceContext; struct ChatCompletionRequest; class ChatSession; class Model; -class GenAIModelInstance; class ActionTracker; struct Request; @@ -38,7 +38,7 @@ class ChatCompletionsHandler : public HttpRequestHandler { /// Look up model in catalog and verify it's loaded. Sets output pointers. /// Returns an error response on failure, nullptr on success. std::shared_ptr ResolveModel(const std::string& model_name, - Model*& model, GenAIModelInstance*& loaded); + Model*& model, ModelLoadManager::LoadedModelLease& loaded); /// Build a Request with an OPENAI_JSON-tagged TEXT item from the original body string. /// Populates catalog defaults and model-specific options (tool_call_start/end). diff --git a/sdk_v2/cpp/src/service/embeddings_handler.cc b/sdk_v2/cpp/src/service/embeddings_handler.cc index 526e9cc51..59e5aee1f 100644 --- a/sdk_v2/cpp/src/service/embeddings_handler.cc +++ b/sdk_v2/cpp/src/service/embeddings_handler.cc @@ -68,7 +68,7 @@ class EmbeddingsHandler : public HttpRequestHandler { return ErrorResponse(Status::CODE_404, "Model not found", model_name); } - auto* loaded = ctx_.model_load_manager.GetLoadedModel(model->Id()); + auto loaded = ctx_.model_load_manager.AcquireLoadedModel(model->Id()); if (!loaded) { tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Model not loaded", model_name); @@ -81,7 +81,8 @@ class EmbeddingsHandler : public HttpRequestHandler { // 4. Create session and process each input try { - EmbeddingsSession session(*model, *loaded, ctx_.logger, ctx_.telemetry); + EmbeddingsSession session(*model, *loaded, ctx_.logger, ctx_.telemetry, true); + loaded.Release(); { ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); create_tracker.SetModelId(model_name); diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index c20ae9c88..c15296d6c 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -80,13 +80,13 @@ std::shared_ptr ResponsesHandler::ParseAnd } std::shared_ptr ResponsesHandler::ResolveModel( - const std::string& model_name, Model*& model, GenAIModelInstance*& loaded) { + const std::string& model_name, Model*& model, ModelLoadManager::LoadedModelLease& loaded) { model = ctx_.catalog.GetModelVariant(model_name); if (!model) { return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + model_name + "'"); } - loaded = ctx_.model_load_manager.GetLoadedModel(model->Id()); + loaded = ctx_.model_load_manager.AcquireLoadedModel(model->Id()); if (!loaded) { return ErrorResponse(Status::CODE_400, "Model not loaded", "Model '" + model_name + "' must be loaded before inference"); @@ -157,7 +157,7 @@ std::shared_ptr ResponsesHandler::handle( // 2. Resolve model std::string model_name = params.model; Model* model = nullptr; - GenAIModelInstance* loaded = nullptr; + ModelLoadManager::LoadedModelLease loaded; if (auto err = ResolveModel(model_name, model, loaded)) { tracker->SetStatus(ActionStatus::kClientError); return err; @@ -206,7 +206,8 @@ std::shared_ptr ResponsesHandler::handle( if (!session) { ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); create_tracker.SetModelId(model_name); - session = std::make_unique(*model, *loaded, ctx_.logger, ctx_.telemetry); + session = std::make_unique(*model, *loaded, ctx_.logger, ctx_.telemetry, true); + loaded.Release(); create_tracker.SetStatus(ActionStatus::kSuccess); } session->SetRequestContext(session_ctx); diff --git a/sdk_v2/cpp/src/service/responses_handler.h b/sdk_v2/cpp/src/service/responses_handler.h index e455d6203..090445a68 100644 --- a/sdk_v2/cpp/src/service/responses_handler.h +++ b/sdk_v2/cpp/src/service/responses_handler.h @@ -4,6 +4,7 @@ #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE +#include "inferencing/model_load_manager.h" #include "service/handler_utils.h" #include @@ -15,7 +16,6 @@ struct ServiceContext; struct Request; class ChatSession; class Model; -class GenAIModelInstance; class ActionTracker; namespace responses { @@ -44,7 +44,7 @@ class ResponsesHandler : public HttpRequestHandler { /// Look up model in catalog and verify it's loaded. Sets output pointers. /// Returns an error response on failure, nullptr on success. std::shared_ptr ResolveModel(const std::string& model_name, - Model*& model, GenAIModelInstance*& loaded); + Model*& model, ModelLoadManager::LoadedModelLease& loaded); /// Load previous response context when chaining via previous_response_id. /// The json storage objects are passed by reference because the output pointers alias into them. diff --git a/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc b/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc index 695f0e5d7..9d91a94f0 100644 --- a/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc +++ b/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc @@ -213,6 +213,19 @@ TEST_F(ModelLoadManagerUnloadTest, UnloadThrowsWhenSessionsLive) { instance_->ReleaseSession(); } +TEST_F(ModelLoadManagerUnloadTest, AcquiredLoadedModelLeaseBlocksUnloadUntilReleased) { + auto lease = mgr_->AcquireLoadedModel(fl::test::kTestChatModelAlias); + ASSERT_TRUE(lease); + EXPECT_EQ(lease.get(), instance_); + EXPECT_EQ(instance_->SessionRefCount(), 1); + + EXPECT_THROW(mgr_->UnloadModel(fl::test::kTestChatModelAlias), fl::Exception); + + lease.Reset(); + EXPECT_EQ(instance_->SessionRefCount(), 0); + EXPECT_TRUE(mgr_->UnloadModel(fl::test::kTestChatModelAlias)); +} + TEST_F(ModelLoadManagerUnloadTest, UnloadFailsWhenInUseAndSucceedsAfterSessionsReleased) { instance_->AcquireSession(); EXPECT_THROW(mgr_->UnloadModel(fl::test::kTestChatModelAlias), fl::Exception); diff --git a/sdk_v2/js/native/src/catalog.cc b/sdk_v2/js/native/src/catalog.cc index 53c37ef90..f7cb868ce 100644 --- a/sdk_v2/js/native/src/catalog.cc +++ b/sdk_v2/js/native/src/catalog.cc @@ -16,10 +16,19 @@ namespace foundry_local_node { namespace { +std::shared_ptr LockManagerOrThrow( + const std::weak_ptr& manager_keepalive) { + auto manager = manager_keepalive.lock(); + if (!manager) { + throw foundry_local::Error("Manager has been disposed", FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } + return manager; +} + // Wrap a ModelList (rvalue) into a JS array of Model handles, each pinning the // passed-in manager reference. Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, Napi::ObjectReference manager, - std::shared_ptr manager_keepalive) { + std::weak_ptr manager_keepalive) { auto list = std::make_shared(std::move(ml)); auto models = list->Models(); Napi::Array arr = Napi::Array::New(env, models.size()); @@ -37,7 +46,7 @@ Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, Napi::Obje // Wrap an owning unique_ptr into a JS Model (or undefined when null). Napi::Value WrapOwnedModelOrUndefined(Napi::Env env, std::unique_ptr owned, Napi::ObjectReference manager, - std::shared_ptr manager_keepalive) { + std::weak_ptr manager_keepalive) { if (!owned) { return env.Undefined(); } @@ -118,6 +127,8 @@ Catalog::Catalog(const Napi::CallbackInfo& info) : Napi::ObjectWrap(inf Napi::Value Catalog::GetName(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; std::string_view name = impl_->GetName(); return Napi::String::New(env, std::string(name)); }); @@ -130,6 +141,8 @@ Napi::Value Catalog::GetModels(const Napi::CallbackInfo& info) { Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked( env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; return WrapModelList(env, impl_->GetModels(), std::move(mgr), manager_keepalive_); }); } @@ -138,6 +151,8 @@ Napi::Value Catalog::GetCachedModels(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; return WrapModelList(env, impl_->GetCachedModels(), std::move(mgr), manager_keepalive_); }); } @@ -146,6 +161,8 @@ Napi::Value Catalog::GetLoadedModels(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; return WrapModelList(env, impl_->GetLoadedModels(), std::move(mgr), manager_keepalive_); }); } @@ -161,6 +178,8 @@ Napi::Value Catalog::GetModel(const Napi::CallbackInfo& info) { std::string alias = info[0].As(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; auto owned = impl_->GetModel(alias); return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_); }); @@ -175,6 +194,8 @@ Napi::Value Catalog::GetModelVariant(const Napi::CallbackInfo& info) { std::string model_id = info[0].As(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; auto owned = impl_->GetModelVariant(model_id); return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_); }); @@ -193,6 +214,8 @@ Napi::Value Catalog::GetLatestVersion(const Napi::CallbackInfo& info) { } Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; auto owned = impl_->GetLatestVersion(*arg); return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_); }); diff --git a/sdk_v2/js/native/src/catalog.h b/sdk_v2/js/native/src/catalog.h index fd24c7c39..df768ceb6 100644 --- a/sdk_v2/js/native/src/catalog.h +++ b/sdk_v2/js/native/src/catalog.h @@ -24,7 +24,7 @@ namespace foundry_local_node { struct CatalogCtorToken { foundry_local::ICatalog* impl = nullptr; Napi::ObjectReference manager; // pins the owning Manager - std::shared_ptr manager_keepalive; + std::weak_ptr manager_keepalive; }; class Catalog : public Napi::ObjectWrap { @@ -45,7 +45,7 @@ class Catalog : public Napi::ObjectWrap { foundry_local::ICatalog* impl_ = nullptr; Napi::ObjectReference manager_; - std::shared_ptr manager_keepalive_; + std::weak_ptr manager_keepalive_; }; } // namespace foundry_local_node diff --git a/sdk_v2/js/native/src/model.cc b/sdk_v2/js/native/src/model.cc index 185a0e72c..c67e687ca 100644 --- a/sdk_v2/js/native/src/model.cc +++ b/sdk_v2/js/native/src/model.cc @@ -73,6 +73,37 @@ void SetPromptTemplate(Napi::Env env, Napi::Object obj, const foundry_local::Mod obj.Set("promptTemplate", template_obj); } +std::shared_ptr LockManager( + const std::weak_ptr& manager_keepalive) { + return manager_keepalive.lock(); +} + +void ThrowFoundryLocalError(Napi::Env env, int code, const std::string& msg) { + Napi::Error err = Napi::Error::New(env, msg); + Napi::Object value = err.Value(); + value.Set("name", Napi::String::New(env, "FoundryLocalError")); + value.Set("code", Napi::Number::New(env, code)); + err.ThrowAsJavaScriptException(); +} + +std::shared_ptr LockManagerOrThrow( + const std::weak_ptr& manager_keepalive) { + auto manager = LockManager(manager_keepalive); + if (!manager) { + throw foundry_local::Error("Manager has been disposed", FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } + return manager; +} + +std::shared_ptr LockManagerOrThrowJs( + Napi::Env env, const std::weak_ptr& manager_keepalive) { + auto manager = LockManager(manager_keepalive); + if (!manager) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); + } + return manager; +} + void SetModelSettings(Napi::Env env, Napi::Object obj, const foundry_local::ModelInfo& info) { auto settings = info.GetModelSettings(); if (!settings.has_value()) { @@ -145,7 +176,7 @@ Napi::Object SnapshotModelInfo(Napi::Env env, const foundry_local::ModelInfo& in // whose keepalive holds the shared ModelList. Napi::Array WrapModelList(Napi::Env env, std::shared_ptr list, Napi::ObjectReference manager, - std::shared_ptr manager_keepalive) { + std::weak_ptr manager_keepalive) { auto models = list->Models(); Napi::Array arr = Napi::Array::New(env, models.size()); for (size_t i = 0; i < models.size(); ++i) { @@ -207,6 +238,8 @@ Model::Model(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) { Napi::Value Model::GetInfo(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; foundry_local::ModelInfo mi = impl_->GetInfo(); Napi::Object snapshot = SnapshotModelInfo(env, mi); snapshot.Set("cached", Napi::Boolean::New(env, impl_->IsCached())); @@ -217,6 +250,8 @@ Napi::Value Model::GetInfo(const Napi::CallbackInfo& info) { Napi::Value Model::IsCached(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; return Napi::Boolean::New(env, impl_->IsCached()); }); } @@ -224,6 +259,8 @@ Napi::Value Model::IsCached(const Napi::CallbackInfo& info) { Napi::Value Model::IsLoaded(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; return Napi::Boolean::New(env, impl_->IsLoaded()); }); } @@ -231,6 +268,8 @@ Napi::Value Model::IsLoaded(const Napi::CallbackInfo& info) { Napi::Value Model::GetPath(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; std::string_view p = impl_->GetPath(); return Napi::String::New(env, std::string(p)); }); @@ -240,6 +279,8 @@ Napi::Value Model::GetVariants(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference owner_clone = Napi::Reference::New(manager_.Value(), 1); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; auto list = std::make_shared(impl_->GetVariants()); return WrapModelList(env, std::move(list), std::move(owner_clone), manager_keepalive_); }); @@ -261,7 +302,10 @@ Napi::Value Model::Load(const Napi::CallbackInfo& info) { Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); foundry_local::IModel* m = impl_; auto keepalive = keepalive_; - auto manager_keepalive = manager_keepalive_; + auto manager_keepalive = LockManagerOrThrowJs(env, manager_keepalive_); + if (!manager_keepalive) { + return env.Undefined(); + } return PromiseWorkerVoid::Run( env, [m, keepalive = std::move(keepalive), manager_keepalive = std::move(manager_keepalive)]() { m->Load(); }, std::move(owner)); @@ -276,7 +320,10 @@ Napi::Value Model::Unload(const Napi::CallbackInfo& info) { Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); foundry_local::IModel* m = impl_; auto keepalive = keepalive_; - auto manager_keepalive = manager_keepalive_; + auto manager_keepalive = LockManagerOrThrowJs(env, manager_keepalive_); + if (!manager_keepalive) { + return env.Undefined(); + } return PromiseWorkerVoid::Run( env, [m, keepalive = std::move(keepalive), manager_keepalive = std::move(manager_keepalive)]() { m->Unload(); }, std::move(owner)); @@ -392,7 +439,12 @@ Napi::Value Model::Download(const Napi::CallbackInfo& info) { } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - auto* w = new DownloadWorker(env, impl_, keepalive_, manager_keepalive_, std::move(owner), std::move(tsfn)); + auto manager_keepalive = LockManagerOrThrowJs(env, manager_keepalive_); + if (!manager_keepalive) { + return env.Undefined(); + } + auto* w = new DownloadWorker(env, impl_, keepalive_, std::move(manager_keepalive), std::move(owner), + std::move(tsfn)); Napi::Promise p = w->Promise(); w->Queue(); return p; @@ -408,6 +460,8 @@ Napi::Value Model::RemoveFromCache(const Napi::CallbackInfo& info) { // V1's contract is `removeFromCache(): void` so we do not bounce to a // worker. return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + (void)manager_alive; impl_->RemoveFromCache(); return env.Undefined(); }); @@ -435,6 +489,10 @@ Napi::Value Model::SelectVariant(const Napi::CallbackInfo& info) { } return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto variant_manager_alive = LockManagerOrThrow(variant->manager_keepalive_); + (void)manager_alive; + (void)variant_manager_alive; impl_->SelectVariant(*variant->impl_); return env.Undefined(); }); diff --git a/sdk_v2/js/native/src/model.h b/sdk_v2/js/native/src/model.h index 42107a07a..150ed5213 100644 --- a/sdk_v2/js/native/src/model.h +++ b/sdk_v2/js/native/src/model.h @@ -39,7 +39,7 @@ struct ModelCtorToken { // or a std::shared_ptr) alive for the JS Model's // lifetime. std::shared_ptr keepalive; - std::shared_ptr manager_keepalive; + std::weak_ptr manager_keepalive; // Pins the parent Manager so its native handle (and the Catalog's flCatalog* // which the IModel views into) cannot be released first. Napi::ObjectReference manager; @@ -63,7 +63,7 @@ class Model : public Napi::ObjectWrap { // Internal accessor used by Session / ChatSession ctors so they can clone // the parent Manager ObjectReference and pin it for the session lifetime. const Napi::ObjectReference& manager() const noexcept { return manager_; } - const std::shared_ptr& manager_keepalive() const noexcept { return manager_keepalive_; } + std::shared_ptr manager_keepalive() const noexcept { return manager_keepalive_.lock(); } private: Napi::Value GetInfo(const Napi::CallbackInfo& info); @@ -80,7 +80,7 @@ class Model : public Napi::ObjectWrap { foundry_local::IModel* impl_ = nullptr; std::shared_ptr keepalive_; - std::shared_ptr manager_keepalive_; + std::weak_ptr manager_keepalive_; Napi::ObjectReference manager_; }; diff --git a/sdk_v2/js/native/src/session.cc b/sdk_v2/js/native/src/session.cc index b7aabd2f0..ec0f3177e 100644 --- a/sdk_v2/js/native/src/session.cc +++ b/sdk_v2/js/native/src/session.cc @@ -97,7 +97,7 @@ foundry_local::Request* UnwrapRequest(Napi::Env env, const Napi::Value& v) { // Pins both the Manager (so the Model handle the Session holds stays alive) // and the Request (so the C++ Request the worker reads stays alive). template -Napi::Value ProcessRequestOn(Napi::Env env, SessT* sess, const Napi::Value& request_arg, +Napi::Value ProcessRequestOn(Napi::Env env, std::shared_ptr sess, const Napi::Value& request_arg, Napi::ObjectReference manager_ref) { foundry_local::Request* req = UnwrapRequest(env, request_arg); if (req == nullptr) return env.Undefined(); // pending exception @@ -112,7 +112,7 @@ Napi::Value ProcessRequestOn(Napi::Env env, SessT* sess, const Napi::Value& requ return PromiseWorker::Run( env, - [sess, req, pins]() -> Result { + [sess = std::move(sess), req, pins]() -> Result { (void)pins; // keepalive captured by reference count return std::make_shared(sess->ProcessRequest(*req)); }, @@ -180,9 +180,9 @@ void FinalizeStream(Napi::Env env, void* /*data*/, StreamCtx* ctx) { template class StreamWorker : public Napi::AsyncWorker { public: - static Napi::Promise Run(Napi::Env env, SessT* sess, foundry_local::Request* req, + static Napi::Promise Run(Napi::Env env, std::shared_ptr sess, foundry_local::Request* req, Napi::Function jsCallback, StreamCtx* ctx) { - auto* w = new StreamWorker(env, sess, req, jsCallback, ctx); + auto* w = new StreamWorker(env, std::move(sess), req, jsCallback, ctx); Napi::Promise p = ctx->deferred.Promise(); w->Queue(); return p; @@ -239,10 +239,10 @@ class StreamWorker : public Napi::AsyncWorker { void OnError(const Napi::Error& /*unused*/) override { tsfn_.Release(); } private: - StreamWorker(Napi::Env env, SessT* sess, foundry_local::Request* req, + StreamWorker(Napi::Env env, std::shared_ptr sess, foundry_local::Request* req, Napi::Function jsCallback, StreamCtx* ctx) : Napi::AsyncWorker(env), - sess_(sess), + sess_(std::move(sess)), req_(req), ctx_(ctx), tsfn_(Napi::ThreadSafeFunction::New(env, jsCallback, "foundry_local_stream", @@ -250,14 +250,14 @@ class StreamWorker : public Napi::AsyncWorker { FinalizeStream, static_cast(nullptr))) {} - SessT* sess_; + std::shared_ptr sess_; foundry_local::Request* req_; StreamCtx* ctx_; Napi::ThreadSafeFunction tsfn_; }; template -Napi::Value ProcessStreamingRequestOn(Napi::Env env, SessT* sess, const Napi::CallbackInfo& info, +Napi::Value ProcessStreamingRequestOn(Napi::Env env, std::shared_ptr sess, const Napi::CallbackInfo& info, Napi::ObjectReference manager_ref) { if (info.Length() < 2 || !info[1].IsFunction()) { Napi::TypeError::New(env, "processStreamingRequest(request: Request, onItem: (item) => void)") @@ -277,7 +277,7 @@ Napi::Value ProcessStreamingRequestOn(Napi::Env env, SessT* sess, const Napi::Ca 0, false, false}; - return StreamWorker::Run(env, sess, req, info[1].As(), ctx); + return StreamWorker::Run(env, std::move(sess), req, info[1].As(), ctx); } } // namespace @@ -322,7 +322,13 @@ ChatSession::ChatSession(const Napi::CallbackInfo& info) : Napi::ObjectWrap(*native); + auto manager_keepalive = model->manager_keepalive(); + if (!manager_keepalive) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); + return; + } + impl_ = std::make_shared(*native); + manager_keepalive_ = std::move(manager_keepalive); } catch (const foundry_local::Error& e) { ThrowFoundryLocalError(env, static_cast(e.Code()), e.what()); return; @@ -331,7 +337,6 @@ ChatSession::ChatSession(const Napi::CallbackInfo& info) : Napi::ObjectWrap::New(model->manager().Value(), 1); - manager_keepalive_ = model->manager_keepalive(); } bool ChatSession::ThrowIfDisposed(Napi::Env env) { @@ -351,14 +356,14 @@ Napi::Value ChatSession::ProcessRequest(const Napi::CallbackInfo& info) { return env.Undefined(); } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessRequestOn(env, impl_.get(), info[0], std::move(owner)); + return ProcessRequestOn(env, impl_, info[0], std::move(owner)); } Napi::Value ChatSession::ProcessStreamingRequest(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (ThrowIfDisposed(env)) return env.Undefined(); Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessStreamingRequestOn(env, impl_.get(), info, std::move(owner)); + return ProcessStreamingRequestOn(env, impl_, info, std::move(owner)); } Napi::Value ChatSession::SetOptions(const Napi::CallbackInfo& info) { @@ -480,7 +485,13 @@ EmbeddingsSession::EmbeddingsSession(const Napi::CallbackInfo& info) return; } try { - impl_ = std::make_unique(*native); + auto manager_keepalive = model->manager_keepalive(); + if (!manager_keepalive) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); + return; + } + impl_ = std::make_shared(*native); + manager_keepalive_ = std::move(manager_keepalive); } catch (const foundry_local::Error& e) { ThrowFoundryLocalError(env, static_cast(e.Code()), e.what()); return; @@ -489,7 +500,6 @@ EmbeddingsSession::EmbeddingsSession(const Napi::CallbackInfo& info) return; } manager_ = Napi::Reference::New(model->manager().Value(), 1); - manager_keepalive_ = model->manager_keepalive(); } bool EmbeddingsSession::ThrowIfDisposed(Napi::Env env) { @@ -509,7 +519,7 @@ Napi::Value EmbeddingsSession::ProcessRequest(const Napi::CallbackInfo& info) { return env.Undefined(); } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessRequestOn(env, impl_.get(), info[0], std::move(owner)); + return ProcessRequestOn(env, impl_, info[0], std::move(owner)); } Napi::Value EmbeddingsSession::SetOptions(const Napi::CallbackInfo& info) { @@ -575,7 +585,13 @@ AudioSession::AudioSession(const Napi::CallbackInfo& info) return; } try { - impl_ = std::make_unique(*native); + auto manager_keepalive = model->manager_keepalive(); + if (!manager_keepalive) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); + return; + } + impl_ = std::make_shared(*native); + manager_keepalive_ = std::move(manager_keepalive); } catch (const foundry_local::Error& e) { ThrowFoundryLocalError(env, static_cast(e.Code()), e.what()); return; @@ -584,7 +600,6 @@ AudioSession::AudioSession(const Napi::CallbackInfo& info) return; } manager_ = Napi::Reference::New(model->manager().Value(), 1); - manager_keepalive_ = model->manager_keepalive(); } bool AudioSession::ThrowIfDisposed(Napi::Env env) { @@ -604,14 +619,14 @@ Napi::Value AudioSession::ProcessRequest(const Napi::CallbackInfo& info) { return env.Undefined(); } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessRequestOn(env, impl_.get(), info[0], std::move(owner)); + return ProcessRequestOn(env, impl_, info[0], std::move(owner)); } Napi::Value AudioSession::ProcessStreamingRequest(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (ThrowIfDisposed(env)) return env.Undefined(); Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessStreamingRequestOn(env, impl_.get(), info, std::move(owner)); + return ProcessStreamingRequestOn(env, impl_, info, std::move(owner)); } Napi::Value AudioSession::SetOptions(const Napi::CallbackInfo& info) { diff --git a/sdk_v2/js/native/src/session.h b/sdk_v2/js/native/src/session.h index e361ee197..36dfeb254 100644 --- a/sdk_v2/js/native/src/session.h +++ b/sdk_v2/js/native/src/session.h @@ -51,7 +51,7 @@ class ChatSession : public Napi::ObjectWrap { bool ThrowIfDisposed(Napi::Env env); - std::unique_ptr impl_; + std::shared_ptr impl_; Napi::ObjectReference manager_; std::shared_ptr manager_keepalive_; }; @@ -83,7 +83,7 @@ class EmbeddingsSession : public Napi::ObjectWrap { bool ThrowIfDisposed(Napi::Env env); - std::unique_ptr impl_; + std::shared_ptr impl_; Napi::ObjectReference manager_; std::shared_ptr manager_keepalive_; }; @@ -113,7 +113,7 @@ class AudioSession : public Napi::ObjectWrap { bool ThrowIfDisposed(Napi::Env env); - std::unique_ptr impl_; + std::shared_ptr impl_; Napi::ObjectReference manager_; std::shared_ptr manager_keepalive_; }; diff --git a/sdk_v2/js/src/catalog.ts b/sdk_v2/js/src/catalog.ts index 978a0f359..c08c8d579 100644 --- a/sdk_v2/js/src/catalog.ts +++ b/sdk_v2/js/src/catalog.ts @@ -4,6 +4,7 @@ // // The underlying native catalog operations are synchronous; the async surface here is for parity with the C# / // Python SDKs. `getModel`, `getModelVariant`, and `getLatestVersion` throw when the alias / id is not found. +// After the parent manager is disposed, native catalog methods throw a FoundryLocalError. import type { NativeCatalog, NativeModel } from "./detail/native.js"; import type { IModel } from "./imodel.js"; diff --git a/sdk_v2/js/test/manager-dispose.test.ts b/sdk_v2/js/test/manager-dispose.test.ts index d6c9a22af..ee89a6dfa 100644 --- a/sdk_v2/js/test/manager-dispose.test.ts +++ b/sdk_v2/js/test/manager-dispose.test.ts @@ -62,6 +62,21 @@ describeIfBuilt("FoundryLocalManager.dispose", () => { } }); + it("a cached catalog handle throws after manager disposal and does not block a new manager", () => { + const mgr = freshManager("cached-catalog"); + const catalog = mgr.catalog; + mgr.dispose(); + + expect(() => catalog.name).toThrow(/disposed/i); + + const next = freshManager("after-cached-catalog"); + try { + expect(next.disposed).toBe(false); + } finally { + next.dispose(); + } + }); + it("Symbol.dispose is wired and idempotent", () => { const mgr = freshManager("symbol-dispose"); mgr[Symbol.dispose](); From df72097f1026beaf64dc81aaa2934e34439c675f Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 04:18:41 -0500 Subject: [PATCH 56/77] Release disposed JS session keepalives Drop the wrapper-level native manager keepalive when a JS session is disposed while preserving temporary strong keepalives inside in-flight async workers. Files changed: - sdk_v2/js/native/src/session.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/js/native/src/session.cc | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/sdk_v2/js/native/src/session.cc b/sdk_v2/js/native/src/session.cc index ec0f3177e..bc7d8b942 100644 --- a/sdk_v2/js/native/src/session.cc +++ b/sdk_v2/js/native/src/session.cc @@ -98,7 +98,8 @@ foundry_local::Request* UnwrapRequest(Napi::Env env, const Napi::Value& v) { // and the Request (so the C++ Request the worker reads stays alive). template Napi::Value ProcessRequestOn(Napi::Env env, std::shared_ptr sess, const Napi::Value& request_arg, - Napi::ObjectReference manager_ref) { + Napi::ObjectReference manager_ref, + std::shared_ptr manager_keepalive) { foundry_local::Request* req = UnwrapRequest(env, request_arg); if (req == nullptr) return env.Undefined(); // pending exception Napi::ObjectReference req_pin = Napi::Reference::New(request_arg.As(), 1); @@ -107,8 +108,10 @@ Napi::Value ProcessRequestOn(Napi::Env env, std::shared_ptr sess, const N struct Pins { Napi::ObjectReference manager; Napi::ObjectReference request; + std::shared_ptr native_manager; }; - auto pins = std::make_shared(Pins{std::move(manager_ref), std::move(req_pin)}); + auto pins = std::make_shared( + Pins{std::move(manager_ref), std::move(req_pin), std::move(manager_keepalive)}); return PromiseWorker::Run( env, @@ -148,6 +151,7 @@ struct StreamCtx { Napi::Promise::Deferred deferred; Napi::ObjectReference manager; Napi::ObjectReference request; + std::shared_ptr native_manager; std::shared_ptr response; std::string err_msg; int err_code = 0; @@ -258,7 +262,8 @@ class StreamWorker : public Napi::AsyncWorker { template Napi::Value ProcessStreamingRequestOn(Napi::Env env, std::shared_ptr sess, const Napi::CallbackInfo& info, - Napi::ObjectReference manager_ref) { + Napi::ObjectReference manager_ref, + std::shared_ptr manager_keepalive) { if (info.Length() < 2 || !info[1].IsFunction()) { Napi::TypeError::New(env, "processStreamingRequest(request: Request, onItem: (item) => void)") .ThrowAsJavaScriptException(); @@ -272,6 +277,7 @@ Napi::Value ProcessStreamingRequestOn(Napi::Env env, std::shared_ptr sess auto* ctx = new StreamCtx{Napi::Promise::Deferred::New(env), std::move(manager_ref), std::move(req_pin), + std::move(manager_keepalive), nullptr, "", 0, @@ -356,14 +362,14 @@ Napi::Value ChatSession::ProcessRequest(const Napi::CallbackInfo& info) { return env.Undefined(); } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessRequestOn(env, impl_, info[0], std::move(owner)); + return ProcessRequestOn(env, impl_, info[0], std::move(owner), manager_keepalive_); } Napi::Value ChatSession::ProcessStreamingRequest(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (ThrowIfDisposed(env)) return env.Undefined(); Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessStreamingRequestOn(env, impl_, info, std::move(owner)); + return ProcessStreamingRequestOn(env, impl_, info, std::move(owner), manager_keepalive_); } Napi::Value ChatSession::SetOptions(const Napi::CallbackInfo& info) { @@ -444,6 +450,7 @@ Napi::Value ChatSession::UndoTurns(const Napi::CallbackInfo& info) { Napi::Value ChatSession::Dispose(const Napi::CallbackInfo& info) { impl_.reset(); + manager_keepalive_.reset(); return info.Env().Undefined(); } @@ -519,7 +526,7 @@ Napi::Value EmbeddingsSession::ProcessRequest(const Napi::CallbackInfo& info) { return env.Undefined(); } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessRequestOn(env, impl_, info[0], std::move(owner)); + return ProcessRequestOn(env, impl_, info[0], std::move(owner), manager_keepalive_); } Napi::Value EmbeddingsSession::SetOptions(const Napi::CallbackInfo& info) { @@ -539,6 +546,7 @@ Napi::Value EmbeddingsSession::SetOptions(const Napi::CallbackInfo& info) { Napi::Value EmbeddingsSession::Dispose(const Napi::CallbackInfo& info) { impl_.reset(); + manager_keepalive_.reset(); return info.Env().Undefined(); } @@ -619,14 +627,14 @@ Napi::Value AudioSession::ProcessRequest(const Napi::CallbackInfo& info) { return env.Undefined(); } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessRequestOn(env, impl_, info[0], std::move(owner)); + return ProcessRequestOn(env, impl_, info[0], std::move(owner), manager_keepalive_); } Napi::Value AudioSession::ProcessStreamingRequest(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (ThrowIfDisposed(env)) return env.Undefined(); Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessStreamingRequestOn(env, impl_, info, std::move(owner)); + return ProcessStreamingRequestOn(env, impl_, info, std::move(owner), manager_keepalive_); } Napi::Value AudioSession::SetOptions(const Napi::CallbackInfo& info) { @@ -646,6 +654,7 @@ Napi::Value AudioSession::SetOptions(const Napi::CallbackInfo& info) { Napi::Value AudioSession::Dispose(const Napi::CallbackInfo& info) { impl_.reset(); + manager_keepalive_.reset(); return info.Env().Undefined(); } From ae6d8f0f100c10398a8a659f00cc15d7d28c15c5 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 04:57:34 -0500 Subject: [PATCH 57/77] Separate JS disposal state from native leases Use an explicit lifecycle disposed flag for cached Catalog and Model handles, check manager disposal before creating download callbacks, and serialize C API URL snapshot publication with web-service stop. Files changed: - sdk_v2/cpp/src/c_api.cc - sdk_v2/js/native/src/catalog.cc - sdk_v2/js/native/src/catalog.h - sdk_v2/js/native/src/manager.cc - sdk_v2/js/native/src/manager.h - sdk_v2/js/native/src/model.cc - sdk_v2/js/native/src/model.h Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/c_api.cc | 23 +++++++-------- sdk_v2/js/native/src/catalog.cc | 42 ++++++++++++++++----------- sdk_v2/js/native/src/catalog.h | 4 +++ sdk_v2/js/native/src/manager.cc | 2 ++ sdk_v2/js/native/src/manager.h | 6 ++++ sdk_v2/js/native/src/model.cc | 50 +++++++++++++++++++++------------ sdk_v2/js/native/src/model.h | 4 +++ 7 files changed, 86 insertions(+), 45 deletions(-) diff --git a/sdk_v2/cpp/src/c_api.cc b/sdk_v2/cpp/src/c_api.cc index 2987bdc17..c36f55df0 100644 --- a/sdk_v2/cpp/src/c_api.cc +++ b/sdk_v2/cpp/src/c_api.cc @@ -381,7 +381,14 @@ FL_API_STATUS_IMPL(Manager_WebServiceUrlsImpl, const flManager* manager, return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null argument"); } + std::lock_guard lock(manager->urls_cache_mutex); auto urls = manager->impl.GetWebServiceUrls(); + if (urls.empty()) { + *out_urls = nullptr; + *out_num_urls = 0; + return nullptr; + } + auto snapshot = std::make_unique(); snapshot->strings = std::move(urls); snapshot->pointers.reserve(snapshot->strings.size()); @@ -390,13 +397,9 @@ FL_API_STATUS_IMPL(Manager_WebServiceUrlsImpl, const flManager* manager, } auto* snapshot_ptr = snapshot.get(); - { - std::lock_guard lock(manager->urls_cache_mutex); - manager->urls_snapshots.push_back(std::move(snapshot)); - *out_urls = snapshot_ptr->pointers.empty() ? nullptr : snapshot_ptr->pointers.data(); - *out_num_urls = snapshot_ptr->pointers.size(); - } - + manager->urls_snapshots.push_back(std::move(snapshot)); + *out_urls = snapshot_ptr->pointers.data(); + *out_num_urls = snapshot_ptr->pointers.size(); return nullptr; API_IMPL_END } @@ -407,11 +410,9 @@ FL_API_STATUS_IMPL(Manager_WebServiceStopImpl, flManager* manager) { return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null manager"); } + std::lock_guard lock(manager->urls_cache_mutex); manager->impl.StopWebService(); - { - std::lock_guard lock(manager->urls_cache_mutex); - manager->urls_snapshots.clear(); - } + manager->urls_snapshots.clear(); return nullptr; API_IMPL_END } diff --git a/sdk_v2/js/native/src/catalog.cc b/sdk_v2/js/native/src/catalog.cc index f7cb868ce..fbec56b20 100644 --- a/sdk_v2/js/native/src/catalog.cc +++ b/sdk_v2/js/native/src/catalog.cc @@ -5,6 +5,7 @@ #include "addon_data.h" #include "errors.h" #include "model.h" +#include "manager.h" #include @@ -17,7 +18,11 @@ namespace foundry_local_node { namespace { std::shared_ptr LockManagerOrThrow( - const std::weak_ptr& manager_keepalive) { + const std::weak_ptr& manager_keepalive, + const std::shared_ptr& lifecycle) { + if (!lifecycle || lifecycle->disposed.load(std::memory_order_acquire)) { + throw foundry_local::Error("Manager has been disposed", FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } auto manager = manager_keepalive.lock(); if (!manager) { throw foundry_local::Error("Manager has been disposed", FOUNDRY_LOCAL_ERROR_INVALID_USAGE); @@ -28,7 +33,8 @@ std::shared_ptr LockManagerOrThrow( // Wrap a ModelList (rvalue) into a JS array of Model handles, each pinning the // passed-in manager reference. Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, Napi::ObjectReference manager, - std::weak_ptr manager_keepalive) { + std::weak_ptr manager_keepalive, + std::shared_ptr lifecycle) { auto list = std::make_shared(std::move(ml)); auto models = list->Models(); Napi::Array arr = Napi::Array::New(env, models.size()); @@ -38,6 +44,7 @@ Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, Napi::Obje token.keepalive = list; token.manager = Napi::Reference::New(manager.Value(), 1); token.manager_keepalive = manager_keepalive; + token.lifecycle = lifecycle; arr.Set(static_cast(i), Model::NewInstance(env, std::move(token))); } return arr; @@ -46,7 +53,8 @@ Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, Napi::Obje // Wrap an owning unique_ptr into a JS Model (or undefined when null). Napi::Value WrapOwnedModelOrUndefined(Napi::Env env, std::unique_ptr owned, Napi::ObjectReference manager, - std::weak_ptr manager_keepalive) { + std::weak_ptr manager_keepalive, + std::shared_ptr lifecycle) { if (!owned) { return env.Undefined(); } @@ -58,6 +66,7 @@ Napi::Value WrapOwnedModelOrUndefined(Napi::Env env, std::unique_ptr(inf impl_ = token->impl; manager_ = std::move(token->manager); manager_keepalive_ = std::move(token->manager_keepalive); + lifecycle_ = std::move(token->lifecycle); } Napi::Value Catalog::GetName(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; std::string_view name = impl_->GetName(); return Napi::String::New(env, std::string(name)); @@ -141,9 +151,9 @@ Napi::Value Catalog::GetModels(const Napi::CallbackInfo& info) { Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked( env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; - return WrapModelList(env, impl_->GetModels(), std::move(mgr), manager_keepalive_); + return WrapModelList(env, impl_->GetModels(), std::move(mgr), manager_keepalive_, lifecycle_); }); } @@ -151,9 +161,9 @@ Napi::Value Catalog::GetCachedModels(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; - return WrapModelList(env, impl_->GetCachedModels(), std::move(mgr), manager_keepalive_); + return WrapModelList(env, impl_->GetCachedModels(), std::move(mgr), manager_keepalive_, lifecycle_); }); } @@ -161,9 +171,9 @@ Napi::Value Catalog::GetLoadedModels(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; - return WrapModelList(env, impl_->GetLoadedModels(), std::move(mgr), manager_keepalive_); + return WrapModelList(env, impl_->GetLoadedModels(), std::move(mgr), manager_keepalive_, lifecycle_); }); } @@ -178,10 +188,10 @@ Napi::Value Catalog::GetModel(const Napi::CallbackInfo& info) { std::string alias = info[0].As(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; auto owned = impl_->GetModel(alias); - return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_); + return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_, lifecycle_); }); } @@ -194,10 +204,10 @@ Napi::Value Catalog::GetModelVariant(const Napi::CallbackInfo& info) { std::string model_id = info[0].As(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; auto owned = impl_->GetModelVariant(model_id); - return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_); + return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_, lifecycle_); }); } @@ -214,10 +224,10 @@ Napi::Value Catalog::GetLatestVersion(const Napi::CallbackInfo& info) { } Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; auto owned = impl_->GetLatestVersion(*arg); - return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_); + return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_, lifecycle_); }); } diff --git a/sdk_v2/js/native/src/catalog.h b/sdk_v2/js/native/src/catalog.h index df768ceb6..6d8ce8fa3 100644 --- a/sdk_v2/js/native/src/catalog.h +++ b/sdk_v2/js/native/src/catalog.h @@ -21,10 +21,13 @@ namespace foundry_local_node { +struct ManagerLifecycle; + struct CatalogCtorToken { foundry_local::ICatalog* impl = nullptr; Napi::ObjectReference manager; // pins the owning Manager std::weak_ptr manager_keepalive; + std::shared_ptr lifecycle; }; class Catalog : public Napi::ObjectWrap { @@ -46,6 +49,7 @@ class Catalog : public Napi::ObjectWrap { foundry_local::ICatalog* impl_ = nullptr; Napi::ObjectReference manager_; std::weak_ptr manager_keepalive_; + std::shared_ptr lifecycle_; }; } // namespace foundry_local_node diff --git a/sdk_v2/js/native/src/manager.cc b/sdk_v2/js/native/src/manager.cc index 3b11e6d25..5677cc13e 100644 --- a/sdk_v2/js/native/src/manager.cc +++ b/sdk_v2/js/native/src/manager.cc @@ -228,6 +228,7 @@ Napi::Value Manager::GetCatalog(const Napi::CallbackInfo& info) { token.impl = &cat; token.manager = std::move(owner); token.manager_keepalive = impl_; + token.lifecycle = lifecycle_; return Catalog::NewInstance(env, std::move(token)); }); } @@ -235,6 +236,7 @@ Napi::Value Manager::GetCatalog(const Napi::CallbackInfo& info) { Napi::Value Manager::Dispose(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); // Idempotent — releasing an already-null unique_ptr is a no-op. + lifecycle_->disposed.store(true, std::memory_order_release); impl_.reset(); return env.Undefined(); } diff --git a/sdk_v2/js/native/src/manager.h b/sdk_v2/js/native/src/manager.h index 8251b6568..757d05216 100644 --- a/sdk_v2/js/native/src/manager.h +++ b/sdk_v2/js/native/src/manager.h @@ -15,10 +15,15 @@ #include +#include #include namespace foundry_local_node { +struct ManagerLifecycle { + std::atomic disposed{false}; +}; + class Manager : public Napi::ObjectWrap { public: static Napi::Function Init(Napi::Env env); @@ -56,6 +61,7 @@ class Manager : public Napi::ObjectWrap { bool ThrowIfDisposed(Napi::Env env); std::shared_ptr impl_; + std::shared_ptr lifecycle_ = std::make_shared(); }; } // namespace foundry_local_node diff --git a/sdk_v2/js/native/src/model.cc b/sdk_v2/js/native/src/model.cc index c67e687ca..4e89cfbd0 100644 --- a/sdk_v2/js/native/src/model.cc +++ b/sdk_v2/js/native/src/model.cc @@ -4,6 +4,7 @@ #include "addon_data.h" #include "errors.h" +#include "manager.h" #include "promise_worker.h" #include @@ -87,7 +88,11 @@ void ThrowFoundryLocalError(Napi::Env env, int code, const std::string& msg) { } std::shared_ptr LockManagerOrThrow( - const std::weak_ptr& manager_keepalive) { + const std::weak_ptr& manager_keepalive, + const std::shared_ptr& lifecycle) { + if (!lifecycle || lifecycle->disposed.load(std::memory_order_acquire)) { + throw foundry_local::Error("Manager has been disposed", FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } auto manager = LockManager(manager_keepalive); if (!manager) { throw foundry_local::Error("Manager has been disposed", FOUNDRY_LOCAL_ERROR_INVALID_USAGE); @@ -96,7 +101,12 @@ std::shared_ptr LockManagerOrThrow( } std::shared_ptr LockManagerOrThrowJs( - Napi::Env env, const std::weak_ptr& manager_keepalive) { + Napi::Env env, const std::weak_ptr& manager_keepalive, + const std::shared_ptr& lifecycle) { + if (!lifecycle || lifecycle->disposed.load(std::memory_order_acquire)) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); + return nullptr; + } auto manager = LockManager(manager_keepalive); if (!manager) { ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); @@ -176,7 +186,8 @@ Napi::Object SnapshotModelInfo(Napi::Env env, const foundry_local::ModelInfo& in // whose keepalive holds the shared ModelList. Napi::Array WrapModelList(Napi::Env env, std::shared_ptr list, Napi::ObjectReference manager, - std::weak_ptr manager_keepalive) { + std::weak_ptr manager_keepalive, + std::shared_ptr lifecycle) { auto models = list->Models(); Napi::Array arr = Napi::Array::New(env, models.size()); for (size_t i = 0; i < models.size(); ++i) { @@ -186,6 +197,7 @@ Napi::Array WrapModelList(Napi::Env env, std::shared_ptr::New(manager.Value(), 1); token.manager_keepalive = manager_keepalive; + token.lifecycle = lifecycle; arr.Set(static_cast(i), Model::NewInstance(env, std::move(token))); } return arr; @@ -232,13 +244,14 @@ Model::Model(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) { impl_ = token->impl; keepalive_ = std::move(token->keepalive); manager_keepalive_ = std::move(token->manager_keepalive); + lifecycle_ = std::move(token->lifecycle); manager_ = std::move(token->manager); } Napi::Value Model::GetInfo(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; foundry_local::ModelInfo mi = impl_->GetInfo(); Napi::Object snapshot = SnapshotModelInfo(env, mi); @@ -250,7 +263,7 @@ Napi::Value Model::GetInfo(const Napi::CallbackInfo& info) { Napi::Value Model::IsCached(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; return Napi::Boolean::New(env, impl_->IsCached()); }); @@ -259,7 +272,7 @@ Napi::Value Model::IsCached(const Napi::CallbackInfo& info) { Napi::Value Model::IsLoaded(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; return Napi::Boolean::New(env, impl_->IsLoaded()); }); @@ -268,7 +281,7 @@ Napi::Value Model::IsLoaded(const Napi::CallbackInfo& info) { Napi::Value Model::GetPath(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; std::string_view p = impl_->GetPath(); return Napi::String::New(env, std::string(p)); @@ -279,10 +292,10 @@ Napi::Value Model::GetVariants(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference owner_clone = Napi::Reference::New(manager_.Value(), 1); return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; auto list = std::make_shared(impl_->GetVariants()); - return WrapModelList(env, std::move(list), std::move(owner_clone), manager_keepalive_); + return WrapModelList(env, std::move(list), std::move(owner_clone), manager_keepalive_, lifecycle_); }); } @@ -302,7 +315,7 @@ Napi::Value Model::Load(const Napi::CallbackInfo& info) { Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); foundry_local::IModel* m = impl_; auto keepalive = keepalive_; - auto manager_keepalive = LockManagerOrThrowJs(env, manager_keepalive_); + auto manager_keepalive = LockManagerOrThrowJs(env, manager_keepalive_, lifecycle_); if (!manager_keepalive) { return env.Undefined(); } @@ -320,7 +333,7 @@ Napi::Value Model::Unload(const Napi::CallbackInfo& info) { Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); foundry_local::IModel* m = impl_; auto keepalive = keepalive_; - auto manager_keepalive = LockManagerOrThrowJs(env, manager_keepalive_); + auto manager_keepalive = LockManagerOrThrowJs(env, manager_keepalive_, lifecycle_); if (!manager_keepalive) { return env.Undefined(); } @@ -426,6 +439,11 @@ Napi::Value Model::Download(const Napi::CallbackInfo& info) { return env.Undefined(); } + auto manager_keepalive = LockManagerOrThrowJs(env, manager_keepalive_, lifecycle_); + if (!manager_keepalive) { + return env.Undefined(); + } + Napi::ThreadSafeFunction tsfn; if (info.Length() >= 1 && info[0].IsFunction()) { tsfn = Napi::ThreadSafeFunction::New(env, info[0].As(), @@ -439,10 +457,6 @@ Napi::Value Model::Download(const Napi::CallbackInfo& info) { } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - auto manager_keepalive = LockManagerOrThrowJs(env, manager_keepalive_); - if (!manager_keepalive) { - return env.Undefined(); - } auto* w = new DownloadWorker(env, impl_, keepalive_, std::move(manager_keepalive), std::move(owner), std::move(tsfn)); Napi::Promise p = w->Promise(); @@ -460,7 +474,7 @@ Napi::Value Model::RemoveFromCache(const Napi::CallbackInfo& info) { // V1's contract is `removeFromCache(): void` so we do not bounce to a // worker. return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); (void)manager_alive; impl_->RemoveFromCache(); return env.Undefined(); @@ -489,8 +503,8 @@ Napi::Value Model::SelectVariant(const Napi::CallbackInfo& info) { } return CallChecked(env, [&]() -> Napi::Value { - auto manager_alive = LockManagerOrThrow(manager_keepalive_); - auto variant_manager_alive = LockManagerOrThrow(variant->manager_keepalive_); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + auto variant_manager_alive = LockManagerOrThrow(variant->manager_keepalive_, variant->lifecycle_); (void)manager_alive; (void)variant_manager_alive; impl_->SelectVariant(*variant->impl_); diff --git a/sdk_v2/js/native/src/model.h b/sdk_v2/js/native/src/model.h index 150ed5213..e67c7cb44 100644 --- a/sdk_v2/js/native/src/model.h +++ b/sdk_v2/js/native/src/model.h @@ -32,6 +32,8 @@ namespace foundry_local_node { +struct ManagerLifecycle; + struct ModelCtorToken { // The IModel accessor. Never null when the token is constructed. foundry_local::IModel* impl = nullptr; @@ -40,6 +42,7 @@ struct ModelCtorToken { // lifetime. std::shared_ptr keepalive; std::weak_ptr manager_keepalive; + std::shared_ptr lifecycle; // Pins the parent Manager so its native handle (and the Catalog's flCatalog* // which the IModel views into) cannot be released first. Napi::ObjectReference manager; @@ -81,6 +84,7 @@ class Model : public Napi::ObjectWrap { foundry_local::IModel* impl_ = nullptr; std::shared_ptr keepalive_; std::weak_ptr manager_keepalive_; + std::shared_ptr lifecycle_; Napi::ObjectReference manager_; }; From 98d0b7051639878bcd19bbc200e97b9e420ab3e5 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 09:36:32 -0500 Subject: [PATCH 58/77] Guard JS session creation after manager disposal Check the explicit manager lifecycle flag before creating sessions and clear streaming callbacks on all worker exit paths after cancellation changes. Files changed: - sdk_v2/js/native/src/model.cc - sdk_v2/js/native/src/model.h - sdk_v2/js/native/src/session.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/js/native/src/model.cc | 4 ++++ sdk_v2/js/native/src/model.h | 1 + sdk_v2/js/native/src/session.cc | 12 ++++++------ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/sdk_v2/js/native/src/model.cc b/sdk_v2/js/native/src/model.cc index 4e89cfbd0..cfe4793df 100644 --- a/sdk_v2/js/native/src/model.cc +++ b/sdk_v2/js/native/src/model.cc @@ -248,6 +248,10 @@ Model::Model(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) { manager_ = std::move(token->manager); } +bool Model::manager_disposed() const noexcept { + return !lifecycle_ || lifecycle_->disposed.load(std::memory_order_acquire); +} + Napi::Value Model::GetInfo(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { diff --git a/sdk_v2/js/native/src/model.h b/sdk_v2/js/native/src/model.h index e67c7cb44..c02e148b1 100644 --- a/sdk_v2/js/native/src/model.h +++ b/sdk_v2/js/native/src/model.h @@ -67,6 +67,7 @@ class Model : public Napi::ObjectWrap { // the parent Manager ObjectReference and pin it for the session lifetime. const Napi::ObjectReference& manager() const noexcept { return manager_; } std::shared_ptr manager_keepalive() const noexcept { return manager_keepalive_.lock(); } + bool manager_disposed() const noexcept; private: Napi::Value GetInfo(const Napi::CallbackInfo& info); diff --git a/sdk_v2/js/native/src/session.cc b/sdk_v2/js/native/src/session.cc index bc7d8b942..321515de7 100644 --- a/sdk_v2/js/native/src/session.cc +++ b/sdk_v2/js/native/src/session.cc @@ -219,9 +219,6 @@ class StreamWorker : public Napi::AsyncWorker { return 0; }); ctx_->response = std::make_shared(sess_->ProcessRequest(*req_)); - // Drop the callback so any stale shared state in the lambda is released - // before the Session is re-used for a follow-up request. - sess_->SetStreamingCallback(nullptr); } catch (const foundry_local::Error& e) { ctx_->errored = true; ctx_->err_code = static_cast(e.Code()); @@ -234,6 +231,9 @@ class StreamWorker : public Napi::AsyncWorker { ctx_->errored = true; ctx_->err_msg = "Unknown native exception"; } + // Drop the callback so any stale shared state in the lambda is released + // before the Session is re-used for a follow-up request. + sess_->SetStreamingCallback(nullptr); } // Promise resolution happens in FinalizeStream — overriding OnOK/OnError @@ -329,7 +329,7 @@ ChatSession::ChatSession(const Napi::CallbackInfo& info) : Napi::ObjectWrapmanager_keepalive(); - if (!manager_keepalive) { + if (model->manager_disposed() || !manager_keepalive) { ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); return; } @@ -493,7 +493,7 @@ EmbeddingsSession::EmbeddingsSession(const Napi::CallbackInfo& info) } try { auto manager_keepalive = model->manager_keepalive(); - if (!manager_keepalive) { + if (model->manager_disposed() || !manager_keepalive) { ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); return; } @@ -594,7 +594,7 @@ AudioSession::AudioSession(const Napi::CallbackInfo& info) } try { auto manager_keepalive = model->manager_keepalive(); - if (!manager_keepalive) { + if (model->manager_disposed() || !manager_keepalive) { ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); return; } From a816c55dcb8b333c3894d16efb4b7fa4d2fb87b5 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 10:04:53 -0500 Subject: [PATCH 59/77] Validate latest-version model lifecycle Reject Catalog.getLatestVersion calls that pass a Model from a disposed manager before dereferencing the native model pointer. Files changed: - sdk_v2/js/native/src/catalog.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/js/native/src/catalog.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sdk_v2/js/native/src/catalog.cc b/sdk_v2/js/native/src/catalog.cc index fbec56b20..d8a210b90 100644 --- a/sdk_v2/js/native/src/catalog.cc +++ b/sdk_v2/js/native/src/catalog.cc @@ -70,8 +70,8 @@ Napi::Value WrapOwnedModelOrUndefined(Napi::Env env, std::unique_ptr::Unwrap(obj); - return m != nullptr ? m->native_impl() : nullptr; + return Napi::ObjectWrap::Unwrap(obj); } Napi::ObjectReference CloneManager(const Napi::ObjectReference& mgr) { @@ -217,16 +216,19 @@ Napi::Value Catalog::GetLatestVersion(const Napi::CallbackInfo& info) { Napi::TypeError::New(env, "getLatestVersion(model: Model)").ThrowAsJavaScriptException(); return env.Undefined(); } - foundry_local::IModel* arg = ExtractIModel(info[0]); - if (arg == nullptr) { + Model* arg = ExtractModel(info[0]); + if (arg == nullptr || arg->native_impl() == nullptr) { Napi::TypeError::New(env, "getLatestVersion: argument must be a Model").ThrowAsJavaScriptException(); return env.Undefined(); } Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + if (arg->manager_disposed() || !arg->manager_keepalive()) { + throw foundry_local::Error("Manager has been disposed", FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } (void)manager_alive; - auto owned = impl_->GetLatestVersion(*arg); + auto owned = impl_->GetLatestVersion(*arg->native_impl()); return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_, lifecycle_); }); } From b0ea28e693359dd34327e247dc5e75b6509b815b Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 10:53:33 -0500 Subject: [PATCH 60/77] Block manager disposal with live JS sessions Track active JS session leases in the native manager lifecycle so manager disposal fails while sessions or in-flight session workers still hold native state. Files changed: - sdk_v2/js/native/src/manager.cc - sdk_v2/js/native/src/manager.h - sdk_v2/js/native/src/model.h - sdk_v2/js/native/src/session.cc - sdk_v2/js/native/src/session.h - sdk_v2/js/src/foundryLocalManager.ts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/js/native/src/manager.cc | 5 ++ sdk_v2/js/native/src/manager.h | 1 + sdk_v2/js/native/src/model.h | 1 + sdk_v2/js/native/src/session.cc | 79 +++++++++++++++++++++++----- sdk_v2/js/native/src/session.h | 11 ++++ sdk_v2/js/src/foundryLocalManager.ts | 2 +- 6 files changed, 84 insertions(+), 15 deletions(-) diff --git a/sdk_v2/js/native/src/manager.cc b/sdk_v2/js/native/src/manager.cc index 5677cc13e..c98fa0f36 100644 --- a/sdk_v2/js/native/src/manager.cc +++ b/sdk_v2/js/native/src/manager.cc @@ -236,6 +236,11 @@ Napi::Value Manager::GetCatalog(const Napi::CallbackInfo& info) { Napi::Value Manager::Dispose(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); // Idempotent — releasing an already-null unique_ptr is a no-op. + if (lifecycle_->active_sessions.load(std::memory_order_acquire) > 0) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "Manager has active sessions; dispose sessions before disposing the manager"); + return env.Undefined(); + } lifecycle_->disposed.store(true, std::memory_order_release); impl_.reset(); return env.Undefined(); diff --git a/sdk_v2/js/native/src/manager.h b/sdk_v2/js/native/src/manager.h index 757d05216..227631725 100644 --- a/sdk_v2/js/native/src/manager.h +++ b/sdk_v2/js/native/src/manager.h @@ -22,6 +22,7 @@ namespace foundry_local_node { struct ManagerLifecycle { std::atomic disposed{false}; + std::atomic active_sessions{0}; }; class Manager : public Napi::ObjectWrap { diff --git a/sdk_v2/js/native/src/model.h b/sdk_v2/js/native/src/model.h index c02e148b1..e90dc3977 100644 --- a/sdk_v2/js/native/src/model.h +++ b/sdk_v2/js/native/src/model.h @@ -67,6 +67,7 @@ class Model : public Napi::ObjectWrap { // the parent Manager ObjectReference and pin it for the session lifetime. const Napi::ObjectReference& manager() const noexcept { return manager_; } std::shared_ptr manager_keepalive() const noexcept { return manager_keepalive_.lock(); } + std::shared_ptr manager_lifecycle() const noexcept { return lifecycle_; } bool manager_disposed() const noexcept; private: diff --git a/sdk_v2/js/native/src/session.cc b/sdk_v2/js/native/src/session.cc index 321515de7..d71f2896e 100644 --- a/sdk_v2/js/native/src/session.cc +++ b/sdk_v2/js/native/src/session.cc @@ -5,6 +5,7 @@ #include "addon_data.h" #include "errors.h" #include "items.h" +#include "manager.h" #include "model.h" #include "promise_worker.h" #include "request.h" @@ -22,6 +23,36 @@ namespace foundry_local_node { namespace { +struct SessionLease { + explicit SessionLease(std::shared_ptr lifecycle) + : lifecycle_(std::move(lifecycle)) { + if (lifecycle_) { + lifecycle_->active_sessions.fetch_add(1, std::memory_order_acq_rel); + } + } + + ~SessionLease() { + if (lifecycle_) { + lifecycle_->active_sessions.fetch_sub(1, std::memory_order_acq_rel); + } + } + + std::shared_ptr lifecycle_; +}; + +std::shared_ptr MakeSessionLease(std::shared_ptr lifecycle) { + return std::make_shared(std::move(lifecycle)); +} + +template +void ReleaseSessionState(std::shared_ptr& impl, + std::shared_ptr& manager_keepalive, + std::shared_ptr& session_lease) { + impl.reset(); + manager_keepalive.reset(); + session_lease.reset(); +} + const char* FinishReasonToString(flFinishReason r) { switch (r) { case FOUNDRY_LOCAL_FINISH_STOP: @@ -99,7 +130,8 @@ foundry_local::Request* UnwrapRequest(Napi::Env env, const Napi::Value& v) { template Napi::Value ProcessRequestOn(Napi::Env env, std::shared_ptr sess, const Napi::Value& request_arg, Napi::ObjectReference manager_ref, - std::shared_ptr manager_keepalive) { + std::shared_ptr manager_keepalive, + std::shared_ptr session_lease) { foundry_local::Request* req = UnwrapRequest(env, request_arg); if (req == nullptr) return env.Undefined(); // pending exception Napi::ObjectReference req_pin = Napi::Reference::New(request_arg.As(), 1); @@ -109,9 +141,10 @@ Napi::Value ProcessRequestOn(Napi::Env env, std::shared_ptr sess, const N Napi::ObjectReference manager; Napi::ObjectReference request; std::shared_ptr native_manager; + std::shared_ptr session_lease; }; auto pins = std::make_shared( - Pins{std::move(manager_ref), std::move(req_pin), std::move(manager_keepalive)}); + Pins{std::move(manager_ref), std::move(req_pin), std::move(manager_keepalive), std::move(session_lease)}); return PromiseWorker::Run( env, @@ -152,6 +185,7 @@ struct StreamCtx { Napi::ObjectReference manager; Napi::ObjectReference request; std::shared_ptr native_manager; + std::shared_ptr session_lease; std::shared_ptr response; std::string err_msg; int err_code = 0; @@ -263,7 +297,8 @@ class StreamWorker : public Napi::AsyncWorker { template Napi::Value ProcessStreamingRequestOn(Napi::Env env, std::shared_ptr sess, const Napi::CallbackInfo& info, Napi::ObjectReference manager_ref, - std::shared_ptr manager_keepalive) { + std::shared_ptr manager_keepalive, + std::shared_ptr session_lease) { if (info.Length() < 2 || !info[1].IsFunction()) { Napi::TypeError::New(env, "processStreamingRequest(request: Request, onItem: (item) => void)") .ThrowAsJavaScriptException(); @@ -278,6 +313,7 @@ Napi::Value ProcessStreamingRequestOn(Napi::Env env, std::shared_ptr sess std::move(manager_ref), std::move(req_pin), std::move(manager_keepalive), + std::move(session_lease), nullptr, "", 0, @@ -334,6 +370,8 @@ ChatSession::ChatSession(const Napi::CallbackInfo& info) : Napi::ObjectWrap(*native); + lifecycle_ = model->manager_lifecycle(); + session_lease_ = MakeSessionLease(lifecycle_); manager_keepalive_ = std::move(manager_keepalive); } catch (const foundry_local::Error& e) { ThrowFoundryLocalError(env, static_cast(e.Code()), e.what()); @@ -345,6 +383,10 @@ ChatSession::ChatSession(const Napi::CallbackInfo& info) : Napi::ObjectWrap::New(model->manager().Value(), 1); } +ChatSession::~ChatSession() { + ReleaseSessionState(impl_, manager_keepalive_, session_lease_); +} + bool ChatSession::ThrowIfDisposed(Napi::Env env) { if (impl_ == nullptr) { ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -362,14 +404,14 @@ Napi::Value ChatSession::ProcessRequest(const Napi::CallbackInfo& info) { return env.Undefined(); } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessRequestOn(env, impl_, info[0], std::move(owner), manager_keepalive_); + return ProcessRequestOn(env, impl_, info[0], std::move(owner), manager_keepalive_, session_lease_); } Napi::Value ChatSession::ProcessStreamingRequest(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (ThrowIfDisposed(env)) return env.Undefined(); Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessStreamingRequestOn(env, impl_, info, std::move(owner), manager_keepalive_); + return ProcessStreamingRequestOn(env, impl_, info, std::move(owner), manager_keepalive_, session_lease_); } Napi::Value ChatSession::SetOptions(const Napi::CallbackInfo& info) { @@ -449,8 +491,7 @@ Napi::Value ChatSession::UndoTurns(const Napi::CallbackInfo& info) { } Napi::Value ChatSession::Dispose(const Napi::CallbackInfo& info) { - impl_.reset(); - manager_keepalive_.reset(); + ReleaseSessionState(impl_, manager_keepalive_, session_lease_); return info.Env().Undefined(); } @@ -498,6 +539,8 @@ EmbeddingsSession::EmbeddingsSession(const Napi::CallbackInfo& info) return; } impl_ = std::make_shared(*native); + lifecycle_ = model->manager_lifecycle(); + session_lease_ = MakeSessionLease(lifecycle_); manager_keepalive_ = std::move(manager_keepalive); } catch (const foundry_local::Error& e) { ThrowFoundryLocalError(env, static_cast(e.Code()), e.what()); @@ -509,6 +552,10 @@ EmbeddingsSession::EmbeddingsSession(const Napi::CallbackInfo& info) manager_ = Napi::Reference::New(model->manager().Value(), 1); } +EmbeddingsSession::~EmbeddingsSession() { + ReleaseSessionState(impl_, manager_keepalive_, session_lease_); +} + bool EmbeddingsSession::ThrowIfDisposed(Napi::Env env) { if (impl_ == nullptr) { ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -526,7 +573,7 @@ Napi::Value EmbeddingsSession::ProcessRequest(const Napi::CallbackInfo& info) { return env.Undefined(); } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessRequestOn(env, impl_, info[0], std::move(owner), manager_keepalive_); + return ProcessRequestOn(env, impl_, info[0], std::move(owner), manager_keepalive_, session_lease_); } Napi::Value EmbeddingsSession::SetOptions(const Napi::CallbackInfo& info) { @@ -545,8 +592,7 @@ Napi::Value EmbeddingsSession::SetOptions(const Napi::CallbackInfo& info) { } Napi::Value EmbeddingsSession::Dispose(const Napi::CallbackInfo& info) { - impl_.reset(); - manager_keepalive_.reset(); + ReleaseSessionState(impl_, manager_keepalive_, session_lease_); return info.Env().Undefined(); } @@ -599,6 +645,8 @@ AudioSession::AudioSession(const Napi::CallbackInfo& info) return; } impl_ = std::make_shared(*native); + lifecycle_ = model->manager_lifecycle(); + session_lease_ = MakeSessionLease(lifecycle_); manager_keepalive_ = std::move(manager_keepalive); } catch (const foundry_local::Error& e) { ThrowFoundryLocalError(env, static_cast(e.Code()), e.what()); @@ -610,6 +658,10 @@ AudioSession::AudioSession(const Napi::CallbackInfo& info) manager_ = Napi::Reference::New(model->manager().Value(), 1); } +AudioSession::~AudioSession() { + ReleaseSessionState(impl_, manager_keepalive_, session_lease_); +} + bool AudioSession::ThrowIfDisposed(Napi::Env env) { if (impl_ == nullptr) { ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -627,14 +679,14 @@ Napi::Value AudioSession::ProcessRequest(const Napi::CallbackInfo& info) { return env.Undefined(); } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessRequestOn(env, impl_, info[0], std::move(owner), manager_keepalive_); + return ProcessRequestOn(env, impl_, info[0], std::move(owner), manager_keepalive_, session_lease_); } Napi::Value AudioSession::ProcessStreamingRequest(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (ThrowIfDisposed(env)) return env.Undefined(); Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessStreamingRequestOn(env, impl_, info, std::move(owner), manager_keepalive_); + return ProcessStreamingRequestOn(env, impl_, info, std::move(owner), manager_keepalive_, session_lease_); } Napi::Value AudioSession::SetOptions(const Napi::CallbackInfo& info) { @@ -653,8 +705,7 @@ Napi::Value AudioSession::SetOptions(const Napi::CallbackInfo& info) { } Napi::Value AudioSession::Dispose(const Napi::CallbackInfo& info) { - impl_.reset(); - manager_keepalive_.reset(); + ReleaseSessionState(impl_, manager_keepalive_, session_lease_); return info.Env().Undefined(); } diff --git a/sdk_v2/js/native/src/session.h b/sdk_v2/js/native/src/session.h index 36dfeb254..796439e54 100644 --- a/sdk_v2/js/native/src/session.h +++ b/sdk_v2/js/native/src/session.h @@ -32,11 +32,14 @@ namespace foundry_local_node { +struct ManagerLifecycle; + class ChatSession : public Napi::ObjectWrap { public: static Napi::Function Init(Napi::Env env); explicit ChatSession(const Napi::CallbackInfo& info); + ~ChatSession(); private: Napi::Value ProcessRequest(const Napi::CallbackInfo& info); @@ -54,6 +57,8 @@ class ChatSession : public Napi::ObjectWrap { std::shared_ptr impl_; Napi::ObjectReference manager_; std::shared_ptr manager_keepalive_; + std::shared_ptr lifecycle_; + std::shared_ptr session_lease_; }; // Napi::ObjectWrap over foundry_local::EmbeddingsSession. @@ -74,6 +79,7 @@ class EmbeddingsSession : public Napi::ObjectWrap { static Napi::Function Init(Napi::Env env); explicit EmbeddingsSession(const Napi::CallbackInfo& info); + ~EmbeddingsSession(); private: Napi::Value ProcessRequest(const Napi::CallbackInfo& info); @@ -86,6 +92,8 @@ class EmbeddingsSession : public Napi::ObjectWrap { std::shared_ptr impl_; Napi::ObjectReference manager_; std::shared_ptr manager_keepalive_; + std::shared_ptr lifecycle_; + std::shared_ptr session_lease_; }; // Napi::ObjectWrap over foundry_local::AudioSession. @@ -103,6 +111,7 @@ class AudioSession : public Napi::ObjectWrap { static Napi::Function Init(Napi::Env env); explicit AudioSession(const Napi::CallbackInfo& info); + ~AudioSession(); private: Napi::Value ProcessRequest(const Napi::CallbackInfo& info); @@ -116,6 +125,8 @@ class AudioSession : public Napi::ObjectWrap { std::shared_ptr impl_; Napi::ObjectReference manager_; std::shared_ptr manager_keepalive_; + std::shared_ptr lifecycle_; + std::shared_ptr session_lease_; }; } // namespace foundry_local_node diff --git a/sdk_v2/js/src/foundryLocalManager.ts b/sdk_v2/js/src/foundryLocalManager.ts index 2d2772a47..f6fc77782 100644 --- a/sdk_v2/js/src/foundryLocalManager.ts +++ b/sdk_v2/js/src/foundryLocalManager.ts @@ -211,10 +211,10 @@ export class FoundryLocalManager { * (and any method on a `Catalog` or `Model` obtained through this manager) throws a `FoundryLocalError`. */ dispose(): void { + this.#native.dispose(); if (liveManager === this) { liveManager = undefined; } - this.#native.dispose(); this.#catalog = undefined; this.#urls = []; } From 17aa2c984264dd551bc8fefa95cfdda35bb3f0f7 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 11:36:23 -0500 Subject: [PATCH 61/77] Back off failed forced catalog refreshes Respect the retry delay after a forced refresh fails on an already-populated catalog, while preserving the existing catalog until the next retry window. Files changed: - sdk_v2/cpp/src/catalog/base_model_catalog.cc - sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/catalog/base_model_catalog.cc | 5 +- .../internal_api/base_model_catalog_test.cc | 52 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.cc b/sdk_v2/cpp/src/catalog/base_model_catalog.cc index 82de5d9ac..fe653ae5d 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.cc @@ -209,8 +209,9 @@ void BaseModelCatalog::EnsurePopulated(bool allow_refresh) const { // not worth the complexity to optimise.) std::lock_guard lock(mutex_); - bool needs_refresh = force_refresh_ || - (allow_refresh && std::chrono::steady_clock::now() >= next_refresh_at_); + auto now = std::chrono::steady_clock::now(); + bool needs_refresh = (force_refresh_ && now >= next_refresh_at_) || + (allow_refresh && now >= next_refresh_at_); if (populated_ && !needs_refresh) { return; diff --git a/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc b/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc index 8e0c9b434..dc5c93f46 100644 --- a/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc +++ b/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc @@ -9,6 +9,7 @@ // - Variant grouping behavior // #include "catalog/base_model_catalog.h" +#include "exception.h" #include "internal_api/test_helpers.h" #include "logger.h" #include "model.h" @@ -108,6 +109,38 @@ class QueryingTestCatalog : public BaseModelCatalog { mutable std::vector id_fetch_results_; }; +class FailingRefreshCatalog : public BaseModelCatalog { + public: + explicit FailingRefreshCatalog(ILogger& logger) : BaseModelCatalog("failing-refresh-catalog", logger) {} + + void SetModels(std::vector models) { + models_ = std::move(models); + } + + void FailNextFetch() { + fail_next_fetch_ = true; + } + + int FetchCount() const { + return fetch_count_; + } + + protected: + std::vector FetchModels() const override { + ++fetch_count_; + if (fail_next_fetch_) { + fail_next_fetch_ = false; + FL_THROW(FOUNDRY_LOCAL_ERROR_NETWORK, "simulated refresh failure"); + } + return std::move(models_); + } + + private: + mutable std::vector models_; + mutable bool fail_next_fetch_ = false; + mutable int fetch_count_ = 0; +}; + // Helper: create a Model from basic fields. static Model MakeModel(const std::string& model_id, const std::string& name, int version, const std::string& alias, @@ -246,6 +279,25 @@ TEST_F(BaseModelCatalogTest, RefreshPreservesExplicitVariantSelection) { EXPECT_EQ(container->Id(), "phi-3-mini-cpu:1"); } +TEST_F(BaseModelCatalogTest, ForcedRefreshFailureBacksOffWhenCatalogAlreadyPopulated) { + FailingRefreshCatalog catalog(logger_); + std::vector models; + models.push_back(MakeModel("phi-3-mini-cpu:1", "phi-3-mini-cpu", 1, "phi-3")); + catalog.SetModels(std::move(models)); + + ASSERT_NE(catalog.GetModel("phi-3"), nullptr); + EXPECT_EQ(catalog.FetchCount(), 1); + + catalog.FailNextFetch(); + catalog.InvalidateCache(); + + EXPECT_EQ(catalog.ListModels().size(), 1u); + EXPECT_EQ(catalog.FetchCount(), 2); + + EXPECT_EQ(catalog.ListModels().size(), 1u); + EXPECT_EQ(catalog.FetchCount(), 2); +} + TEST_F(BaseModelCatalogTest, GetModel_NotFound_ReturnsNullptr) { TestCatalog catalog(logger_); catalog.AddModel(MakeModel("phi-3-mini:1", "phi-3-mini", 1, "phi-3")); From 0092ff4dc0be04e1de3b2192ba15873819decc10 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 12:23:08 -0500 Subject: [PATCH 62/77] Preserve progress monotonicity on resume reset Reset complete-but-unfinalized sidecars before seeding progress, so discarded resume state never emits a high initial progress value before a fresh download pass. Files changed: - sdk_v2/cpp/src/download/blob_downloader.cc - sdk_v2/cpp/test/internal_api/download_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/download/blob_downloader.cc | 15 ++++---- sdk_v2/cpp/test/internal_api/download_test.cc | 36 +++++++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/sdk_v2/cpp/src/download/blob_downloader.cc b/sdk_v2/cpp/src/download/blob_downloader.cc index efd09200f..6ea57940c 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.cc +++ b/sdk_v2/cpp/src/download/blob_downloader.cc @@ -226,13 +226,6 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, } } - // Track cumulative bytes for progress reporting; seed with bytes already - // present on disk so percent stays monotonic across resume. - std::atomic bytes_completed{state->CalculateDownloadedSize()}; - if (bytes_written_cb && bytes_completed.load() > 0) { - bytes_written_cb(bytes_completed.load()); - } - auto pending = state->GetPendingChunks(); if (pending.empty()) { // A complete sidecar means a previous attempt finished all chunks but did not reach finalization. Do not trust @@ -249,10 +242,16 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "failed to persist reset download state for '" + local_path + "'"); } - bytes_completed.store(0); pending = state->GetPendingChunks(); } + // Track cumulative bytes for progress reporting; seed with bytes already + // present on disk so percent stays monotonic across resume. + std::atomic bytes_completed{state->CalculateDownloadedSize()}; + if (bytes_written_cb && bytes_completed.load() > 0) { + bytes_written_cb(bytes_completed.load()); + } + // Open the file writer once for the whole download. Open() pre-allocates // the file to blob_size if needed, preserving any existing bytes from a // resume. Concurrent WriteAt calls to disjoint ranges are thread-safe — the diff --git a/sdk_v2/cpp/test/internal_api/download_test.cc b/sdk_v2/cpp/test/internal_api/download_test.cc index fe4dc17c2..92fbf1931 100644 --- a/sdk_v2/cpp/test/internal_api/download_test.cc +++ b/sdk_v2/cpp/test/internal_api/download_test.cc @@ -1846,6 +1846,42 @@ TEST(AzureBlobDownloaderResumeTest, CleansUpSidecarOnEmptyBlob) { EXPECT_EQ(d.chunk_call_count.load(), 0); } +TEST(AzureBlobDownloaderResumeTest, CompleteUnfinalizedSidecarDoesNotEmitRegressingProgress) { + TempDir tmpdir; + auto local = tmpdir.path() / "blob.bin"; + + constexpr int32_t kChunkSize = 2 * 1024 * 1024; + constexpr int32_t kNumChunks = 2; + constexpr int64_t kBlobSize = static_cast(kNumChunks) * kChunkSize; + + { + std::ofstream f(local, std::ios::binary); + f.seekp(kBlobSize - 1); + f.put('\0'); + } + { + auto state = BlobDownloadState::CreateNew("blob", local, kBlobSize, kChunkSize, kNumChunks); + for (int32_t i = 0; i < kNumChunks; ++i) { + state->MarkChunkComplete(i); + } + ASSERT_TRUE(state->SaveState(fl::test::NullLog())); + } + + FakeChunkAzureDownloader d; + d.blob_size = kBlobSize; + std::vector progress; + + d.DownloadBlob(/*sas_uri=*/"", "blob", local.string(), /*max_concurrency=*/2, + [&](int64_t bytes) { progress.push_back(bytes); }); + + ASSERT_FALSE(progress.empty()); + EXPECT_LT(progress.front(), kBlobSize); + for (size_t i = 1; i < progress.size(); ++i) { + EXPECT_GE(progress[i], progress[i - 1]); + } + EXPECT_EQ(progress.back(), kBlobSize); +} + TEST(AzureBlobDownloaderResumeTest, ChunkFailureCancelsInFlightPeersFast) { TempDir tmpdir; auto local = tmpdir.path() / "blob.bin"; From 83343ba8738b41d10c08e51f5b282c90c4d3cd0e Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 13:26:32 -0500 Subject: [PATCH 63/77] Align shutdown disposal and telemetry input handling Reject web-service start during shutdown, avoid unmatched session-end telemetry, block manager disposal while native workers run, and sanitize/cap client User-Agent before telemetry. Files changed: - sdk_v2/cpp/src/manager.cc - sdk_v2/cpp/src/manager.h - sdk_v2/cpp/src/service/handler_utils.h - sdk_v2/cpp/src/service/web_service.cc - sdk_v2/js/native/src/manager.cc - sdk_v2/js/native/src/manager.h - sdk_v2/js/native/src/model.cc - sdk_v2/js/test/manager-dispose.test.ts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/manager.cc | 20 ++++++++++----- sdk_v2/cpp/src/manager.h | 1 + sdk_v2/cpp/src/service/handler_utils.h | 12 ++++++++- sdk_v2/cpp/src/service/web_service.cc | 6 +---- sdk_v2/js/native/src/manager.cc | 30 ++++++++++++++++++++-- sdk_v2/js/native/src/manager.h | 1 + sdk_v2/js/native/src/model.cc | 35 ++++++++++++++++++++++---- sdk_v2/js/test/manager-dispose.test.ts | 5 ++-- 8 files changed, 89 insertions(+), 21 deletions(-) diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index 7f22ca451..901d8edc6 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -510,6 +510,9 @@ void Manager::StartWebService() { FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "cannot start local web service when external_service_url is configured"); } + if (shutdown_requested_.load(std::memory_order_acquire)) { + FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "cannot start web service during shutdown"); + } web_service_ = std::make_unique(*catalog_, *logger_, *config_.model_cache_dir, *model_load_manager_, *session_manager_, *telemetry_, @@ -530,6 +533,7 @@ void Manager::StartWebService() { // carry ext.app.sesId and the backend gets session duration. try { telemetry_->StartSession(); + web_service_session_started_ = true; } catch (const std::exception& ex) { logger_->Log(LogLevel::Warning, std::string("telemetry StartSession failed: ") + ex.what()); } catch (...) { @@ -549,6 +553,7 @@ void Manager::StartWebService() { if (stop_after_start) { StopWebService(); + FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "web service stopped because shutdown was requested"); } tracker.SetStatus(ActionStatus::kSuccess); @@ -582,12 +587,15 @@ void Manager::StopWebService() { web_service_.reset(); web_service_running_ = false; bound_urls_.clear(); - try { - telemetry_->EndSession(); - } catch (const std::exception& ex) { - logger_->Log(LogLevel::Warning, std::string("telemetry EndSession failed: ") + ex.what()); - } catch (...) { - logger_->Log(LogLevel::Warning, "telemetry EndSession failed with unknown error"); + if (web_service_session_started_) { + web_service_session_started_ = false; + try { + telemetry_->EndSession(); + } catch (const std::exception& ex) { + logger_->Log(LogLevel::Warning, std::string("telemetry EndSession failed: ") + ex.what()); + } catch (...) { + logger_->Log(LogLevel::Warning, "telemetry EndSession failed with unknown error"); + } } tracker.SetStatus(ActionStatus::kSuccess); #else diff --git a/sdk_v2/cpp/src/manager.h b/sdk_v2/cpp/src/manager.h index 9b71a9410..37e1ce82d 100644 --- a/sdk_v2/cpp/src/manager.h +++ b/sdk_v2/cpp/src/manager.h @@ -150,6 +150,7 @@ class Manager { std::atomic shutdown_requested_{false}; std::atomic web_service_running_{false}; mutable std::mutex web_service_mutex_; + bool web_service_session_started_ = false; std::vector bound_urls_; #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE diff --git a/sdk_v2/cpp/src/service/handler_utils.h b/sdk_v2/cpp/src/service/handler_utils.h index c54fd7e91..23c016728 100644 --- a/sdk_v2/cpp/src/service/handler_utils.h +++ b/sdk_v2/cpp/src/service/handler_utils.h @@ -10,6 +10,8 @@ #include #include +#include "telemetry/telemetry_redaction.h" + #include #include #include @@ -69,7 +71,15 @@ inline std::string GetUserAgent(const std::shared_ptrgetHeader("User-Agent"); - return ua ? *ua : std::string{}; + if (!ua) { + return {}; + } + auto scrubbed = ScrubTelemetryErrorMessage(*ua); + constexpr size_t kMaxUserAgentLength = 256; + if (scrubbed.size() > kMaxUserAgentLength) { + scrubbed.resize(kMaxUserAgentLength); + } + return scrubbed; } // ======================================================================== diff --git a/sdk_v2/cpp/src/service/web_service.cc b/sdk_v2/cpp/src/service/web_service.cc index e743ee411..8672bf25c 100644 --- a/sdk_v2/cpp/src/service/web_service.cc +++ b/sdk_v2/cpp/src/service/web_service.cc @@ -227,13 +227,9 @@ class UnmatchedRouteInterceptor : public oatpp::web::server::interceptor::Reques return nullptr; // a handler will serve this request } - std::string user_agent; - if (auto ua = request->getHeader("User-Agent")) { - user_agent = *ua; - } { ActionTracker tracker(Action::kServiceRequestUnmatched, telemetry_, - InvocationContext::Direct(user_agent)); + InvocationContext::Direct(GetUserAgent(request))); tracker.SetStatus(ActionStatus::kClientError); } diff --git a/sdk_v2/js/native/src/manager.cc b/sdk_v2/js/native/src/manager.cc index c98fa0f36..84ca2db89 100644 --- a/sdk_v2/js/native/src/manager.cc +++ b/sdk_v2/js/native/src/manager.cc @@ -20,6 +20,24 @@ namespace foundry_local_node { namespace { +struct WorkerLease { + explicit WorkerLease(std::shared_ptr lifecycle) : lifecycle_(std::move(lifecycle)) { + if (lifecycle_) { + lifecycle_->active_workers.fetch_add(1, std::memory_order_acq_rel); + } + } + ~WorkerLease() { + if (lifecycle_) { + lifecycle_->active_workers.fetch_sub(1, std::memory_order_acq_rel); + } + } + std::shared_ptr lifecycle_; +}; + +std::shared_ptr MakeWorkerLease(std::shared_ptr lifecycle) { + return std::make_shared(std::move(lifecycle)); +} + Napi::Value ConvertEndpoints(Napi::Env env, std::vector& endpoints) { Napi::Array out = Napi::Array::New(env, endpoints.size()); for (size_t i = 0; i < endpoints.size(); ++i) { @@ -241,6 +259,11 @@ Napi::Value Manager::Dispose(const Napi::CallbackInfo& info) { "Manager has active sessions; dispose sessions before disposing the manager"); return env.Undefined(); } + if (lifecycle_->active_workers.load(std::memory_order_acquire) > 0) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "Manager has active native workers; await them before disposing the manager"); + return env.Undefined(); + } lifecycle_->disposed.store(true, std::memory_order_release); impl_.reset(); return env.Undefined(); @@ -296,12 +319,13 @@ namespace { // progress callback. Mirrors the pattern in model.cc's DownloadWorker. class EpDownloadWorker : public Napi::AsyncWorker { public: - EpDownloadWorker(Napi::Env env, std::shared_ptr impl, + EpDownloadWorker(Napi::Env env, std::shared_ptr impl, std::shared_ptr worker_lease, std::vector ep_names, Napi::ObjectReference owner, Napi::ThreadSafeFunction tsfn) : Napi::AsyncWorker(env), deferred_(Napi::Promise::Deferred::New(env)), impl_(impl), + worker_lease_(std::move(worker_lease)), ep_names_(std::move(ep_names)), owner_(std::move(owner)), tsfn_(std::move(tsfn)) {} @@ -366,6 +390,7 @@ class EpDownloadWorker : public Napi::AsyncWorker { Napi::Promise::Deferred deferred_; std::shared_ptr impl_; + std::shared_ptr worker_lease_; std::vector ep_names_; Napi::ObjectReference owner_; Napi::ThreadSafeFunction tsfn_; @@ -422,7 +447,8 @@ Napi::Value Manager::DownloadAndRegisterEps(const Napi::CallbackInfo& info) { } Napi::ObjectReference owner = Napi::Reference::New(info.This().As(), 1); - auto* w = new EpDownloadWorker(env, impl_, std::move(ep_names), std::move(owner), std::move(tsfn)); + auto* w = new EpDownloadWorker(env, impl_, MakeWorkerLease(lifecycle_), std::move(ep_names), std::move(owner), + std::move(tsfn)); Napi::Promise p = w->Promise(); w->Queue(); return p; diff --git a/sdk_v2/js/native/src/manager.h b/sdk_v2/js/native/src/manager.h index 227631725..c321a1c12 100644 --- a/sdk_v2/js/native/src/manager.h +++ b/sdk_v2/js/native/src/manager.h @@ -23,6 +23,7 @@ namespace foundry_local_node { struct ManagerLifecycle { std::atomic disposed{false}; std::atomic active_sessions{0}; + std::atomic active_workers{0}; }; class Manager : public Napi::ObjectWrap { diff --git a/sdk_v2/js/native/src/model.cc b/sdk_v2/js/native/src/model.cc index cfe4793df..a6605acdc 100644 --- a/sdk_v2/js/native/src/model.cc +++ b/sdk_v2/js/native/src/model.cc @@ -19,6 +19,24 @@ namespace foundry_local_node { namespace { +struct WorkerLease { + explicit WorkerLease(std::shared_ptr lifecycle) : lifecycle_(std::move(lifecycle)) { + if (lifecycle_) { + lifecycle_->active_workers.fetch_add(1, std::memory_order_acq_rel); + } + } + ~WorkerLease() { + if (lifecycle_) { + lifecycle_->active_workers.fetch_sub(1, std::memory_order_acq_rel); + } + } + std::shared_ptr lifecycle_; +}; + +std::shared_ptr MakeWorkerLease(std::shared_ptr lifecycle) { + return std::make_shared(std::move(lifecycle)); +} + const char* DeviceTypeToString(flDeviceType dt) { switch (dt) { case FOUNDRY_LOCAL_DEVICE_CPU: @@ -323,8 +341,10 @@ Napi::Value Model::Load(const Napi::CallbackInfo& info) { if (!manager_keepalive) { return env.Undefined(); } + auto worker_lease = MakeWorkerLease(lifecycle_); return PromiseWorkerVoid::Run( - env, [m, keepalive = std::move(keepalive), manager_keepalive = std::move(manager_keepalive)]() { m->Load(); }, + env, [m, keepalive = std::move(keepalive), manager_keepalive = std::move(manager_keepalive), + worker_lease = std::move(worker_lease)]() { m->Load(); }, std::move(owner)); } @@ -341,8 +361,10 @@ Napi::Value Model::Unload(const Napi::CallbackInfo& info) { if (!manager_keepalive) { return env.Undefined(); } + auto worker_lease = MakeWorkerLease(lifecycle_); return PromiseWorkerVoid::Run( - env, [m, keepalive = std::move(keepalive), manager_keepalive = std::move(manager_keepalive)]() { m->Unload(); }, + env, [m, keepalive = std::move(keepalive), manager_keepalive = std::move(manager_keepalive), + worker_lease = std::move(worker_lease)]() { m->Unload(); }, std::move(owner)); } @@ -355,13 +377,14 @@ namespace { class DownloadWorker : public Napi::AsyncWorker { public: DownloadWorker(Napi::Env env, foundry_local::IModel* impl, std::shared_ptr keepalive, - std::shared_ptr manager_keepalive, Napi::ObjectReference owner, - Napi::ThreadSafeFunction tsfn) + std::shared_ptr manager_keepalive, std::shared_ptr worker_lease, + Napi::ObjectReference owner, Napi::ThreadSafeFunction tsfn) : Napi::AsyncWorker(env), deferred_(Napi::Promise::Deferred::New(env)), impl_(impl), keepalive_(std::move(keepalive)), manager_keepalive_(std::move(manager_keepalive)), + worker_lease_(std::move(worker_lease)), owner_(std::move(owner)), tsfn_(std::move(tsfn)) {} @@ -427,6 +450,7 @@ class DownloadWorker : public Napi::AsyncWorker { foundry_local::IModel* impl_; std::shared_ptr keepalive_; std::shared_ptr manager_keepalive_; + std::shared_ptr worker_lease_; Napi::ObjectReference owner_; Napi::ThreadSafeFunction tsfn_; std::string err_msg_; @@ -461,7 +485,8 @@ Napi::Value Model::Download(const Napi::CallbackInfo& info) { } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - auto* w = new DownloadWorker(env, impl_, keepalive_, std::move(manager_keepalive), std::move(owner), + auto* w = new DownloadWorker(env, impl_, keepalive_, std::move(manager_keepalive), MakeWorkerLease(lifecycle_), + std::move(owner), std::move(tsfn)); Napi::Promise p = w->Promise(); w->Queue(); diff --git a/sdk_v2/js/test/manager-dispose.test.ts b/sdk_v2/js/test/manager-dispose.test.ts index ee89a6dfa..f7628695e 100644 --- a/sdk_v2/js/test/manager-dispose.test.ts +++ b/sdk_v2/js/test/manager-dispose.test.ts @@ -85,11 +85,12 @@ describeIfBuilt("FoundryLocalManager.dispose", () => { expect(mgr.disposed).toBe(true); }); - it("dispose() does not invalidate an in-flight native EP worker", async () => { + it("dispose() rejects while a native EP worker is in flight", async () => { const mgr = freshManager("async-worker"); const pending = mgr.downloadAndRegisterEps(["__not-a-provider__"]); - mgr.dispose(); + expect(() => mgr.dispose()).toThrow(/active native workers/i); await expect(pending).resolves.toMatchObject({ success: true }); + mgr.dispose(); expect(mgr.disposed).toBe(true); }); From 6dc0d8e027bf6a4d9ee65fc915b5594b8fa3efe5 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 14:02:04 -0500 Subject: [PATCH 64/77] Tighten catalog download and EP edge cases Return latest versions by model name, fail unknown requested EP names, and keep suspect complete blob sidecars durable until reset state is saved. Files changed: - sdk_v2/cpp/src/catalog/base_model_catalog.cc - sdk_v2/cpp/src/download/blob_downloader.cc - sdk_v2/cpp/src/ep_detection/ep_detector.cc - sdk_v2/cpp/test/internal_api/ep_detector_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/catalog/base_model_catalog.cc | 6 ++++-- sdk_v2/cpp/src/download/blob_downloader.cc | 5 ++++- sdk_v2/cpp/src/ep_detection/ep_detector.cc | 10 ++++++++++ sdk_v2/cpp/test/internal_api/ep_detector_test.cc | 12 +++++++----- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.cc b/sdk_v2/cpp/src/catalog/base_model_catalog.cc index fe653ae5d..0bf3a29f0 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.cc @@ -332,8 +332,10 @@ Model* BaseModelCatalog::GetLatestVersion(const Model* model) const { auto alias_it = idx->alias_index.find(std::string(model->Info().alias)); if (alias_it != idx->alias_index.end()) { auto variants = alias_it->second->Variants(); - if (!variants.empty()) { - return variants.front(); + for (auto* variant : variants) { + if (variant != nullptr && variant->Info().name == model->Info().name) { + return variant; + } } } diff --git a/sdk_v2/cpp/src/download/blob_downloader.cc b/sdk_v2/cpp/src/download/blob_downloader.cc index 6ea57940c..d04539d28 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.cc +++ b/sdk_v2/cpp/src/download/blob_downloader.cc @@ -234,7 +234,10 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, "Resume sidecar for '" + local_path + "' is complete but unfinalized; starting fresh"); std::error_code remove_ec; std::filesystem::remove(local_path, remove_ec); - BlobDownloadState::DeleteState(local_path, logger_); + if (remove_ec && std::filesystem::exists(local_path)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "failed to remove suspect completed blob before restart: " + local_path); + } state = BlobDownloadState::CreateNew(blob_name, local_path, blob_size, static_cast(kChunkSize), num_chunks, chunk_ctx.blob_identity); diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.cc b/sdk_v2/cpp/src/ep_detection/ep_detector.cc index 7fbd72ebd..5dbee4533 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.cc @@ -173,6 +173,7 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector unmatched_requested_names = names ? *names : std::vector{}; if (progress_cb) { wrapped_cb = [&progress_cb, &cancelled](const std::string& ep_name, float percent) -> bool { @@ -202,6 +203,9 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vectorend()) { continue; } + unmatched_requested_names.erase(std::remove(unmatched_requested_names.begin(), unmatched_requested_names.end(), + bs->Name()), + unmatched_requested_names.end()); } ++telemetry_num_providers; @@ -270,6 +274,12 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector names = {"CUDAExecutionProvider", "NonExistentProvider"}; auto result = detector->DownloadAndRegisterEps(&names, nullptr); - EXPECT_TRUE(result.success); + EXPECT_FALSE(result.success); ASSERT_EQ(result.registered_eps.size(), 1u); EXPECT_EQ(result.registered_eps[0], "CUDAExecutionProvider"); - EXPECT_TRUE(result.failed_eps.empty()); + ASSERT_EQ(result.failed_eps.size(), 1u); + EXPECT_EQ(result.failed_eps[0], "NonExistentProvider"); EXPECT_TRUE(mocks[0]->download_called_); } -TEST_F(EpDetectorTest, DownloadFiltered_AllNamesUnknown_SucceedsWithNothing) { +TEST_F(EpDetectorTest, DownloadFiltered_AllNamesUnknownFailsWithRequestedName) { std::vector mocks; auto detector = MakeDetector(mocks, {{"CUDAExecutionProvider", true}}); std::vector names = {"FakeProvider"}; auto result = detector->DownloadAndRegisterEps(&names, nullptr); - EXPECT_TRUE(result.success); + EXPECT_FALSE(result.success); EXPECT_TRUE(result.registered_eps.empty()); - EXPECT_TRUE(result.failed_eps.empty()); + ASSERT_EQ(result.failed_eps.size(), 1u); + EXPECT_EQ(result.failed_eps[0], "FakeProvider"); EXPECT_FALSE(mocks[0]->download_called_); } \ No newline at end of file From 7ce6e37bbca5c9441ee534748d0c92d62e4f1678 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 15:58:31 -0500 Subject: [PATCH 65/77] Expect unknown EP requests to fail in JS tests Keep the manager-dispose worker test aligned with native behavior now that unmatched requested EP names fail instead of reporting success. Files changed: - sdk_v2/js/test/manager-dispose.test.ts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/js/test/manager-dispose.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk_v2/js/test/manager-dispose.test.ts b/sdk_v2/js/test/manager-dispose.test.ts index f7628695e..26bbe1c4e 100644 --- a/sdk_v2/js/test/manager-dispose.test.ts +++ b/sdk_v2/js/test/manager-dispose.test.ts @@ -89,7 +89,7 @@ describeIfBuilt("FoundryLocalManager.dispose", () => { const mgr = freshManager("async-worker"); const pending = mgr.downloadAndRegisterEps(["__not-a-provider__"]); expect(() => mgr.dispose()).toThrow(/active native workers/i); - await expect(pending).resolves.toMatchObject({ success: true }); + await expect(pending).rejects.toThrow(/not available/i); mgr.dispose(); expect(mgr.disposed).toBe(true); }); From e5ab67dc30bb5b7994b2df2ff2434b0330100556 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 19:00:50 -0500 Subject: [PATCH 66/77] Close final material edge cases Skip finalized blobs during model retry downloads, avoid no-op EP snapshot growth, preserve live model instances for invalid late sessions during teardown, and treat failed terminal SSE pushes as cancellation before success/storage. Files changed: - sdk_v2/cpp/src/download/download_manager.cc - sdk_v2/cpp/src/ep_detection/ep_detector.cc - sdk_v2/cpp/src/inferencing/model_load_manager.cc - sdk_v2/cpp/src/service/audio_transcriptions_handler.cc - sdk_v2/cpp/src/service/chat_completions_handler.cc - sdk_v2/cpp/src/service/responses_handler.cc - sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc - sdk_v2/cpp/test/internal_api/download_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/download/download_manager.cc | 2 +- sdk_v2/cpp/src/ep_detection/ep_detector.cc | 7 +++- .../cpp/src/inferencing/model_load_manager.cc | 18 +++++++- .../service/audio_transcriptions_handler.cc | 4 +- .../src/service/chat_completions_handler.cc | 8 +++- sdk_v2/cpp/src/service/responses_handler.cc | 17 ++++++-- .../internal_api/base_model_catalog_test.cc | 13 ++++++ sdk_v2/cpp/test/internal_api/download_test.cc | 41 +++++++++++++++++++ 8 files changed, 100 insertions(+), 10 deletions(-) diff --git a/sdk_v2/cpp/src/download/download_manager.cc b/sdk_v2/cpp/src/download/download_manager.cc index 862c00c81..c322cde72 100644 --- a/sdk_v2/cpp/src/download/download_manager.cc +++ b/sdk_v2/cpp/src/download/download_manager.cc @@ -376,7 +376,7 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, // and the blob container has all files at the root or in variant subdirectories. download_opts.path_prefix = ""; download_opts.max_concurrency = max_concurrency_; - download_opts.skip_completed_files = false; + download_opts.skip_completed_files = true; if (progress_cb) { download_opts.progress = [&progress_cb](float percent) { diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.cc b/sdk_v2/cpp/src/ep_detection/ep_detector.cc index 5dbee4533..befad97ae 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.cc @@ -246,9 +246,12 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector cache_lock(cache_mutex_); + const bool registration_changed = !cached_eps_[i].is_registered; cached_eps_[i].is_registered = true; - PublishEpSnapshotLocked(); - PublishCApiSnapshotLocked(); + if (registration_changed) { + PublishEpSnapshotLocked(); + PublishCApiSnapshotLocked(); + } if (tracker) { tracker->RecordDownloadComplete(ActionStatus::kSuccess, "Installed"); diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.cc b/sdk_v2/cpp/src/inferencing/model_load_manager.cc index f44527c07..b67ab2326 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.cc +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.cc @@ -57,8 +57,24 @@ ModelLoadManager::ModelLoadManager(IEpDetector& ep_detector, ILogger& logger) : ep_detector_(ep_detector), logger_(logger) {} ModelLoadManager::~ModelLoadManager() { - // Destroy all loaded models under the lock. + // Destroy all loaded models under the lock. If a caller violates the public + // lifetime contract and keeps direct sessions alive past Manager destruction, + // keep those model instances alive rather than leaving sessions with dangling + // GenAIModelInstance references. std::lock_guard lock(mutex_); + for (auto it = loaded_models_.begin(); it != loaded_models_.end();) { + auto live_sessions = it->second->SessionRefCount(); + if (live_sessions > 0) { + logger_.Log(LogLevel::Warning, + fmt::format("ModelLoadManager destroyed while model '{}' still has {} live session(s); " + "leaving model instance allocated to avoid dangling session references", + it->first, live_sessions)); + it->second.release(); + it = loaded_models_.erase(it); + } else { + ++it; + } + } loaded_models_.clear(); } diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc index 870f0a557..436431fee 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc @@ -240,7 +240,9 @@ std::shared_ptr AudioTranscriptionsHandler } // Send terminal event - body_ptr->Push("data: [DONE]\n\n"); + if (!body_ptr->Push("data: [DONE]\n\n")) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "audio transcription stream cancelled"); + } if (route_tracker) { route_tracker->SetStatus(ActionStatus::kSuccess); diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.cc b/sdk_v2/cpp/src/service/chat_completions_handler.cc index 289309d9d..ca45c5e15 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.cc +++ b/sdk_v2/cpp/src/service/chat_completions_handler.cc @@ -271,10 +271,14 @@ std::shared_ptr ChatCompletionsHandler::Ha usage.total_tokens = static_cast(bg_response.usage.total_tokens); usage_chunk.usage = std::move(usage); - body_ptr->Push("data: " + nlohmann::json(usage_chunk).dump() + "\n\n"); + if (!body_ptr->Push("data: " + nlohmann::json(usage_chunk).dump() + "\n\n")) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "chat completion stream cancelled"); + } } - body_ptr->Push("data: [DONE]\n\n"); + if (!body_ptr->Push("data: [DONE]\n\n")) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "chat completion stream cancelled"); + } // Inference streamed to completion — record the route action as a success. if (route_tracker) { diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index c15296d6c..7a9885aa1 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -346,6 +346,7 @@ std::shared_ptr ResponsesHandler::HandleSt route_tracker = std::move(route_tracker), &tracker, stream_done]() mutable { + bool terminal_sent = false; int seq = 2; std::string full_text; // concatenation of all visible runs, used for output_text in completed_response @@ -560,6 +561,9 @@ std::shared_ptr ResponsesHandler::HandleSt // Close whatever item is still open at end-of-generation so the SSE stream is well-formed. close_current(); + if (req->canceled.load(std::memory_order_relaxed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "response stream cancelled"); + } auto completed_response = ResponseConverter::BuildResponseObject( response_id, created_at, model_name, params_copy, std::move(closed_items), full_text, bg_response.usage); @@ -568,7 +572,13 @@ std::shared_ptr ResponsesHandler::HandleSt completed.type = StreamEventType::kResponseCompleted; completed.sequence_number = seq++; completed.response = completed_response; - push_event("response.completed", completed); + if (!push_event("response.completed", completed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "response stream cancelled"); + } + if (!body_ptr->Push("data: [DONE]\n\n")) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "response stream cancelled"); + } + terminal_sent = true; // Store if requested if (should_store) { @@ -610,8 +620,9 @@ std::shared_ptr ResponsesHandler::HandleSt } } - // Terminal event per spec - body_ptr->Push("data: [DONE]\n\n"); + if (!terminal_sent) { + body_ptr->Push("data: [DONE]\n\n"); + } body_ptr->Finish(); stream_done->store(true, std::memory_order_release); diff --git a/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc b/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc index dc5c93f46..204bf3d62 100644 --- a/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc +++ b/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc @@ -279,6 +279,19 @@ TEST_F(BaseModelCatalogTest, RefreshPreservesExplicitVariantSelection) { EXPECT_EQ(container->Id(), "phi-3-mini-cpu:1"); } +TEST_F(BaseModelCatalogTest, GetLatestVersionReturnsLatestSameModelName) { + TestCatalog catalog(logger_); + catalog.AddModel(MakeModel("phi-3-mini-cpu:1", "phi-3-mini-cpu", 1, "phi-3")); + catalog.AddModel(MakeModel("phi-3-mini-gpu:2", "phi-3-mini-gpu", 2, "phi-3")); + + Model* cpu_variant = catalog.GetModelVariant("phi-3-mini-cpu:1"); + ASSERT_NE(cpu_variant, nullptr); + + Model* latest = catalog.GetLatestVersion(cpu_variant); + ASSERT_NE(latest, nullptr); + EXPECT_EQ(latest->Info().name, "phi-3-mini-cpu"); +} + TEST_F(BaseModelCatalogTest, ForcedRefreshFailureBacksOffWhenCatalogAlreadyPopulated) { FailingRefreshCatalog catalog(logger_); std::vector models; diff --git a/sdk_v2/cpp/test/internal_api/download_test.cc b/sdk_v2/cpp/test/internal_api/download_test.cc index 92fbf1931..4bfadc50c 100644 --- a/sdk_v2/cpp/test/internal_api/download_test.cc +++ b/sdk_v2/cpp/test/internal_api/download_test.cc @@ -927,6 +927,47 @@ TEST(DownloadManagerTest, FullDownloadFlow) { EXPECT_FALSE(progress_values.empty()); } +TEST(DownloadManagerTest, RetrySkipsCompletedBlobWithoutSidecar) { + TempDir tmpdir; + + auto manager = std::make_unique(tmpdir.string(), "eastus", 64, fl::test::NullLog()); + + auto registry = std::make_unique( + "eastus", fl::test::NullLog(), std::make_unique(fl::test::NullLog(), false), + [](const std::string&) { + return MakeRegistryResponse( + R"({"blobSasUri": "https://storage.blob.core.windows.net/container?sig=test"})"); + }); + manager->SetModelRegistryClient(std::move(registry)); + + auto mock_downloader = std::make_unique(); + auto* mock_downloader_ptr = mock_downloader.get(); + mock_downloader->blobs_to_return = { + {"weights.safetensors", 1024}, + {"config.json", 100}, + }; + manager->SetBlobDownloader(std::move(mock_downloader)); + + ModelInfo info; + info.model_id = "test-model:1"; + info.name = "test-model"; + info.uri = "azureml://registries/test/models/test-model/versions/1"; + info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR] = "TestPublisher"; + + auto model_dir = fs::path(tmpdir.string()) / "TestPublisher" / "test-model-1"; + fs::create_directories(model_dir); + { + std::ofstream f(model_dir / "weights.safetensors", std::ios::binary); + f.seekp(1023); + f.put('\0'); + } + + manager->DownloadModel(info); + + ASSERT_EQ(mock_downloader_ptr->downloaded_blobs.size(), 1u); + EXPECT_EQ(mock_downloader_ptr->downloaded_blobs[0], "config.json"); +} + // --- Region resolution: detected region drives the download endpoint --- // Run one download and return the registry URL the manager hit. From cf6fa2e4a7002d5c1db9b301732d788e5b39be99 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 21:09:12 -0500 Subject: [PATCH 67/77] Keep response continuations on matching models Store model IDs with cached Responses sessions and only reuse them when the continuation requests the same resolved model, avoiding silent cross-model inference reuse. Files changed: - sdk_v2/cpp/src/inferencing/session/session_manager.cc - sdk_v2/cpp/src/inferencing/session/session_manager.h - sdk_v2/cpp/src/service/responses_handler.cc - sdk_v2/cpp/src/service/responses_handler.h - sdk_v2/cpp/test/internal_api/session_manager_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .../src/inferencing/session/session_manager.cc | 12 +++++++++--- .../src/inferencing/session/session_manager.h | 7 +++++-- sdk_v2/cpp/src/service/responses_handler.cc | 16 +++++++++------- sdk_v2/cpp/src/service/responses_handler.h | 8 ++++---- .../test/internal_api/session_manager_test.cc | 14 ++++++++++++++ 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/session/session_manager.cc b/sdk_v2/cpp/src/inferencing/session/session_manager.cc index ae0df19e3..59f1a86d7 100644 --- a/sdk_v2/cpp/src/inferencing/session/session_manager.cc +++ b/sdk_v2/cpp/src/inferencing/session/session_manager.cc @@ -85,13 +85,19 @@ size_t SessionManager::ActiveCount() const { // --- Session cache --- -std::unique_ptr SessionManager::CheckOut(const std::string& key) { +std::unique_ptr SessionManager::CheckOut(const std::string& key, const std::string& expected_model_id) { std::lock_guard lock(mutex_); auto it = cache_.find(key); if (it == cache_.end()) { return nullptr; } + if (!expected_model_id.empty() && it->second.model_id != expected_model_id) { + logger_.Log(LogLevel::Information, + fmt::format("SessionManager: cached session for '{}' uses model '{}', not requested model '{}'", + key, it->second.model_id, expected_model_id)); + return nullptr; + } auto session = std::move(it->second.session); lru_order_.erase(it->second.lru_iter); @@ -101,7 +107,7 @@ std::unique_ptr SessionManager::CheckOut(const std::string& key) { return session; } -void SessionManager::CheckIn(const std::string& key, std::unique_ptr session) { +void SessionManager::CheckIn(const std::string& key, std::unique_ptr session, std::string model_id) { // Collect evicted sessions to destroy outside the lock. std::vector> evicted; @@ -133,7 +139,7 @@ void SessionManager::CheckIn(const std::string& key, std::unique_ptr CheckOut(const std::string& key); + std::unique_ptr CheckOut(const std::string& key, const std::string& expected_model_id = {}); /// Insert a session into the cache under the given key. /// May evict the least-recently-used entry if at capacity. /// The session remains tracked (registered) while cached. - void CheckIn(const std::string& key, std::unique_ptr session); + void CheckIn(const std::string& key, std::unique_ptr session, std::string model_id = {}); /// Remove and destroy a cached session by key. Returns true if an entry was removed. /// @@ -94,6 +96,7 @@ class SessionManager : public ISessionManager { private: struct CacheEntry { std::unique_ptr session; + std::string model_id; std::list::iterator lru_iter; }; diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index 7a9885aa1..b6a02ca77 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -178,9 +178,10 @@ std::shared_ptr ResponsesHandler::handle( std::string response_id = ResponseConverter::GenerateId("resp"); std::unique_ptr session; + const std::string resolved_model_id = model->Id(); if (params.previous_response_id) { - session = ctx_.session_manager.CheckOut(*params.previous_response_id); + session = ctx_.session_manager.CheckOut(*params.previous_response_id, resolved_model_id); if (session) { ctx_.logger.Log(LogLevel::Information, @@ -225,13 +226,13 @@ std::shared_ptr ResponsesHandler::handle( // The route action is recorded by the streaming thread when the stream // finishes — move the tracker in rather than marking success now. - return HandleStreaming(std::move(session), std::move(session_request), model_name, + return HandleStreaming(std::move(session), std::move(session_request), model_name, resolved_model_id, response_id, created_at, params, req_json, std::move(tracker)); } else { ctx_.logger.Log(LogLevel::Debug, fmt::format("Creating response {} for model {}", response_id, model_name)); - auto response = HandleNonStreaming(std::move(session), session_request, model_name, + auto response = HandleNonStreaming(std::move(session), session_request, model_name, resolved_model_id, response_id, created_at, params, req_json); tracker->SetStatus(ActionStatus::kSuccess); @@ -253,7 +254,7 @@ std::shared_ptr ResponsesHandler::handle( std::shared_ptr ResponsesHandler::HandleNonStreaming( std::unique_ptr session, Request& session_request, - const std::string& model_name, const std::string& response_id, + const std::string& model_name, const std::string& model_id, const std::string& response_id, int64_t created_at, const ResponseCreateParams& params, const nlohmann::json& req_json) { SessionRegistration reg(ctx_.session_manager, *session); @@ -285,7 +286,7 @@ std::shared_ptr ResponsesHandler::HandleNo session->SetStreamingCallback(nullptr); // Cache the session for potential reuse on the next turn - ctx_.session_manager.CheckIn(response_id, std::move(session)); + ctx_.session_manager.CheckIn(response_id, std::move(session), model_id); } return JsonResponse(Status::CODE_200, response_json); @@ -293,7 +294,7 @@ std::shared_ptr ResponsesHandler::HandleNo std::shared_ptr ResponsesHandler::HandleStreaming( std::unique_ptr session, Request session_request, - const std::string& model_name, const std::string& response_id, + const std::string& model_name, const std::string& model_id, const std::string& response_id, int64_t created_at, const ResponseCreateParams& params, const nlohmann::json& req_json, std::unique_ptr route_tracker) { auto body = std::make_shared(); @@ -340,6 +341,7 @@ std::shared_ptr ResponsesHandler::HandleSt session = std::move(session), req = stream_request, model_name, response_id, created_at, + model_id, should_store, &store, req_copy = std::move(req_copy), params_copy = std::move(params_copy), @@ -593,7 +595,7 @@ std::shared_ptr ResponsesHandler::HandleSt session->SetStreamingCallback(nullptr); // Cache the session for potential reuse on the next turn - session_manager.CheckIn(response_id, std::move(session)); + session_manager.CheckIn(response_id, std::move(session), model_id); } // Streamed to completion — record the route action as a success. diff --git a/sdk_v2/cpp/src/service/responses_handler.h b/sdk_v2/cpp/src/service/responses_handler.h index 090445a68..7ce6af59c 100644 --- a/sdk_v2/cpp/src/service/responses_handler.h +++ b/sdk_v2/cpp/src/service/responses_handler.h @@ -57,14 +57,14 @@ class ResponsesHandler : public HttpRequestHandler { // --- Inference dispatch --- std::shared_ptr HandleNonStreaming(std::unique_ptr session, Request& session_request, - const std::string& model_name, const std::string& response_id, - int64_t created_at, + const std::string& model_name, const std::string& model_id, + const std::string& response_id, int64_t created_at, const responses::ResponseCreateParams& params, const nlohmann::json& req_json); std::shared_ptr HandleStreaming(std::unique_ptr session, Request session_request, - const std::string& model_name, const std::string& response_id, - int64_t created_at, + const std::string& model_name, const std::string& model_id, + const std::string& response_id, int64_t created_at, const responses::ResponseCreateParams& params, const nlohmann::json& req_json, std::unique_ptr route_tracker); diff --git a/sdk_v2/cpp/test/internal_api/session_manager_test.cc b/sdk_v2/cpp/test/internal_api/session_manager_test.cc index cbd3e988b..914fb5cc4 100644 --- a/sdk_v2/cpp/test/internal_api/session_manager_test.cc +++ b/sdk_v2/cpp/test/internal_api/session_manager_test.cc @@ -145,6 +145,20 @@ TEST_F(SessionManagerTest, CheckInAndCheckOutRoundTrip) { EXPECT_EQ(mgr.CacheSize(), 0u); } +TEST(SessionManagerStandaloneTest, CheckOutModelMismatchLeavesCachedSession) { + StderrLogger logger; + SessionManager mgr(logger); + + mgr.CheckIn("resp-1", nullptr, "model-a"); + + EXPECT_EQ(mgr.CheckOut("resp-1", "model-b"), nullptr); + EXPECT_EQ(mgr.CacheSize(), 1u); + + auto checked_out = mgr.CheckOut("resp-1", "model-a"); + EXPECT_EQ(checked_out, nullptr); + EXPECT_EQ(mgr.CacheSize(), 0u); +} + TEST(SessionManagerStandaloneTest, CheckInAfterCancelAllDropsSession) { StderrLogger logger; SessionManager mgr(logger); From 7c6a6bf4920df01175d9992d00aaf4a6a45b4f5c Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 22:19:52 -0500 Subject: [PATCH 68/77] Harden shutdown and streaming cleanup Avoid raw model pointers during shutdown unload, cache web-service URL snapshots without per-poll growth, and reap completed streaming threads after producer completion. Files changed: - sdk_v2/cpp/src/c_api.cc - sdk_v2/cpp/src/inferencing/model_load_manager.cc - sdk_v2/cpp/src/service/audio_transcriptions_handler.cc - sdk_v2/cpp/src/service/chat_completions_handler.cc - sdk_v2/cpp/src/service/responses_handler.cc - sdk_v2/cpp/src/service/web_service.h Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/c_api.cc | 21 ++++--- .../cpp/src/inferencing/model_load_manager.cc | 46 ++++++++------- .../service/audio_transcriptions_handler.cc | 1 + .../src/service/chat_completions_handler.cc | 1 + sdk_v2/cpp/src/service/responses_handler.cc | 1 + sdk_v2/cpp/src/service/web_service.h | 59 +++++++++++++++++++ 6 files changed, 98 insertions(+), 31 deletions(-) diff --git a/sdk_v2/cpp/src/c_api.cc b/sdk_v2/cpp/src/c_api.cc index c36f55df0..719f11f0d 100644 --- a/sdk_v2/cpp/src/c_api.cc +++ b/sdk_v2/cpp/src/c_api.cc @@ -81,7 +81,7 @@ struct flManager { fl::Manager& impl; std::unique_ptr catalog; // stores the flCatalog wrapper around impl.GetCatalog() mutable std::mutex urls_cache_mutex; - mutable std::vector> urls_snapshots; + mutable std::unique_ptr urls_snapshot; }; // ======================================================================== @@ -384,20 +384,23 @@ FL_API_STATUS_IMPL(Manager_WebServiceUrlsImpl, const flManager* manager, std::lock_guard lock(manager->urls_cache_mutex); auto urls = manager->impl.GetWebServiceUrls(); if (urls.empty()) { + manager->urls_snapshot.reset(); *out_urls = nullptr; *out_num_urls = 0; return nullptr; } - auto snapshot = std::make_unique(); - snapshot->strings = std::move(urls); - snapshot->pointers.reserve(snapshot->strings.size()); - for (const auto& u : snapshot->strings) { - snapshot->pointers.push_back(u.c_str()); + if (!manager->urls_snapshot || manager->urls_snapshot->strings != urls) { + auto snapshot = std::make_unique(); + snapshot->strings = std::move(urls); + snapshot->pointers.reserve(snapshot->strings.size()); + for (const auto& u : snapshot->strings) { + snapshot->pointers.push_back(u.c_str()); + } + manager->urls_snapshot = std::move(snapshot); } - auto* snapshot_ptr = snapshot.get(); - manager->urls_snapshots.push_back(std::move(snapshot)); + auto* snapshot_ptr = manager->urls_snapshot.get(); *out_urls = snapshot_ptr->pointers.data(); *out_num_urls = snapshot_ptr->pointers.size(); return nullptr; @@ -412,7 +415,7 @@ FL_API_STATUS_IMPL(Manager_WebServiceStopImpl, flManager* manager) { std::lock_guard lock(manager->urls_cache_mutex); manager->impl.StopWebService(); - manager->urls_snapshots.clear(); + manager->urls_snapshot.reset(); return nullptr; API_IMPL_END } diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.cc b/sdk_v2/cpp/src/inferencing/model_load_manager.cc index b67ab2326..6254a51cb 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.cc +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.cc @@ -243,24 +243,21 @@ void ModelLoadManager::RejectNewLoads() { } void ModelLoadManager::UnloadAll(std::chrono::milliseconds timeout) { - // Snapshot ids+pointers under the lock; the GenAIModelInstance pointers stay valid - // because (a) only this method or UnloadModel can erase entries, and (b) we serialize - // the per-id erase below. - std::vector> snapshot; + std::vector ids; { std::lock_guard lock(mutex_); - snapshot.reserve(loaded_models_.size()); + ids.reserve(loaded_models_.size()); for (auto& [id, instance] : loaded_models_) { - snapshot.emplace_back(id, instance.get()); + ids.emplace_back(id); } } - if (snapshot.empty()) { + if (ids.empty()) { return; } logger_.Log(LogLevel::Information, - fmt::format("Shutdown: unloading {} model(s)", snapshot.size())); + fmt::format("Shutdown: unloading {} model(s)", ids.size())); using clock = std::chrono::steady_clock; constexpr auto kPollInterval = std::chrono::milliseconds(50); @@ -270,27 +267,32 @@ void ModelLoadManager::UnloadAll(std::chrono::milliseconds timeout) { // time linearly with model count. auto deadline = clock::now() + timeout; - for (auto& [id, instance] : snapshot) { - while (instance->SessionRefCount() > 0 && clock::now() < deadline) { + for (const auto& id : ids) { + int remaining = 0; + while (clock::now() < deadline) { + { + std::lock_guard lock(mutex_); + auto it = loaded_models_.find(id); + if (it == loaded_models_.end()) { + remaining = 0; + break; + } + + remaining = it->second->SessionRefCount(); + if (remaining == 0) { + logger_.Log(LogLevel::Information, fmt::format("unloading model: {}", id)); + loaded_models_.erase(it); + break; + } + } + std::this_thread::sleep_for(kPollInterval); } - auto remaining = instance->SessionRefCount(); if (remaining > 0) { logger_.Log(LogLevel::Warning, fmt::format("Shutdown: model '{}' still has {} session(s) after overall {}ms deadline; leaving loaded", id, remaining, timeout.count())); - continue; - } - - try { - UnloadModel(id); - } catch (const std::exception& ex) { - // A new session attached between our refcount poll and the lock acquisition inside - // UnloadModel. Log and move on — IsShutdownRequested-gated callers shouldn't be - // creating new sessions, but we don't crash shutdown over it. - logger_.Log(LogLevel::Warning, - fmt::format("Shutdown: failed to unload '{}': {}", id, ex.what())); } } } diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc index 436431fee..1ac716623 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc @@ -261,6 +261,7 @@ std::shared_ptr AudioTranscriptionsHandler body_ptr->Finish(); stream_done->store(true, std::memory_order_release); + thread_tracker.NotifyCompleted(); }); thread_tracker.Track(std::move(streaming_thread), stream_done, [body, stream_request] { diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.cc b/sdk_v2/cpp/src/service/chat_completions_handler.cc index ca45c5e15..b4079ba5f 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.cc +++ b/sdk_v2/cpp/src/service/chat_completions_handler.cc @@ -301,6 +301,7 @@ std::shared_ptr ChatCompletionsHandler::Ha // route_tracker is destroyed with this closure once the thread completes, // recording the route action with the full streaming duration and final status. stream_done->store(true, std::memory_order_release); + thread_tracker.NotifyCompleted(); }); thread_tracker.Track(std::move(streaming_thread), stream_done, [body, stream_request] { diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index b6a02ca77..09bb46d3a 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -627,6 +627,7 @@ std::shared_ptr ResponsesHandler::HandleSt } body_ptr->Finish(); stream_done->store(true, std::memory_order_release); + tracker.NotifyCompleted(); }); diff --git a/sdk_v2/cpp/src/service/web_service.h b/sdk_v2/cpp/src/service/web_service.h index 5f2fb89b8..33d25f476 100644 --- a/sdk_v2/cpp/src/service/web_service.h +++ b/sdk_v2/cpp/src/service/web_service.h @@ -30,9 +30,19 @@ class StreamingThreadTracker { }; public: + StreamingThreadTracker() { + StartReaperLocked(); + } + + ~StreamingThreadTracker() { + StopReaper(); + JoinAll(); + } + /// Take ownership of a streaming thread. void Track(std::thread t, std::shared_ptr> done, std::function abort) { std::lock_guard lock(mutex_); + StartReaperLocked(); ReapCompletedLocked(); if (stopping_) { if (abort) { @@ -42,6 +52,10 @@ class StreamingThreadTracker { threads_.push_back(TrackedThread{std::move(t), std::move(done), std::move(abort)}); } + void NotifyCompleted() { + completion_cv_.notify_one(); + } + void BeginStopping() { std::lock_guard lock(mutex_); stopping_ = true; @@ -56,6 +70,7 @@ class StreamingThreadTracker { void Reset() { std::lock_guard lock(mutex_); stopping_ = false; + StartReaperLocked(); } void AbortAllLocked() { @@ -69,6 +84,8 @@ class StreamingThreadTracker { /// Join all remaining threads. Called by WebService::Stop(). /// Moves entries out before joining so completed streaming threads are cleaned up safely. void JoinAll() { + StopReaper(); + std::vector local; { std::lock_guard lock(mutex_); @@ -83,6 +100,45 @@ class StreamingThreadTracker { } private: + bool HasCompletedLocked() const { + for (const auto& tracked : threads_) { + if (tracked.done && tracked.done->load(std::memory_order_acquire)) { + return true; + } + } + return false; + } + + void StartReaperLocked() { + if (reaper_thread_.joinable()) { + return; + } + reaper_stop_ = false; + reaper_thread_ = std::thread([this]() { ReaperLoop(); }); + } + + void StopReaper() { + { + std::lock_guard lock(mutex_); + reaper_stop_ = true; + } + completion_cv_.notify_all(); + if (reaper_thread_.joinable()) { + reaper_thread_.join(); + } + } + + void ReaperLoop() { + while (true) { + std::unique_lock lock(mutex_); + completion_cv_.wait(lock, [this]() { return reaper_stop_ || HasCompletedLocked(); }); + if (reaper_stop_) { + return; + } + ReapCompletedLocked(); + } + } + void ReapCompletedLocked() { for (auto it = threads_.begin(); it != threads_.end();) { if (it->done && it->done->load(std::memory_order_acquire)) { @@ -97,8 +153,11 @@ class StreamingThreadTracker { } std::mutex mutex_; + std::condition_variable completion_cv_; std::vector threads_; + std::thread reaper_thread_; bool stopping_ = false; + bool reaper_stop_ = false; }; /// Context shared with all HTTP controllers. From bb6842e2a7f2446a42b4efd9e9467029559c6834 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Sun, 12 Jul 2026 23:16:35 -0500 Subject: [PATCH 69/77] Persist streaming responses before done signal Commit streaming Responses state before sending the terminal [DONE] marker so continuations cannot race ahead of response/session persistence. Files changed: - sdk_v2/cpp/src/service/responses_handler.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/service/responses_handler.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index 09bb46d3a..cd9730227 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -577,10 +577,6 @@ std::shared_ptr ResponsesHandler::HandleSt if (!push_event("response.completed", completed)) { FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "response stream cancelled"); } - if (!body_ptr->Push("data: [DONE]\n\n")) { - FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "response stream cancelled"); - } - terminal_sent = true; // Store if requested if (should_store) { @@ -598,6 +594,11 @@ std::shared_ptr ResponsesHandler::HandleSt session_manager.CheckIn(response_id, std::move(session), model_id); } + if (!body_ptr->Push("data: [DONE]\n\n")) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "response stream cancelled"); + } + terminal_sent = true; + // Streamed to completion — record the route action as a success. if (route_tracker) { route_tracker->SetStatus(ActionStatus::kSuccess); From f70c003b75e5a7449286b311eef07e65155ab095 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 13 Jul 2026 00:09:39 -0500 Subject: [PATCH 70/77] Finish unload and wheel artifact hardening Ensure shutdown unload checks each model at least once after the deadline and fail Windows Python wheel staging when any required native runtime artifact is missing. Files changed: - .pipelines/v2/templates/steps-build-python.yml - sdk_v2/cpp/src/inferencing/model_load_manager.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .pipelines/v2/templates/steps-build-python.yml | 10 ++++++++++ sdk_v2/cpp/src/inferencing/model_load_manager.cc | 11 +++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.pipelines/v2/templates/steps-build-python.yml b/.pipelines/v2/templates/steps-build-python.yml index c65c72b3d..84201a682 100644 --- a/.pipelines/v2/templates/steps-build-python.yml +++ b/.pipelines/v2/templates/steps-build-python.yml @@ -122,6 +122,16 @@ steps: if ($files.Count -eq 0) { throw "No native artifacts found under ${{ parameters.nativeArtifactDir }}" } + if ($rid -like 'win-*') { + $found = @{} + foreach ($f in $files) { + $found[$f.Name] = $true + } + $missing = @($allowed | Where-Object { -not $found.ContainsKey($_) }) + if ($missing.Count -gt 0) { + throw "Missing required Windows native artifact(s): $($missing -join ', ') under ${{ parameters.nativeArtifactDir }}" + } + } foreach ($f in $files) { Copy-Item $f.FullName -Destination $dest -Force } diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.cc b/sdk_v2/cpp/src/inferencing/model_load_manager.cc index 6254a51cb..dccb87605 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.cc +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.cc @@ -269,12 +269,14 @@ void ModelLoadManager::UnloadAll(std::chrono::milliseconds timeout) { for (const auto& id : ids) { int remaining = 0; - while (clock::now() < deadline) { + bool unloaded = false; + while (true) { { std::lock_guard lock(mutex_); auto it = loaded_models_.find(id); if (it == loaded_models_.end()) { remaining = 0; + unloaded = true; break; } @@ -282,14 +284,19 @@ void ModelLoadManager::UnloadAll(std::chrono::milliseconds timeout) { if (remaining == 0) { logger_.Log(LogLevel::Information, fmt::format("unloading model: {}", id)); loaded_models_.erase(it); + unloaded = true; break; } } + if (clock::now() >= deadline) { + break; + } + std::this_thread::sleep_for(kPollInterval); } - if (remaining > 0) { + if (!unloaded && remaining > 0) { logger_.Log(LogLevel::Warning, fmt::format("Shutdown: model '{}' still has {} session(s) after overall {}ms deadline; leaving loaded", id, remaining, timeout.count())); From ad5315f3706ac2f97a9f3c80944ef15d9faf9c66 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 13 Jul 2026 01:06:03 -0500 Subject: [PATCH 71/77] Close shutdown load and JS packaging gaps Re-check shutdown under the load-manager lock and require WinML/DirectML siblings when staging Windows JS prebuilds. Files changed: - sdk_v2/cpp/src/inferencing/model_load_manager.cc - sdk_v2/js/script/pack-prebuilds.mjs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/inferencing/model_load_manager.cc | 4 ++++ sdk_v2/js/script/pack-prebuilds.mjs | 10 +++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.cc b/sdk_v2/cpp/src/inferencing/model_load_manager.cc index dccb87605..f10e09e91 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.cc +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.cc @@ -130,6 +130,10 @@ ModelLoadManager::LoadResult ModelLoadManager::LoadModel(std::string_view model_ std::string path_str(model_path); std::string id_str(model_id); std::lock_guard lock(mutex_); + if (shutdown_.load()) { + FL_LOG_AND_THROW(logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "cannot load model during shutdown"); + } // Check if model is already loaded auto it = loaded_models_.find(id_str); diff --git a/sdk_v2/js/script/pack-prebuilds.mjs b/sdk_v2/js/script/pack-prebuilds.mjs index 8dad4a509..5dbf558aa 100644 --- a/sdk_v2/js/script/pack-prebuilds.mjs +++ b/sdk_v2/js/script/pack-prebuilds.mjs @@ -43,18 +43,14 @@ const destDir = resolve(pkgRoot, "prebuilds", `${process.platform}-${process.arc mkdirSync(destDir, { recursive: true }); // foundry_local is required; ORT/GenAI siblings are excluded (fetched at install -// time). The WinML EP catalog DLL is an optional sibling bundled on Windows. +// time). The WinML EP catalog and DirectML DLLs are required siblings on Windows. const wanted = (() => { - if (process.platform === "win32") return ["foundry_local.dll"]; + if (process.platform === "win32") return ["foundry_local.dll", "Microsoft.Windows.AI.MachineLearning.dll", "DirectML.dll"]; if (process.platform === "darwin") return ["libfoundry_local.dylib"]; return ["libfoundry_local.so"]; })(); -// Optional native siblings — copied when present, skipped (with a warning) when -// not. The reg-free WinML 2.x runtime ships next to foundry_local.dll on Windows -// so WinML hardware EPs work out of the box without an install-time download. -const optional = - process.platform === "win32" ? ["Microsoft.Windows.AI.MachineLearning.dll", "DirectML.dll"] : []; +const optional = []; let copied = 0; const available = new Set(readdirSync(sourceDir)); From f8c2e2a4097ef291db9a9dc8823f2f76852b1134 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 13 Jul 2026 03:21:35 -0500 Subject: [PATCH 72/77] Gate blob skipping to sidecar-safe retries Only skip full-size blobs during model retry downloads when the existing download marker proves the directory was created by the sidecar-safe downloader; legacy incomplete directories redownload full-size blobs. Files changed: - sdk_v2/cpp/src/download/download_manager.cc - sdk_v2/cpp/test/internal_api/download_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/download/download_manager.cc | 20 ++++++- sdk_v2/cpp/test/internal_api/download_test.cc | 55 +++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/sdk_v2/cpp/src/download/download_manager.cc b/sdk_v2/cpp/src/download/download_manager.cc index c322cde72..fde97d855 100644 --- a/sdk_v2/cpp/src/download/download_manager.cc +++ b/sdk_v2/cpp/src/download/download_manager.cc @@ -28,6 +28,7 @@ namespace fl { namespace { const char* kDownloadSignalFileName = "download.tmp"; +const char* kDownloadSignalSidecarSafeMarker = "foundry-local-sidecar-safe-v1\n"; const char* kGenAIConfigFileName = "genai_config.json"; const char* kInferenceModelFileName = "inference_model.json"; const char* kDefaultRegistryRegion = "centralus"; @@ -45,7 +46,6 @@ bool HasInferenceModelJson(const std::string& model_path) { if (ec) { return false; } - for (const auto& entry : it) { if (entry.is_directory(ec)) { if (std::filesystem::exists(entry.path() / kInferenceModelFileName)) { @@ -57,6 +57,17 @@ bool HasInferenceModelJson(const std::string& model_path) { return false; } +bool HasSidecarSafeDownloadSignal(const std::filesystem::path& signal_path) { + std::ifstream signal(signal_path); + if (!signal.is_open()) { + return false; + } + + std::string marker; + std::getline(signal, marker); + return marker == "foundry-local-sidecar-safe-v1"; +} + /// Resolve the effective model path — the directory containing genai_config.json. /// For single-variant models this is model_path itself. /// For multi-variant models it's the first subdirectory containing genai_config.json. @@ -349,10 +360,13 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, return ResolveEffectiveModelPath(model_path); } + const bool can_skip_completed_files = + !std::filesystem::exists(model_path) || HasSidecarSafeDownloadSignal(signal_path); + // Create download signal file { std::ofstream signal(signal_path); - // Empty file — its presence indicates download is in progress + signal << kDownloadSignalSidecarSafeMarker; } // Emit 0% immediately so callers know the download process has started. @@ -376,7 +390,7 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, // and the blob container has all files at the root or in variant subdirectories. download_opts.path_prefix = ""; download_opts.max_concurrency = max_concurrency_; - download_opts.skip_completed_files = true; + download_opts.skip_completed_files = can_skip_completed_files; if (progress_cb) { download_opts.progress = [&progress_cb](float percent) { diff --git a/sdk_v2/cpp/test/internal_api/download_test.cc b/sdk_v2/cpp/test/internal_api/download_test.cc index 4bfadc50c..360775532 100644 --- a/sdk_v2/cpp/test/internal_api/download_test.cc +++ b/sdk_v2/cpp/test/internal_api/download_test.cc @@ -956,11 +956,16 @@ TEST(DownloadManagerTest, RetrySkipsCompletedBlobWithoutSidecar) { auto model_dir = fs::path(tmpdir.string()) / "TestPublisher" / "test-model-1"; fs::create_directories(model_dir); + { + std::ofstream signal(model_dir / "download.tmp"); + signal << "foundry-local-sidecar-safe-v1\n"; + } { std::ofstream f(model_dir / "weights.safetensors", std::ios::binary); f.seekp(1023); f.put('\0'); } + ASSERT_EQ(fs::file_size(model_dir / "weights.safetensors"), static_cast(1024)); manager->DownloadModel(info); @@ -968,6 +973,56 @@ TEST(DownloadManagerTest, RetrySkipsCompletedBlobWithoutSidecar) { EXPECT_EQ(mock_downloader_ptr->downloaded_blobs[0], "config.json"); } +TEST(DownloadManagerTest, LegacyIncompleteRetryRedownloadsFullSizeBlobWithoutSidecar) { + TempDir tmpdir; + + auto manager = std::make_unique(tmpdir.string(), "eastus", 64, fl::test::NullLog()); + + auto registry = std::make_unique( + "eastus", fl::test::NullLog(), std::make_unique(fl::test::NullLog(), false), + [](const std::string&) { + return MakeRegistryResponse( + R"({"blobSasUri": "https://storage.blob.core.windows.net/container?sig=test"})"); + }); + manager->SetModelRegistryClient(std::move(registry)); + + auto mock_downloader = std::make_unique(); + auto* mock_downloader_ptr = mock_downloader.get(); + mock_downloader->blobs_to_return = { + {"weights.safetensors", 1024}, + {"config.json", 100}, + }; + manager->SetBlobDownloader(std::move(mock_downloader)); + + ModelInfo info; + info.model_id = "test-model:1"; + info.name = "test-model"; + info.uri = "azureml://registries/test/models/test-model/versions/1"; + info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR] = "TestPublisher"; + + auto model_dir = fs::path(tmpdir.string()) / "TestPublisher" / "test-model-1"; + fs::create_directories(model_dir); + { + std::ofstream signal(model_dir / "download.tmp"); + signal << "legacy-incomplete-download"; + } + { + std::ofstream f(model_dir / "weights.safetensors", std::ios::binary); + f.seekp(1023); + f.put('\0'); + } + + manager->DownloadModel(info); + + ASSERT_EQ(mock_downloader_ptr->downloaded_blobs.size(), 2u); + EXPECT_NE(std::find(mock_downloader_ptr->downloaded_blobs.begin(), mock_downloader_ptr->downloaded_blobs.end(), + "weights.safetensors"), + mock_downloader_ptr->downloaded_blobs.end()); + EXPECT_NE(std::find(mock_downloader_ptr->downloaded_blobs.begin(), mock_downloader_ptr->downloaded_blobs.end(), + "config.json"), + mock_downloader_ptr->downloaded_blobs.end()); +} + // --- Region resolution: detected region drives the download endpoint --- // Run one download and return the registry URL the manager hit. From a82a88569495b34c4fe9aaceafd49bc0ded7bbd1 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 13 Jul 2026 04:06:59 -0500 Subject: [PATCH 73/77] Keep legacy download repairs untrusted after failure Leave unsafe legacy retry markers in place until a full repair completes so failed retries cannot promote corrupt full-size blobs to sidecar-safe skipping. Files changed: - sdk_v2/cpp/src/download/download_manager.cc - sdk_v2/cpp/test/internal_api/download_test.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/download/download_manager.cc | 3 +- sdk_v2/cpp/test/internal_api/download_test.cc | 54 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/sdk_v2/cpp/src/download/download_manager.cc b/sdk_v2/cpp/src/download/download_manager.cc index fde97d855..deb5df250 100644 --- a/sdk_v2/cpp/src/download/download_manager.cc +++ b/sdk_v2/cpp/src/download/download_manager.cc @@ -29,6 +29,7 @@ namespace { const char* kDownloadSignalFileName = "download.tmp"; const char* kDownloadSignalSidecarSafeMarker = "foundry-local-sidecar-safe-v1\n"; +const char* kDownloadSignalLegacyRepairMarker = "foundry-local-legacy-repair\n"; const char* kGenAIConfigFileName = "genai_config.json"; const char* kInferenceModelFileName = "inference_model.json"; const char* kDefaultRegistryRegion = "centralus"; @@ -366,7 +367,7 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, // Create download signal file { std::ofstream signal(signal_path); - signal << kDownloadSignalSidecarSafeMarker; + signal << (can_skip_completed_files ? kDownloadSignalSidecarSafeMarker : kDownloadSignalLegacyRepairMarker); } // Emit 0% immediately so callers know the download process has started. diff --git a/sdk_v2/cpp/test/internal_api/download_test.cc b/sdk_v2/cpp/test/internal_api/download_test.cc index 360775532..435bff39f 100644 --- a/sdk_v2/cpp/test/internal_api/download_test.cc +++ b/sdk_v2/cpp/test/internal_api/download_test.cc @@ -1023,6 +1023,60 @@ TEST(DownloadManagerTest, LegacyIncompleteRetryRedownloadsFullSizeBlobWithoutSid mock_downloader_ptr->downloaded_blobs.end()); } +TEST(DownloadManagerTest, FailedLegacyRepairDoesNotPromoteToSidecarSafeRetry) { + TempDir tmpdir; + + int registry_calls = 0; + auto manager = std::make_unique(tmpdir.string(), "eastus", 64, fl::test::NullLog()); + auto registry = std::make_unique( + "eastus", fl::test::NullLog(), std::make_unique(fl::test::NullLog(), false), + [®istry_calls](const std::string&) { + ++registry_calls; + if (registry_calls == 1) { + return MakeRegistryResponse("temporary failure", 503); + } + return MakeRegistryResponse( + R"({"blobSasUri": "https://storage.blob.core.windows.net/container?sig=test"})"); + }); + manager->SetModelRegistryClient(std::move(registry)); + + auto mock_downloader = std::make_unique(); + auto* mock_downloader_ptr = mock_downloader.get(); + mock_downloader->blobs_to_return = { + {"weights.safetensors", 1024}, + {"config.json", 100}, + }; + manager->SetBlobDownloader(std::move(mock_downloader)); + + ModelInfo info; + info.model_id = "test-model:1"; + info.name = "test-model"; + info.uri = "azureml://registries/test/models/test-model/versions/1"; + info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR] = "TestPublisher"; + + auto model_dir = fs::path(tmpdir.string()) / "TestPublisher" / "test-model-1"; + fs::create_directories(model_dir); + { + std::ofstream signal(model_dir / "download.tmp"); + signal << "legacy-incomplete-download"; + } + { + std::ofstream f(model_dir / "weights.safetensors", std::ios::binary); + f.seekp(1023); + f.put('\0'); + } + + EXPECT_THROW(manager->DownloadModel(info), fl::Exception); + EXPECT_TRUE(mock_downloader_ptr->downloaded_blobs.empty()); + + manager->DownloadModel(info); + + ASSERT_EQ(mock_downloader_ptr->downloaded_blobs.size(), 2u); + EXPECT_NE(std::find(mock_downloader_ptr->downloaded_blobs.begin(), mock_downloader_ptr->downloaded_blobs.end(), + "weights.safetensors"), + mock_downloader_ptr->downloaded_blobs.end()); +} + // --- Region resolution: detected region drives the download endpoint --- // Run one download and return the registry URL the manager hit. From 515e28522846f3c4b0a8c3e28aa25cdab24d5bdc Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 13 Jul 2026 04:58:04 -0500 Subject: [PATCH 74/77] Preserve fresh retry trust and catalog names Use pre-creation directory state for sidecar-safe retry markers and keep Azure catalog public names as configured catalog URIs. Files changed: - sdk_v2/cpp/src/catalog/azure_model_catalog.cc - sdk_v2/cpp/src/download/download_manager.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/catalog/azure_model_catalog.cc | 3 +-- sdk_v2/cpp/src/download/download_manager.cc | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index cc4342dac..2135a6fe7 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -94,8 +94,7 @@ AzureModelCatalog::AzureModelCatalog(std::vector Date: Mon, 13 Jul 2026 08:51:03 -0500 Subject: [PATCH 75/77] Retain URL snapshots and harden user agents Keep web-service URL snapshots valid until manager release and sanitize client User-Agent telemetry with a conservative allowlist and length cap. Files changed: - sdk_v2/cpp/src/c_api.cc - sdk_v2/cpp/src/service/handler_utils.h Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/c_api.cc | 22 +++++++++------------ sdk_v2/cpp/src/service/handler_utils.h | 27 +++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/sdk_v2/cpp/src/c_api.cc b/sdk_v2/cpp/src/c_api.cc index 719f11f0d..61315004a 100644 --- a/sdk_v2/cpp/src/c_api.cc +++ b/sdk_v2/cpp/src/c_api.cc @@ -81,7 +81,7 @@ struct flManager { fl::Manager& impl; std::unique_ptr catalog; // stores the flCatalog wrapper around impl.GetCatalog() mutable std::mutex urls_cache_mutex; - mutable std::unique_ptr urls_snapshot; + mutable std::vector> urls_snapshots; }; // ======================================================================== @@ -384,23 +384,20 @@ FL_API_STATUS_IMPL(Manager_WebServiceUrlsImpl, const flManager* manager, std::lock_guard lock(manager->urls_cache_mutex); auto urls = manager->impl.GetWebServiceUrls(); if (urls.empty()) { - manager->urls_snapshot.reset(); *out_urls = nullptr; *out_num_urls = 0; return nullptr; } - if (!manager->urls_snapshot || manager->urls_snapshot->strings != urls) { - auto snapshot = std::make_unique(); - snapshot->strings = std::move(urls); - snapshot->pointers.reserve(snapshot->strings.size()); - for (const auto& u : snapshot->strings) { - snapshot->pointers.push_back(u.c_str()); - } - manager->urls_snapshot = std::move(snapshot); + auto snapshot = std::make_unique(); + snapshot->strings = std::move(urls); + snapshot->pointers.reserve(snapshot->strings.size()); + for (const auto& u : snapshot->strings) { + snapshot->pointers.push_back(u.c_str()); } - auto* snapshot_ptr = manager->urls_snapshot.get(); + auto* snapshot_ptr = snapshot.get(); + manager->urls_snapshots.push_back(std::move(snapshot)); *out_urls = snapshot_ptr->pointers.data(); *out_num_urls = snapshot_ptr->pointers.size(); return nullptr; @@ -412,10 +409,9 @@ FL_API_STATUS_IMPL(Manager_WebServiceStopImpl, flManager* manager) { if (!manager) { return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null manager"); } - std::lock_guard lock(manager->urls_cache_mutex); manager->impl.StopWebService(); - manager->urls_snapshot.reset(); + manager->impl.StopWebService(); return nullptr; API_IMPL_END } diff --git a/sdk_v2/cpp/src/service/handler_utils.h b/sdk_v2/cpp/src/service/handler_utils.h index 23c016728..0eda909b6 100644 --- a/sdk_v2/cpp/src/service/handler_utils.h +++ b/sdk_v2/cpp/src/service/handler_utils.h @@ -12,6 +12,8 @@ #include "telemetry/telemetry_redaction.h" +#include +#include #include #include #include @@ -74,7 +76,30 @@ inline std::string GetUserAgent(const std::shared_ptr(ua->size(), 256)); + size_t run_length = 0; + for (char ch : *ua) { + unsigned char c = static_cast(ch); + bool safe = std::isalnum(c) || ch == ' ' || ch == '.' || ch == '-' || ch == '_' || ch == '/' || + ch == '(' || ch == ')' || ch == ';'; + if (!safe) { + scrubbed.push_back('?'); + run_length = 0; + continue; + } + + if (std::isalnum(c)) { + ++run_length; + if (run_length > 32) { + scrubbed.push_back('?'); + continue; + } + } else { + run_length = 0; + } + scrubbed.push_back(ch); + } constexpr size_t kMaxUserAgentLength = 256; if (scrubbed.size() > kMaxUserAgentLength) { scrubbed.resize(kMaxUserAgentLength); From 34ba2bcf0b3334ab2a96112161e67ff79ce09580 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 13 Jul 2026 10:41:34 -0500 Subject: [PATCH 76/77] Reuse URL snapshots and notify completed streams Reuse immutable web-service URL snapshots for unchanged URL generations and notify the streaming reaper when a thread completes before tracking observes it. Files changed: - sdk_v2/cpp/src/c_api.cc - sdk_v2/cpp/src/service/web_service.h Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- sdk_v2/cpp/src/c_api.cc | 21 +++++++++++++-------- sdk_v2/cpp/src/service/web_service.h | 4 ++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/sdk_v2/cpp/src/c_api.cc b/sdk_v2/cpp/src/c_api.cc index 61315004a..71b385322 100644 --- a/sdk_v2/cpp/src/c_api.cc +++ b/sdk_v2/cpp/src/c_api.cc @@ -82,6 +82,7 @@ struct flManager { std::unique_ptr catalog; // stores the flCatalog wrapper around impl.GetCatalog() mutable std::mutex urls_cache_mutex; mutable std::vector> urls_snapshots; + mutable UrlSnapshot* current_urls_snapshot = nullptr; }; // ======================================================================== @@ -386,18 +387,22 @@ FL_API_STATUS_IMPL(Manager_WebServiceUrlsImpl, const flManager* manager, if (urls.empty()) { *out_urls = nullptr; *out_num_urls = 0; + manager->current_urls_snapshot = nullptr; return nullptr; } - auto snapshot = std::make_unique(); - snapshot->strings = std::move(urls); - snapshot->pointers.reserve(snapshot->strings.size()); - for (const auto& u : snapshot->strings) { - snapshot->pointers.push_back(u.c_str()); + if (manager->current_urls_snapshot == nullptr || manager->current_urls_snapshot->strings != urls) { + auto snapshot = std::make_unique(); + snapshot->strings = std::move(urls); + snapshot->pointers.reserve(snapshot->strings.size()); + for (const auto& u : snapshot->strings) { + snapshot->pointers.push_back(u.c_str()); + } + manager->current_urls_snapshot = snapshot.get(); + manager->urls_snapshots.push_back(std::move(snapshot)); } - auto* snapshot_ptr = snapshot.get(); - manager->urls_snapshots.push_back(std::move(snapshot)); + auto* snapshot_ptr = manager->current_urls_snapshot; *out_urls = snapshot_ptr->pointers.data(); *out_num_urls = snapshot_ptr->pointers.size(); return nullptr; @@ -411,7 +416,7 @@ FL_API_STATUS_IMPL(Manager_WebServiceStopImpl, flManager* manager) { } std::lock_guard lock(manager->urls_cache_mutex); manager->impl.StopWebService(); - manager->impl.StopWebService(); + manager->current_urls_snapshot = nullptr; return nullptr; API_IMPL_END } diff --git a/sdk_v2/cpp/src/service/web_service.h b/sdk_v2/cpp/src/service/web_service.h index 33d25f476..fb44cfea2 100644 --- a/sdk_v2/cpp/src/service/web_service.h +++ b/sdk_v2/cpp/src/service/web_service.h @@ -5,6 +5,7 @@ #include "logger.h" #include +#include #include #include #include @@ -50,6 +51,9 @@ class StreamingThreadTracker { } } threads_.push_back(TrackedThread{std::move(t), std::move(done), std::move(abort)}); + if (threads_.back().done && threads_.back().done->load(std::memory_order_acquire)) { + completion_cv_.notify_one(); + } } void NotifyCompleted() { From bc4bf4d8ccea91005da3d9648e1df402d8ff81e1 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 13 Jul 2026 12:05:37 -0500 Subject: [PATCH 77/77] Stage Linux GenAI provider runtimes Copy and stage Linux GenAI CUDA and ORT provider runtime libraries so standalone C++ SDK archives include the native dependencies needed by GPU execution providers. Files changed: - sdk_v2/cpp/CMakeLists.txt - .pipelines/v2/templates/steps-build-linux.yml Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 297bc539-5f7f-4a70-9411-48e91a5bf532 --- .pipelines/v2/templates/steps-build-linux.yml | 6 ++++++ sdk_v2/cpp/CMakeLists.txt | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/.pipelines/v2/templates/steps-build-linux.yml b/.pipelines/v2/templates/steps-build-linux.yml index 20d246d08..7d9e76ef6 100644 --- a/.pipelines/v2/templates/steps-build-linux.yml +++ b/.pipelines/v2/templates/steps-build-linux.yml @@ -101,6 +101,12 @@ steps: echo " staged $(basename "$f")" fi done + for f in "$src"/libonnxruntime-genai*.so; do + if [ -e "$f" ] && [ "$(basename "$f")" != "libonnxruntime-genai.so" ]; then + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + fi + done for f in "$src"/libonnxruntime_providers_*.so; do if [ -e "$f" ]; then cp -P "$f" "$dst/" diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index 72ebf6316..d0677dd75 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -536,6 +536,21 @@ if(TARGET OnnxRuntime::OnnxRuntime) $ ) endif() + + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + file(GLOB _ORT_GENAI_RUNTIME_LIBS CONFIGURE_DEPENDS + "${ORT_GENAI_LIB_DIR}/libonnxruntime-genai*.so") + file(GLOB _ORT_PROVIDER_RUNTIME_LIBS CONFIGURE_DEPENDS + "${ORT_LIB_DIR}/libonnxruntime_providers_*.so") + set(_LINUX_OPTIONAL_RUNTIME_LIBS ${_ORT_GENAI_RUNTIME_LIBS} ${_ORT_PROVIDER_RUNTIME_LIBS}) + if(_LINUX_OPTIONAL_RUNTIME_LIBS) + add_custom_command(TARGET foundry_local POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${_LINUX_OPTIONAL_RUNTIME_LIBS} + $ + ) + endif() + endif() endif() endif()