Read Tool & Reasoning Tokens from GenAI API instead of Catalog Metadata

sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc

@@ -84,7 +84,37 @@ std::string OnnxChatGenerator::Decode() {
 
   int32_t token_id = next_tokens[0];
 
-  // Decode through the normal tokenizer stream
+  // Fast path: use tag token IDs for efficient special-token detection.
+  // When tag IDs are configured (>= 0), we detect tool-call and reasoning tokens
+  // with a simple integer comparison, avoiding the expensive double-decode + string search.
+  const auto& tag_info = model_.GetTagInfo();
+  bool has_tag_ids = (tag_info.bot_id.has_value() || tag_info.eot_id.has_value() ||
+                      tag_info.bor_id.has_value() || tag_info.eor_id.has_value());
+
+  if (has_tag_ids) {
+    if (tag_info.bot_id.has_value() && token_id == *tag_info.bot_id) {
+      stream_->Decode(token_id);  // keep normal stream in sync
+      return tag_info.bot_str;
+    }
+    if (tag_info.eot_id.has_value() && token_id == *tag_info.eot_id) {
+      stream_->Decode(token_id);
+      return tag_info.eot_str;
+    }
+    if (tag_info.bor_id.has_value() && token_id == *tag_info.bor_id) {
+      stream_->Decode(token_id);
+      return tag_info.bor_str;
+    }
+    if (tag_info.eor_id.has_value() && token_id == *tag_info.eor_id) {
+      stream_->Decode(token_id);
+      return tag_info.eor_str;
+    }
+
+    // Not a tag token — decode normally (no special stream needed)
+    const char* token_text = stream_->Decode(token_id);
+    return token_text ? std::string(token_text) : "";
+  }
+
+  // Slow path: no tag IDs configured — fall back to double-decode + string matching.
   const char* token_text = stream_->Decode(token_id);
 
   // Also decode through the special-token stream to detect tool call and think tokens.

sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc

@@ -140,4 +140,37 @@ const std::vector<int32_t>& GenAIModelInstance::GetEosTokenIds() {
   return eos_token_ids_;
 }
 
+const GenAIModelInstance::TagInfo& GenAIModelInstance::GetTagInfo() {
+  std::call_once(tag_info_init_flag_, [this]() {
+    if (!tokenizer_) return;
+
+    // Get tag IDs from the tokenizer (reads from config, with fallback vocab lookup).
+    // These throw if the model doesn't define the token, so we catch and leave as nullopt.
+    auto try_get_id = [](auto& getter) -> std::optional<int32_t> {
+      try { return getter(); } catch (...) { return std::nullopt; }
+    };
+    tag_info_.bot_id = try_get_id([&] { return tokenizer_->GetBotTokenId(); });
+    tag_info_.eot_id = try_get_id([&] { return tokenizer_->GetEotTokenId(); });
+    tag_info_.bor_id = try_get_id([&] { return tokenizer_->GetBorTokenId(); });
+    tag_info_.eor_id = try_get_id([&] { return tokenizer_->GetEorTokenId(); });
+
+    // Decode each valid ID once through the special tokenizer to get the string.
+    // Uses tokenizer_with_special_ so that special token text (e.g., "<tool_call>") is produced.
+    auto decode_id = [this](std::optional<int32_t> id) -> std::string {
+      if (!id.has_value() || !tokenizer_with_special_) return {};
+      int32_t val = *id;
+      OgaString text = tokenizer_with_special_->Decode(&val, 1);
+      const char* p = text;
+      return p ? std::string(p) : std::string();
+    };
+
+    tag_info_.bot_str = decode_id(tag_info_.bot_id);
+    tag_info_.eot_str = decode_id(tag_info_.eot_id);
+    tag_info_.bor_str = decode_id(tag_info_.bor_id);
+    tag_info_.eor_str = decode_id(tag_info_.eor_id);
+  });
+
+  return tag_info_;
+}
+
 }  // namespace fl

sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h

@@ -10,6 +10,7 @@
 #include <chrono>
 #include <memory>
 #include <mutex>
+#include <optional>
 #include <string>
 
 // Forward declarations for ORT GenAI types (defined in ort_genai.h)
@@ -36,6 +37,25 @@ class GenAIModelInstance {
   ExecutionProvider EP() const { return ep_; }
   bool IsMultiModal() const;
 
+  /// Cached tag token IDs and their decoded strings for efficient detection.
+  /// IDs are used for integer comparison in the decode loop (fast path).
+  /// Strings are used by ToolCallContext/Accumulator for text-based processing.
+  /// Populated once at first access via OgaTokenizer getters + tokenizer decode.
+  /// Naming follows bos/eos/pad convention:
+  ///   bot = beginning of tool (call), eot = end of tool (call)
+  ///   bor = beginning of reasoning,   eor = end of reasoning
+  struct TagInfo {
+    std::optional<int32_t> bot_id;
+    std::optional<int32_t> eot_id;
+    std::optional<int32_t> bor_id;
+    std::optional<int32_t> eor_id;
+    std::string bot_str;
+    std::string eot_str;
+    std::string bor_str;
+    std::string eor_str;
+  };
+  const TagInfo& GetTagInfo();
+
   /// Access the underlying OGA objects (for future chat generation work).
   OgaModel& GetOgaModel();
   OgaTokenizer& GetOgaTokenizer();
@@ -76,6 +96,8 @@ class GenAIModelInstance {
   std::unique_ptr<OgaMultiModalProcessor> processor_;  // nullptr if not multimodal
   std::vector<int32_t> eos_token_ids_;                 // cached; populated on first GetEosTokenIds() call
   std::once_flag eos_token_ids_init_flag_;
+  TagInfo tag_info_;                                    // cached; populated on first GetTagInfo() call
+  std::once_flag tag_info_init_flag_;
   std::chrono::steady_clock::time_point last_activity_;
   mutable std::atomic<int> session_ref_count_{0};
 };

sdk_v2/cpp/src/model.cc

@@ -316,6 +316,36 @@ void Model::Load(ExecutionProvider ep) {
   if (result.status == ModelLoadManager::LoadStatus::kModelNotFound) {
     FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "model not found at path: " + local_path_);
   }
+
+  // Enrich ModelInfo with metadata from the loaded GenAI model (genai_config.json + fallback map).
+  // This makes tool/reasoning tags available via ModelInfo regardless of whether they came from
+  // catalog metadata, so downstream code doesn't need to know about multiple metadata sources.
+  if (result.model && result.status == ModelLoadManager::LoadStatus::kSuccess) {
+    const auto& tag_info = result.model->GetTagInfo();
+
+    auto enrich = [&](const std::string& val, const char* prop_key) {
+      if (!val.empty() && !info_.GetPropertyStr(prop_key)) {
+        info_.string_properties[prop_key] = val;
+      }
+    };
+
+    enrich(tag_info.bot_str, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR);
+    enrich(tag_info.eot_str, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR);
+    enrich(tag_info.bor_str, FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR);
+    enrich(tag_info.eor_str, FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR);
+
+    // Infer support flags from the presence of valid tag IDs
+    if (!info_.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT)) {
+      if (tag_info.bot_id.has_value()) {
+        info_.int_properties[FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT] = 1;
+      }
+    }
+    if (!info_.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT)) {
+      if (tag_info.bor_id.has_value()) {
+        info_.int_properties[FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT] = 1;
+      }
+    }
+  }
 }
 
 void Model::Unload() {