diff --git a/app/controllers/ruby_llm/agents/requests_controller.rb b/app/controllers/ruby_llm/agents/requests_controller.rb new file mode 100644 index 00000000..987be19d --- /dev/null +++ b/app/controllers/ruby_llm/agents/requests_controller.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +module RubyLLM + module Agents + # Controller for browsing tracked request groups + # + # Provides listing and detail views for executions grouped by + # request_id, as set by RubyLLM::Agents.track blocks. + # + # @api private + class RequestsController < ApplicationController + include Paginatable + + # Lists all tracked requests with aggregated stats + # + # @return [void] + def index + @sort_column = sanitize_sort_column(params[:sort]) + @sort_direction = (params[:direction] == "asc") ? "asc" : "desc" + + scope = Execution + .where.not(request_id: [nil, ""]) + .select( + "request_id", + "COUNT(*) AS call_count", + "SUM(total_cost) AS total_cost", + "SUM(total_tokens) AS total_tokens", + "MIN(started_at) AS started_at", + "MAX(completed_at) AS completed_at", + "SUM(duration_ms) AS total_duration_ms", + "GROUP_CONCAT(DISTINCT agent_type) AS agent_types_list", + "GROUP_CONCAT(DISTINCT status) AS statuses_list", + "MAX(created_at) AS latest_created_at" + ) + .group(:request_id) + + # Apply time filter + days = params[:days].to_i + scope = scope.where("created_at >= ?", days.days.ago) if days > 0 + + result = paginate_requests(scope) + @requests = result[:records] + @pagination = result[:pagination] + + # Stats + total_scope = Execution.where.not(request_id: [nil, ""]) + @stats = { + total_requests: total_scope.distinct.count(:request_id), + total_cost: total_scope.sum(:total_cost) || 0 + } + end + + # Shows a single tracked request with all its executions + # + # @return [void] + def show + @request_id = params[:id] + @executions = Execution + .where(request_id: @request_id) + .order(started_at: :asc) + + if @executions.empty? + redirect_to ruby_llm_agents.requests_path, + alert: "Request not found: #{@request_id}" + return + end + + @summary = { + call_count: @executions.count, + total_cost: @executions.sum(:total_cost) || 0, + total_tokens: @executions.sum(:total_tokens) || 0, + started_at: @executions.minimum(:started_at), + completed_at: @executions.maximum(:completed_at), + agent_types: @executions.distinct.pluck(:agent_type), + models_used: @executions.distinct.pluck(:model_id), + all_successful: @executions.where.not(status: "success").count.zero?, + error_count: @executions.where(status: "error").count + } + + if @summary[:started_at] && @summary[:completed_at] + @summary[:duration_ms] = ((@summary[:completed_at] - @summary[:started_at]) * 1000).to_i + end + end + + private + + ALLOWED_SORT_COLUMNS = %w[latest_created_at call_count total_cost total_tokens total_duration_ms].freeze + + def sanitize_sort_column(column) + ALLOWED_SORT_COLUMNS.include?(column) ? column : "latest_created_at" + end + + def paginate_requests(scope) + page = [(params[:page] || 1).to_i, 1].max + per_page = RubyLLM::Agents.configuration.per_page + + total_count = Execution + .where.not(request_id: [nil, ""]) + .distinct + .count(:request_id) + + sorted = scope.order("#{@sort_column} #{@sort_direction.upcase}") + offset = (page - 1) * per_page + + { + records: sorted.offset(offset).limit(per_page), + pagination: { + current_page: page, + per_page: per_page, + total_count: total_count, + total_pages: (total_count.to_f / per_page).ceil + } + } + end + end + end +end diff --git a/app/views/layouts/ruby_llm/agents/application.html.erb b/app/views/layouts/ruby_llm/agents/application.html.erb index fc34a5e9..0cad5873 100644 --- a/app/views/layouts/ruby_llm/agents/application.html.erb +++ b/app/views/layouts/ruby_llm/agents/application.html.erb @@ -295,7 +295,8 @@ <% nav_items = [ [ruby_llm_agents.root_path, "dashboard"], [ruby_llm_agents.agents_path, "agents"], - [ruby_llm_agents.executions_path, "executions"] + [ruby_llm_agents.executions_path, "executions"], + [ruby_llm_agents.requests_path, "requests"] ] nav_items << [ruby_llm_agents.tenants_path, "tenants"] if tenant_filter_enabled? nav_items.each do |path, label| %> @@ -345,7 +346,8 @@ <% mobile_nav_items = [ [ruby_llm_agents.root_path, "dashboard"], [ruby_llm_agents.agents_path, "agents"], - [ruby_llm_agents.executions_path, "executions"] + [ruby_llm_agents.executions_path, "executions"], + [ruby_llm_agents.requests_path, "requests"] ] mobile_nav_items << [ruby_llm_agents.tenants_path, "tenants"] if tenant_filter_enabled? mobile_nav_items.each do |path, label| %> diff --git a/app/views/ruby_llm/agents/requests/index.html.erb b/app/views/ruby_llm/agents/requests/index.html.erb new file mode 100644 index 00000000..8d2f37bc --- /dev/null +++ b/app/views/ruby_llm/agents/requests/index.html.erb @@ -0,0 +1,153 @@ +
+ requests +
+ + <%= number_with_delimiter(@stats[:total_requests]) %> tracked + · + $<%= number_with_precision(@stats[:total_cost] || 0, precision: 4) %> total + +
+ +<% + sort_column = @sort_column + sort_direction = @sort_direction + + sort_link = ->(column, label, extra_class: "") { + is_active = column == sort_column + next_dir = is_active && sort_direction == "asc" ? "desc" : "asc" + url = url_for(request.query_parameters.merge(sort: column, direction: next_dir, page: 1)) + + arrow = if is_active && sort_direction == "asc" + raw('') + elsif is_active + raw('') + else + "" + end + + active_class = is_active ? "text-gray-700 dark:text-gray-300" : "" + raw(%(#{label}#{arrow})) + } +%> + +<% if @requests.empty? %> +
+ No tracked requests found. Use RubyLLM::Agents.track { ... } to start tracking. +
+<% else %> + +
+ request_id + agents + <%= sort_link.call("call_count", "calls", extra_class: "justify-end w-full") %> + + + + <%= sort_link.call("total_cost", "cost", extra_class: "justify-end w-full") %> + <%= sort_link.call("latest_created_at", "time", extra_class: "justify-end w-full") %> +
+ + +
+ <% @requests.each do |req| %> + <% + statuses = (req.statuses_list || "").split(",") + has_errors = statuses.include?("error") || statuses.include?("timeout") + all_success = statuses == ["success"] + status_class = if has_errors + "badge-error" + elsif all_success + "badge-success" + else + "badge-running" + end + status_label = if has_errors + "errors" + elsif all_success + "ok" + else + "mixed" + end + agent_names = (req.agent_types_list || "").split(",").map { |a| a.gsub(/Agent$/, "") } + %> +
+ + <%= truncate(req.request_id, length: 20) %> + + + <%= agent_names.first(3).join(", ") %><%= agent_names.size > 3 ? " +#{agent_names.size - 3}" : "" %> + + <%= req.call_count %> + + + + $<%= number_with_precision(req.total_cost || 0, precision: 4) %> + + <% if req.respond_to?(:latest_created_at) && req.latest_created_at %> + <%= time_ago_in_words(req.latest_created_at) %> + <% else %> + — + <% end %> + +
+ <% end %> +
+ + <%# Pagination %> + <% if @pagination && @pagination[:total_pages] > 1 %> + <% + current_page = @pagination[:current_page] + total_pages = @pagination[:total_pages] + total_count = @pagination[:total_count] + per_page = @pagination[:per_page] + from_record = ((current_page - 1) * per_page) + 1 + to_record = [current_page * per_page, total_count].min + %> +
+ <%= from_record %>-<%= to_record %> of <%= number_with_delimiter(total_count) %> + +
+ <% end %> +<% end %> diff --git a/app/views/ruby_llm/agents/requests/show.html.erb b/app/views/ruby_llm/agents/requests/show.html.erb new file mode 100644 index 00000000..c8c3b864 --- /dev/null +++ b/app/views/ruby_llm/agents/requests/show.html.erb @@ -0,0 +1,136 @@ +
+ <%= link_to ruby_llm_agents.requests_path, class: "text-gray-400 dark:text-gray-600 hover:text-gray-700 dark:hover:text-gray-300 transition-colors" do %> + + + + <% end %> + request + <%= @request_id %> +
+ <% if @summary[:all_successful] %> + all ok + <% elsif @summary[:error_count] > 0 %> + <%= @summary[:error_count] %> error<%= @summary[:error_count] > 1 ? "s" : "" %> + <% end %> +
+ + +
+
+
calls
+
<%= @summary[:call_count] %>
+
+
+
total cost
+
$<%= number_with_precision(@summary[:total_cost] || 0, precision: 4) %>
+
+
+
tokens
+
<%= number_with_delimiter(@summary[:total_tokens] || 0) %>
+
+
+
duration
+
+ <%= @summary[:duration_ms] ? format_duration_ms(@summary[:duration_ms]) : "—" %> +
+
+
+ + +
+ <% @summary[:agent_types].each do |agent_type| %> + <%= link_to agent_type.gsub(/Agent$/, ""), + ruby_llm_agents.agent_path(agent_type), + class: "font-mono text-xs px-2 py-0.5 rounded bg-gray-100 dark:bg-gray-800 text-gray-600 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-200 transition-colors" %> + <% end %> + + <% @summary[:models_used].each do |model| %> + <%= model %> + <% end %> +
+ + +
+ execution timeline +
+ +
+ +
+ # + agent + status + + duration + + +
+ +
+ <% @executions.each_with_index do |execution, idx| %> + <% + status_badge_class = case execution.status.to_s + when "running" then "badge-running" + when "success" then "badge-success" + when "error" then "badge-error" + when "timeout" then "badge-timeout" + else "badge-timeout" + end + %> +
+ <%= idx + 1 %> + + <%= link_to execution.agent_type.gsub(/Agent$/, ""), + ruby_llm_agents.agent_path(execution.agent_type), + class: "text-gray-900 dark:text-gray-200 hover:underline", + onclick: "event.stopPropagation()" %> + + + <%= execution.status %> + + + + <% if execution.status_running? %> + ... + <% else %> + <%= execution.duration_ms ? format_duration_ms(execution.duration_ms) : "—" %> + <% end %> + + + +
+ <% if execution.status_error? && execution.error_class.present? %> +
+ └ + <%= execution.error_class.split("::").last %><%= execution.error_message.present? ? ": #{truncate(execution.error_message, length: 100)}" : "" %> +
+ <% end %> + <% end %> +
+
+ + +<% if @summary[:started_at] %> +
+ timing +
+
+ started + <%= @summary[:started_at].strftime("%Y-%m-%d %H:%M:%S.%L") %> +
+ <% if @summary[:completed_at] %> +
+ completed + <%= @summary[:completed_at].strftime("%Y-%m-%d %H:%M:%S.%L") %> +
+ <% end %> + <% if @summary[:duration_ms] %> +
+ wall clock + <%= format_duration_ms(@summary[:duration_ms]) %> +
+ <% end %> +
+
+<% end %> diff --git a/config/routes.rb b/config/routes.rb index 3c15f7d9..08fd0138 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -13,6 +13,8 @@ end end + resources :requests, only: [:index, :show] + resources :tenants, only: [:index, :show, :edit, :update] # Redirect old analytics route to dashboard diff --git a/example/README.md b/example/README.md index 88ee31b6..ac1806d6 100644 --- a/example/README.md +++ b/example/README.md @@ -130,6 +130,25 @@ result.classification # Classification details result.content # Response from routed agent ``` +### Multi-Agent Tracking + +```ruby +# Track multiple agent calls, get aggregated cost/tokens/timing +report = RubyLLM::Agents.track(request_id: "support_flow") do + summary = SummarizeAgent.call(text: "Long customer complaint...") + category = ClassifyAgent.call(text: summary.content) + { summary: summary.content, category: category.content } +end + +report.value[:summary] # => "Customer reports billing issue" +report.total_cost # => 0.0023 (combined cost) +report.call_count # => 2 +report.duration_ms # => 1250 +report.request_id # => "support_flow" + +# View in dashboard: http://localhost:3000/agents/requests +``` + ### Image Analyzers (Vision AI) ```ruby diff --git a/example/app/agents/tracking_example.rb b/example/app/agents/tracking_example.rb new file mode 100644 index 00000000..5cea11f3 --- /dev/null +++ b/example/app/agents/tracking_example.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +# Tracking Example - Demonstrates RubyLLM::Agents.track +# +# The `track` block collects results from multiple agent calls, +# aggregates cost/tokens/timing, and links executions in the +# dashboard via a shared request_id. +# +# @example Basic tracking +# report = RubyLLM::Agents.track do +# SummarizeAgent.call(text: "Long article...") +# ClassifyAgent.call(text: "Customer complaint") +# end +# +# report.call_count # => 2 +# report.total_cost # => 0.0023 +# report.total_tokens # => 450 +# report.value # => return value of the block +# +# @example With shared tenant (injected into every call) +# report = RubyLLM::Agents.track(tenant: current_user) do +# AgentA.call(query: "hello") # gets tenant: current_user +# AgentB.call(query: "world") # gets tenant: current_user +# end +# +# @example With request_id and tags +# report = RubyLLM::Agents.track( +# request_id: "voice_chat_#{SecureRandom.hex(4)}", +# tags: { feature: "voice-chat", session_id: session.id } +# ) do +# transcript = TranscribeAgent.call(with: audio_path) +# reply = ChatAgent.call(message: transcript.content) +# audio = SpeakAgent.call(text: reply.content) +# { transcript: transcript.content, reply: reply.content } +# end +# +# report.value[:reply] # => "Here's my response..." +# report.total_cost # => cost of all 3 calls +# report.request_id # => "voice_chat_a1b2c3d4" +# +# @example Error handling (track never raises) +# report = RubyLLM::Agents.track do +# AgentA.call(query: "ok") +# raise "something broke" +# end +# +# report.failed? # => true +# report.error # => # +# report.call_count # => 1 (call before the raise) +# +# @example Nesting (inner results bubble to outer) +# outer = RubyLLM::Agents.track do +# AgentA.call(query: "outer") +# inner = RubyLLM::Agents.track do +# AgentB.call(query: "inner") +# end +# inner.total_cost # cost of inner calls only +# end +# outer.call_count # => 2 (both outer + inner) +# +# @example TrackReport API +# report.successful? # block completed without raising +# report.failed? # block raised +# report.value # block return value (nil if raised) +# report.error # exception (nil if successful) +# report.results # [Result, ...] in call order +# report.call_count # number of agent calls +# report.request_id # shared request_id +# report.total_cost # sum of all costs +# report.total_tokens # sum of all tokens +# report.duration_ms # wall clock time of block +# report.models_used # ["gpt-4o", "whisper-1"] +# report.cost_breakdown # per-call breakdown +# report.all_successful? # all results ok? +# report.any_errors? # any result errors? +# report.to_h # everything as a hash +# +# Dashboard: tracked requests appear on the "requests" page, +# grouped by request_id with aggregate cost/token/timing stats. + +# Helper agents used in tracking examples +class SummarizeAgent < ApplicationAgent + description "Summarizes text content" + model "gpt-4o-mini" + + param :text, required: true + + user "Summarize this in one sentence: {text}" + + def metadata + {showcase: "tracking", role: "summarizer"} + end +end + +class ClassifyAgent < ApplicationAgent + description "Classifies text into categories" + model "gpt-4o-mini" + + param :text, required: true + + user "Classify this text into one of: positive, negative, neutral. Text: {text}" + + def metadata + {showcase: "tracking", role: "classifier"} + end +end diff --git a/example/db/seeds.rb b/example/db/seeds.rb index f3ea475a..09bc1864 100644 --- a/example/db/seeds.rb +++ b/example/db/seeds.rb @@ -2229,7 +2229,290 @@ def create_moderator_execution(attrs = {}) puts " #{generator}: (class definition error - skipping)" end +# ============================================================================= +# TRACKED REQUESTS (multi-agent workflows grouped by request_id) +# ============================================================================= +puts "\n#{"=" * 60}" +puts "Creating tracked requests..." +puts "=" * 60 + +# Request 1: Customer support workflow — classify then respond +req1_id = "track_customer_support_001" +req1_base = Time.current - 15.minutes +create_execution( + request_id: req1_id, + tenant_id: acme.llm_tenant_id, + agent_type: "ClassifyAgent", + model_id: "gpt-4o-mini", + status: "success", + started_at: req1_base, + input_tokens: 320, + output_tokens: 45, + parameters: {message: "My order hasn't arrived and it's been 2 weeks"}, + response: {content: "shipping_issue"}, + metadata: {tags: {"workflow" => "customer_support", "priority" => "high"}}, + created_at: req1_base +) +create_execution( + request_id: req1_id, + tenant_id: acme.llm_tenant_id, + agent_type: "SummarizeAgent", + model_id: "gpt-4o", + status: "success", + started_at: req1_base + 2, + input_tokens: 850, + output_tokens: 320, + parameters: {text: "Customer reports delayed order..."}, + response: {content: "I apologize for the delay. Let me look into your order status..."}, + metadata: {tags: {"workflow" => "customer_support", "priority" => "high"}}, + created_at: req1_base + 2 +) +puts " Created request #{req1_id} (2 agents: classify + summarize)" + +# Request 2: Content pipeline — summarize, classify, then generate +req2_id = "track_content_pipeline_002" +req2_base = Time.current - 45.minutes +create_execution( + request_id: req2_id, + tenant_id: enterprise.llm_tenant_id, + agent_type: "SummarizeAgent", + model_id: "gpt-4o", + status: "success", + started_at: req2_base, + input_tokens: 2400, + output_tokens: 450, + parameters: {text: "Long article about AI safety..."}, + response: {content: "AI safety research focuses on alignment, interpretability..."}, + metadata: {tags: {"pipeline" => "content", "source" => "blog"}}, + created_at: req2_base +) +create_execution( + request_id: req2_id, + tenant_id: enterprise.llm_tenant_id, + agent_type: "ClassifyAgent", + model_id: "gpt-4o-mini", + status: "success", + started_at: req2_base + 3, + input_tokens: 480, + output_tokens: 35, + parameters: {message: "AI safety research focuses on alignment..."}, + response: {content: "technology"}, + metadata: {tags: {"pipeline" => "content", "source" => "blog"}}, + created_at: req2_base + 3 +) +create_execution( + request_id: req2_id, + tenant_id: enterprise.llm_tenant_id, + agent_type: "FullFeaturedAgent", + model_id: "gpt-4o", + status: "success", + started_at: req2_base + 5, + input_tokens: 1200, + output_tokens: 800, + parameters: {query: "Write a social media post about this AI safety summary"}, + response: {content: "🤖 New insights on AI safety: alignment and interpretability..."}, + metadata: {tags: {"pipeline" => "content", "source" => "blog"}}, + created_at: req2_base + 5 +) +puts " Created request #{req2_id} (3 agents: summarize + classify + generate)" + +# Request 3: Failed workflow — classify succeeds, response fails +req3_id = "track_failed_response_003" +req3_base = Time.current - 90.minutes +create_execution( + request_id: req3_id, + tenant_id: startup.llm_tenant_id, + agent_type: "ClassifyAgent", + model_id: "gpt-4o-mini", + status: "success", + started_at: req3_base, + input_tokens: 250, + output_tokens: 30, + parameters: {message: "How do I integrate your API?"}, + response: {content: "technical_question"}, + metadata: {tags: {"workflow" => "support"}}, + created_at: req3_base +) +create_execution( + request_id: req3_id, + tenant_id: startup.llm_tenant_id, + agent_type: "ReliabilityAgent", + model_id: "gpt-4o", + status: "error", + error_class: "RubyLLM::Error", + error_message: "Rate limit exceeded (429)", + started_at: req3_base + 1.5, + input_tokens: 0, + output_tokens: 0, + parameters: {query: "Explain API integration steps"}, + metadata: {tags: {"workflow" => "support"}}, + created_at: req3_base + 1.5 +) +puts " Created request #{req3_id} (2 agents: classify ok + response error)" + +# Request 4: Large batch processing workflow +req4_id = "track_batch_process_004" +req4_base = Time.current - 2.hours +%w[SummarizeAgent SummarizeAgent SummarizeAgent ClassifyAgent ClassifyAgent SchemaAgent].each_with_index do |agent, i| + create_execution( + request_id: req4_id, + tenant_id: acme.llm_tenant_id, + agent_type: agent, + model_id: (agent == "SchemaAgent") ? "gpt-4o" : "gpt-4o-mini", + status: "success", + started_at: req4_base + (i * 2), + input_tokens: rand(400..1200), + output_tokens: rand(100..500), + parameters: {text: "Batch document #{i + 1}"}, + response: {content: "Processed batch item #{i + 1}"}, + metadata: {tags: {"workflow" => "batch_processing", "batch_size" => "6"}}, + created_at: req4_base + (i * 2) + ) +end +puts " Created request #{req4_id} (6 agents: batch processing)" + +# Request 5: Real-time streaming conversation +req5_id = "track_conversation_005" +req5_base = Time.current - 30.minutes +create_execution( + request_id: req5_id, + tenant_id: enterprise.llm_tenant_id, + agent_type: "ConversationAgent", + model_id: "claude-sonnet-4-20250514", + model_provider: "anthropic", + status: "success", + streaming: true, + started_at: req5_base, + input_tokens: 500, + output_tokens: 200, + parameters: {message: "Hello, I need help with my project"}, + response: {content: "Hi! I'd be happy to help. What kind of project are you working on?"}, + metadata: {tags: {"session" => "chat_abc123"}}, + created_at: req5_base +) +create_execution( + request_id: req5_id, + tenant_id: enterprise.llm_tenant_id, + agent_type: "ConversationAgent", + model_id: "claude-sonnet-4-20250514", + model_provider: "anthropic", + status: "success", + streaming: true, + started_at: req5_base + 30, + input_tokens: 900, + output_tokens: 350, + parameters: {message: "It's a Rails app that needs AI features"}, + response: {content: "Great choice! RubyLLM::Agents is perfect for Rails AI integration..."}, + metadata: {tags: {"session" => "chat_abc123"}}, + created_at: req5_base + 30 +) +create_execution( + request_id: req5_id, + tenant_id: enterprise.llm_tenant_id, + agent_type: "ConversationAgent", + model_id: "claude-sonnet-4-20250514", + model_provider: "anthropic", + status: "success", + streaming: true, + started_at: req5_base + 75, + input_tokens: 1400, + output_tokens: 500, + parameters: {message: "Can you show me an example?"}, + response: {content: "Here's a simple agent example..."}, + metadata: {tags: {"session" => "chat_abc123"}}, + created_at: req5_base + 75 +) +puts " Created request #{req5_id} (3 conversation turns with Claude)" + +# Request 6: Multi-model workflow with Gemini +req6_id = "track_multi_model_006" +req6_base = Time.current - 5.minutes +create_execution( + request_id: req6_id, + tenant_id: acme.llm_tenant_id, + agent_type: "ClassifyAgent", + model_id: "gemini-2.0-flash", + model_provider: "google", + status: "success", + started_at: req6_base, + input_tokens: 300, + output_tokens: 25, + parameters: {message: "Translate this document to French"}, + response: {content: "translation_request"}, + metadata: {tags: {"workflow" => "translation"}}, + created_at: req6_base +) +create_execution( + request_id: req6_id, + tenant_id: acme.llm_tenant_id, + agent_type: "FullFeaturedAgent", + model_id: "gpt-4o", + status: "success", + started_at: req6_base + 1, + input_tokens: 1800, + output_tokens: 1600, + parameters: {query: "Translate the following to French: ..."}, + response: {content: "Voici la traduction..."}, + metadata: {tags: {"workflow" => "translation"}}, + created_at: req6_base + 1 +) +puts " Created request #{req6_id} (2 agents: Gemini classify + GPT-4o translate)" + +# Request 7: Running request (still in progress) +req7_id = "track_in_progress_007" +req7_base = Time.current - 10.seconds +create_execution( + request_id: req7_id, + tenant_id: acme.llm_tenant_id, + agent_type: "SummarizeAgent", + model_id: "gpt-4o", + status: "success", + started_at: req7_base, + input_tokens: 600, + output_tokens: 150, + parameters: {text: "Summarize this report..."}, + response: {content: "The quarterly report shows..."}, + metadata: {tags: {"workflow" => "reporting"}}, + created_at: req7_base +) +create_execution( + request_id: req7_id, + tenant_id: acme.llm_tenant_id, + agent_type: "FullFeaturedAgent", + model_id: "gpt-4o", + status: "running", + started_at: req7_base + 2, + input_tokens: 0, + output_tokens: 0, + parameters: {query: "Generate executive brief from summary"}, + metadata: {tags: {"workflow" => "reporting"}}, + created_at: req7_base + 2 +) +puts " Created request #{req7_id} (1 done + 1 running)" + +# Request 8: Single-agent tracked request +req8_id = "track_single_agent_008" +req8_base = Time.current - 3.hours +create_execution( + request_id: req8_id, + tenant_id: demo.llm_tenant_id, + agent_type: "CachingAgent", + model_id: "gpt-4o-mini", + status: "success", + started_at: req8_base, + input_tokens: 200, + output_tokens: 100, + parameters: {query: "What is the capital of France?"}, + response: {content: "Paris"}, + metadata: {tags: {"source" => "demo"}, cached: true}, + created_at: req8_base +) +puts " Created request #{req8_id} (single cached agent)" + +puts " Total: 8 tracked requests created" + puts "\nTotal: #{Organization.count} organizations, #{RubyLLM::Agents::Execution.count} executions" puts "\nStart the server with: bin/rails server" puts "Then visit: http://localhost:3000/agents" puts "\nTo test tenant filtering, append ?tenant_id=acme-corp to the URL" +puts "To view tracked requests, visit: http://localhost:3000/agents/requests" diff --git a/lib/ruby_llm/agents.rb b/lib/ruby_llm/agents.rb index 48f106c1..164e2b8f 100644 --- a/lib/ruby_llm/agents.rb +++ b/lib/ruby_llm/agents.rb @@ -37,7 +37,12 @@ require_relative "agents/infrastructure/budget/forecaster" require_relative "agents/infrastructure/budget/spend_recorder" +# Tracking +require_relative "agents/tracker" +require_relative "agents/track_report" + # Results +require_relative "agents/results/trackable" require_relative "agents/results/base" require_relative "agents/results/embedding_result" require_relative "agents/results/transcription_result" @@ -115,6 +120,69 @@ module RubyLLM # @see RubyLLM::Agents::Configuration module Agents class << self + # Wraps a block of agent calls, collecting all Results and + # returning an aggregated TrackReport. + # + # Shared options (tenant, tags, request_id) are injected into + # every agent instantiated inside the block unless overridden. + # + # @param tenant [Hash, Object, nil] Shared tenant for all calls + # @param request_id [String, nil] Shared request ID (auto-generated if nil) + # @param tags [Hash] Tags merged into each execution's metadata + # @param defaults [Hash] Additional shared options for agents + # @yield Block containing agent calls to track + # @return [TrackReport] Aggregated report of all calls + # + # @example Basic usage + # report = RubyLLM::Agents.track do + # ChatAgent.call(query: "hello") + # SummaryAgent.call(text: "...") + # end + # report.total_cost # => 0.015 + # + # @example With shared tenant + # report = RubyLLM::Agents.track(tenant: current_user) do + # AgentA.call(query: "test") + # end + def track(tenant: nil, request_id: nil, tags: {}, **defaults) + defaults[:tenant] = tenant if tenant + tracker = Tracker.new(defaults: defaults, request_id: request_id, tags: tags) + + # Stack trackers for nesting support + previous_tracker = Thread.current[:ruby_llm_agents_tracker] + Thread.current[:ruby_llm_agents_tracker] = tracker + + started_at = Time.current + value = nil + error = nil + + begin + value = yield + rescue => e + error = e + end + + completed_at = Time.current + + report = TrackReport.new( + value: value, + error: error, + results: tracker.results, + request_id: tracker.request_id, + started_at: started_at, + completed_at: completed_at + ) + + # Bubble results up to parent tracker if nested + if previous_tracker + tracker.results.each { |r| previous_tracker << r } + end + + report + ensure + Thread.current[:ruby_llm_agents_tracker] = previous_tracker + end + # Returns the global configuration instance # # @return [Configuration] The configuration object diff --git a/lib/ruby_llm/agents/base_agent.rb b/lib/ruby_llm/agents/base_agent.rb index 35e143ac..64b3651c 100644 --- a/lib/ruby_llm/agents/base_agent.rb +++ b/lib/ruby_llm/agents/base_agent.rb @@ -332,6 +332,14 @@ def default_temperature # @param temperature [Float] Override the class-level temperature # @param options [Hash] Agent parameters defined via the param DSL def initialize(model: self.class.model, temperature: self.class.temperature, **options) + # Merge tracker defaults (shared options like tenant) — explicit opts win + tracker = Thread.current[:ruby_llm_agents_tracker] + if tracker + options = tracker.defaults.merge(options) + @_track_request_id = tracker.request_id + @_track_tags = tracker.tags + end + @ask_message = options.delete(:_ask_message) @parent_execution_id = options.delete(:_parent_execution_id) @root_execution_id = options.delete(:_root_execution_id) @@ -885,6 +893,7 @@ def find_model_info(model_id) def build_result(content, response, context) Result.new( content: content, + agent_class_name: self.class.name, input_tokens: context.input_tokens, output_tokens: context.output_tokens, input_cost: context.input_cost, diff --git a/lib/ruby_llm/agents/pipeline/middleware/instrumentation.rb b/lib/ruby_llm/agents/pipeline/middleware/instrumentation.rb index b5cea616..6bbcdcb5 100644 --- a/lib/ruby_llm/agents/pipeline/middleware/instrumentation.rb +++ b/lib/ruby_llm/agents/pipeline/middleware/instrumentation.rb @@ -289,6 +289,9 @@ def build_running_execution_data(context) data[:root_execution_id] = context.root_execution_id || context.parent_execution_id end + # Inject tracker request_id and tags + inject_tracker_data(context, data) + data end @@ -548,6 +551,34 @@ def safe_agent_metadata(context) {} end + # Injects tracker request_id and tags into execution data + # + # Reads @_track_request_id and @_track_tags from the agent instance, + # which are set by BaseAgent#initialize when a Tracker is active. + # + # @param context [Context] The execution context + # @param data [Hash] The execution data hash to modify + def inject_tracker_data(context, data) + agent = context.agent_instance + return unless agent + + # Inject request_id + track_request_id = agent.instance_variable_get(:@_track_request_id) + if track_request_id && data[:request_id].blank? + data[:request_id] = track_request_id + end + + # Merge tracker tags into metadata + track_tags = agent.instance_variable_get(:@_track_tags) + if track_tags.is_a?(Hash) && track_tags.any? + data[:metadata] = (data[:metadata] || {}).merge( + "tags" => track_tags.transform_keys(&:to_s) + ) + end + rescue + # Never let tracker data injection break execution + end + # Sensitive parameter keys that should be redacted SENSITIVE_KEYS = %w[ password token api_key secret credential auth key diff --git a/lib/ruby_llm/agents/results/background_removal_result.rb b/lib/ruby_llm/agents/results/background_removal_result.rb index b0cd5ecc..7f0c7124 100644 --- a/lib/ruby_llm/agents/results/background_removal_result.rb +++ b/lib/ruby_llm/agents/results/background_removal_result.rb @@ -15,6 +15,8 @@ module Agents # result.success? # => true # class BackgroundRemovalResult + include Trackable + attr_reader :foreground, :mask, :source_image, :model_id, :output_format, :alpha_matting, :refine_edges, :started_at, :completed_at, :tenant_id, :remover_class, @@ -38,7 +40,7 @@ class BackgroundRemovalResult # @param error_message [String, nil] Error message if failed def initialize(foreground:, mask:, source_image:, model_id:, output_format:, alpha_matting:, refine_edges:, started_at:, completed_at:, - tenant_id:, remover_class:, error_class: nil, error_message: nil) + tenant_id:, remover_class:, error_class: nil, error_message: nil, agent_class_name: nil) @foreground = foreground @mask = mask @source_image = source_image @@ -53,6 +55,10 @@ def initialize(foreground:, mask:, source_image:, model_id:, output_format:, @error_class = error_class @error_message = error_message @execution_id = nil + + # Tracking + @agent_class_name = agent_class_name + register_with_tracker end # Loads the associated Execution record from the database diff --git a/lib/ruby_llm/agents/results/base.rb b/lib/ruby_llm/agents/results/base.rb index 6731deb6..c26bf982 100644 --- a/lib/ruby_llm/agents/results/base.rb +++ b/lib/ruby_llm/agents/results/base.rb @@ -107,6 +107,11 @@ class Result # @return [Integer, nil] Database ID of the associated Execution record attr_reader :execution_id + # @!group Tracking + # @!attribute [r] agent_class_name + # @return [String, nil] The agent class that produced this result + attr_reader :agent_class_name + # Creates a new Result instance # # @param content [Hash, String] The processed response content @@ -159,6 +164,13 @@ def initialize(content:, **options) # Execution record @execution_id = options[:execution_id] + + # Tracking + @agent_class_name = options[:agent_class_name] + + # Register with active tracker + tracker = Thread.current[:ruby_llm_agents_tracker] + tracker << self if tracker end # Loads the associated Execution record from the database @@ -262,7 +274,8 @@ def to_h thinking_text: thinking_text, thinking_signature: thinking_signature, thinking_tokens: thinking_tokens, - execution_id: execution_id + execution_id: execution_id, + agent_class_name: agent_class_name } end diff --git a/lib/ruby_llm/agents/results/embedding_result.rb b/lib/ruby_llm/agents/results/embedding_result.rb index c649cbae..12be0568 100644 --- a/lib/ruby_llm/agents/results/embedding_result.rb +++ b/lib/ruby_llm/agents/results/embedding_result.rb @@ -25,6 +25,8 @@ module Agents # # @api public class EmbeddingResult + include Trackable + # @!attribute [r] vectors # @return [Array>] The embedding vectors attr_reader :vectors @@ -106,6 +108,8 @@ def initialize(attributes = {}) @error_class = attributes[:error_class] @error_message = attributes[:error_message] @execution_id = attributes[:execution_id] + @agent_class_name = attributes[:agent_class_name] + register_with_tracker end # Loads the associated Execution record from the database diff --git a/lib/ruby_llm/agents/results/image_analysis_result.rb b/lib/ruby_llm/agents/results/image_analysis_result.rb index 7bcf2b6b..18928b02 100644 --- a/lib/ruby_llm/agents/results/image_analysis_result.rb +++ b/lib/ruby_llm/agents/results/image_analysis_result.rb @@ -17,6 +17,8 @@ module Agents # result.success? # => true # class ImageAnalysisResult + include Trackable + attr_reader :image, :model_id, :analysis_type, :caption, :description, :tags, :objects, :colors, :text, :raw_response, :started_at, :completed_at, :tenant_id, :analyzer_class, @@ -43,7 +45,7 @@ class ImageAnalysisResult # @param error_message [String, nil] Error message if failed def initialize(image:, model_id:, analysis_type:, caption:, description:, tags:, objects:, colors:, text:, raw_response:, started_at:, completed_at:, - tenant_id:, analyzer_class:, error_class: nil, error_message: nil) + tenant_id:, analyzer_class:, error_class: nil, error_message: nil, agent_class_name: nil) @image = image @model_id = model_id @analysis_type = analysis_type @@ -61,6 +63,10 @@ def initialize(image:, model_id:, analysis_type:, caption:, description:, tags:, @error_class = error_class @error_message = error_message @execution_id = nil + + # Tracking + @agent_class_name = agent_class_name + register_with_tracker end # Loads the associated Execution record from the database diff --git a/lib/ruby_llm/agents/results/image_edit_result.rb b/lib/ruby_llm/agents/results/image_edit_result.rb index db7a0f91..877b63be 100644 --- a/lib/ruby_llm/agents/results/image_edit_result.rb +++ b/lib/ruby_llm/agents/results/image_edit_result.rb @@ -17,6 +17,8 @@ module Agents # result.success? # => true # class ImageEditResult + include Trackable + attr_reader :images, :source_image, :mask, :prompt, :model_id, :size, :started_at, :completed_at, :tenant_id, :editor_class, :error_class, :error_message @@ -38,7 +40,7 @@ class ImageEditResult # @param error_message [String, nil] Error message if failed def initialize(images:, source_image:, mask:, prompt:, model_id:, size:, started_at:, completed_at:, tenant_id:, editor_class:, - error_class: nil, error_message: nil) + error_class: nil, error_message: nil, agent_class_name: nil) @images = images @source_image = source_image @mask = mask @@ -52,6 +54,10 @@ def initialize(images:, source_image:, mask:, prompt:, model_id:, size:, @error_class = error_class @error_message = error_message @execution_id = nil + + # Tracking + @agent_class_name = agent_class_name + register_with_tracker end # Loads the associated Execution record from the database diff --git a/lib/ruby_llm/agents/results/image_generation_result.rb b/lib/ruby_llm/agents/results/image_generation_result.rb index f6591d25..de0b36b5 100644 --- a/lib/ruby_llm/agents/results/image_generation_result.rb +++ b/lib/ruby_llm/agents/results/image_generation_result.rb @@ -20,6 +20,8 @@ module Agents # result.save_all("./logos") # class ImageGenerationResult + include Trackable + attr_reader :images, :prompt, :model_id, :size, :quality, :style, :started_at, :completed_at, :tenant_id, :generator_class, :error_class, :error_message, :execution_id @@ -40,7 +42,7 @@ class ImageGenerationResult # @param error_message [String, nil] Error message if failed def initialize(images:, prompt:, model_id:, size:, quality:, style:, started_at:, completed_at:, tenant_id:, generator_class:, - error_class: nil, error_message: nil, execution_id: nil) + error_class: nil, error_message: nil, execution_id: nil, agent_class_name: nil) @images = images @prompt = prompt @model_id = model_id @@ -54,6 +56,10 @@ def initialize(images:, prompt:, model_id:, size:, quality:, style:, @error_class = error_class @error_message = error_message @execution_id = execution_id + + # Tracking + @agent_class_name = agent_class_name + register_with_tracker end # Loads the associated Execution record from the database diff --git a/lib/ruby_llm/agents/results/image_pipeline_result.rb b/lib/ruby_llm/agents/results/image_pipeline_result.rb index 7df61d36..863e5110 100644 --- a/lib/ruby_llm/agents/results/image_pipeline_result.rb +++ b/lib/ruby_llm/agents/results/image_pipeline_result.rb @@ -21,6 +21,8 @@ module Agents # result.analysis # => Shortcut to analyzer step result # class ImagePipelineResult + include Trackable + attr_reader :step_results, :started_at, :completed_at, :tenant_id, :pipeline_class, :context, :error_class, :error_message attr_accessor :execution_id @@ -36,7 +38,7 @@ class ImagePipelineResult # @param error_class [String, nil] Error class name if failed # @param error_message [String, nil] Error message if failed def initialize(step_results:, started_at:, completed_at:, tenant_id:, - pipeline_class:, context:, error_class: nil, error_message: nil) + pipeline_class:, context:, error_class: nil, error_message: nil, agent_class_name: nil) @step_results = step_results @started_at = started_at @completed_at = completed_at @@ -46,6 +48,10 @@ def initialize(step_results:, started_at:, completed_at:, tenant_id:, @error_class = error_class @error_message = error_message @execution_id = nil + + # Tracking + @agent_class_name = agent_class_name + register_with_tracker end # Loads the associated Execution record from the database diff --git a/lib/ruby_llm/agents/results/image_transform_result.rb b/lib/ruby_llm/agents/results/image_transform_result.rb index ef503b7c..47a61621 100644 --- a/lib/ruby_llm/agents/results/image_transform_result.rb +++ b/lib/ruby_llm/agents/results/image_transform_result.rb @@ -17,6 +17,8 @@ module Agents # result.success? # => true # class ImageTransformResult + include Trackable + attr_reader :images, :source_image, :prompt, :model_id, :size, :strength, :started_at, :completed_at, :tenant_id, :transformer_class, :error_class, :error_message @@ -38,7 +40,7 @@ class ImageTransformResult # @param error_message [String, nil] Error message if failed def initialize(images:, source_image:, prompt:, model_id:, size:, strength:, started_at:, completed_at:, tenant_id:, transformer_class:, - error_class: nil, error_message: nil) + error_class: nil, error_message: nil, agent_class_name: nil) @images = images @source_image = source_image @prompt = prompt @@ -52,6 +54,10 @@ def initialize(images:, source_image:, prompt:, model_id:, size:, strength:, @error_class = error_class @error_message = error_message @execution_id = nil + + # Tracking + @agent_class_name = agent_class_name + register_with_tracker end # Loads the associated Execution record from the database diff --git a/lib/ruby_llm/agents/results/image_upscale_result.rb b/lib/ruby_llm/agents/results/image_upscale_result.rb index 843c9be6..b29fcf45 100644 --- a/lib/ruby_llm/agents/results/image_upscale_result.rb +++ b/lib/ruby_llm/agents/results/image_upscale_result.rb @@ -15,6 +15,8 @@ module Agents # result.success? # => true # class ImageUpscaleResult + include Trackable + attr_reader :image, :source_image, :model_id, :scale, :output_size, :face_enhance, :started_at, :completed_at, :tenant_id, :upscaler_class, :error_class, :error_message @@ -36,7 +38,7 @@ class ImageUpscaleResult # @param error_message [String, nil] Error message if failed def initialize(image:, source_image:, model_id:, scale:, output_size:, face_enhance:, started_at:, completed_at:, tenant_id:, upscaler_class:, - error_class: nil, error_message: nil) + error_class: nil, error_message: nil, agent_class_name: nil) @image = image @source_image = source_image @model_id = model_id @@ -50,6 +52,10 @@ def initialize(image:, source_image:, model_id:, scale:, output_size:, face_enha @error_class = error_class @error_message = error_message @execution_id = nil + + # Tracking + @agent_class_name = agent_class_name + register_with_tracker end # Loads the associated Execution record from the database diff --git a/lib/ruby_llm/agents/results/image_variation_result.rb b/lib/ruby_llm/agents/results/image_variation_result.rb index abc2c020..023b14ab 100644 --- a/lib/ruby_llm/agents/results/image_variation_result.rb +++ b/lib/ruby_llm/agents/results/image_variation_result.rb @@ -14,6 +14,8 @@ module Agents # result.success? # => true # class ImageVariationResult + include Trackable + attr_reader :images, :source_image, :model_id, :size, :variation_strength, :started_at, :completed_at, :tenant_id, :variator_class, :error_class, :error_message @@ -34,7 +36,7 @@ class ImageVariationResult # @param error_message [String, nil] Error message if failed def initialize(images:, source_image:, model_id:, size:, variation_strength:, started_at:, completed_at:, tenant_id:, variator_class:, - error_class: nil, error_message: nil) + error_class: nil, error_message: nil, agent_class_name: nil) @images = images @source_image = source_image @model_id = model_id @@ -47,6 +49,10 @@ def initialize(images:, source_image:, model_id:, size:, variation_strength:, @error_class = error_class @error_message = error_message @execution_id = nil + + # Tracking + @agent_class_name = agent_class_name + register_with_tracker end # Loads the associated Execution record from the database diff --git a/lib/ruby_llm/agents/results/speech_result.rb b/lib/ruby_llm/agents/results/speech_result.rb index 4406e217..dca60306 100644 --- a/lib/ruby_llm/agents/results/speech_result.rb +++ b/lib/ruby_llm/agents/results/speech_result.rb @@ -23,6 +23,8 @@ module Agents # # @api public class SpeechResult + include Trackable + # @!group Audio Content # @!attribute [r] audio @@ -229,6 +231,10 @@ def initialize(attributes = {}) # Execution record @execution_id = attributes[:execution_id] + + # Tracking + @agent_class_name = attributes[:agent_class_name] + register_with_tracker end # Loads the associated Execution record from the database diff --git a/lib/ruby_llm/agents/results/trackable.rb b/lib/ruby_llm/agents/results/trackable.rb new file mode 100644 index 00000000..9da03473 --- /dev/null +++ b/lib/ruby_llm/agents/results/trackable.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module RubyLLM + module Agents + # Mixin that registers a result object with the active Tracker. + # + # Included in every result class so that RubyLLM::Agents.track + # can collect results automatically. + # + # @api private + module Trackable + def self.included(base) + base.attr_reader :agent_class_name unless base.method_defined?(:agent_class_name) + end + + private + + # Call from the end of initialize to register with the active tracker + def register_with_tracker + tracker = Thread.current[:ruby_llm_agents_tracker] + tracker << self if tracker + end + end + end +end diff --git a/lib/ruby_llm/agents/results/transcription_result.rb b/lib/ruby_llm/agents/results/transcription_result.rb index 20a39d0f..5911b1de 100644 --- a/lib/ruby_llm/agents/results/transcription_result.rb +++ b/lib/ruby_llm/agents/results/transcription_result.rb @@ -26,6 +26,8 @@ module Agents # # @api public class TranscriptionResult + include Trackable + # @!group Content # @!attribute [r] text @@ -250,6 +252,10 @@ def initialize(attributes = {}) # Execution record @execution_id = attributes[:execution_id] + + # Tracking + @agent_class_name = attributes[:agent_class_name] + register_with_tracker end # Loads the associated Execution record from the database diff --git a/lib/ruby_llm/agents/track_report.rb b/lib/ruby_llm/agents/track_report.rb new file mode 100644 index 00000000..eb39163f --- /dev/null +++ b/lib/ruby_llm/agents/track_report.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +module RubyLLM + module Agents + # Aggregated read-only report returned by RubyLLM::Agents.track. + # + # Provides totals and breakdowns across all agent calls made + # inside the tracked block. + # + # @example + # report = RubyLLM::Agents.track do + # TranscribeAgent.call(with: audio_path) + # ChatAgent.call(message: "hello") + # end + # report.total_cost # => 0.0078 + # report.call_count # => 2 + # + # @api public + class TrackReport + attr_reader :value, :error, :results, :request_id + attr_reader :started_at, :completed_at + + def initialize(value:, error:, results:, request_id:, started_at:, completed_at:) + @value = value + @error = error + @results = results.freeze + @request_id = request_id + @started_at = started_at + @completed_at = completed_at + end + + def successful? + @error.nil? + end + + def failed? + !successful? + end + + def call_count + @results.size + end + + def total_cost + @results.sum { |r| r.total_cost || 0 } + end + + def input_cost + @results.sum { |r| r.input_cost || 0 } + end + + def output_cost + @results.sum { |r| r.output_cost || 0 } + end + + def total_tokens + @results.sum { |r| r.total_tokens } + end + + def input_tokens + @results.sum { |r| r.input_tokens || 0 } + end + + def output_tokens + @results.sum { |r| r.output_tokens || 0 } + end + + def duration_ms + return nil unless @started_at && @completed_at + ((@completed_at - @started_at) * 1000).to_i + end + + def all_successful? + @results.all?(&:success?) + end + + def any_errors? + @results.any?(&:error?) + end + + def errors + @results.select(&:error?) + end + + def successful + @results.select(&:success?) + end + + def models_used + @results.filter_map(&:chosen_model_id).uniq + end + + def cost_breakdown + @results.map do |r| + { + agent: r.respond_to?(:agent_class_name) ? r.agent_class_name : nil, + model: r.chosen_model_id, + cost: r.total_cost || 0, + tokens: r.total_tokens, + duration_ms: r.duration_ms + } + end + end + + def to_h + { + successful: successful?, + value: value, + error: error&.message, + request_id: request_id, + call_count: call_count, + total_cost: total_cost, + input_cost: input_cost, + output_cost: output_cost, + total_tokens: total_tokens, + input_tokens: input_tokens, + output_tokens: output_tokens, + duration_ms: duration_ms, + started_at: started_at, + completed_at: completed_at, + models_used: models_used, + cost_breakdown: cost_breakdown + } + end + end + end +end diff --git a/lib/ruby_llm/agents/tracker.rb b/lib/ruby_llm/agents/tracker.rb new file mode 100644 index 00000000..3fcaa390 --- /dev/null +++ b/lib/ruby_llm/agents/tracker.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module RubyLLM + module Agents + # Internal collector used by RubyLLM::Agents.track to accumulate + # Result objects produced during a tracked block. + # + # Not part of the public API — users interact with TrackReport instead. + # + # @api private + class Tracker + attr_reader :results, :defaults, :request_id, :tags + + def initialize(defaults: {}, request_id: nil, tags: {}) + @results = [] + @defaults = defaults + @request_id = request_id || generate_request_id + @tags = tags + end + + def <<(result) + @results << result + end + + private + + def generate_request_id + "track_#{SecureRandom.hex(8)}" + end + end + end +end diff --git a/spec/agents/result_tracking_spec.rb b/spec/agents/result_tracking_spec.rb new file mode 100644 index 00000000..1f689ed3 --- /dev/null +++ b/spec/agents/result_tracking_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe "Result tracking integration" do + describe "agent_class_name" do + it "stores agent_class_name when provided" do + result = RubyLLM::Agents::Result.new( + content: "test", + input_tokens: 100, + output_tokens: 50, + agent_class_name: "MyAgent" + ) + expect(result.agent_class_name).to eq("MyAgent") + end + + it "defaults to nil when not provided" do + result = RubyLLM::Agents::Result.new(content: "test", input_tokens: 10, output_tokens: 5) + expect(result.agent_class_name).to be_nil + end + end + + describe "tracker registration" do + it "registers with active tracker on creation" do + tracker = RubyLLM::Agents::Tracker.new + Thread.current[:ruby_llm_agents_tracker] = tracker + + result = RubyLLM::Agents::Result.new(content: "test", input_tokens: 10, output_tokens: 5) + expect(tracker.results).to eq([result]) + ensure + Thread.current[:ruby_llm_agents_tracker] = nil + end + + it "does nothing when no tracker is active" do + Thread.current[:ruby_llm_agents_tracker] = nil + + result = RubyLLM::Agents::Result.new(content: "test", input_tokens: 10, output_tokens: 5) + expect(result).to be_a(RubyLLM::Agents::Result) + end + + it "registers subclass results with tracker" do + tracker = RubyLLM::Agents::Tracker.new + Thread.current[:ruby_llm_agents_tracker] = tracker + + result = RubyLLM::Agents::EmbeddingResult.new( + vectors: [[0.1, 0.2, 0.3]], + input_tokens: 10, + model_id: "text-embedding-3-small" + ) + expect(tracker.results).to eq([result]) + ensure + Thread.current[:ruby_llm_agents_tracker] = nil + end + + it "registers multiple results in order" do + tracker = RubyLLM::Agents::Tracker.new + Thread.current[:ruby_llm_agents_tracker] = tracker + + r1 = RubyLLM::Agents::Result.new(content: "first", input_tokens: 10, output_tokens: 5) + r2 = RubyLLM::Agents::Result.new(content: "second", input_tokens: 20, output_tokens: 10) + + expect(tracker.results).to eq([r1, r2]) + ensure + Thread.current[:ruby_llm_agents_tracker] = nil + end + end + + describe "agent_class_name in to_h" do + it "includes agent_class_name in hash output" do + result = RubyLLM::Agents::Result.new( + content: "test", + input_tokens: 10, + output_tokens: 5, + agent_class_name: "SearchAgent" + ) + expect(result.to_h).to include(agent_class_name: "SearchAgent") + end + end +end diff --git a/spec/agents/track_instrumentation_spec.rb b/spec/agents/track_instrumentation_spec.rb new file mode 100644 index 00000000..872c68c7 --- /dev/null +++ b/spec/agents/track_instrumentation_spec.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require "spec_helper" + +# Test agent for instrumentation tests +class InstrumentTrackTestAgent < RubyLLM::Agents::BaseAgent + model "gpt-4o" + param :query, required: true + + user "Answer: {query}" +end + +RSpec.describe "Track instrumentation integration" do + before do + setup_agent_mocks(content: "response", input_tokens: 100, output_tokens: 50) + end + + describe "request_id injection" do + it "sets request_id on execution records" do + report = RubyLLM::Agents.track(request_id: "req_abc") do + InstrumentTrackTestAgent.call(query: "hello") + end + + expect(report).to be_successful + expect(report.call_count).to eq(1) + + # The execution should have the request_id + execution = RubyLLM::Agents::Execution.last + expect(execution.request_id).to eq("req_abc") + end + + it "auto-generates request_id when none provided" do + RubyLLM::Agents.track do + InstrumentTrackTestAgent.call(query: "hello") + end + + execution = RubyLLM::Agents::Execution.last + expect(execution.request_id).to start_with("track_") + end + + it "sets same request_id on all executions in block" do + RubyLLM::Agents.track(request_id: "req_multi") do + InstrumentTrackTestAgent.call(query: "first") + InstrumentTrackTestAgent.call(query: "second") + end + + executions = RubyLLM::Agents::Execution.where(request_id: "req_multi") + expect(executions.count).to eq(2) + end + end + + describe "tags injection" do + it "merges tags into execution metadata" do + RubyLLM::Agents.track(tags: {feature: "voice-chat", session_id: "sess_1"}) do + InstrumentTrackTestAgent.call(query: "hello") + end + + execution = RubyLLM::Agents::Execution.last + metadata = execution.metadata || {} + tags = metadata["tags"] || metadata[:tags] + expect(tags).to include("feature" => "voice-chat") + expect(tags).to include("session_id" => "sess_1") + end + end + + describe "request_id grouping" do + it "allows querying executions by request_id" do + RubyLLM::Agents.track(request_id: "group_test") do + InstrumentTrackTestAgent.call(query: "a") + InstrumentTrackTestAgent.call(query: "b") + end + + # Also create one outside the track block + InstrumentTrackTestAgent.call(query: "outside") + + grouped = RubyLLM::Agents::Execution.where(request_id: "group_test") + expect(grouped.count).to eq(2) + end + end +end diff --git a/spec/agents/track_integration_spec.rb b/spec/agents/track_integration_spec.rb new file mode 100644 index 00000000..66496b56 --- /dev/null +++ b/spec/agents/track_integration_spec.rb @@ -0,0 +1,214 @@ +# frozen_string_literal: true + +require "spec_helper" + +# Minimal agent for track integration tests +class TrackTestAgent < RubyLLM::Agents::BaseAgent + model "gpt-4o" + param :query, required: true + + user "Answer: {query}" +end + +RSpec.describe "RubyLLM::Agents.track" do + before { setup_agent_mocks(content: "response", input_tokens: 100, output_tokens: 50) } + + describe "basic tracking" do + it "collects results from agent calls" do + report = RubyLLM::Agents.track do + TrackTestAgent.call(query: "hello") + TrackTestAgent.call(query: "world") + end + + expect(report).to be_a(RubyLLM::Agents::TrackReport) + expect(report.call_count).to eq(2) + expect(report.results.size).to eq(2) + end + + it "captures block return value" do + report = RubyLLM::Agents.track do + r = TrackTestAgent.call(query: "hello") + {answer: r.content} + end + + expect(report.value).to eq({answer: "response"}) + end + + it "returns successful report for successful block" do + report = RubyLLM::Agents.track do + TrackTestAgent.call(query: "hello") + end + + expect(report).to be_successful + expect(report).not_to be_failed + end + + it "generates a request_id when not provided" do + report = RubyLLM::Agents.track do + TrackTestAgent.call(query: "hello") + end + + expect(report.request_id).to start_with("track_") + end + + it "uses provided request_id" do + report = RubyLLM::Agents.track(request_id: "req_abc") do + TrackTestAgent.call(query: "hello") + end + + expect(report.request_id).to eq("req_abc") + end + end + + describe "shared defaults" do + it "injects shared tenant into agent options" do + tenant_hash = {id: "tenant_1", object: nil} + + report = RubyLLM::Agents.track(tenant: tenant_hash) do + TrackTestAgent.call(query: "hello") + end + + expect(report.call_count).to eq(1) + expect(report).to be_successful + end + + it "allows explicit options to override shared defaults" do + report = RubyLLM::Agents.track(tenant: {id: "tenant_a"}) do + TrackTestAgent.call(query: "default") + TrackTestAgent.call(query: "override", tenant: {id: "tenant_b"}) + end + + expect(report.call_count).to eq(2) + end + end + + describe "error handling" do + it "captures errors without raising" do + report = RubyLLM::Agents.track do + TrackTestAgent.call(query: "hello") + raise "boom" + end + + expect(report).to be_failed + expect(report.error).to be_a(RuntimeError) + expect(report.error.message).to eq("boom") + expect(report.call_count).to eq(1) + end + + it "sets value to nil on error" do + report = RubyLLM::Agents.track do + TrackTestAgent.call(query: "hello") + raise "boom" + end + + expect(report.value).to be_nil + end + + it "captures partial results before error" do + report = RubyLLM::Agents.track do + TrackTestAgent.call(query: "first") + TrackTestAgent.call(query: "second") + raise "boom" + end + + expect(report.call_count).to eq(2) + expect(report).to be_failed + end + end + + describe "nesting" do + it "supports nested track blocks" do + outer = RubyLLM::Agents.track do + TrackTestAgent.call(query: "outer") + + inner = RubyLLM::Agents.track do + TrackTestAgent.call(query: "inner") + end + + expect(inner.call_count).to eq(1) + "outer_done" + end + + expect(outer.call_count).to eq(2) # both outer and inner calls + expect(outer.value).to eq("outer_done") + end + + it "inner block has its own report" do + inner_report = nil + + RubyLLM::Agents.track do + inner_report = RubyLLM::Agents.track do + TrackTestAgent.call(query: "inner_only") + end + end + + expect(inner_report.call_count).to eq(1) + end + + it "inner results bubble to outer tracker" do + outer = RubyLLM::Agents.track do + RubyLLM::Agents.track do + TrackTestAgent.call(query: "inner1") + TrackTestAgent.call(query: "inner2") + end + TrackTestAgent.call(query: "outer1") + end + + expect(outer.call_count).to eq(3) + end + end + + describe "edge cases" do + it "handles empty block" do + report = RubyLLM::Agents.track {} + expect(report.call_count).to eq(0) + expect(report.total_cost).to eq(0) + expect(report.value).to be_nil + end + + it "handles block with no agent calls" do + report = RubyLLM::Agents.track { 1 + 1 } + expect(report.call_count).to eq(0) + expect(report.value).to eq(2) + end + + it "cleans up thread-local on normal completion" do + RubyLLM::Agents.track { TrackTestAgent.call(query: "hello") } + expect(Thread.current[:ruby_llm_agents_tracker]).to be_nil + end + + it "cleans up thread-local on exception in block" do + RubyLLM::Agents.track { raise "boom" } + expect(Thread.current[:ruby_llm_agents_tracker]).to be_nil + end + + it "restores previous tracker after nested block" do + outer_tracker = nil + + RubyLLM::Agents.track do + outer_tracker = Thread.current[:ruby_llm_agents_tracker] + + RubyLLM::Agents.track do + # inner block + end + + # After inner block, outer tracker should be restored + expect(Thread.current[:ruby_llm_agents_tracker]).to eq(outer_tracker) + end + end + end + + describe "tracker defaults merging" do + it "stores track request_id and tags on agent instance" do + report = RubyLLM::Agents.track( + request_id: "req_xyz", + tags: {feature: "test"} + ) do + TrackTestAgent.call(query: "hello") + end + + expect(report.request_id).to eq("req_xyz") + expect(report).to be_successful + end + end +end diff --git a/spec/agents/track_report_spec.rb b/spec/agents/track_report_spec.rb new file mode 100644 index 00000000..4a0ff4e2 --- /dev/null +++ b/spec/agents/track_report_spec.rb @@ -0,0 +1,286 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe RubyLLM::Agents::TrackReport do + let(:started_at) { 1.second.ago } + let(:completed_at) { Time.current } + + def build_result(**overrides) + defaults = { + content: "response", + input_tokens: 100, + output_tokens: 50, + input_cost: 0.001, + output_cost: 0.002, + total_cost: 0.003, + model_id: "gpt-4o", + chosen_model_id: "gpt-4o", + started_at: started_at, + completed_at: completed_at, + duration_ms: 500, + agent_class_name: "TestAgent" + } + RubyLLM::Agents::Result.new(**defaults.merge(overrides)) + end + + let(:results) do + [ + build_result(input_tokens: 100, output_tokens: 50, input_cost: 0.001, output_cost: 0.002, total_cost: 0.003, chosen_model_id: "gpt-4o", duration_ms: 400, agent_class_name: "AgentA"), + build_result(input_tokens: 500, output_tokens: 200, input_cost: 0.005, output_cost: 0.010, total_cost: 0.015, chosen_model_id: "gpt-4o-mini", duration_ms: 600, agent_class_name: "AgentB"), + build_result(input_tokens: 300, output_tokens: 100, input_cost: 0.003, output_cost: 0.004, total_cost: 0.007, chosen_model_id: "gpt-4o", duration_ms: 500, agent_class_name: "AgentC") + ] + end + + let(:report) do + described_class.new( + value: "done", + error: nil, + results: results, + request_id: "req_123", + started_at: started_at, + completed_at: completed_at + ) + end + + describe "#successful?" do + it "returns true when no error" do + expect(report).to be_successful + end + + it "returns false when error is present" do + error_report = described_class.new( + value: nil, error: RuntimeError.new("boom"), + results: [], request_id: "req_1", + started_at: started_at, completed_at: completed_at + ) + expect(error_report).not_to be_successful + end + end + + describe "#failed?" do + it "is the inverse of successful?" do + expect(report).not_to be_failed + end + end + + describe "#call_count" do + it "returns number of results" do + expect(report.call_count).to eq(3) + end + + it "returns 0 for empty results" do + empty = described_class.new( + value: nil, error: nil, results: [], + request_id: "req_1", started_at: started_at, completed_at: completed_at + ) + expect(empty.call_count).to eq(0) + end + end + + describe "cost aggregation" do + it "sums total_cost" do + expect(report.total_cost).to eq(0.025) + end + + it "sums input_cost" do + expect(report.input_cost).to be_within(0.0001).of(0.009) + end + + it "sums output_cost" do + expect(report.output_cost).to eq(0.016) + end + + it "handles nil costs gracefully" do + results_with_nil = [ + build_result(total_cost: 0.01, input_cost: 0.005, output_cost: 0.005), + build_result(total_cost: nil, input_cost: nil, output_cost: nil) + ] + r = described_class.new( + value: nil, error: nil, results: results_with_nil, + request_id: "req_1", started_at: started_at, completed_at: completed_at + ) + expect(r.total_cost).to eq(0.01) + expect(r.input_cost).to eq(0.005) + expect(r.output_cost).to eq(0.005) + end + end + + describe "token aggregation" do + it "sums total_tokens" do + expect(report.total_tokens).to eq(1250) + end + + it "sums input_tokens" do + expect(report.input_tokens).to eq(900) + end + + it "sums output_tokens" do + expect(report.output_tokens).to eq(350) + end + + it "handles nil tokens gracefully" do + results_with_nil = [ + build_result(input_tokens: 100, output_tokens: 50), + build_result(input_tokens: nil, output_tokens: nil) + ] + r = described_class.new( + value: nil, error: nil, results: results_with_nil, + request_id: "req_1", started_at: started_at, completed_at: completed_at + ) + expect(r.input_tokens).to eq(100) + expect(r.output_tokens).to eq(50) + end + end + + describe "#duration_ms" do + it "calculates wall clock time" do + expect(report.duration_ms).to be_a(Integer) + expect(report.duration_ms).to be >= 0 + end + + it "returns nil if timestamps missing" do + r = described_class.new( + value: nil, error: nil, results: [], + request_id: "req_1", started_at: nil, completed_at: nil + ) + expect(r.duration_ms).to be_nil + end + end + + describe "#value" do + it "returns the block return value" do + expect(report.value).to eq("done") + end + end + + describe "#error" do + it "returns nil for successful report" do + expect(report.error).to be_nil + end + + it "returns the captured error" do + err = RuntimeError.new("something broke") + r = described_class.new( + value: nil, error: err, results: [], + request_id: "req_1", started_at: started_at, completed_at: completed_at + ) + expect(r.error).to eq(err) + end + end + + describe "#request_id" do + it "returns the request_id" do + expect(report.request_id).to eq("req_123") + end + end + + describe "#all_successful?" do + it "returns true when all results succeeded" do + expect(report).to be_all_successful + end + + it "returns false when any result has an error" do + results_with_error = [ + build_result, + build_result(error_class: "RuntimeError", error_message: "boom") + ] + r = described_class.new( + value: nil, error: nil, results: results_with_error, + request_id: "req_1", started_at: started_at, completed_at: completed_at + ) + expect(r).not_to be_all_successful + end + end + + describe "#any_errors?" do + it "returns false when all succeeded" do + expect(report.any_errors?).to be false + end + + it "returns true when any result has an error" do + results_with_error = [ + build_result, + build_result(error_class: "RuntimeError", error_message: "boom") + ] + r = described_class.new( + value: nil, error: nil, results: results_with_error, + request_id: "req_1", started_at: started_at, completed_at: completed_at + ) + expect(r.any_errors?).to be true + end + end + + describe "#errors" do + it "returns only error results" do + error_result = build_result(error_class: "RuntimeError", error_message: "boom") + results_mixed = [build_result, error_result] + r = described_class.new( + value: nil, error: nil, results: results_mixed, + request_id: "req_1", started_at: started_at, completed_at: completed_at + ) + expect(r.errors).to eq([error_result]) + end + end + + describe "#successful" do + it "returns only successful results" do + ok_result = build_result + error_result = build_result(error_class: "RuntimeError", error_message: "boom") + results_mixed = [ok_result, error_result] + r = described_class.new( + value: nil, error: nil, results: results_mixed, + request_id: "req_1", started_at: started_at, completed_at: completed_at + ) + expect(r.successful).to eq([ok_result]) + end + end + + describe "#models_used" do + it "returns unique model IDs" do + expect(report.models_used).to contain_exactly("gpt-4o", "gpt-4o-mini") + end + end + + describe "#cost_breakdown" do + it "returns per-result cost data" do + breakdown = report.cost_breakdown + expect(breakdown.size).to eq(3) + expect(breakdown.first).to include( + model: "gpt-4o", + cost: 0.003, + tokens: 150, + duration_ms: 400 + ) + end + end + + describe "#to_h" do + it "returns a complete hash" do + hash = report.to_h + expect(hash).to include( + successful: true, + value: "done", + error: nil, + request_id: "req_123", + call_count: 3, + total_cost: 0.025, + total_tokens: 1250 + ) + end + + it "includes error message when failed" do + r = described_class.new( + value: nil, error: RuntimeError.new("boom"), results: [], + request_id: "req_1", started_at: started_at, completed_at: completed_at + ) + expect(r.to_h[:error]).to eq("boom") + end + end + + describe "results freezing" do + it "freezes results array" do + expect(report.results).to be_frozen + end + end +end diff --git a/spec/agents/tracker_spec.rb b/spec/agents/tracker_spec.rb new file mode 100644 index 00000000..9e5fa633 --- /dev/null +++ b/spec/agents/tracker_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe RubyLLM::Agents::Tracker do + describe "#initialize" do + it "starts with empty results" do + tracker = described_class.new + expect(tracker.results).to eq([]) + end + + it "generates a request_id when none provided" do + tracker = described_class.new + expect(tracker.request_id).to start_with("track_") + expect(tracker.request_id.length).to be > 10 + end + + it "uses provided request_id" do + tracker = described_class.new(request_id: "req_123") + expect(tracker.request_id).to eq("req_123") + end + + it "stores defaults" do + tracker = described_class.new(defaults: {tenant: "user_1"}) + expect(tracker.defaults).to eq({tenant: "user_1"}) + end + + it "defaults to empty defaults hash" do + tracker = described_class.new + expect(tracker.defaults).to eq({}) + end + + it "stores tags" do + tracker = described_class.new(tags: {feature: "voice-chat"}) + expect(tracker.tags).to eq({feature: "voice-chat"}) + end + + it "defaults to empty tags hash" do + tracker = described_class.new + expect(tracker.tags).to eq({}) + end + end + + describe "#<<" do + it "collects results pushed to it" do + tracker = described_class.new + result = RubyLLM::Agents::Result.new(content: "test", total_cost: 0.01, input_tokens: 100, output_tokens: 50) + tracker << result + expect(tracker.results).to eq([result]) + end + + it "maintains insertion order" do + tracker = described_class.new + r1 = RubyLLM::Agents::Result.new(content: "first", input_tokens: 10, output_tokens: 5) + r2 = RubyLLM::Agents::Result.new(content: "second", input_tokens: 20, output_tokens: 10) + r3 = RubyLLM::Agents::Result.new(content: "third", input_tokens: 30, output_tokens: 15) + + tracker << r1 + tracker << r2 + tracker << r3 + + expect(tracker.results).to eq([r1, r2, r3]) + end + end + + describe "unique request_ids" do + it "generates different request_ids for different trackers" do + ids = 10.times.map { described_class.new.request_id } + expect(ids.uniq.size).to eq(10) + end + end +end diff --git a/spec/concerns/llm_tenant_spec.rb b/spec/concerns/llm_tenant_spec.rb index 446c283b..4fbe8762 100644 --- a/spec/concerns/llm_tenant_spec.rb +++ b/spec/concerns/llm_tenant_spec.rb @@ -23,6 +23,11 @@ include RubyLLM::Agents::LLMTenant + # Define llm_executions so tests can stub it even with verify_partial_doubles + def llm_executions + RubyLLM::Agents::Execution.where(tenant_id: llm_tenant_id) + end + def to_s name || "Organization ##{id}" end diff --git a/spec/controllers/agents_controller_spec.rb b/spec/controllers/agents_controller_spec.rb index ea1c6b4f..88c851bb 100644 --- a/spec/controllers/agents_controller_spec.rb +++ b/spec/controllers/agents_controller_spec.rb @@ -134,12 +134,6 @@ def show describe "GET #show" do let!(:execution) { create(:execution, agent_type: "TestAgent") } - # avg_time_to_first_token queries time_to_first_token_ms column which has been - # moved to the metadata JSON column. Stub it to avoid SQLite errors. - before do - allow_any_instance_of(ActiveRecord::Relation).to receive(:avg_time_to_first_token).and_return(nil) - end - it "returns http success" do get :show, params: {id: "TestAgent"} expect(response).to have_http_status(:success) diff --git a/spec/controllers/requests_controller_spec.rb b/spec/controllers/requests_controller_spec.rb new file mode 100644 index 00000000..8247c5df --- /dev/null +++ b/spec/controllers/requests_controller_spec.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe RubyLLM::Agents::RequestsController, type: :request do + let(:engine_routes) { RubyLLM::Agents::Engine.routes } + + before do + RubyLLM::Agents.reset_configuration! + RubyLLM::Agents.configure do |c| + c.track_executions = true + end + end + + describe "GET #index" do + it "renders successfully with no requests" do + get engine_routes.url_helpers.requests_path + expect(response).to have_http_status(:ok) + end + + it "lists requests grouped by request_id" do + create(:execution, request_id: "req_001", agent_type: "AgentA") + create(:execution, request_id: "req_001", agent_type: "AgentB") + create(:execution, request_id: "req_002", agent_type: "AgentC") + + get engine_routes.url_helpers.requests_path + expect(response).to have_http_status(:ok) + expect(response.body).to include("req_001") + expect(response.body).to include("req_002") + end + + it "excludes executions without request_id" do + create(:execution, request_id: nil) + create(:execution, request_id: "req_with_id") + + get engine_routes.url_helpers.requests_path + expect(response).to have_http_status(:ok) + expect(response.body).to include("req_with_id") + end + + it "supports sorting by cost" do + create(:execution, request_id: "cheap_req", total_cost: 0.001) + create(:execution, request_id: "expensive_req", total_cost: 1.0) + + get engine_routes.url_helpers.requests_path, params: {sort: "total_cost", direction: "desc"} + expect(response).to have_http_status(:ok) + end + end + + describe "GET #show" do + it "shows a request detail page" do + create(:execution, request_id: "req_abc", agent_type: "AgentA") + create(:execution, request_id: "req_abc", agent_type: "AgentB") + + get engine_routes.url_helpers.request_path("req_abc") + expect(response).to have_http_status(:ok) + expect(response.body).to include("req_abc") + expect(response.body).to include("AgentA") + expect(response.body).to include("AgentB") + end + + it "shows summary statistics" do + create(:execution, request_id: "req_stats", total_cost: 0.005, total_tokens: 100) + create(:execution, request_id: "req_stats", total_cost: 0.010, total_tokens: 200) + + get engine_routes.url_helpers.request_path("req_stats") + expect(response).to have_http_status(:ok) + expect(response.body).to include("300") # total tokens + end + + it "redirects when request_id not found" do + get engine_routes.url_helpers.request_path("nonexistent_req") + expect(response).to redirect_to(engine_routes.url_helpers.requests_path) + end + + it "shows error status when executions have errors" do + create(:execution, request_id: "req_err", status: "success") + create(:execution, :failed, request_id: "req_err") + + get engine_routes.url_helpers.request_path("req_err") + expect(response).to have_http_status(:ok) + expect(response.body).to include("error") + end + end +end