Scope Prometheus dashboard queries by environment

app/models/concerns/upright/services/live_status.rb

@@ -26,18 +26,23 @@ def live_up_fraction
 
     def live_down_fraction
       response = Upright.prometheus_client.query(
-        query: "max(upright:probe_down_fraction{probe_service=\"#{code}\"}) or vector(0)"
+        query: live_down_query
       ).deep_symbolize_keys
       response.dig(:result, 0, :value, 1).to_f
     end
 
     def live_down_history(now:)
       response = Upright.prometheus_client.query_range(
-        query: "max(upright:probe_down_fraction{probe_service=\"#{code}\"}) or vector(0)",
+        query: live_down_query,
         start: (now - OUTAGE_LOOKBACK).iso8601,
         end:   now.iso8601,
         step:  "300s"
       ).deep_symbolize_keys
       response.dig(:result, 0, :values) || []
     end
+
+    def live_down_query
+      matchers = [ %(probe_service="#{code}"), %(environment="#{Rails.env}") ]
+      %(max(upright:probe_down_fraction{#{matchers.join(",")}}) or vector(0))
+    end
 end

app/models/upright/probes/status.rb

@@ -17,7 +17,7 @@ def query(probe_type)
       end
 
       def label_selector(probe_type)
-        matchers = [ "alert_severity!=\"\"" ]
+        matchers = [ "alert_severity!=\"\"", "environment=\"#{Rails.env}\"" ]
         matchers << "type=\"#{probe_type}\"" if probe_type.present?
         "{#{matchers.join(",")}}"
       end

app/models/upright/probes/uptime.rb

@@ -21,7 +21,7 @@ def query(probe_type)
       end
 
       def label_selector(probe_type)
-        matchers = [ "alert_severity!=\"\"" ]
+        matchers = [ "alert_severity!=\"\"", "environment=\"#{Rails.env}\"" ]
         matchers << "type=\"#{probe_type}\"" if probe_type.present?
         "{#{matchers.join(",")}}"
       end

app/models/upright/rollups/probe_rollup.rb

@@ -23,7 +23,7 @@ def self.rollup_day(day)
   def self.fetch_uptime_for(day)
     query_time = [ day.end_of_day, Time.current ].min
 
-    response = Upright.prometheus_client.query(query: PROMETHEUS_METRIC, time: query_time.iso8601).deep_symbolize_keys
+    response = Upright.prometheus_client.query(query: uptime_query, time: query_time.iso8601).deep_symbolize_keys
 
     Array(response[:result]).map do |series|
       {
@@ -34,6 +34,10 @@ def self.fetch_uptime_for(day)
     end
   end
 
+  def self.uptime_query
+    %(#{PROMETHEUS_METRIC}{environment="#{Rails.env}"})
+  end
+
   def service
     Upright::Service.find_by(code: probe_service) if probe_service.present?
   end

test/dummy/bin/seed-prometheus

@@ -92,7 +92,7 @@ echo "Generating seed data..."
         esac
       fi
 
-      echo "upright:probe_uptime_daily{name=\"${pname}\",type=\"${ptype}\",probe_target=\"${ptarget}\",probe_service=\"${pservice}\"} ${uptime} ${ts}.0"
+      echo "upright:probe_uptime_daily{name=\"${pname}\",type=\"${ptype}\",probe_target=\"${ptarget}\",probe_service=\"${pservice}\",environment=\"development\"} ${uptime} ${ts}.0"
     done
   done
 
@@ -116,7 +116,7 @@ echo "Generating seed data..."
           val=0
         fi
 
-        echo "upright_probe_up{name=\"${pname}\",type=\"${ptype}\",probe_target=\"${ptarget}\",probe_service=\"${pservice}\",alert_severity=\"high\",site_code=\"${scode}\",site_city=\"${scity}\",site_country=\"${scountry}\",site_geohash=\"${sgeohash}\",site_provider=\"${sprovider}\"} ${val}.0 ${ts}.0"
+        echo "upright_probe_up{name=\"${pname}\",type=\"${ptype}\",probe_target=\"${ptarget}\",probe_service=\"${pservice}\",alert_severity=\"high\",site_code=\"${scode}\",site_city=\"${scity}\",site_country=\"${scountry}\",site_geohash=\"${sgeohash}\",site_provider=\"${sprovider}\",environment=\"development\"} ${val}.0 ${ts}.0"
       done
     done
   done

test/lib/helpers/rails_env_helper.rb

@@ -0,0 +1,8 @@
+module RailsEnvHelper
+  def with_rails_env(name)
+    Rails.stubs(:env).returns(ActiveSupport::EnvironmentInquirer.new(name))
+    yield
+  ensure
+    Rails.unstub(:env)
+  end
+end

test/models/upright/probes/status_test.rb

@@ -25,6 +25,18 @@ class Upright::Probes::StatusTest < ActiveSupport::TestCase
     assert_equal [], Upright::Probes::Status.for_type(:http)
   end
 
+  test ".for_type scopes the query to the deployment environment" do
+    with_rails_env("staging") do
+      stub_prometheus_query_range([])
+
+      Upright::Probes::Status.for_type(:http)
+
+      assert_requested :get, /localhost:9090.*query_range/ do |request|
+        request.uri.query_values["query"].include?(%(environment="staging"))
+      end
+    end
+  end
+
   test "probe exposes site statuses with up/down state" do
     stub_prometheus_query_range([
       { "metric" => { "name" => "example.com", "type" => "http", "probe_target" => "https://example.com", "site_code" => "iad" },

test/test_helper.rb

@@ -30,6 +30,7 @@ class TestCase
 
     include IpApiHelper
     include MtrHelper
+    include RailsEnvHelper
     include SiteHelper
     include YabedaTestHelper