Aggregate internal task queues under __temporal_sys__ instead of noop handler

Instead of suppressing all metrics for /temporal-sys/ queues with a noop
handler, aggregate them under a synthetic "__temporal_sys__" task queue tag.
This preserves operational observability while preventing cardinality explosion.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: rkannan82

common/metrics/task_queues.go

@@ -7,9 +7,13 @@ import (
 )
 
 const (
-	omitted = "__omitted__"
-	normal  = "__normal__"
-	sticky  = "__sticky__"
+	omitted     = "__omitted__"
+	normal      = "__normal__"
+	sticky      = "__sticky__"
+	temporalSys = "__temporal_sys__"
+
+	// NormalPartitionTagValue is the partition tag value for normal (non-sticky) partitions.
+	NormalPartitionTagValue = normal
 )
 
 // GetPerTaskQueueFamilyScope returns "namespace" and "taskqueue" tags. "taskqueue" will be "__omitted__" if
@@ -90,3 +94,9 @@ func GetPerTaskQueuePartitionTypeScope(
 	return GetPerTaskQueueScope(handler, namespaceName, partition.TaskQueue(), taskQueueBreakdown,
 		append(tags, PartitionTag(value))...)
 }
+
+// TemporalSysTaskQueueTag returns a taskqueue tag with the "__temporal_sys__" synthetic value,
+// used to aggregate server-internal task queues under a single label.
+func TemporalSysTaskQueueTag() Tag {
+	return UnsafeTaskQueueTag(temporalSys)
+}

service/matching/matching_engine.go

@@ -547,7 +547,16 @@ func (e *matchingEngineImpl) loggerAndMetricsForPartition(
 		tag.WorkflowNamespace(nsName))
 	var metricsHandler metrics.Handler
 	if strings.HasPrefix(partition.TaskQueue().Name(), internalTaskQueuePrefix) {
-		metricsHandler = metrics.NoopMetricsHandler
+		// Aggregate all internal task queues under "__temporal_sys__" to avoid
+		// cardinality explosion from per-worker queue names.
+		metricsHandler = e.metricsHandler.WithTags(
+			metrics.NamespaceTag(nsName),
+			metrics.TemporalSysTaskQueueTag(),
+			metrics.TaskQueueTypeTag(partition.TaskType()),
+			metrics.PartitionTag(metrics.NormalPartitionTagValue),
+			metrics.OperationTag(metrics.MatchingTaskQueuePartitionManagerScope),
+			metrics.NamespaceStateTag(nsState),
+		)
 	} else {
 		metricsHandler = metrics.GetPerTaskQueuePartitionIDScope(
 			e.metricsHandler,

service/matching/matching_engine_test.go

@@ -5230,35 +5230,45 @@ func TestLoggerAndMetricsForPartition_InternalTaskQueue(t *testing.T) {
 	ns, mockNamespaceCache := createMockNamespaceCache(controller, matchingTestNamespace)
 	config := defaultTestConfig()
 	e := createTestMatchingEngine(log.NewTestLogger(), controller, config, nil, mockNamespaceCache)
-	e.metricsHandler = metricstest.NewCaptureHandler()
+	captureHandler := metricstest.NewCaptureHandler()
+	e.metricsHandler = captureHandler
 
 	tests := []struct {
-		name       string
-		tqName     string
-		expectNoop bool
+		name          string
+		tqName        string
+		expectTQValue string
 	}{
 		{
-			name:       "normal task queue gets real metrics handler",
-			tqName:     "my-task-queue",
-			expectNoop: false,
+			name:          "normal task queue uses actual queue name",
+			tqName:        "my-task-queue",
+			expectTQValue: "my-task-queue",
 		},
 		{
-			name:       "internal nexus task queue gets noop metrics handler",
-			tqName:     "/temporal-sys/worker-commands/ns/key",
-			expectNoop: true,
+			name:          "internal task queue gets __temporal_sys__ taskqueue tag",
+			tqName:        "/temporal-sys/worker-commands/ns/key",
+			expectTQValue: "__temporal_sys__",
 		},
 	}
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
+			capture := captureHandler.StartCapture()
 			prtn := newRootPartition(ns.ID().String(), tc.tqName, enumspb.TASK_QUEUE_TYPE_NEXUS)
 			tqConfig := newTaskQueueConfig(prtn.TaskQueue(), config, matchingTestNamespace)
 			_, _, handler := e.loggerAndMetricsForPartition(ns, prtn, tqConfig)
-			if tc.expectNoop {
-				assert.Equal(t, metrics.NoopMetricsHandler, handler)
-			} else {
-				assert.NotEqual(t, metrics.NoopMetricsHandler, handler)
+			// Emit a test metric through the handler and verify the taskqueue tag value.
+			metrics.PollSuccessPerTaskQueueCounter.With(handler).Record(1)
+			snap := capture.Snapshot()
+			captureHandler.StopCapture(capture)
+			recordings := snap["poll_success"]
+			require.NotEmpty(t, recordings, "expected poll_success metric to be recorded")
+			found := false
+			for _, rec := range recordings {
+				if rec.Tags["taskqueue"] == tc.expectTQValue {
+					found = true
+				}
 			}
+			assert.True(t, found, "expected taskqueue tag value %q", tc.expectTQValue)
 		})
 	}
 }