libpcp_web, pmproxy: support for optional labels

qa/1744

@@ -0,0 +1,199 @@
+#!/bin/sh
+# PCP QA Test No. 1744
+# Test pmproxy / pmseries label storing mechanism
+#
+# Copyright (c) 2026 Red Hat.  All Rights Reserved.
+#
+seq=`basename $0`
+echo "QA output created by $seq"
+
+# get standard environment, filters and checks
+. ./common.openmetrics
+. ./common.python
+. ./common.keys
+
+_pmdaopenmetrics_check || _notrun "openmetrics pmda not installed"
+
+test -x $PCP_BINADM_DIR/pmseries_import || _notrun "No pmseries_import script"
+_check_series
+
+status=1	# failure is the default!
+
+_cleanup()
+{
+    cd $here
+    [ -n "$pmloggerpid" ] && $sudo kill -TERM $pmloggerpid 2>/dev/null
+    if [ -n "$key_server_port" ]; then
+        echo "Shutting down key server on port $key_server_port..." >>$here/$seq.full
+        $keys_cli -p $key_server_port shutdown >>$here/$seq.full 2>&1
+        sleep 1
+        # Verify it's actually dead
+        if $keys_cli -p $key_server_port PING >>$here/$seq.full 2>&1; then
+            echo "WARNING: Key server still running after shutdown!" >>$here/$seq.full
+        else
+            echo "Key server shutdown confirmed" >>$here/$seq.full
+        fi
+    fi
+    $sudo rm -rf $PCP_ETC_DIR/pcp/labels/*
+    _restore_config $PCP_ETC_DIR/pcp/labels
+    _sighup_pmcd
+    _pmdaopenmetrics_cleanup
+    _restore_config $PCP_SYSCONF_DIR/pmseries
+    $sudo rm -rf $tmp $tmp.*
+}
+
+_prepare_pmda openmetrics
+trap "_cleanup; exit \$status" 0 1 2 3 15
+_stop_auto_restart pmcd
+
+_filter()
+{
+    # Replace machine-specific label values with placeholders
+    hostname=`hostname`
+    machineid=`_machine_id`
+    domainid=`_domain_name`
+    sed \
+    -e "s/\"hostname\":\"$hostname\"/\"hostname\":\"HOSTNAME\"/g" \
+    -e "s/\"domainname\":\"$domainid\"/\"domainname\":\"DOMAINNAME\"/g" \
+    -e "s/\"machineid\":\"$machineid\"/\"machineid\":\"MACHINEID\"/g" \
+    -e 's/"groupid":[0-9]*/"groupid":GROUPID/g' \
+    -e 's/"userid":[0-9]*/"userid":USERID/g'
+}
+
+_filter_series()
+{
+    sed \
+	-e 's/[0-9a-z]\{40\}/TIMESERIES/g' \
+    #end
+}
+
+# real QA test starts here
+
+key_server_port=`_find_free_port`
+_save_config $PCP_SYSCONF_DIR/pmseries
+$sudo rm -f $PCP_SYSCONF_DIR/pmseries/*
+
+# Create pmseries config pointing to our test key server
+$sudo tee $PCP_SYSCONF_DIR/pmseries/pmseries.conf > /dev/null <<EOF
+[pmseries]
+servers = localhost:$key_server_port
+EOF
+
+echo "Start test key server on port $key_server_port..." >>$seq.full
+$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 &
+_check_key_server_ping $key_server_port
+_check_key_server $key_server_port
+echo
+
+_check_key_server_version $key_server_port
+
+_pmdaopenmetrics_save_config
+_save_config $PCP_ETC_DIR/pcp/labels
+$sudo rm -rf $PCP_ETC_DIR/pcp/labels/*
+
+# add the URL for this test
+# need to be a place the user $PCP_USER (pmcd) can read
+#
+file=pmseries_label_test.txt
+cp $here/openmetrics/samples/$file $tmp.$file
+urlbase=`basename "$file" .txt | tr .- _`
+echo 'file://'$tmp.$file >$tmp.tmp
+
+# add a bunch of label filters to exercise various filtering options
+cat <<EOF >>$tmp.tmp
+
+FILTER: OPTIONAL LABEL some_optional_label
+
+EOF
+$sudo cp $tmp.tmp $PCP_PMDAS_DIR/openmetrics/config.d/$urlbase.url
+
+_pmdaopenmetrics_install
+
+if ! _pmdaopenmetrics_wait_for_metric openmetrics.control.calls
+then
+    status=1
+    exit
+fi
+
+# Verify metrics are available from PMCD
+echo "Verify openmetrics metrics are available ..."
+pminfo -f openmetrics.pmseries_label_test.test_metric1 #>>$seq_full 2>&1
+pminfo -f openmetrics.pmseries_label_test.test_metric2 #>>$seq_full 2>&1
+echo
+
+# Create a test archive with pmlogger
+echo "Creating test archive with pmlogger ..."
+cat > $tmp.pmlogger.config <<EOF
+log mandatory on once {
+    openmetrics.pmseries_label_test
+}
+EOF
+pmlogger -c $tmp.pmlogger.config -l $tmp.pmlogger.log -t 1sec $tmp.archive &
+pmloggerpid=$!
+echo "=== pmlogger started, PID=$pmloggerpid ===" >>$seq_full
+
+# Wait for pmlogger to capture some data
+pmsleep 3
+
+# Stop pmlogger
+$sudo kill -TERM $pmloggerpid 2>/dev/null
+pmsleep 1
+
+# Verify archive was created
+echo "Archive files created:" >>$seq_full
+ls -l $tmp.archive* >>$seq_full 2>&1
+
+# Check key server port and flush
+echo "Using key server on port $key_server_port" >>$seq_full
+echo "Flushing key server on port $key_server_port ..."
+flush_result=`echo "FLUSHALL" | $keys_cli -p $key_server_port 2>&1`
+echo "Flush result: $flush_result" >>$seq_full
+if [ "$flush_result" != "OK" ]; then
+    echo "ERROR: Failed to flush key server, got: $flush_result"
+    status=1
+    exit
+fi
+echo "Key server flushed successfully"
+
+# Load the archive into pmseries
+echo "Loading test archive into pmseries ..."
+pmseries $options --load "{source.path: \"$tmp.archive\"}" 2>&1 | \
+    sed "s|$tmp.archive|ARCHIVE|g" | tee -a $seq_full
+
+# Wait for metrics to be indexed
+echo "Waiting for metrics to be indexed ..."
+pmsleep 2
+
+# Query for the first metric (identifying labels only)
+echo
+echo "=== test_metric1 (identifying labels only) ==="
+series1=`pmseries $options 'openmetrics.pmseries_label_test.test_metric1'`
+if [ -z "$series1" ]; then
+    echo "ERROR: No series found for test_metric1"
+    status=1
+else
+    pmseries $series1 | _filter_series | _filter
+fi
+
+# Query for the second metric (with optional label)
+echo
+echo "=== test_metric2 (with optional label) ==="
+series2=`pmseries $options 'openmetrics.pmseries_label_test.test_metric2'`
+if [ -z "$series2" ]; then
+    echo "ERROR: No series found for test_metric2"
+    status=1
+else
+    pmseries $series2 | _filter_series | _filter
+fi
+
+echo == Note: check $seq.full for details
+echo == pmdaopenmetrics LOG == >>$seq_full
+cat $PCP_LOG_DIR/pmcd/openmetrics.log >>$seq_full
+echo == pmlogger LOG == >>$seq_full
+cat $tmp.pmlogger.log >>$seq_full 2>&1
+
+_pmdaopenmetrics_remove
+
+# success, all done
+status=0
+exit

qa/1744.out

@@ -0,0 +1,50 @@
+QA output created by 1744
+PING
+PONG
+
+
+=== openmetrics agent installation ===
+Verify openmetrics metrics are available ...
+
+openmetrics.pmseries_label_test.test_metric1
+    inst [0 or "0 label1:somelabel"] value 1
+
+openmetrics.pmseries_label_test.test_metric2
+    inst [0 or "0 label2:somelabel optional_label:some_optional_label"] value 2
+
+Creating test archive with pmlogger ...
+Flushing key server on port 54321 ...
+Key server flushed successfully
+Loading test archive into pmseries ...
+pmseries: [Info] processed 3 archive records from ARCHIVE
+Waiting for metrics to be indexed ...
+
+=== test_metric1 (identifying labels only) ===
+
+TIMESERIES
+    PMID: 144.1.0
+    Data Type: double  InDom: 144.5120 0x24001400
+    Semantics: instant  Units: none
+    Source: TIMESERIES
+    Metric: openmetrics.pmseries_label_test.test_metric1
+    inst [0 or "0 label1:somelabel"] series TIMESERIES
+    inst [0 or "0 label1:somelabel"] labels {"agent":"openmetrics","domainname":"DOMAINNAME","groupid":GROUPID,"hostname":"HOSTNAME","machineid":"MACHINEID","source":"pmseries_label_test","userid":USERID}
+
+=== test_metric2 (with optional label) ===
+
+TIMESERIES
+    PMID: 144.1.1
+    Data Type: double  InDom: 144.5121 0x24001401
+    Semantics: instant  Units: none
+    Source: TIMESERIES
+    Metric: openmetrics.pmseries_label_test.test_metric2
+    inst [0 or "0 label2:somelabel optional_label:some_optional_label"] series TIMESERIES
+    inst [0 or "0 label2:somelabel optional_label:some_optional_label"] labels {"agent":"openmetrics","domainname":"DOMAINNAME","groupid":GROUPID,"hostname":"HOSTNAME","machineid":"MACHINEID","source":"pmseries_label_test","userid":USERID}
+== Note: check 1744.full for details
+
+=== remove openmetrics agent ===
+Culling the Performance Metrics Name Space ...
+openmetrics ... done
+Updating the PMCD control file, and notifying PMCD ...
+[...removing files...]
+Check openmetrics metrics have gone away ... OK

qa/common.check

@@ -1559,12 +1559,12 @@ _check_key_server_version()
 	return
     fi
 
-    
-    if which valkey-cli >/dev/null 2>&1 && test -n `_get_pids_by_name valkey-server`
+
+    if which valkey-cli >/dev/null 2>&1 && test -n "`_get_pids_by_name valkey-server`"
     then
 	keys_cli=valkey-cli
 	keys_ver=valkey_version
-    elif which redis-cli >/dev/null 2>&1 && test -n _get_pids_by_name redis-server
+    elif which redis-cli >/dev/null 2>&1 && test -n "`_get_pids_by_name redis-server`"
     then
 	keys_cli=redis-cli
 	keys_ver=redis_version

qa/group

@@ -2248,6 +2248,7 @@ suse
 1727 pmproxy libpcp_web pmda.openmetrics local
 1735 python pmimport local
 1740 pmda.proc local
+1744 pmseries local
 1745 pmlogger libpcp pmval local pmda.sample pmda.simple pmlogdump
 1747 pmlogger labels local
 1748 atop local

qa/openmetrics/samples/pmseries_label_test.txt

@@ -0,0 +1,8 @@
+# HELP test_metric1 local metric
+# Type test_metric1 gauge
+test_metric1 {label1="somelabel"} 1
+
+
+# HELP test_metric2 local metric
+# Type test_metric2 gauge
+test_metric2 {label2="somelabel", optional_label="some_optional_label"} 2

src/libpcp_web/src/util.c

@@ -244,6 +244,13 @@ instance_labelsets(indom_t *indom, instance_t *inst, char *buffer, int length,
     return pmMergeLabelSets(sets, nsets, buffer, length, filter, arg);
 }
 
+/* extract all labels (identifying and optional) */
+static int
+all_labels(const pmLabel *label, const char *json, void *arg)
+{
+    return 1;
+}
+
 /* extract only the identifying labels (not optional) */
 static int
 labels(const pmLabel *label, const char *json, void *arg)
@@ -276,7 +283,13 @@ pmwebapi_source_meta(context_t *c, char *buffer, int length)
 static int
 context_labels_str(context_t *c, char *buffer, int length)
 {
-    return pmMergeLabelSets(&c->labelset, 1, buffer, length, NULL, NULL);
+    return pmMergeLabelSets(&c->labelset, 1, buffer, length, all_labels, NULL);
+}
+
+static int
+context_labels_str_identifying(context_t *c, char *buffer, int length)
+{
+    return pmMergeLabelSets(&c->labelset, 1, buffer, length, labels, NULL);
 }
 
 int
@@ -447,14 +460,17 @@ int
 pmwebapi_context_hash(context_t *context)
 {
     char		labels[PM_MAXLABELJSONLEN];
+    char        idlabels[PM_MAXLABELJSONLEN];
     int			sts;
 
     if (context->labels == NULL) {
 	if ((sts = context_labels_str(context, labels, sizeof(labels))) < 0)
 	    return sts;
 	context->labels = sdsnewlen(labels, sts);
     }
-    return pmwebapi_source_hash(context->name.hash, context->labels, sdslen(context->labels));
+    if ((sts = context_labels_str_identifying(context, idlabels, sizeof(idlabels))) < 0)
+	    return sts;
+    return pmwebapi_source_hash(context->name.hash, idlabels, sts);
 }
 
 void
@@ -464,22 +480,27 @@ pmwebapi_metric_hash(metric_t *metric)
     pmDesc		*desc = &metric->desc;
     sds			identifier;
     char		buf[PM_MAXLABELJSONLEN];
+    char        idbuf[PM_MAXLABELJSONLEN];
     char		sem[32], type[32], units[64];
     int			len, i;
 
     if (metric->labels == NULL) {
-	len = metric_labelsets(metric, buf, sizeof(buf), labels, NULL);
+	len = metric_labelsets(metric, buf, sizeof(buf), all_labels, NULL);
 	if (len <= 0)
 	    len = pmsprintf(buf, sizeof(buf), "null");
 	metric->labels = sdsnewlen(buf, len);
     }
 
+    len = metric_labelsets(metric, idbuf, sizeof(idbuf), labels, NULL);
+	if (len <= 0)
+	    len = pmsprintf(idbuf, sizeof(idbuf), "null");
+
     identifier = sdsempty();
     for (i = 0; i < metric->numnames; i++) {
 	identifier = sdscatfmt(identifier,
-		"{\"series\":\"metric\",\"name\":\"%S\",\"labels\":%S,"
+		"{\"series\":\"metric\",\"name\":\"%S\",\"labels\":%s,"
 		 "\"semantics\":\"%s\",\"type\":\"%s\",\"units\":\"%s\"}",
-		metric->names[i].sds, metric->labels,
+		metric->names[i].sds, idbuf,
 		pmSemStr_r(desc->sem, sem, sizeof(sem)),
 		pmTypeStr_r(desc->type, type, sizeof(type)),
 		pmUnitsStr_r(&desc->units, units, sizeof(units)));
@@ -501,7 +522,7 @@ pmwebapi_add_indom_labels(indom_t *indom)
     int			len;
 
     if (indom->labels == NULL) {
-	len = instance_labelsets(indom, NULL, buf, sizeof(buf), labels, NULL);
+	len = instance_labelsets(indom, NULL, buf, sizeof(buf), all_labels, NULL);
 	if (len <= 0)
 	    len = pmsprintf(buf, sizeof(buf), "null");
 	indom->labels = sdsnewlen(buf, len);
@@ -514,18 +535,23 @@ pmwebapi_instance_hash(indom_t *ip, instance_t *instance)
     SHA1_CTX		shactx;
     sds			identifier;
     char		buf[PM_MAXLABELJSONLEN];
+    char        idbuf[PM_MAXLABELJSONLEN];
     int			len;
 
     if (instance->labels == NULL) {
-	len = instance_labelsets(ip, instance, buf, sizeof(buf), labels, NULL);
+	len = instance_labelsets(ip, instance, buf, sizeof(buf), all_labels, NULL);
 	if (len <= 0)
 	    len = pmsprintf(buf, sizeof(buf), "null");
 	instance->labels = sdsnewlen(buf, len);
     }
 
+    len = instance_labelsets(ip, instance, idbuf, sizeof(idbuf), labels, NULL);
+	if (len <= 0)
+	    len = pmsprintf(idbuf, sizeof(idbuf), "null");
+
     identifier = sdscatfmt(sdsempty(),
-		"{\"series\":\"instance\",\"name\":\"%S\",\"labels\":%S}",
-		instance->name.sds, instance->labels);
+		"{\"series\":\"instance\",\"name\":\"%S\",\"labels\":%s}",
+		instance->name.sds, idbuf);
     /* Calculate unique instance identifier 20-byte SHA1 hash */
     SHA1Init(&shactx);
     SHA1Update(&shactx, (unsigned char *)identifier, sdslen(identifier));