From 2c22ced136c45aacb82c8c60677b6f7d1ae6f30f Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sun, 12 Jul 2026 16:01:24 -0600 Subject: [PATCH 1/2] lib: cmetrics: upgrade to v2.2.1 Signed-off-by: Eduardo Silva --- lib/cmetrics/.github/workflows/build.yaml | 7 +- lib/cmetrics/AGENTS.md | 148 ++++++ lib/cmetrics/CLAUDE.md | 1 + lib/cmetrics/CMakeLists.txt | 15 +- lib/cmetrics/README.md | 4 + lib/cmetrics/benchmarks/CMakeLists.txt | 6 + lib/cmetrics/benchmarks/README.md | 38 ++ lib/cmetrics/benchmarks/benchmark.c | 346 ++++++++++++++ lib/cmetrics/benchmarks/run-perf.sh | 35 ++ lib/cmetrics/docs/ai/bug-fix.md | 42 ++ lib/cmetrics/docs/ai/code-review.md | 36 ++ lib/cmetrics/docs/ai/cross-repository.md | 44 ++ lib/cmetrics/docs/ai/dependency-update.md | 40 ++ lib/cmetrics/docs/ai/investigate.md | 42 ++ lib/cmetrics/docs/ai/memory-safety-review.md | 39 ++ lib/cmetrics/docs/ai/performance-review.md | 37 ++ lib/cmetrics/docs/architecture.md | 48 ++ lib/cmetrics/docs/dependencies.md | 34 ++ lib/cmetrics/docs/label-value-handling.md | 117 +++++ .../include/cmetrics/cmt_decode_prometheus.h | 1 + lib/cmetrics/include/cmetrics/cmt_map.h | 12 + lib/cmetrics/include/cmetrics/cmt_metric.h | 5 + .../include/cmetrics/cmt_mpack_utils_defs.h | 3 +- lib/cmetrics/scripts/agent-build.sh | 14 + lib/cmetrics/scripts/agent-test.sh | 21 + lib/cmetrics/scripts/agent-verify.sh | 11 + lib/cmetrics/src/cmt_cat.c | 105 ++++- lib/cmetrics/src/cmt_decode_msgpack.c | 2 + lib/cmetrics/src/cmt_decode_prometheus.c | 27 +- lib/cmetrics/src/cmt_encode_opentelemetry.c | 327 ++++++------- .../src/cmt_encode_prometheus_remote_write.c | 54 ++- lib/cmetrics/src/cmt_filter.c | 22 +- lib/cmetrics/src/cmt_map.c | 298 ++++++++++-- lib/cmetrics/src/cmt_mpack_utils.c | 45 +- lib/cmetrics/src/cmt_summary.c | 12 +- lib/cmetrics/tests/CMakeLists.txt | 1 + lib/cmetrics/tests/cat.c | 121 +++++ lib/cmetrics/tests/counter.c | 84 ++++ lib/cmetrics/tests/encoding.c | 199 ++++++++ lib/cmetrics/tests/expire.c | 75 +++ lib/cmetrics/tests/filter.c | 51 ++ lib/cmetrics/tests/format_conversion.c | 440 ++++++++++++++++++ lib/cmetrics/tests/issues.c | 89 ++++ lib/cmetrics/tests/opentelemetry.c | 97 ++++ lib/cmetrics/tests/prometheus_parser.c | 69 ++- lib/cmetrics/tests/summary.c | 32 ++ 46 files changed, 2971 insertions(+), 325 deletions(-) create mode 100644 lib/cmetrics/AGENTS.md create mode 100644 lib/cmetrics/CLAUDE.md create mode 100644 lib/cmetrics/benchmarks/CMakeLists.txt create mode 100644 lib/cmetrics/benchmarks/README.md create mode 100644 lib/cmetrics/benchmarks/benchmark.c create mode 100755 lib/cmetrics/benchmarks/run-perf.sh create mode 100644 lib/cmetrics/docs/ai/bug-fix.md create mode 100644 lib/cmetrics/docs/ai/code-review.md create mode 100644 lib/cmetrics/docs/ai/cross-repository.md create mode 100644 lib/cmetrics/docs/ai/dependency-update.md create mode 100644 lib/cmetrics/docs/ai/investigate.md create mode 100644 lib/cmetrics/docs/ai/memory-safety-review.md create mode 100644 lib/cmetrics/docs/ai/performance-review.md create mode 100644 lib/cmetrics/docs/architecture.md create mode 100644 lib/cmetrics/docs/dependencies.md create mode 100644 lib/cmetrics/docs/label-value-handling.md create mode 100755 lib/cmetrics/scripts/agent-build.sh create mode 100755 lib/cmetrics/scripts/agent-test.sh create mode 100755 lib/cmetrics/scripts/agent-verify.sh create mode 100644 lib/cmetrics/tests/format_conversion.c diff --git a/lib/cmetrics/.github/workflows/build.yaml b/lib/cmetrics/.github/workflows/build.yaml index 2724c5acd26..2b29104f941 100644 --- a/lib/cmetrics/.github/workflows/build.yaml +++ b/lib/cmetrics/.github/workflows/build.yaml @@ -59,15 +59,12 @@ jobs: provenance: false build-debian: - name: Debian Buster build to confirm no issues once used downstream + name: Debian Bookworm build to confirm no issues once used downstream runs-on: ubuntu-latest - container: debian:buster + container: debian:bookworm steps: - name: Set up base image dependencies run: | - # Update sources to use archive.debian.org (Buster reached end-of-life) - sed -i 's/deb.debian.org/archive.debian.org/g' /etc/apt/sources.list - sed -i 's/security.debian.org/archive.debian.org/g' /etc/apt/sources.list apt-get update apt-get install -y build-essential wget make gcc g++ git libcurl4-openssl-dev diff --git a/lib/cmetrics/AGENTS.md b/lib/cmetrics/AGENTS.md new file mode 100644 index 00000000000..c640e3e9593 --- /dev/null +++ b/lib/cmetrics/AGENTS.md @@ -0,0 +1,148 @@ +# CMetrics repository guide + +This is the canonical, vendor-neutral operating guide for automated tools and +human contributors. Tool-specific files must point here instead of duplicating +project rules. + +## Project role + +CMetrics is a standalone C library for creating, mutating, aggregating, +encoding, and decoding metrics contexts. It is consumed by downstream projects, +including Fluent Bit, so changes can affect public C APIs and serialized data. + +## Repository map + +- `include/cmetrics/`: installed public headers; treat layout and declarations + as compatibility-sensitive. +- `src/`: metric implementations, map/index ownership, filters, and codecs. +- `tests/`: Acutest unit tests, one CTest executable per source file. +- `benchmarks/`: opt-in benchmark executable and Linux `perf stat` runner. +- `docs/`: design and behavior notes; agent-neutral workflows are in `docs/ai/`. +- `cmake/`, `CMakeLists.txt`: build, dependency, install, and packaging rules. +- `lib/cfl`: CFL Git submodule; owns containers, SDS strings, arenas, and + supporting primitives. +- `lib/fluent-otel-proto`: generated OpenTelemetry protobuf support submodule. +- `.github/workflows/`: compiler/platform builds, tests, linting, and packages. + +Start architecture investigation at `include/cmetrics/cmetrics.h`, the +metric-specific public header, and its matching `src/cmt_*.c` implementation. +For encoding or decoding, follow the corresponding `cmt_encode_*` or +`cmt_decode_*` pair and its tests. See `docs/architecture.md` for the component +map and `docs/dependencies.md` for repository boundaries. + +## Dependency boundaries + +Initialize submodules before building: + +```sh +git submodule update --init --recursive +``` + +Changes owned by CFL or fluent-otel-proto should normally land in their source +repository first, then update the CMetrics submodule revision separately. +Validate CMetrics standalone before updating a downstream consumer such as +Fluent Bit. See `docs/ai/cross-repository.md` and +`docs/ai/dependency-update.md`. + +## Build and test + +The supported build system is CMake 3.20 or newer. Local builds require Git, a +C compiler, and the initialized submodules. Flex 2 and Bison 3 enable the +Prometheus text decoder and its tests; CMake omits that decoder if they are not +found. A normal development build: + +```sh +cmake -S . -B build/agent -DCMT_TESTS=On -DCMT_INSTALL_TARGETS=Off +cmake --build build/agent +ctest --test-dir build/agent --output-on-failure +``` + +Repository wrappers provide the same flow: + +```sh +scripts/agent-build.sh +scripts/agent-test.sh +scripts/agent-verify.sh +``` + +Use `BUILD_DIR=/path` to select another build directory. Pass extra CMake +configuration arguments to `agent-build.sh`. Pass a CTest regular expression +to `agent-test.sh`, for example: + +```sh +scripts/agent-test.sh '^cmt-test-opentelemetry$' +``` + +When making code changes, run the related unit test when one is available. +Before handoff, run `scripts/agent-verify.sh` unless the change is documentation +only or the environment cannot build the project; report any omitted check. + +There is no repository-defined C formatter or C static-lint command. Preserve +the surrounding four-space, no-tab style. Shell changes must pass `sh -n`; CI +also runs ShellCheck. Do not claim sanitizer coverage unless the binary was +actually compiled and executed with the requested sanitizer. + +## Memory and ownership + +- Check every allocation and preserve cleanup for partial initialization. +- Match allocation families (`malloc`/`free`, CFL SDS create/destroy, arena + lifetime) and make ownership transfers explicit in code structure. +- Exercise failure and cleanup paths for codec and container changes. +- Treat map mutation, metric indexing, expiration, and destruction as + concurrency-sensitive. +- Use AddressSanitizer or Valgrind for ownership changes; follow + `docs/ai/memory-safety-review.md`. + +## Compatibility-sensitive changes + +- Public declarations under `include/cmetrics/` can affect source or ABI + compatibility. Avoid changing established public structure layout without + explicit review of downstream users. +- Internal MessagePack, OTLP protobuf, Prometheus formats, and remote write are + wire-sensitive. Add round-trip and malformed-input coverage where relevant. +- Preserve integer value types, timestamps, start timestamps, label ordering, + and metric identity unless the change intentionally revises their contract. +- Decoders process untrusted lengths and values: prevent overflow, oversized + allocation from incomplete input, desynchronization, and partial-result leaks. + +## Generated and vendored content + +- Do not edit CMake-generated `include/cmetrics/cmt_info.h` or + `include/cmetrics/cmt_version.h`; edit their `.in` templates or CMake version. +- Flex/Bison outputs are generated in the build directory from + `src/cmt_decode_prometheus.l` and `.y`; edit the grammar sources. +- Files marked generated under `src/external/` and + `include/prometheus_remote_write/` must be regenerated from their source + schema/toolchain, not hand-edited. +- Do not edit submodule contents as part of a CMetrics-only change. Make the + change in the owning repository and update the recorded revision. +- Build outputs and generated payloads do not belong in commits unless an + existing tracked fixture is intentionally updated and reviewed. + +## Performance changes + +Build benchmarks with `CMT_BENCHMARKS=ON` and an optimized Release build. Use +`benchmarks/run-perf.sh` for standard workloads and Linux hardware counters. +Capture at least five before and five after runs on the same machine with +identical compiler flags and workload parameters. Keep only repeatable gains +with no relevant regression. See `docs/ai/performance-review.md`. + +## Change discipline and definition of done + +- Keep changes scoped; preserve unrelated user work and avoid drive-by cleanup. +- Add a regression test for a bug when practical. +- Verify every documented command against repository files before adding it. +- Review error paths, compatibility, generated files, and dependency ownership. +- Run targeted tests plus the strongest practical complete validation. +- Report root cause or intent, files changed, exact checks, compatibility + impact, unresolved risks, and anything not validated. + +Detailed workflows: + +- `docs/ai/investigate.md` +- `docs/ai/bug-fix.md` +- `docs/ai/code-review.md` +- `docs/ai/cross-repository.md` +- `docs/ai/dependency-update.md` +- `docs/ai/memory-safety-review.md` +- `docs/ai/performance-review.md` diff --git a/lib/cmetrics/CLAUDE.md b/lib/cmetrics/CLAUDE.md new file mode 100644 index 00000000000..43c994c2d36 --- /dev/null +++ b/lib/cmetrics/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/lib/cmetrics/CMakeLists.txt b/lib/cmetrics/CMakeLists.txt index 80fa4e905cf..1012829c282 100644 --- a/lib/cmetrics/CMakeLists.txt +++ b/lib/cmetrics/CMakeLists.txt @@ -6,7 +6,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # CMetrics Version set(CMT_VERSION_MAJOR 2) set(CMT_VERSION_MINOR 2) -set(CMT_VERSION_PATCH 0) +set(CMT_VERSION_PATCH 1) set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}") # Include helpers @@ -63,17 +63,14 @@ if(NOT MSVC) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") endif() -# Define __CMT_FILENAME__ consistently across Operating Systems -if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__CMT_FILENAME__='\"$$(subst ${CMAKE_SOURCE_DIR}/,,$$(abspath $$<))\"'") -else() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__CMT_FILENAME__=__FILE__") -endif() +# Define __CMT_FILENAME__ +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__CMT_FILENAME__=__FILE__") # Configuration options option(CMT_DEV "Enable development mode" No) option(CMT_DEBUG "Enable debug mode" No) option(CMT_TESTS "Enable unit testing" No) +option(CMT_BENCHMARKS "Build performance benchmarks" No) option(CMT_INSTALL_TARGETS "Enable subdirectory library installations" Yes) option(CMT_PROMETHEUS_TEXT_DECODER "Enable prometheus text format decoder (requires Flex/Bison)" Yes) @@ -302,6 +299,10 @@ endif() add_subdirectory(include) add_subdirectory(src) +if(CMT_BENCHMARKS) + add_subdirectory(benchmarks) +endif() + # Tests if(CMT_TESTS) enable_testing() diff --git a/lib/cmetrics/README.md b/lib/cmetrics/README.md index 191d6b1c4e5..649f96c9526 100644 --- a/lib/cmetrics/README.md +++ b/lib/cmetrics/README.md @@ -141,6 +141,10 @@ CMetrics is heavily inspired by the Go Prometheus Client API design: - https://pkg.go.dev/github.com/prometheus/client_golang/prometheus#section-documentation +Additional design notes: + +- [Long metric label handling](docs/label-value-handling.md) + ## License This program is under the terms of the diff --git a/lib/cmetrics/benchmarks/CMakeLists.txt b/lib/cmetrics/benchmarks/CMakeLists.txt new file mode 100644 index 00000000000..59c1ae38d3b --- /dev/null +++ b/lib/cmetrics/benchmarks/CMakeLists.txt @@ -0,0 +1,6 @@ +add_executable(cmt-benchmark benchmark.c) +target_link_libraries(cmt-benchmark cmetrics-static cfl-static fluent-otel-proto) + +if(NOT CMT_SYSTEM_WINDOWS) + target_link_libraries(cmt-benchmark pthread) +endif() diff --git a/lib/cmetrics/benchmarks/README.md b/lib/cmetrics/benchmarks/README.md new file mode 100644 index 00000000000..1aa43c558e8 --- /dev/null +++ b/lib/cmetrics/benchmarks/README.md @@ -0,0 +1,38 @@ +# CMetrics benchmarks + +Benchmarks are opt-in and are intended for before/after comparisons on the +same machine. Build an optimized binary: + +```sh +cmake -S . -B build-perf \ + -DCMT_BENCHMARKS=ON \ + -DCMT_INSTALL_TARGETS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS_RELEASE='-O3 -DNDEBUG' +cmake --build build-perf -j --target cmt-benchmark +``` + +Run the standard repeated workloads and Linux hardware counters: + +```sh +REPETITIONS=5 benchmarks/run-perf.sh \ + ./build-perf/benchmarks/cmt-benchmark +``` + +The executable also accepts individual workloads: + +```text +cmt-benchmark lookup|update|prometheus|opentelemetry|opentelemetry-mixed CARDINALITY OPERATIONS +``` + +The `opentelemetry` workload repeatedly encodes a labeled counter with the +requested number of series. The `opentelemetry-mixed` workload creates that +many counter, gauge, and histogram series to exercise scalar and aggregate +protobuf data points in the same request. + +Compare medians from at least five alternating before/after runs. Keep CPU +frequency policy, compiler, flags, machine load, and input parameters fixed. +Use the reported in-process `elapsed_ns` for the operation itself and `perf +stat` for whole-process hardware counters. Whole-process counters include +series construction and teardown by design, exposing setup complexity as well +as steady-state behavior. diff --git a/lib/cmetrics/benchmarks/benchmark.c b/lib/cmetrics/benchmarks/benchmark.c new file mode 100644 index 00000000000..13a8661d18c --- /dev/null +++ b/lib/cmetrics/benchmarks/benchmark.c @@ -0,0 +1,346 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +static uint64_t monotonic_ns(void) +{ + struct timespec timestamp; + + clock_gettime(CLOCK_MONOTONIC, ×tamp); + + return (uint64_t) timestamp.tv_sec * 1000000000ULL + timestamp.tv_nsec; +} + +static size_t parse_size(const char *text, const char *name) +{ + char *end; + unsigned long long value; + + errno = 0; + value = strtoull(text, &end, 10); + if (errno != 0 || *text == '\0' || *end != '\0' || value == 0) { + fprintf(stderr, "invalid %s: %s\n", name, text); + exit(EXIT_FAILURE); + } + + return (size_t) value; +} + +static struct cmt_counter *create_series(struct cmt *cmt, size_t cardinality) +{ + size_t index; + int result; + char label[32]; + char *values[] = {label}; + struct cmt_counter *counter; + + counter = cmt_counter_create(cmt, "bench", "", "counter", "benchmark", + 1, (char *[]) {"series"}); + if (counter == NULL) { + return NULL; + } + + for (index = 0; index < cardinality; index++) { + snprintf(label, sizeof(label), "series-%zu", index); + result = cmt_counter_set(counter, 1, 1.0, 1, values); + if (result != 0) { + return NULL; + } + } + + return counter; +} + +static int create_mixed_series(struct cmt *cmt, size_t cardinality) +{ + size_t index; + char label[32]; + char *values[] = {label}; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_histogram *histogram; + struct cmt_histogram_buckets *buckets; + + counter = cmt_counter_create(cmt, "bench", "", "requests_total", + "benchmark counter", 1, + (char *[]) {"series"}); + gauge = cmt_gauge_create(cmt, "bench", "", "queue_depth", + "benchmark gauge", 1, + (char *[]) {"series"}); + buckets = cmt_histogram_buckets_create(4, 0.01, 0.1, 1.0, 10.0); + histogram = cmt_histogram_create(cmt, "bench", "", "latency_seconds", + "benchmark histogram", buckets, 1, + (char *[]) {"series"}); + if (counter == NULL || gauge == NULL || buckets == NULL || + histogram == NULL) { + return -1; + } + + for (index = 0; index < cardinality; index++) { + snprintf(label, sizeof(label), "series-%zu", index); + if (cmt_counter_set(counter, index + 1, 1.0, 1, values) != 0 || + cmt_gauge_set(gauge, index + 1, (double) index, 1, values) != 0 || + cmt_histogram_observe(histogram, index + 1, + (double) (index % 100) / 10.0, + 1, values) != 0) { + return -1; + } + } + + return 0; +} + +static int benchmark_lookup(size_t cardinality, size_t operations) +{ + size_t index; + int result; + double value; + uint64_t start; + uint64_t elapsed; + char label[32]; + char *values[] = {label}; + struct cmt *cmt; + struct cmt_counter *counter; + + cmt = cmt_create(); + if (cmt == NULL) { + return -1; + } + + counter = create_series(cmt, cardinality); + if (counter == NULL) { + cmt_destroy(cmt); + return -1; + } + + start = monotonic_ns(); + for (index = 0; index < operations; index++) { + snprintf(label, sizeof(label), "series-%zu", index % cardinality); + result = cmt_counter_get_val(counter, 1, values, &value); + if (result != 0 || value != 1.0) { + cmt_destroy(cmt); + return -1; + } + } + elapsed = monotonic_ns() - start; + + printf("benchmark=lookup cardinality=%zu operations=%zu elapsed_ns=%" PRIu64 + " ns_per_op=%.2f ops_per_second=%.2f\n", + cardinality, operations, elapsed, (double) elapsed / operations, + (double) operations * 1000000000.0 / elapsed); + cmt_destroy(cmt); + return 0; +} + +static int benchmark_update(size_t cardinality, size_t operations) +{ + size_t index; + int result; + uint64_t start; + uint64_t elapsed; + char label[32]; + char *values[] = {label}; + struct cmt *cmt; + struct cmt_counter *counter; + + cmt = cmt_create(); + if (cmt == NULL) { + return -1; + } + + counter = create_series(cmt, cardinality); + if (counter == NULL) { + cmt_destroy(cmt); + return -1; + } + + start = monotonic_ns(); + for (index = 0; index < operations; index++) { + snprintf(label, sizeof(label), "series-%zu", index % cardinality); + result = cmt_counter_inc(counter, index + 2, 1, values); + if (result != 0) { + cmt_destroy(cmt); + return -1; + } + } + elapsed = monotonic_ns() - start; + + printf("benchmark=update cardinality=%zu operations=%zu elapsed_ns=%" PRIu64 + " ns_per_op=%.2f ops_per_second=%.2f\n", + cardinality, operations, elapsed, (double) elapsed / operations, + (double) operations * 1000000000.0 / elapsed); + cmt_destroy(cmt); + return 0; +} + +static int benchmark_prometheus(size_t cardinality, size_t operations) +{ + size_t index; + size_t bytes = 0; + uint64_t start; + uint64_t elapsed; + cfl_sds_t output; + struct cmt *cmt; + + cmt = cmt_create(); + if (cmt == NULL) { + return -1; + } + + if (create_series(cmt, cardinality) == NULL) { + cmt_destroy(cmt); + return -1; + } + + start = monotonic_ns(); + for (index = 0; index < operations; index++) { + output = cmt_encode_prometheus_create(cmt, CMT_FALSE); + if (output == NULL) { + cmt_destroy(cmt); + return -1; + } + bytes += cfl_sds_len(output); + cmt_encode_prometheus_destroy(output); + } + elapsed = monotonic_ns() - start; + + printf("benchmark=prometheus cardinality=%zu operations=%zu bytes=%zu " + "elapsed_ns=%" PRIu64 " ns_per_op=%.2f mb_per_second=%.2f\n", + cardinality, operations, bytes, elapsed, (double) elapsed / operations, + ((double) bytes / (1024.0 * 1024.0)) / + ((double) elapsed / 1000000000.0)); + cmt_destroy(cmt); + return 0; +} + +static int benchmark_opentelemetry(size_t cardinality, size_t operations) +{ + size_t index; + size_t bytes; + uint64_t start; + uint64_t elapsed; + cfl_sds_t output; + struct cmt *cmt; + + bytes = 0; + cmt = cmt_create(); + if (cmt == NULL) { + return -1; + } + + if (create_series(cmt, cardinality) == NULL) { + cmt_destroy(cmt); + return -1; + } + + start = monotonic_ns(); + for (index = 0; index < operations; index++) { + output = cmt_encode_opentelemetry_create(cmt); + if (output == NULL) { + cmt_destroy(cmt); + return -1; + } + bytes += cfl_sds_len(output); + cmt_encode_opentelemetry_destroy(output); + } + elapsed = monotonic_ns() - start; + + printf("benchmark=opentelemetry cardinality=%zu operations=%zu bytes=%zu " + "elapsed_ns=%" PRIu64 " ns_per_op=%.2f mb_per_second=%.2f\n", + cardinality, operations, bytes, elapsed, (double) elapsed / operations, + ((double) bytes / (1024.0 * 1024.0)) / + ((double) elapsed / 1000000000.0)); + cmt_destroy(cmt); + return 0; +} + +static int benchmark_opentelemetry_mixed(size_t cardinality, size_t operations) +{ + size_t index; + size_t bytes; + uint64_t start; + uint64_t elapsed; + cfl_sds_t output; + struct cmt *cmt; + + bytes = 0; + cmt = cmt_create(); + if (cmt == NULL || create_mixed_series(cmt, cardinality) != 0) { + cmt_destroy(cmt); + return -1; + } + + start = monotonic_ns(); + for (index = 0; index < operations; index++) { + output = cmt_encode_opentelemetry_create(cmt); + if (output == NULL) { + cmt_destroy(cmt); + return -1; + } + bytes += cfl_sds_len(output); + cmt_encode_opentelemetry_destroy(output); + } + elapsed = monotonic_ns() - start; + + printf("benchmark=opentelemetry-mixed cardinality=%zu operations=%zu " + "bytes=%zu elapsed_ns=%" PRIu64 " ns_per_op=%.2f " + "mb_per_second=%.2f\n", + cardinality, operations, bytes, elapsed, + (double) elapsed / operations, + ((double) bytes / (1024.0 * 1024.0)) / + ((double) elapsed / 1000000000.0)); + cmt_destroy(cmt); + return 0; +} + +int main(int argc, char **argv) +{ + size_t cardinality; + size_t operations; + + if (argc != 4) { + fprintf(stderr, "usage: %s lookup|update|prometheus|opentelemetry|" + "opentelemetry-mixed " + "CARDINALITY OPERATIONS\n", + argv[0]); + return EXIT_FAILURE; + } + + cardinality = parse_size(argv[2], "cardinality"); + operations = parse_size(argv[3], "operations"); + cmt_initialize(); + + if (strcmp(argv[1], "lookup") == 0) { + return benchmark_lookup(cardinality, operations) == 0 ? + EXIT_SUCCESS : EXIT_FAILURE; + } + if (strcmp(argv[1], "update") == 0) { + return benchmark_update(cardinality, operations) == 0 ? + EXIT_SUCCESS : EXIT_FAILURE; + } + if (strcmp(argv[1], "prometheus") == 0) { + return benchmark_prometheus(cardinality, operations) == 0 ? + EXIT_SUCCESS : EXIT_FAILURE; + } + if (strcmp(argv[1], "opentelemetry") == 0) { + return benchmark_opentelemetry(cardinality, operations) == 0 ? + EXIT_SUCCESS : EXIT_FAILURE; + } + if (strcmp(argv[1], "opentelemetry-mixed") == 0) { + return benchmark_opentelemetry_mixed(cardinality, operations) == 0 ? + EXIT_SUCCESS : EXIT_FAILURE; + } + + fprintf(stderr, "unknown benchmark: %s\n", argv[1]); + return EXIT_FAILURE; +} diff --git a/lib/cmetrics/benchmarks/run-perf.sh b/lib/cmetrics/benchmarks/run-perf.sh new file mode 100755 index 00000000000..49a7b6bbdc6 --- /dev/null +++ b/lib/cmetrics/benchmarks/run-perf.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env sh +set -eu + +benchmark=${1:-./build/benchmarks/cmt-benchmark} +repetitions=${REPETITIONS:-5} + +run_repeated() +{ + mode=$1 + cardinality=$2 + operations=$3 + iteration=1 + + while [ "$iteration" -le "$repetitions" ]; do + "$benchmark" "$mode" "$cardinality" "$operations" + iteration=$((iteration + 1)) + done +} + +run_repeated lookup 5000 100000 +run_repeated update 5000 100000 +run_repeated update 1 5000000 +run_repeated prometheus 5000 100 +run_repeated opentelemetry 5000 100 +run_repeated opentelemetry-mixed 2000 100 + +perf stat \ + -e cycles,instructions,branches,branch-misses,cache-misses \ + -r "$repetitions" \ + "$benchmark" lookup 5000 100000 + +perf stat \ + -e task-clock,cycles,instructions,branches,branch-misses,cache-misses \ + -r "$repetitions" \ + "$benchmark" opentelemetry 5000 500 diff --git a/lib/cmetrics/docs/ai/bug-fix.md b/lib/cmetrics/docs/ai/bug-fix.md new file mode 100644 index 00000000000..c32c8cb8228 --- /dev/null +++ b/lib/cmetrics/docs/ai/bug-fix.md @@ -0,0 +1,42 @@ +# Bug-fix workflow + +## Purpose + +Produce a minimal, regression-tested fix with explicit compatibility and memory +ownership review. + +## When to use + +Use after an observed defect has enough evidence to justify changing CMetrics. + +## Investigation + +1. Restate observed behavior and separate facts from assumptions. +2. Identify the owning subsystem and repository. +3. Trace the relevant public entry point, internal path, and cleanup path. +4. Locate existing tests and reproduce the defect when practical. + +## Implementation + +1. Add a focused regression test that fails for the reported reason. +2. Implement the smallest reasonable fix; avoid unrelated refactoring. +3. Check allocation failure, partial initialization, cleanup, concurrency, and + malformed-input behavior where relevant. +4. Review public headers and encoded formats for compatibility impact. + +## Validation + +Run the focused CTest target, then `scripts/agent-verify.sh`. Ownership fixes +also require the applicable checks in `memory-safety-review.md`. Format changes +require round-trip and malformed-input coverage. + +## Expected report + +Report root cause, fix, regression test, exact commands/results, compatibility +impact, dependency impact, and unresolved risks. + +## Stop conditions + +Stop when the intended behavior requires a maintainer decision, the fix belongs +in a dependency, a public or wire contract must change without agreement, or a +safe reproducer cannot be created. diff --git a/lib/cmetrics/docs/ai/code-review.md b/lib/cmetrics/docs/ai/code-review.md new file mode 100644 index 00000000000..b8af87b55d4 --- /dev/null +++ b/lib/cmetrics/docs/ai/code-review.md @@ -0,0 +1,36 @@ +# Code-review workflow + +## Purpose + +Find actionable defects and regression risks in a proposed change. + +## When to use + +Use for pull requests, local diffs, dependency updates, and pre-release audits. + +## Investigation + +1. Establish the diff base, intended behavior, and affected public entry points. +2. Read surrounding code and tests; verify findings against the current tree. +3. Prioritize correctness, memory/resource ownership, cleanup paths, + concurrency, compatibility, malformed input, missing tests, regressions, + unnecessary complexity, and measurable performance impact. +4. Trace callers before claiming a nullability, lifetime, or locking defect. +5. Distinguish confirmed findings from questions and optional improvements. + +## Validation + +Run focused tests that can confirm or reject each high-confidence finding. For +codec changes, inspect both encoding and decoding and relevant format +conversions. Use sanitizer or benchmark evidence where the claim depends on it. + +## Expected report + +List findings by severity with file/line, failure scenario, evidence, and the +smallest corrective action. Then list validation gaps and a short overall risk +assessment. Do not bury findings in a general summary. + +## Stop conditions + +Do not request speculative changes without a concrete failure mode. Escalate +public API, ABI, wire-format, or cross-repository policy decisions. diff --git a/lib/cmetrics/docs/ai/cross-repository.md b/lib/cmetrics/docs/ai/cross-repository.md new file mode 100644 index 00000000000..5d2e98ec9c7 --- /dev/null +++ b/lib/cmetrics/docs/ai/cross-repository.md @@ -0,0 +1,44 @@ +# Cross-repository workflow + +## Purpose + +Coordinate changes whose behavior or validation spans CMetrics, its submodules, +or a downstream consumer. + +## When to use + +Use for CFL or fluent-otel-proto changes, CMetrics submodule updates, and +consumer integrations such as Fluent Bit. + +## Investigation + +1. Identify which repository owns the behavior and which repositories consume + its API, ABI, generated code, or wire output. +2. Determine compatibility impact and the required landing order. +3. Record exact base branches, revisions, and temporary dependency revisions. +4. Reproduce in the owning repository before changing the consumer when + practical. + +## Implementation + +1. Keep separate commits and pull requests per repository. +2. Land the owning-library fix first unless a coordinated transition is needed. +3. Update submodule or vendored revisions in a focused consumer change. +4. Avoid copying an unmerged implementation into multiple repositories. + +## Validation + +Run standalone validation in the owning repository. Then build and test each +consumer against the exact dependency branch/revision, including its relevant +integration and memory-safety jobs. + +## Expected report + +Report ownership, dependency direction, revisions, landing order, standalone +results, consumer integration results, and temporary coordination steps. + +## Stop conditions + +Stop when a required repository or CI environment is unavailable, the landing +order could break a supported branch, or ownership/compatibility policy is +unclear. diff --git a/lib/cmetrics/docs/ai/dependency-update.md b/lib/cmetrics/docs/ai/dependency-update.md new file mode 100644 index 00000000000..3a21bcad621 --- /dev/null +++ b/lib/cmetrics/docs/ai/dependency-update.md @@ -0,0 +1,40 @@ +# Dependency-update workflow + +## Purpose + +Update a Git submodule revision without mixing dependency implementation changes +into CMetrics. + +## When to use + +Use for `lib/cfl` and `lib/fluent-otel-proto` revisions. + +## Investigation + +1. Review `.gitmodules`, the current recorded revision, candidate commits/tags, + and the dependency's release notes or diff. +2. Identify public API, generated protobuf, allocator, threading, compiler, and + platform changes consumed by CMetrics. +3. Confirm the candidate revision exists in the dependency's upstream remote. + +## Implementation + +Update only the gitlink unless CMetrics requires a separate adaptation. Keep +adaptations explicit and avoid editing the submodule as uncommitted content. + +## Validation + +Run `scripts/agent-verify.sh`, relevant codec tests, and compiler/platform checks +affected by the dependency. For memory or performance changes, follow the +specialized workflows. Validate downstream consumers after CMetrics passes. + +## Expected report + +Report old/new revisions, included upstream changes, CMetrics adaptations, +standalone results, downstream results, and landing order. + +## Stop conditions + +Stop if the revision is not available upstream, includes unexplained generated +changes, requires an undocumented compatibility break, or cannot be tested in +the owning dependency first. diff --git a/lib/cmetrics/docs/ai/investigate.md b/lib/cmetrics/docs/ai/investigate.md new file mode 100644 index 00000000000..4568976a1b5 --- /dev/null +++ b/lib/cmetrics/docs/ai/investigate.md @@ -0,0 +1,42 @@ +# Investigation workflow + +## Purpose + +Establish an evidence-backed explanation of CMetrics behavior without assuming +that a code change is required. + +## When to use + +Use for bug reports, unexpected codec output, crashes, compatibility questions, +and performance reports before selecting a fix. + +## Investigation + +1. Restate the observed and expected behavior, including platform and input. +2. Separate reproduced facts, repository evidence, and working assumptions. +3. Identify the owning layer: CMetrics, CFL, fluent-otel-proto, generated code, + or a downstream consumer. +4. Trace from the public entry point through allocation, mutation, codec, and + cleanup paths. Include error exits and ownership transfers. +5. Locate focused tests and recent changes in the same subsystem. +6. Reproduce with the smallest representative input when practical. +7. For malformed input, retain a non-sensitive reproducer or describe its exact + construction rather than relying on an external artifact. + +## Validation + +Run the closest existing CTest target. Use a sanitizer, Valgrind, or benchmarks +only when relevant to the reported behavior. Do not infer results from CI job +names. + +## Expected report + +Report reproduction status, owning subsystem/repository, traced code path, +root-cause confidence, affected versions or formats when known, and remaining +unknowns. Propose a minimal fix and test plan separately. + +## Stop conditions + +Stop and escalate when the reproducer depends on unavailable private data, the +behavior belongs to another repository, the expected contract is ambiguous, or +a compatibility decision is required before implementation. diff --git a/lib/cmetrics/docs/ai/memory-safety-review.md b/lib/cmetrics/docs/ai/memory-safety-review.md new file mode 100644 index 00000000000..ac37fce80e4 --- /dev/null +++ b/lib/cmetrics/docs/ai/memory-safety-review.md @@ -0,0 +1,39 @@ +# Memory-safety review + +## Purpose + +Validate allocation, ownership, lifetime, and cleanup changes in this C library. + +## When to use + +Use for constructors/destructors, containers, codecs, arenas, metadata, +submodule allocator changes, and crash or leak reports. + +## Investigation + +1. Inventory allocations and their matching destructors by allocation family. +2. Trace ownership transfers, shared/borrowed pointers, arena lifetime, and + partial initialization. +3. Audit every normal and error exit, including empty collections and nested + values. +4. Check concurrent mutation/destruction and untrusted length arithmetic. +5. Add focused zero/one/many and failure-path regression cases where practical. + +## Validation + +First run the focused test normally. For AddressSanitizer, use a separate build +with compiler/linker sanitizer flags and execute CTest with leak detection. For +Valgrind, use a non-sanitized debug build and treat definite leaks/errors as +failures. Record the exact compiler, flags, commands, and test count; sanitizer +and Valgrind runs are complementary, not interchangeable. + +## Expected report + +Report the ownership model, confirmed defect, cleanup coverage, sanitizer and +Valgrind results, allocation-failure coverage, and untested paths. + +## Stop conditions + +Stop when ownership cannot be established from callers, allocator injection is +required but unavailable, or a proposed lifetime change affects a public +structure without compatibility review. diff --git a/lib/cmetrics/docs/ai/performance-review.md b/lib/cmetrics/docs/ai/performance-review.md new file mode 100644 index 00000000000..a6c33d69141 --- /dev/null +++ b/lib/cmetrics/docs/ai/performance-review.md @@ -0,0 +1,37 @@ +# Performance-review workflow + +## Purpose + +Accept only repeatable CMetrics performance improvements without correctness or +workload regressions. + +## When to use + +Use for map lookup/update, codec allocation, cardinality, and hot-path changes. + +## Investigation and implementation + +1. Identify the hot path and representative existing benchmark workload. +2. Establish a correctness baseline and a performance baseline before editing. +3. Change one material factor at a time and preserve ownership/concurrency rules. +4. Add a benchmark workload only when existing modes cannot represent the path. + +## Validation + +Follow `benchmarks/README.md`. Build Release with `CMT_BENCHMARKS=ON`; run at +least five alternating before/after samples on the same idle machine with the +same compiler, flags, CPU policy, cardinality, and operations. Compare medians, +variability, allocations when relevant, and `perf stat` counters. Run +`scripts/agent-verify.sh` for correctness. + +## Expected report + +Report revisions, hardware/software environment, exact commands, raw samples, +medians, percentage changes, counter changes, correctness results, and affected +workloads. Explain any regression. + +## Stop conditions + +Do not ship noise-level, single-run, differently configured, or correctness- +regressing results. Escalate tradeoffs that improve one supported workload while +materially degrading another. diff --git a/lib/cmetrics/docs/architecture.md b/lib/cmetrics/docs/architecture.md new file mode 100644 index 00000000000..179eea88e96 --- /dev/null +++ b/lib/cmetrics/docs/architecture.md @@ -0,0 +1,48 @@ +# CMetrics architecture + +CMetrics is a static C library that owns metric contexts and converts them to +and from several metrics protocols. The installed API is declared under +`include/cmetrics/`; implementations live under `src/`. + +## Core model + +`struct cmt` is the top-level context. Metric-family modules implement counters, +gauges, untyped metrics, summaries, histograms, and exponential histograms. +Each family owns a `struct cmt_map`, whose static or labeled datapoints are +represented by `struct cmt_metric`. Labels, options, timestamps, values, and +family-specific storage are shared by codecs and filters. + +The main entry points are: + +- `cmetrics.c`: context initialization and destruction. +- `cmt_map.c`, `cmt_metric.c`: datapoint lookup, storage, indexing, expiration, + and value representation. +- `cmt_.c`: metric-family creation and mutation. +- `cmt_cat.c`, `cmt_filter.c`: context combination and selection. +- `cmt_encode_*.c`, `cmt_decode_*.c`: protocol boundaries. + +## Protocol boundaries + +OTLP uses generated protobuf definitions supplied by fluent-otel-proto. +Prometheus remote write uses generated protobuf-C files in the repository. +Prometheus text decoding uses Flex/Bison sources and build-directory generated +parsers. CMetrics MessagePack is an internal serialized representation used by +format-conversion and downstream flows. + +Changes at these boundaries should preserve metric identity, label ordering, +numeric type, timestamps, aggregation fields, and decoder synchronization unless +the format contract intentionally changes. Tests under `tests/encoding.c`, +`tests/decoding.c`, `tests/opentelemetry.c`, `tests/format_conversion.c`, and +the Prometheus-specific test files cover these paths. + +## Ownership and concurrency + +Metric families own their maps; maps own dynamic metrics and label storage. +Some encoders create temporary heap or arena-backed protobuf structures before +packing them into an SDS result. Allocation family and lifetime must remain +consistent across success and partial-initialization cleanup. + +Map lookup, mutation, indexing, expiration, and destruction share internal +state and must be reviewed together for concurrent access. Public structures in +installed headers also constrain internal layout changes because downstream C +code can compile against them. diff --git a/lib/cmetrics/docs/dependencies.md b/lib/cmetrics/docs/dependencies.md new file mode 100644 index 00000000000..101f333eb16 --- /dev/null +++ b/lib/cmetrics/docs/dependencies.md @@ -0,0 +1,34 @@ +# Repository dependencies + +## Build dependencies + +- CMake 3.20 or newer configures the project. +- A platform C compiler builds the static library and tests. +- Flex 2 and Bison 3 generate the optional Prometheus text decoder. +- CTest runs the Acutest executables registered by `tests/CMakeLists.txt`. +- Linux `perf` is required only for the standard hardware-counter benchmarks. + +## Repository relationships + +CMetrics records two Git submodules: + +- `lib/cfl` → `fluent/cfl`: containers, SDS strings, variants, arenas, hashes, + atomic helpers, and other foundational C utilities. +- `lib/fluent-otel-proto` → `fluent/fluent-otel-proto`: generated OpenTelemetry + protobuf-C definitions and runtime integration used by the OTLP codec. + +The top-level build can use system-detected copies of these libraries; otherwise +it builds the recorded submodules. Behavior owned by either dependency should +be fixed and validated there before updating the CMetrics gitlink. + +Fluent Bit is an evidenced downstream consumer of CMetrics. It is not part of +this source tree, so consumer integration validation must use a separate Fluent +Bit checkout with the intended CMetrics revision. A normal landing order is: + +1. Land and validate a dependency change in its owning repository. +2. Update and validate CMetrics against that dependency revision. +3. Update the CMetrics revision or bundled copy in the downstream consumer. +4. Run the consumer's focused integration tests and applicable CI checks. + +Keep commits and pull requests separate per repository so each project can be +built, reviewed, and reverted independently. diff --git a/lib/cmetrics/docs/label-value-handling.md b/lib/cmetrics/docs/label-value-handling.md new file mode 100644 index 00000000000..1f5413a4f62 --- /dev/null +++ b/lib/cmetrics/docs/label-value-handling.md @@ -0,0 +1,117 @@ +# Long metric label handling + +## Context + +Before this fix, CMetrics rejected any string longer than 1024 bytes while +decoding its internal MessagePack representation. The validation was performed +by the generic string decoder, so it applied not only to label values, but also +to metric names, namespaces, subsystems, descriptions, and label names. + +This behavior caused +[fluent/fluent-bit#9297](https://github.com/fluent/fluent-bit/issues/9297): a +valid Prometheus scrape containing a `process_command_line` label longer than +1024 bytes was accepted by the Prometheus parser, but failed during the +subsequent CMetrics MessagePack round trip. The failure discarded all metrics +from the scrape without a useful diagnostic. + +[PR #224](https://github.com/fluent/cmetrics/pull/224) proposed retaining the +first 1024 bytes and appending `...` during MessagePack decoding. The issue is +valid, but that behavior should not be implemented in the generic decoder. + +## Why silent truncation is unsafe + +Prometheus identifies a time series using its metric name and complete label +set. Consider two label values with the same 1024-byte prefix: + +```text +process_command_line="A" +process_command_line="B" +``` + +Blindly replacing both suffixes with `...` gives both samples the same series +identity. This can merge unrelated series and produce incorrect results. + +Truncating at a fixed byte offset can also split a multi-byte UTF-8 character. +Because the generic MessagePack helper decodes every CMetrics string, the same +policy could silently alter metric names, descriptions, and label names. + +Presentation requirements must not be implemented by mutating the internal +data model during deserialization. An output encoder may abbreviate a value for +display, but the stored value must remain unchanged. + +## Ecosystem behavior + +There is no universal 1024-byte limit for Prometheus label values: + +- Prometheus accepts label values without a length limit by default. When a + `label_value_length_limit` is configured, one violation fails the scrape + instead of silently rewriting the value. +- VictoriaMetrics provides a configurable label-value limit. It ignores an + oversized series and reports the event through logs and an internal metric. +- Grafana Mimir provides configurable `error`, `drop`, and `truncate` + strategies. Its truncation strategy includes a hash of the original value so + that values with a common prefix remain distinct. +- OpenTelemetry metric attributes are currently exempt from the general SDK + attribute-length limits. + +References: + +- [Prometheus data model](https://prometheus.io/docs/concepts/data_model/) +- [Prometheus scrape configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/) +- [VictoriaMetrics ingestion limits](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) +- [Grafana Mimir configuration](https://grafana.com/docs/mimir/latest/configure/configuration-parameters/) +- [OpenTelemetry attribute limits](https://opentelemetry.io/docs/specs/otel/common/#attribute-limits) + +## Recommended CMetrics behavior + +The internal MessagePack decoder should: + +1. Decode valid strings losslessly by default. +2. Treat resource limits separately from string contents and metric semantics. +3. Prevent integer overflow when calculating allocation sizes. +4. Bound allocations made from untrusted length fields. +5. Return a distinct resource-limit error with enough context for callers to + emit a useful diagnostic. +6. Never silently truncate metric identifiers or labels. + +This fix implements those requirements for data-backed CMetrics MessagePack +decoding. It verifies that all declared string bytes are present before making +an allocation, then copies the complete value into CMetrics-owned storage. A +malicious length field therefore cannot trigger an allocation larger than its +available input, while valid long strings round-trip without modification. + +A configurable safety ceiling may be appropriate for callers that process +untrusted MessagePack. Such a ceiling should apply to decoder resources, be +documented in bytes, and reject input explicitly. It must not reinterpret an +oversized string as valid data with different contents. + +If Fluent Bit or another caller needs an ingestion policy, it should be +implemented above the MessagePack decoder. Useful policies are: + +- `preserve`: retain the complete value; this matches Prometheus defaults. +- `reject`: reject the scrape or batch with a diagnostic. +- `drop`: discard only the offending series and increment an error counter. +- `truncate_hash`: truncate on a valid UTF-8 boundary and append a hash derived + from the complete value to preserve series identity as far as practical. + +The selected policy and limit should be configurable by the component that +owns ingestion, because CMetrics is also used with OTLP and other formats whose +requirements differ. + +## Regression coverage + +An implementation should include tests for: + +- Values of 1023, 1024, 1025, 2048, and 65536 bytes. +- Two values with an identical 1024-byte prefix and different suffixes. +- UTF-8 characters crossing any configured boundary. +- Long metric names, descriptions, label names, and label values independently. +- Multiple fields and metrics after a long string, proving that the MessagePack + reader remains synchronized. +- A declared string length larger than the available input. +- A declared length near the integer and configured allocation limits. +- Allocation failure and cleanup paths. +- End-to-end Prometheus scrape, CMetrics MessagePack encode/decode, and output. + +Memory-safety validation should include AddressSanitizer, UndefinedBehaviorSanitizer, +and Valgrind in addition to the focused unit tests. diff --git a/lib/cmetrics/include/cmetrics/cmt_decode_prometheus.h b/lib/cmetrics/include/cmetrics/cmt_decode_prometheus.h index a565a4715b0..cd0bfb0efe7 100644 --- a/lib/cmetrics/include/cmetrics/cmt_decode_prometheus.h +++ b/lib/cmetrics/include/cmetrics/cmt_decode_prometheus.h @@ -68,6 +68,7 @@ struct cmt_decode_prometheus_context_metric { size_t label_count; cfl_sds_t labels[CMT_DECODE_PROMETHEUS_MAX_LABEL_COUNT]; struct cfl_list samples; + char *name_buf; }; struct cmt_decode_prometheus_parse_opts { diff --git a/lib/cmetrics/include/cmetrics/cmt_map.h b/lib/cmetrics/include/cmetrics/cmt_map.h index 00ab7ae6800..6a3c7e79620 100644 --- a/lib/cmetrics/include/cmetrics/cmt_map.h +++ b/lib/cmetrics/include/cmetrics/cmt_map.h @@ -43,12 +43,23 @@ struct cmt_map { int label_count; /* Number of labels */ struct cfl_list label_keys; /* Linked list of labels */ void *parent; + + /* Internal lock. Keep this after the established public fields. */ + uint64_t metric_lock; + struct cfl_list *metric_buckets; + size_t metric_bucket_count; + size_t indexed_metric_count; + /* Most recently created metric; only changed with the metric list. */ + struct cmt_metric *last_metric; }; struct cmt_map *cmt_map_create(int type, struct cmt_opts *opts, int count, char **labels, void *parent); void cmt_map_destroy(struct cmt_map *map); +/* Concurrent lookups and metric creation are serialized internally. The + * returned metric remains caller-usable only while the map is not expired or + * destroyed. */ struct cmt_metric *cmt_map_metric_get(struct cmt_opts *opts, struct cmt_map *map, int labels_count, char **labels_val, int write_op); @@ -57,6 +68,7 @@ int cmt_map_metric_get_val(struct cmt_opts *opts, struct cmt_map *map, double *out_val); void cmt_map_metric_destroy(struct cmt_metric *metric); +/* Expiration requires external coordination with metric users. */ void cmt_map_metrics_expire(struct cmt_map *, uint64_t); void destroy_label_list(struct cfl_list *label_list); diff --git a/lib/cmetrics/include/cmetrics/cmt_metric.h b/lib/cmetrics/include/cmetrics/cmt_metric.h index 35ca44f8407..a45eaba5b50 100644 --- a/lib/cmetrics/include/cmetrics/cmt_metric.h +++ b/lib/cmetrics/include/cmetrics/cmt_metric.h @@ -70,6 +70,11 @@ struct cmt_metric { uint64_t exp_hist_lock; struct cfl_list labels; struct cfl_list _head; + + /* Internal lookup index. Keep these after the established public fields. */ + int hash_indexed; + struct cmt_map *map; + struct cfl_list _hash_head; }; struct cmt_exp_histogram_snapshot { diff --git a/lib/cmetrics/include/cmetrics/cmt_mpack_utils_defs.h b/lib/cmetrics/include/cmetrics/cmt_mpack_utils_defs.h index 361886c06af..19eabd101eb 100644 --- a/lib/cmetrics/include/cmetrics/cmt_mpack_utils_defs.h +++ b/lib/cmetrics/include/cmetrics/cmt_mpack_utils_defs.h @@ -35,6 +35,5 @@ #define CMT_MPACK_MAX_ARRAY_ENTRY_COUNT 65535 #define CMT_MPACK_MAX_MAP_ENTRY_COUNT 10 -#define CMT_MPACK_MAX_STRING_LENGTH 1024 -#endif \ No newline at end of file +#endif diff --git a/lib/cmetrics/scripts/agent-build.sh b/lib/cmetrics/scripts/agent-build.sh new file mode 100755 index 00000000000..db001d0177b --- /dev/null +++ b/lib/cmetrics/scripts/agent-build.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +build_dir=${BUILD_DIR:-"$repository_root/build/agent"} + +git -C "$repository_root" submodule update --init --recursive + +cmake -S "$repository_root" -B "$build_dir" \ + -DCMT_TESTS=On \ + -DCMT_INSTALL_TARGETS=Off \ + "$@" + +cmake --build "$build_dir" diff --git a/lib/cmetrics/scripts/agent-test.sh b/lib/cmetrics/scripts/agent-test.sh new file mode 100755 index 00000000000..3515652963a --- /dev/null +++ b/lib/cmetrics/scripts/agent-test.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +build_dir=${BUILD_DIR:-"$repository_root/build/agent"} + +if [ ! -f "$build_dir/CTestTestfile.cmake" ]; then + echo "error: $build_dir is not configured for tests; run scripts/agent-build.sh first" >&2 + exit 1 +fi + +if [ "$#" -gt 1 ]; then + echo "usage: $0 [ctest-regular-expression]" >&2 + exit 2 +fi + +if [ "$#" -eq 1 ]; then + ctest --test-dir "$build_dir" --output-on-failure -R "$1" +else + ctest --test-dir "$build_dir" --output-on-failure +fi diff --git a/lib/cmetrics/scripts/agent-verify.sh b/lib/cmetrics/scripts/agent-verify.sh new file mode 100755 index 00000000000..ce4af194e03 --- /dev/null +++ b/lib/cmetrics/scripts/agent-verify.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) + +for script in "$repository_root"/scripts/agent-*.sh; do + sh -n "$script" +done + +"$repository_root/scripts/agent-build.sh" "$@" +"$repository_root/scripts/agent-test.sh" diff --git a/lib/cmetrics/src/cmt_cat.c b/lib/cmetrics/src/cmt_cat.c index bd01fc01f81..0e3f0841955 100644 --- a/lib/cmetrics/src/cmt_cat.c +++ b/lib/cmetrics/src/cmt_cat.c @@ -658,6 +658,50 @@ static struct cmt_exp_histogram *exp_histogram_lookup(struct cmt *cmt, struct cm return NULL; } +static struct cmt_summary *summary_lookup(struct cmt *cmt, struct cmt_opts *opts) +{ + struct cmt_summary *summary; + struct cfl_list *head; + + cfl_list_foreach(head, &cmt->summaries) { + summary = cfl_list_entry(head, struct cmt_summary, _head); + if (cmt_opts_compare(&summary->opts, opts) == 0) { + return summary; + } + } + + return NULL; +} + +static int summary_label_keys_match(struct cmt_map *left, + struct cmt_map *right) +{ + struct cfl_list *left_head; + struct cfl_list *right_head; + struct cmt_map_label *left_label; + struct cmt_map_label *right_label; + + left_head = left->label_keys.next; + right_head = right->label_keys.next; + + while (left_head != &left->label_keys && + right_head != &right->label_keys) { + left_label = cfl_list_entry(left_head, struct cmt_map_label, _head); + right_label = cfl_list_entry(right_head, struct cmt_map_label, _head); + + if (left_label->name == NULL || right_label->name == NULL || + strcmp(left_label->name, right_label->name) != 0) { + return CMT_FALSE; + } + + left_head = left_head->next; + right_head = right_head->next; + } + + return left_head == &left->label_keys && + right_head == &right->label_keys; +} + int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, struct cmt_map *filtered_map) { @@ -856,48 +900,65 @@ int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, struct cmt_map *filtered_map) { - int i; + size_t i; int ret; char **labels = NULL; struct cmt_map *map; struct cmt_opts *opts; struct cmt_summary *sum; double *quantiles; - uint64_t timestamp; - double summary_sum; map = summary->map; opts = map->opts; - timestamp = cmt_metric_get_timestamp(&map->metric); - ret = cmt_cat_copy_label_keys(map, (char **) &labels); if (ret == -1) { return -1; } - quantiles = calloc(1, sizeof(double) * summary->quantiles_count); - for (i = 0; i < summary->quantiles_count; i++) { - quantiles[i] = summary->quantiles[i]; + sum = summary_lookup(cmt, opts); + if (sum != NULL) { + if (!summary_label_keys_match(sum->map, map)) { + free(labels); + return -1; + } + + if (sum->quantiles_count != summary->quantiles_count) { + free(labels); + return -1; + } + + for (i = 0; i < summary->quantiles_count; i++) { + if (sum->quantiles[i] != summary->quantiles[i]) { + free(labels); + return -1; + } + } } + else { + quantiles = NULL; + if (summary->quantiles_count > 0) { + quantiles = calloc(summary->quantiles_count, sizeof(double)); + if (quantiles == NULL) { + free(labels); + return -1; + } + memcpy(quantiles, summary->quantiles, + summary->quantiles_count * sizeof(double)); + } - /* create summary */ - sum = cmt_summary_create(cmt, - opts->ns, opts->subsystem, - opts->name, opts->description, - summary->quantiles_count, - quantiles, - map->label_count, labels); - if (!sum) { - free(labels); + sum = cmt_summary_create(cmt, + opts->ns, opts->subsystem, + opts->name, opts->description, + summary->quantiles_count, + quantiles, + map->label_count, labels); free(quantiles); - return -1; } - summary_sum = cmt_summary_get_sum_value(&summary->map->metric); - - cmt_summary_set_default(sum, timestamp, quantiles, summary_sum, summary->quantiles_count, map->label_count, labels); free(labels); - free(quantiles); + if (!sum) { + return -1; + } if (filtered_map != NULL) { ret = cmt_cat_copy_map(&sum->opts, sum->map, filtered_map); diff --git a/lib/cmetrics/src/cmt_decode_msgpack.c b/lib/cmetrics/src/cmt_decode_msgpack.c index abd179570a4..70fab9e879d 100644 --- a/lib/cmetrics/src/cmt_decode_msgpack.c +++ b/lib/cmetrics/src/cmt_decode_msgpack.c @@ -1336,6 +1336,8 @@ static int unpack_metric_array_entry(mpack_reader_t *reader, size_t index, void else if (decode_context->map->type == CMT_SUMMARY) { cmt_atomic_store(&decode_context->map->metric.sum_quantiles_set, cmt_atomic_load(&metric->sum_quantiles_set)); decode_context->map->metric.sum_quantiles = metric->sum_quantiles; + decode_context->map->metric.sum_quantiles_count = + metric->sum_quantiles_count; cmt_atomic_store(&decode_context->map->metric.sum_count, cmt_atomic_load(&metric->sum_count)); cmt_atomic_store(&decode_context->map->metric.sum_sum, diff --git a/lib/cmetrics/src/cmt_decode_prometheus.c b/lib/cmetrics/src/cmt_decode_prometheus.c index 25fb83e39c3..13b08ac5b6b 100644 --- a/lib/cmetrics/src/cmt_decode_prometheus.c +++ b/lib/cmetrics/src/cmt_decode_prometheus.c @@ -56,23 +56,7 @@ static void reset_context(struct cmt_decode_prometheus_context *context, cfl_sds_destroy(context->metric.labels[i]); } - if (context->metric.ns) { - if ((void *) context->metric.ns != (void *) "") { - /* when namespace is empty, "name" contains a pointer to the - * allocated string. - * - * Note : When the metric name doesn't include the namespace - * ns is set to a constant empty string and we need to - * differentiate that case from the case where an empty - * namespace is provided. - */ - - free(context->metric.ns); - } - else { - free(context->metric.name); - } - } + free(context->metric.name_buf); cfl_sds_destroy(context->strbuf); context->strbuf = NULL; @@ -166,13 +150,18 @@ static int split_metric_name(struct cmt_decode_prometheus_context *context, cfl_sds_t metric_name, char **ns, char **subsystem, char **name) { + char *name_buf; + /* split the name */ - *ns = strdup(metric_name); - if (!*ns) { + name_buf = strdup(metric_name); + if (name_buf == NULL) { return report_error(context, CMT_DECODE_PROMETHEUS_ALLOCATION_ERROR, "memory allocation failed"); } + + context->metric.name_buf = name_buf; + *ns = name_buf; *subsystem = strchr(*ns, '_'); if (!(*subsystem)) { *name = *ns; diff --git a/lib/cmetrics/src/cmt_encode_opentelemetry.c b/lib/cmetrics/src/cmt_encode_opentelemetry.c index 434746c08c0..8622b2f00aa 100644 --- a/lib/cmetrics/src/cmt_encode_opentelemetry.c +++ b/lib/cmetrics/src/cmt_encode_opentelemetry.c @@ -26,8 +26,44 @@ #include #include #include +#include #include +#define CMT_OTLP_ARENA_INITIAL_CHUNK_SIZE 4096 +#define CMT_OTLP_ARENA_MAX_CHUNK_SIZE 65536 + +struct cmt_opentelemetry_encoder { + struct cmt_opentelemetry_context context; + struct cfl_arena *arena; +}; + +/* + * Data points and their label attributes remain immutable and live until the + * protobuf request has been packed. Allocate that temporary object graph from + * one arena and release it after packing. The returned SDS and metadata-owned + * objects remain heap-backed and retain their existing ownership rules. + */ + +static struct cfl_arena *get_context_arena(struct cmt_opentelemetry_context *context) +{ + struct cmt_opentelemetry_encoder *encoder; + + encoder = (struct cmt_opentelemetry_encoder *) context; + + return encoder->arena; +} + +static struct cfl_arena *create_encoder_arena(void) +{ + struct cfl_arena_options options; + + cfl_arena_options_init(&options); + options.chunk_size = CMT_OTLP_ARENA_INITIAL_CHUNK_SIZE; + options.maximum_chunk_size = CMT_OTLP_ARENA_MAX_CHUNK_SIZE; + + return cfl_arena_create_with_options(&options); +} + static Opentelemetry__Proto__Metrics__V1__ScopeMetrics ** initialize_scope_metrics_list( size_t element_count); @@ -746,15 +782,12 @@ static int is_metric_empty(struct cmt_map *map); static size_t get_metric_count(struct cmt *cmt); static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile * - initialize_summary_value_at_quantile( - double quantile, double value); - -static void destroy_summary_value_at_quantile( - Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile *value); + initialize_summary_value_at_quantile(struct cfl_arena *arena, + double quantile, double value); static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile ** - initialize_summary_value_at_quantile_list( - size_t element_count); + initialize_summary_value_at_quantile_list(struct cfl_arena *arena, + size_t element_count); static void destroy_summary_value_at_quantile_list( Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile **list); @@ -810,14 +843,15 @@ static void destroy_attribute( Opentelemetry__Proto__Common__V1__KeyValue *attribute); static Opentelemetry__Proto__Common__V1__KeyValue * - initialize_string_attribute(char *key, char *value); + initialize_string_attribute(struct cfl_arena *arena, + char *key, char *value); static void destroy_attribute_list( Opentelemetry__Proto__Common__V1__KeyValue **attribute_list); static Opentelemetry__Proto__Common__V1__KeyValue ** - initialize_attribute_list( - size_t element_count); + initialize_attribute_list(struct cfl_arena *arena, + size_t element_count); static void destroy_numerical_data_point( Opentelemetry__Proto__Metrics__V1__NumberDataPoint *data_point); @@ -848,16 +882,16 @@ static void destroy_exponential_histogram_data_point_list( Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint **data_point_list); static Opentelemetry__Proto__Metrics__V1__NumberDataPoint * - initialize_numerical_data_point( - uint64_t start_time, + initialize_numerical_data_point(struct cfl_arena *arena, + uint64_t start_time, uint64_t timestamp, struct cmt_metric *sample, Opentelemetry__Proto__Common__V1__KeyValue **attribute_list, size_t attribute_count); static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * - initialize_summary_data_point( - uint64_t start_time, + initialize_summary_data_point(struct cfl_arena *arena, + uint64_t start_time, uint64_t timestamp, uint64_t count, double sum, @@ -869,8 +903,8 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * size_t attribute_count); static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * - initialize_histogram_data_point( - uint64_t start_time, + initialize_histogram_data_point(struct cfl_arena *arena, + uint64_t start_time, uint64_t timestamp, uint64_t count, double sum, @@ -1726,7 +1760,7 @@ void destroy_instrumentation_scope(Opentelemetry__Proto__Common__V1__Instrumenta } if (scope->attributes != NULL) { - destroy_attribute_list(scope->attributes); + otlp_kvpair_list_destroy(scope->attributes, scope->n_attributes); } free(scope); @@ -1904,33 +1938,18 @@ static Opentelemetry__Proto__Metrics__V1__ScopeMetrics **initialize_scope_metric static void destroy_attribute(Opentelemetry__Proto__Common__V1__KeyValue *attribute) { - if (attribute != NULL) { - if (attribute->value != NULL) { - if (attribute->value->value_case == \ - OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE) { - if (is_string_releaseable(attribute->value->string_value)) { - cfl_sds_destroy(attribute->value->string_value); - } - } - - free(attribute->value); - } - - if (is_string_releaseable(attribute->key)) { - cfl_sds_destroy(attribute->key); - } - - free(attribute); - } + (void) attribute; } static Opentelemetry__Proto__Common__V1__KeyValue * - initialize_string_attribute(char *key, char *value) + initialize_string_attribute(struct cfl_arena *arena, + char *key, char *value) { Opentelemetry__Proto__Common__V1__KeyValue *attribute; - attribute = calloc(1, - sizeof(Opentelemetry__Proto__Common__V1__KeyValue)); + attribute = cfl_arena_calloc( + arena, 1, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue)); if (attribute == NULL) { return NULL; @@ -1938,8 +1957,9 @@ static Opentelemetry__Proto__Common__V1__KeyValue * opentelemetry__proto__common__v1__key_value__init(attribute); - attribute->value = calloc(1, - sizeof(Opentelemetry__Proto__Common__V1__AnyValue)); + attribute->value = cfl_arena_calloc( + arena, 1, + sizeof(Opentelemetry__Proto__Common__V1__AnyValue)); if (attribute->value == NULL) { destroy_attribute(attribute); @@ -1949,7 +1969,8 @@ static Opentelemetry__Proto__Common__V1__KeyValue * opentelemetry__proto__common__v1__any_value__init(attribute->value); - attribute->value->string_value = cfl_sds_create(value); + attribute->value->string_value = cfl_arena_strndup(arena, value, + strlen(value)); if (attribute->value->string_value == NULL) { destroy_attribute(attribute); @@ -1959,7 +1980,7 @@ static Opentelemetry__Proto__Common__V1__KeyValue * attribute->value->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE; - attribute->key = cfl_sds_create(key); + attribute->key = cfl_arena_strndup(arena, key, strlen(key)); if (attribute->key == NULL) { destroy_attribute(attribute); @@ -1973,29 +1994,18 @@ static Opentelemetry__Proto__Common__V1__KeyValue * static void destroy_attribute_list( Opentelemetry__Proto__Common__V1__KeyValue **attribute_list) { - size_t element_index; - - if (attribute_list != NULL) { - for (element_index = 0 ; - attribute_list[element_index] != NULL ; - element_index++) { - destroy_attribute(attribute_list[element_index]); - - attribute_list[element_index] = NULL; - } - - free(attribute_list); - } + (void) attribute_list; } static Opentelemetry__Proto__Common__V1__KeyValue ** - initialize_attribute_list( - size_t element_count) + initialize_attribute_list(struct cfl_arena *arena, + size_t element_count) { Opentelemetry__Proto__Common__V1__KeyValue **attribute_list; - attribute_list = calloc(element_count + 1, - sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); + attribute_list = cfl_arena_calloc( + arena, element_count + 1, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); return attribute_list; } @@ -2017,23 +2027,13 @@ static void destroy_numerical_data_point( free(data_point->exemplars); } - - free(data_point); } } static void destroy_summary_data_point( Opentelemetry__Proto__Metrics__V1__SummaryDataPoint *data_point) { - if (data_point != NULL) { - destroy_attribute_list(data_point->attributes); - - if (data_point->quantile_values != NULL) { - destroy_summary_value_at_quantile_list(data_point->quantile_values); - } - - free(data_point); - } + (void) data_point; } static void destroy_histogram_data_point( @@ -2042,16 +2042,6 @@ static void destroy_histogram_data_point( size_t index; if (data_point != NULL) { - destroy_attribute_list(data_point->attributes); - - if (data_point->bucket_counts != NULL) { - free(data_point->bucket_counts); - } - - if (data_point->explicit_bounds != NULL) { - free(data_point->explicit_bounds); - } - if (data_point->exemplars != NULL) { for (index = 0; index < data_point->n_exemplars; index++) { if (data_point->exemplars[index] != NULL) { @@ -2061,8 +2051,6 @@ static void destroy_histogram_data_point( free(data_point->exemplars); } - - free(data_point); } } @@ -2072,22 +2060,6 @@ static void destroy_exponential_histogram_data_point( size_t index; if (data_point != NULL) { - destroy_attribute_list(data_point->attributes); - - if (data_point->positive != NULL) { - if (data_point->positive->bucket_counts != NULL) { - free(data_point->positive->bucket_counts); - } - free(data_point->positive); - } - - if (data_point->negative != NULL) { - if (data_point->negative->bucket_counts != NULL) { - free(data_point->negative->bucket_counts); - } - free(data_point->negative); - } - if (data_point->exemplars != NULL) { for (index = 0; index < data_point->n_exemplars; index++) { if (data_point->exemplars[index] != NULL) { @@ -2097,8 +2069,6 @@ static void destroy_exponential_histogram_data_point( free(data_point->exemplars); } - - free(data_point); } } @@ -2192,8 +2162,8 @@ static void destroy_exponential_histogram_data_point_list( } static Opentelemetry__Proto__Metrics__V1__NumberDataPoint * - initialize_numerical_data_point( - uint64_t start_time, + initialize_numerical_data_point(struct cfl_arena *arena, + uint64_t start_time, uint64_t timestamp, struct cmt_metric *sample, Opentelemetry__Proto__Common__V1__KeyValue **attribute_list, @@ -2201,8 +2171,9 @@ static Opentelemetry__Proto__Metrics__V1__NumberDataPoint * { Opentelemetry__Proto__Metrics__V1__NumberDataPoint *data_point; - data_point = calloc(1, - sizeof(Opentelemetry__Proto__Metrics__V1__NumberDataPoint)); + data_point = cfl_arena_calloc( + arena, 1, + sizeof(Opentelemetry__Proto__Metrics__V1__NumberDataPoint)); if (data_point == NULL) { return NULL; @@ -2237,12 +2208,14 @@ static Opentelemetry__Proto__Metrics__V1__NumberDataPoint * } static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile * - initialize_summary_value_at_quantile( - double quantile, double value) + initialize_summary_value_at_quantile(struct cfl_arena *arena, + double quantile, double value) { Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile *instance; - instance = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile)); + instance = cfl_arena_calloc( + arena, 1, + sizeof(Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile)); if (instance != NULL) { opentelemetry__proto__metrics__v1__summary_data_point__value_at_quantile__init(instance); @@ -2254,22 +2227,15 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile * return instance; } -static void destroy_summary_value_at_quantile( - Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile *value) -{ - if (value != NULL) { - free(value); - } -} - static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile ** - initialize_summary_value_at_quantile_list( - size_t element_count) + initialize_summary_value_at_quantile_list(struct cfl_arena *arena, + size_t element_count) { Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile **list; - list = calloc(element_count + 1, - sizeof(Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile *)); + list = cfl_arena_calloc( + arena, element_count + 1, + sizeof(Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile *)); return list; } @@ -2277,26 +2243,14 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile ** static void destroy_summary_value_at_quantile_list( Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile **list) { - size_t index; - - if (list != NULL) { - for (index = 0 ; - list[index] != NULL ; - index++) { - destroy_summary_value_at_quantile(list[index]); - - list[index] = NULL; - } - - free(list); - } + (void) list; } static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * - initialize_summary_data_point( - uint64_t start_time, + initialize_summary_data_point(struct cfl_arena *arena, + uint64_t start_time, uint64_t timestamp, uint64_t count, double sum, @@ -2310,8 +2264,9 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * Opentelemetry__Proto__Metrics__V1__SummaryDataPoint *data_point; size_t index; - data_point = calloc(1, - sizeof(Opentelemetry__Proto__Metrics__V1__SummaryDataPoint)); + data_point = cfl_arena_calloc( + arena, 1, + sizeof(Opentelemetry__Proto__Metrics__V1__SummaryDataPoint)); if (data_point == NULL) { return NULL; @@ -2326,13 +2281,12 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * data_point->sum = sum; data_point->n_quantile_values = quantile_count; - data_point->quantile_values = initialize_summary_value_at_quantile_list(quantile_count); + data_point->quantile_values = + initialize_summary_value_at_quantile_list(arena, quantile_count); if (data_point->quantile_values == NULL) { cmt_errno(); - free(data_point); - return NULL; } @@ -2340,14 +2294,13 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * if (value_list != NULL) { for (index = 0 ; index < quantile_count ; index++) { data_point->quantile_values[index] = - initialize_summary_value_at_quantile(quantile_list[index], + initialize_summary_value_at_quantile(arena, + quantile_list[index], cmt_math_uint64_to_d64(value_list[index])); if (data_point->quantile_values[index] == NULL) { destroy_summary_value_at_quantile_list(data_point->quantile_values); - free(data_point); - return NULL; } } @@ -2361,8 +2314,8 @@ static Opentelemetry__Proto__Metrics__V1__SummaryDataPoint * } static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * - initialize_histogram_data_point( - uint64_t start_time, + initialize_histogram_data_point(struct cfl_arena *arena, + uint64_t start_time, uint64_t timestamp, uint64_t count, double sum, @@ -2376,8 +2329,9 @@ static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *data_point; size_t index; - data_point = calloc(1, - sizeof(Opentelemetry__Proto__Metrics__V1__HistogramDataPoint)); + data_point = cfl_arena_calloc( + arena, 1, + sizeof(Opentelemetry__Proto__Metrics__V1__HistogramDataPoint)); if (data_point == NULL) { return NULL; @@ -2402,13 +2356,12 @@ static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * if (bucket_count > 0) { - data_point->bucket_counts = calloc(bucket_count, sizeof(uint64_t)); + data_point->bucket_counts = cfl_arena_calloc(arena, bucket_count, + sizeof(uint64_t)); if (data_point->bucket_counts == NULL) { cmt_errno(); - free(data_point); - return NULL; } @@ -2422,17 +2375,12 @@ static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * data_point->n_explicit_bounds = boundary_count; if (boundary_count > 0) { - data_point->explicit_bounds = calloc(boundary_count, sizeof(double)); + data_point->explicit_bounds = cfl_arena_calloc(arena, boundary_count, + sizeof(double)); if (data_point->explicit_bounds == NULL) { cmt_errno(); - if (data_point->bucket_counts != NULL) { - free(data_point->bucket_counts); - } - - free(data_point); - return NULL; } @@ -2944,7 +2892,11 @@ static Opentelemetry__Proto__Metrics__V1__Metric ** static void destroy_opentelemetry_context( struct cmt_opentelemetry_context *context) { + struct cmt_opentelemetry_encoder *encoder; + if (context != NULL) { + encoder = (struct cmt_opentelemetry_encoder *) context; + if (context->scope_metrics_list != NULL) { free(context->scope_metrics_list); } @@ -2953,7 +2905,8 @@ static void destroy_opentelemetry_context( destroy_metrics_data(context->metrics_data); } - free(context); + cfl_arena_destroy(encoder->arena); + free(encoder); } } @@ -3023,6 +2976,7 @@ static struct cmt_opentelemetry_context *initialize_opentelemetry_context( struct cfl_kvlist *scope_root; Opentelemetry__Proto__Resource__V1__Resource *resource; struct cmt_opentelemetry_context *context; + struct cmt_opentelemetry_encoder *encoder; size_t resource_count; size_t scope_count; size_t total_scope_count; @@ -3039,17 +2993,26 @@ static struct cmt_opentelemetry_context *initialize_opentelemetry_context( result = CMT_ENCODE_OPENTELEMETRY_SUCCESS; total_scope_count = 0; + context = NULL; + encoder = NULL; - context = calloc(1, sizeof(struct cmt_opentelemetry_context)); - - if (context == NULL) { + encoder = calloc(1, sizeof(struct cmt_opentelemetry_encoder)); + if (encoder == NULL) { result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; goto cleanup; } - memset(context, 0, sizeof(struct cmt_opentelemetry_context)); + encoder->arena = create_encoder_arena(); + if (encoder->arena == NULL) { + free(encoder); + context = NULL; + result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + + goto cleanup; + } + context = &encoder->context; context->cmt = cmt; if (resource_metrics_list != NULL && resource_metrics_list->entry_count > 0) { @@ -3215,7 +3178,8 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, start_timestamp = cmt_metric_get_start_timestamp(sample); } - attribute_list = initialize_attribute_list(attribute_count); + attribute_list = initialize_attribute_list(get_context_arena(context), + attribute_count); if (attribute_list == NULL) { return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; @@ -3224,7 +3188,8 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, if (map->type == CMT_COUNTER || map->type == CMT_GAUGE || map->type == CMT_UNTYPED ) { - data_point = initialize_numerical_data_point(start_timestamp, + data_point = initialize_numerical_data_point(get_context_arena(context), + start_timestamp, cmt_metric_get_timestamp(sample), sample, attribute_list, @@ -3233,7 +3198,8 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, else if (map->type == CMT_SUMMARY) { summary = (struct cmt_summary *) map->parent; - data_point = initialize_summary_data_point(start_timestamp, + data_point = initialize_summary_data_point(get_context_arena(context), + start_timestamp, cmt_metric_get_timestamp(sample), cmt_summary_get_count_value(sample), cmt_summary_get_sum_value(sample), @@ -3247,7 +3213,8 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, else if (map->type == CMT_HISTOGRAM) { histogram = (struct cmt_histogram *) map->parent; - data_point = initialize_histogram_data_point(start_timestamp, + data_point = initialize_histogram_data_point(get_context_arena(context), + start_timestamp, cmt_metric_get_timestamp(sample), cmt_metric_hist_get_count_value(sample), cmt_metric_hist_get_sum_value(sample), @@ -3266,7 +3233,9 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; } - exponential_data_point = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint)); + exponential_data_point = cfl_arena_calloc( + get_context_arena(context), 1, + sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint)); if (exponential_data_point == NULL) { cmt_metric_exp_hist_snapshot_destroy(&snapshot); destroy_attribute_list(attribute_list); @@ -3290,7 +3259,9 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, } if (snapshot.positive_count > 0) { - positive = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets)); + positive = cfl_arena_calloc( + get_context_arena(context), 1, + sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets)); if (positive == NULL) { cmt_metric_exp_hist_snapshot_destroy(&snapshot); destroy_data_point(exponential_data_point, map->type); @@ -3300,10 +3271,12 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, opentelemetry__proto__metrics__v1__exponential_histogram_data_point__buckets__init(positive); positive->offset = snapshot.positive_offset; positive->n_bucket_counts = snapshot.positive_count; - positive->bucket_counts = calloc(snapshot.positive_count, sizeof(uint64_t)); + positive->bucket_counts = cfl_arena_calloc( + get_context_arena(context), + snapshot.positive_count, + sizeof(uint64_t)); if (positive->bucket_counts == NULL) { cmt_metric_exp_hist_snapshot_destroy(&snapshot); - free(positive); destroy_data_point(exponential_data_point, map->type); return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; } @@ -3313,7 +3286,9 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, } if (snapshot.negative_count > 0) { - negative = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets)); + negative = cfl_arena_calloc( + get_context_arena(context), 1, + sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets)); if (negative == NULL) { cmt_metric_exp_hist_snapshot_destroy(&snapshot); destroy_data_point(exponential_data_point, map->type); @@ -3323,10 +3298,12 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, opentelemetry__proto__metrics__v1__exponential_histogram_data_point__buckets__init(negative); negative->offset = snapshot.negative_offset; negative->n_bucket_counts = snapshot.negative_count; - negative->bucket_counts = calloc(snapshot.negative_count, sizeof(uint64_t)); + negative->bucket_counts = cfl_arena_calloc( + get_context_arena(context), + snapshot.negative_count, + sizeof(uint64_t)); if (negative->bucket_counts == NULL) { cmt_metric_exp_hist_snapshot_destroy(&snapshot); - free(negative); destroy_data_point(exponential_data_point, map->type); return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; } @@ -3350,7 +3327,8 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, cfl_list_foreach(head, &context->cmt->static_labels->list) { static_label = cfl_list_entry(head, struct cmt_label, _head); - attribute = initialize_string_attribute(static_label->key, + attribute = initialize_string_attribute(get_context_arena(context), + static_label->key, static_label->val); if (attribute == NULL) { @@ -3410,7 +3388,8 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, return CMT_ENCODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; } - attribute = initialize_string_attribute(label_name->name, + attribute = initialize_string_attribute(get_context_arena(context), + label_name->name, label_value->name); if (attribute == NULL) { diff --git a/lib/cmetrics/src/cmt_encode_prometheus_remote_write.c b/lib/cmetrics/src/cmt_encode_prometheus_remote_write.c index 63ce463c696..e09c216211a 100644 --- a/lib/cmetrics/src/cmt_encode_prometheus_remote_write.c +++ b/lib/cmetrics/src/cmt_encode_prometheus_remote_write.c @@ -735,6 +735,11 @@ static int check_staled_timestamp(struct cmt_metric *metric, uint64_t now, uint6 uint64_t diff; ts = cmt_metric_get_timestamp(metric); + + if (ts >= now) { + return CMT_FALSE; + } + diff = now - ts; return diff > cutoff; @@ -751,6 +756,7 @@ int pack_basic_type(struct cmt_prometheus_remote_write_context *context, context->sequence_number++; add_metadata = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_ADD_METADATA; + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; now = cfl_time_now(); @@ -758,13 +764,14 @@ int pack_basic_type(struct cmt_prometheus_remote_write_context *context, if (check_staled_timestamp(&map->metric, now, CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_THRESHOLD)) { /* Skip processing metrics which are staled over the threshold */ - return CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR; + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; } - - result = pack_basic_metric_sample(context, map, &map->metric, add_metadata); - - if (result != CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { - return result; + else { + result = pack_basic_metric_sample(context, map, &map->metric, add_metadata); + if (result != CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + return result; + } + add_metadata = CMT_FALSE; } } @@ -774,7 +781,7 @@ int pack_basic_type(struct cmt_prometheus_remote_write_context *context, if (check_staled_timestamp(metric, now, CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_THRESHOLD)) { /* Skip processing metrics which are staled over over the threshold */ - return CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR; + continue; } result = pack_basic_metric_sample(context, map, metric, add_metadata); @@ -1208,6 +1215,12 @@ int pack_complex_type(struct cmt_prometheus_remote_write_context *context, if (map->metric_static_set == CMT_TRUE) { result = pack_complex_metric_sample(context, map, &map->metric, add_metadata); + if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; + } + else if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + add_metadata = CMT_FALSE; + } } if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { @@ -1216,10 +1229,17 @@ int pack_complex_type(struct cmt_prometheus_remote_write_context *context, result = pack_complex_metric_sample(context, map, metric, add_metadata); - if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { - if (add_metadata == CMT_TRUE) { - add_metadata = CMT_FALSE; - } + if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; + continue; + } + + if (result != CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + break; + } + + if (add_metadata == CMT_TRUE) { + add_metadata = CMT_FALSE; } } } @@ -1266,6 +1286,7 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) result = pack_basic_type(&context, counter->map); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; continue; } @@ -1281,6 +1302,7 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) result = pack_basic_type(&context, gauge->map); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; continue; } @@ -1294,11 +1316,16 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) /* Untyped */ cfl_list_foreach(head, &cmt->untypeds) { untyped = cfl_list_entry(head, struct cmt_untyped, _head); - pack_basic_type(&context, untyped->map); + result = pack_basic_type(&context, untyped->map); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; continue; } + + if (result != CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + break; + } } } @@ -1309,6 +1336,7 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) result = pack_complex_type(&context, summary->map); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; continue; } @@ -1325,6 +1353,7 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) result = pack_complex_type(&context, histogram->map); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; continue; } @@ -1341,6 +1370,7 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) result = pack_complex_type(&context, exp_histogram->map); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; continue; } diff --git a/lib/cmetrics/src/cmt_filter.c b/lib/cmetrics/src/cmt_filter.c index 6b7c220443d..0124f57d03d 100644 --- a/lib/cmetrics/src/cmt_filter.c +++ b/lib/cmetrics/src/cmt_filter.c @@ -186,7 +186,7 @@ static int filter_get_label_index(struct cmt_map *src, const char *label_key) cfl_list_foreach(head, &src->label_keys) { label = cfl_list_entry(head, struct cmt_map_label, _head); - if (strncmp(label->name, label_key, strlen(label->name)) == 0) { + if (strcmp(label->name, label_key) == 0) { return index; } @@ -228,7 +228,7 @@ int metrics_check_label_value_existence(struct cmt_metric *metric, return CMT_FALSE; } - if (strncmp(label->name, label_value, strlen(label->name)) == 0) { + if (strcmp(label->name, label_value) == 0) { return CMT_TRUE; } @@ -239,29 +239,27 @@ static int metrics_map_drop_label_value_pairs(struct cmt_map *map, size_t label_index, const char *label_value) { + struct cfl_list *tmp; struct cfl_list *head; struct cmt_metric *metric; + int match; int result; result = CMT_FALSE; - cfl_list_foreach(head, &map->metrics) { + cfl_list_foreach_safe(head, tmp, &map->metrics) { metric = cfl_list_entry(head, struct cmt_metric, _head); - result = metrics_check_label_value_existence(metric, - label_index, - label_value); + match = metrics_check_label_value_existence(metric, + label_index, + label_value); - if (result == CMT_TRUE) { + if (match == CMT_TRUE) { result = CMT_TRUE; - break; + cmt_map_metric_destroy(metric); } } - if (result == CMT_TRUE) { - cmt_map_metric_destroy(metric); - } - return result; } diff --git a/lib/cmetrics/src/cmt_map.c b/lib/cmetrics/src/cmt_map.c index 3442abda502..fb4fe9a150e 100644 --- a/lib/cmetrics/src/cmt_map.c +++ b/lib/cmetrics/src/cmt_map.c @@ -21,8 +21,91 @@ #include #include #include +#include +#include +#include #include +#define CMT_MAP_INITIAL_BUCKET_COUNT 64 +#define CMT_MAP_BUCKET_LOAD_FACTOR 4 + +static void map_lock(struct cmt_map *map) +{ + while (cmt_atomic_compare_exchange(&map->metric_lock, 0, 1) == 0) { + } +} + +static void map_unlock(struct cmt_map *map) +{ + cmt_atomic_store(&map->metric_lock, 0); +} + +static void metric_release_storage(struct cmt_metric *metric) +{ + free(metric->hist_buckets); + free(metric->exp_hist_positive_buckets); + free(metric->exp_hist_negative_buckets); + free(metric->sum_quantiles); + + metric->hist_buckets = NULL; + metric->exp_hist_positive_buckets = NULL; + metric->exp_hist_negative_buckets = NULL; + metric->sum_quantiles = NULL; +} + +static int metric_index_resize(struct cmt_map *map, size_t bucket_count) +{ + size_t index; + struct cfl_list *head; + struct cfl_list *buckets; + struct cmt_metric *metric; + + buckets = calloc(bucket_count, sizeof(struct cfl_list)); + if (buckets == NULL) { + return -1; + } + for (index = 0; index < bucket_count; index++) { + cfl_list_init(&buckets[index]); + } + + cfl_list_foreach(head, &map->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + if (metric->hash_indexed) { + cfl_list_del(&metric->_hash_head); + cfl_list_add(&metric->_hash_head, + &buckets[metric->hash % bucket_count]); + } + } + + free(map->metric_buckets); + map->metric_buckets = buckets; + map->metric_bucket_count = bucket_count; + return 0; +} + +static void metric_index_add(struct cmt_map *map, struct cmt_metric *metric) +{ + if (metric->hash_indexed) { + return; + } + + if (map->metric_buckets == NULL && + metric_index_resize(map, CMT_MAP_INITIAL_BUCKET_COUNT) != 0) { + return; + } + + if (map->indexed_metric_count >= + map->metric_bucket_count * CMT_MAP_BUCKET_LOAD_FACTOR) { + metric_index_resize(map, map->metric_bucket_count * 2); + } + + cfl_list_add(&metric->_hash_head, + &map->metric_buckets[metric->hash % map->metric_bucket_count]); + metric->hash_indexed = CMT_TRUE; + metric->map = map; + map->indexed_metric_count++; +} + struct cmt_map *cmt_map_create(int type, struct cmt_opts *opts, int count, char **labels, void *parent) { @@ -48,6 +131,13 @@ struct cmt_map *cmt_map_create(int type, struct cmt_opts *opts, int count, char cfl_list_init(&map->metrics); cfl_list_init(&map->metric.labels); + if (count > 0 && + metric_index_resize(map, CMT_MAP_INITIAL_BUCKET_COUNT) != 0) { + cmt_errno(); + cmt_map_destroy(map); + return NULL; + } + if (count == 0) { map->metric_static_set = 1; } @@ -76,7 +166,75 @@ struct cmt_map *cmt_map_create(int type, struct cmt_opts *opts, int count, char return NULL; } -static struct cmt_metric *metric_hash_lookup(struct cmt_map *map, uint64_t hash) +static int metric_labels_match(struct cmt_metric *metric, + int labels_count, char **labels_val) +{ + int index = 0; + struct cfl_list *head; + struct cmt_map_label *label; + + cfl_list_foreach(head, &metric->labels) { + if (index >= labels_count) { + return CMT_FALSE; + } + + label = cfl_list_entry(head, struct cmt_map_label, _head); + if ((label->name == NULL) != (labels_val[index] == NULL)) { + return CMT_FALSE; + } + if (label->name != NULL && strcmp(label->name, labels_val[index]) != 0) { + return CMT_FALSE; + } + index++; + } + + return index == labels_count; +} + +static struct cmt_metric *metric_prepare_storage(struct cmt_map *map, + struct cmt_metric *metric, + int write_op) +{ + struct cmt_summary *summary; + struct cmt_histogram *histogram; + + if (metric == NULL || !write_op) { + return metric; + } + + if (map->type == CMT_HISTOGRAM && metric->hist_buckets == NULL) { + histogram = map->parent; + if (histogram == NULL || histogram->buckets == NULL) { + return NULL; + } + metric->hist_buckets = calloc(histogram->buckets->count + 1, + sizeof(uint64_t)); + if (metric->hist_buckets == NULL) { + cmt_errno(); + return NULL; + } + } + else if (map->type == CMT_SUMMARY && metric->sum_quantiles == NULL) { + summary = map->parent; + if (summary == NULL) { + return NULL; + } + if (summary->quantiles_count > 0) { + metric->sum_quantiles = calloc(summary->quantiles_count, + sizeof(uint64_t)); + if (metric->sum_quantiles == NULL) { + cmt_errno(); + return NULL; + } + } + metric->sum_quantiles_count = summary->quantiles_count; + } + + return metric; +} + +static struct cmt_metric *metric_hash_lookup(struct cmt_map *map, uint64_t hash, + int labels_count, char **labels_val) { struct cfl_list *head; struct cmt_metric *metric; @@ -85,9 +243,30 @@ static struct cmt_metric *metric_hash_lookup(struct cmt_map *map, uint64_t hash) return &map->metric; } + metric = map->last_metric; + if (metric != NULL && metric->hash == hash && + metric_labels_match(metric, labels_count, labels_val)) { + return metric; + } + + if (map->metric_buckets != NULL) { + cfl_list_foreach(head, + &map->metric_buckets[hash % map->metric_bucket_count]) { + metric = cfl_list_entry(head, struct cmt_metric, _hash_head); + if (metric->hash == hash && + metric_labels_match(metric, labels_count, labels_val)) { + return metric; + } + } + } + + /* Decoders can populate the public metric list directly. Search only + * entries that have not yet been indexed, then index a successful match. */ cfl_list_foreach(head, &map->metrics) { metric = cfl_list_entry(head, struct cmt_metric, _head); - if (metric->hash == hash) { + if (!metric->hash_indexed && metric->hash == hash && + metric_labels_match(metric, labels_count, labels_val)) { + metric_index_add(map, metric); return metric; } } @@ -95,7 +274,7 @@ static struct cmt_metric *metric_hash_lookup(struct cmt_map *map, uint64_t hash) return NULL; } -static struct cmt_metric *map_metric_create(uint64_t hash, +static struct cmt_metric *map_metric_create(struct cmt_map *map, uint64_t hash, int labels_count, char **labels_val) { int i; @@ -109,8 +288,10 @@ static struct cmt_metric *map_metric_create(uint64_t hash, return NULL; } cfl_list_init(&metric->labels); + cfl_list_init(&metric->_hash_head); cmt_metric_set_double(metric, 0, 0.0); metric->hash = hash; + metric->map = map; for (i = 0; i < labels_count; i++) { label = malloc(sizeof(struct cmt_map_label)); @@ -137,16 +318,20 @@ static struct cmt_metric *map_metric_create(uint64_t hash, return metric; error: + destroy_label_list(&metric->labels); free(metric); return NULL; } -void cmt_map_metric_destroy(struct cmt_metric *metric) +static void map_metric_destroy_unlocked(struct cmt_metric *metric) { + struct cmt_map *map; struct cfl_list *tmp; struct cfl_list *head; struct cmt_map_label *label; + map = metric->map; + cfl_list_foreach_safe(head, tmp, &metric->labels) { label = cfl_list_entry(head, struct cmt_map_label, _head); cfl_sds_destroy(label->name); @@ -154,29 +339,49 @@ void cmt_map_metric_destroy(struct cmt_metric *metric) free(label); } - if (metric->hist_buckets) { - free(metric->hist_buckets); - } - if (metric->exp_hist_positive_buckets) { - free(metric->exp_hist_positive_buckets); - } - if (metric->exp_hist_negative_buckets) { - free(metric->exp_hist_negative_buckets); + metric_release_storage(metric); + + if (map != NULL && map->last_metric == metric) { + map->last_metric = NULL; } - if (metric->sum_quantiles) { - free(metric->sum_quantiles); + + if (metric->hash_indexed) { + if (map != NULL) { + if (map->indexed_metric_count > 0) { + map->indexed_metric_count--; + } + } + cfl_list_del(&metric->_hash_head); } cfl_list_del(&metric->_head); free(metric); } -struct cmt_metric *cmt_map_metric_get(struct cmt_opts *opts, struct cmt_map *map, - int labels_count, char **labels_val, - int write_op) +void cmt_map_metric_destroy(struct cmt_metric *metric) +{ + struct cmt_map *map; + + map = metric->map; + if (map != NULL) { + map_lock(map); + } + + map_metric_destroy_unlocked(metric); + + if (map != NULL) { + map_unlock(map); + } +} + +static struct cmt_metric *map_metric_get_unlocked(struct cmt_opts *opts, + struct cmt_map *map, + int labels_count, + char **labels_val, + int write_op) { int i; - int len; + size_t len; char *ptr; uint64_t hash; cfl_hash_state_t state; @@ -207,7 +412,7 @@ struct cmt_metric *cmt_map_metric_get(struct cmt_opts *opts, struct cmt_map *map } /* return the proper context or NULL */ - return metric; + return metric_prepare_storage(map, metric, write_op); } /* Lookup the metric */ @@ -225,10 +430,10 @@ struct cmt_metric *cmt_map_metric_get(struct cmt_opts *opts, struct cmt_map *map } hash = cfl_hash_64bits_digest(&state); - metric = metric_hash_lookup(map, hash); + metric = metric_hash_lookup(map, hash, labels_count, labels_val); if (metric) { - return metric; + return metric_prepare_storage(map, metric, write_op); } /* @@ -240,11 +445,27 @@ struct cmt_metric *cmt_map_metric_get(struct cmt_opts *opts, struct cmt_map *map } /* If the metric has not been found, just create it */ - metric = map_metric_create(hash, labels_count, labels_val); + metric = map_metric_create(map, hash, labels_count, labels_val); if (!metric) { return NULL; } cfl_list_add(&metric->_head, &map->metrics); + metric_index_add(map, metric); + map->last_metric = metric; + return metric_prepare_storage(map, metric, write_op); +} + +struct cmt_metric *cmt_map_metric_get(struct cmt_opts *opts, struct cmt_map *map, + int labels_count, char **labels_val, + int write_op) +{ + struct cmt_metric *metric; + + map_lock(map); + metric = map_metric_get_unlocked(opts, map, labels_count, labels_val, + write_op); + map_unlock(map); + return metric; } @@ -288,30 +509,15 @@ void cmt_map_destroy(struct cmt_map *map) if (map->metric_static_set) { metric = &map->metric; - if (map->type == CMT_HISTOGRAM) { - if (metric->hist_buckets) { - free(metric->hist_buckets); - } - } - else if (map->type == CMT_EXP_HISTOGRAM) { - if (metric->exp_hist_positive_buckets) { - free(metric->exp_hist_positive_buckets); - } - if (metric->exp_hist_negative_buckets) { - free(metric->exp_hist_negative_buckets); - } - } - else if (map->type == CMT_SUMMARY) { - if (metric->sum_quantiles) { - free(metric->sum_quantiles); - } - } + metric_release_storage(metric); } if (map->unit != NULL) { cfl_sds_destroy(map->unit); } + free(map->metric_buckets); + free(map); } @@ -347,10 +553,20 @@ void cmt_map_metrics_expire(struct cmt_map *map, uint64_t expiration) struct cfl_list *head; struct cmt_metric *metric; + map_lock(map); + + if (map->metric_static_set && map->metric.timestamp < expiration) { + metric_release_storage(&map->metric); + memset(&map->metric, 0, sizeof(struct cmt_metric)); + cfl_list_init(&map->metric.labels); + map->metric_static_set = CMT_FALSE; + } + cfl_list_foreach_safe(head, tmp, &map->metrics) { metric = cfl_list_entry(head, struct cmt_metric, _head); if (metric->timestamp < expiration) { - cmt_map_metric_destroy(metric); + map_metric_destroy_unlocked(metric); } } + map_unlock(map); } diff --git a/lib/cmetrics/src/cmt_mpack_utils.c b/lib/cmetrics/src/cmt_mpack_utils.c index ce639b356f7..eeac6f87b3d 100644 --- a/lib/cmetrics/src/cmt_mpack_utils.c +++ b/lib/cmetrics/src/cmt_mpack_utils.c @@ -114,6 +114,7 @@ int cmt_mpack_consume_uint_tag(mpack_reader_t *reader, uint64_t *output_buffer) int cmt_mpack_consume_string_tag(mpack_reader_t *reader, cfl_sds_t *output_buffer) { + const char *string_data; uint32_t string_length; mpack_tag_t tag; @@ -137,43 +138,39 @@ int cmt_mpack_consume_string_tag(mpack_reader_t *reader, cfl_sds_t *output_buffe string_length = mpack_tag_str_length(&tag); - /* This validation only applies to cmetrics and its use cases, we know - * for a fact that our label names and values are not supposed to be really - * long so a huge value here probably means that the data stream got corrupted. - */ - - if (CMT_MPACK_MAX_STRING_LENGTH < string_length) { - return CMT_MPACK_CORRUPT_INPUT_DATA_ERROR; - } - - *output_buffer = cfl_sds_create_size(string_length + 1); + /* Validate that the complete string is present before allocating memory + * based on its untrusted length field. The data-backed reader used by the + * decoder can return the bytes in place without an intermediate copy. */ + string_data = mpack_read_bytes_inplace(reader, (size_t) string_length); - if (NULL == *output_buffer) { - return CMT_MPACK_ALLOCATION_ERROR; + if (mpack_ok != mpack_reader_error(reader)) { + return CMT_MPACK_ENGINE_ERROR; } - cfl_sds_set_len(*output_buffer, string_length); - - mpack_read_cstr(reader, *output_buffer, string_length + 1, string_length); + mpack_done_str(reader); if (mpack_ok != mpack_reader_error(reader)) { - cfl_sds_destroy(*output_buffer); - - *output_buffer = NULL; - return CMT_MPACK_ENGINE_ERROR; } - mpack_done_str(reader); + /* CMetrics strings are represented as C strings. Preserve the existing + * rejection of embedded NUL bytes while allowing arbitrary valid lengths. */ + if (NULL != memchr(string_data, '\0', (size_t) string_length)) { + return CMT_MPACK_CORRUPT_INPUT_DATA_ERROR; + } - if (mpack_ok != mpack_reader_error(reader)) { - cfl_sds_destroy(*output_buffer); + *output_buffer = cfl_sds_create_size((size_t) string_length); - *output_buffer = NULL; + if (NULL == *output_buffer) { + return CMT_MPACK_ALLOCATION_ERROR; + } - return CMT_MPACK_ENGINE_ERROR; + if (string_length > 0) { + memcpy(*output_buffer, string_data, (size_t) string_length); } + cfl_sds_set_len(*output_buffer, (size_t) string_length); + return CMT_MPACK_SUCCESS; } diff --git a/lib/cmetrics/src/cmt_summary.c b/lib/cmetrics/src/cmt_summary.c index 014ad0bbd02..ab117ddf51a 100644 --- a/lib/cmetrics/src/cmt_summary.c +++ b/lib/cmetrics/src/cmt_summary.c @@ -94,7 +94,7 @@ struct cmt_summary *cmt_summary_create(struct cmt *cmt, if (quantiles_count > 0) { s->quantiles_count = quantiles_count; s->quantiles = calloc(1, sizeof(double) * quantiles_count); - if (!s->quantiles_count) { + if (!s->quantiles) { cmt_errno(); cmt_summary_destroy(s); return NULL; @@ -130,7 +130,8 @@ double cmt_summary_quantile_get_value(struct cmt_metric *metric, int quantile_id { uint64_t val; - if (quantile_id < 0 /*|| quantile_id > metric->sum_quantiles_count*/) { + if (metric == NULL || metric->sum_quantiles == NULL || quantile_id < 0 || + (size_t) quantile_id >= metric->sum_quantiles_count) { return 0; } @@ -220,6 +221,11 @@ void cmt_summary_quantile_set(struct cmt_metric *metric, uint64_t timestamp, double new; int result; + if (metric == NULL || metric->sum_quantiles == NULL || quantile_id < 0 || + (size_t) quantile_id >= metric->sum_quantiles_count) { + return; + } + do { old = cmt_summary_quantile_get_value(metric, quantile_id); new = val; @@ -303,4 +309,4 @@ int cmt_summary_set_default(struct cmt_summary *summary, cmt_summary_count_set(metric, timestamp, count); return 0; -} \ No newline at end of file +} diff --git a/lib/cmetrics/tests/CMakeLists.txt b/lib/cmetrics/tests/CMakeLists.txt index 850e310cfda..e7ae66d67e3 100644 --- a/lib/cmetrics/tests/CMakeLists.txt +++ b/lib/cmetrics/tests/CMakeLists.txt @@ -16,6 +16,7 @@ set(UNIT_TESTS_FILES exp_histogram.c msgpack_abi.c msgpack_temporality.c + format_conversion.c expire.c ) diff --git a/lib/cmetrics/tests/cat.c b/lib/cmetrics/tests/cat.c index dce4385cee7..86469bdeedd 100644 --- a/lib/cmetrics/tests/cat.c +++ b/lib/cmetrics/tests/cat.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -668,6 +669,123 @@ void test_exp_histogram_preserves_aggregation_type() cmt_destroy(dst); } +void test_summary_concatenation_preserves_series() +{ + int ret; + int found_first; + int found_second; + double quantiles[] = {0.5, 0.9}; + double values[] = {5.0, 9.0}; + struct cfl_list *head; + struct cfl_list *label_head; + struct cmt *dst; + struct cmt *src_first; + struct cmt *src_second; + struct cmt_summary *summary; + struct cmt_metric *metric; + struct cmt_map_label *label; + + dst = cmt_create(); + src_first = cmt_create(); + src_second = cmt_create(); + TEST_ASSERT(dst != NULL); + TEST_ASSERT(src_first != NULL); + TEST_ASSERT(src_second != NULL); + + summary = cmt_summary_create(src_first, "test", "cat", "summary", + "Summary concatenation", 2, quantiles, + 1, (char *[]) {"kind"}); + TEST_ASSERT(summary != NULL); + ret = cmt_summary_set_default(summary, 10, values, 14.0, 7, + 1, (char *[]) {"first"}); + TEST_ASSERT(ret == 0); + + ret = cmt_cat(dst, src_first); + TEST_ASSERT(ret == 0); + TEST_CHECK(cfl_list_size(&dst->summaries) == 1); + + summary = cfl_list_entry_first(&dst->summaries, + struct cmt_summary, _head); + TEST_CHECK(cfl_list_size(&summary->map->metrics) == 1); + + metric = cfl_list_entry_first(&summary->map->metrics, + struct cmt_metric, _head); + label = cfl_list_entry_first(&metric->labels, + struct cmt_map_label, _head); + TEST_CHECK(strcmp(label->name, "first") == 0); + TEST_CHECK(cmt_summary_get_count_value(metric) == 7); + + summary = cmt_summary_create(src_second, "test", "cat", "summary", + "Summary concatenation", 2, quantiles, + 1, (char *[]) {"kind"}); + TEST_ASSERT(summary != NULL); + ret = cmt_summary_set_default(summary, 20, values, 18.0, 9, + 1, (char *[]) {"second"}); + TEST_ASSERT(ret == 0); + + ret = cmt_cat(dst, src_second); + TEST_ASSERT(ret == 0); + TEST_CHECK(cfl_list_size(&dst->summaries) == 1); + + summary = cfl_list_entry_first(&dst->summaries, + struct cmt_summary, _head); + TEST_CHECK(cfl_list_size(&summary->map->metrics) == 2); + + found_first = CMT_FALSE; + found_second = CMT_FALSE; + cfl_list_foreach(head, &summary->map->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + label_head = metric->labels.next; + label = cfl_list_entry(label_head, struct cmt_map_label, _head); + + TEST_CHECK(strcmp(label->name, "kind") != 0); + if (strcmp(label->name, "first") == 0) { + found_first = CMT_TRUE; + } + else if (strcmp(label->name, "second") == 0) { + found_second = CMT_TRUE; + } + } + + TEST_CHECK(found_first == CMT_TRUE); + TEST_CHECK(found_second == CMT_TRUE); + + cmt_destroy(src_second); + cmt_destroy(src_first); + cmt_destroy(dst); +} + +void test_summary_concatenation_rejects_mismatched_label_schema() +{ + int ret; + double quantiles[] = {0.5, 0.9}; + struct cmt *dst; + struct cmt *src; + struct cmt_summary *summary; + + dst = cmt_create(); + src = cmt_create(); + TEST_ASSERT(dst != NULL); + TEST_ASSERT(src != NULL); + + summary = cmt_summary_create(dst, "test", "cat", "schema", + "schema validation", 2, quantiles, 2, + (char *[]) {"method", "status"}); + TEST_ASSERT(summary != NULL); + + summary = cmt_summary_create(src, "test", "cat", "schema", + "schema validation", 2, quantiles, 2, + (char *[]) {"status", "method"}); + TEST_ASSERT(summary != NULL); + + ret = cmt_cat(dst, src); + TEST_CHECK(ret == -1); + TEST_CHECK(cfl_list_size(&dst->summaries) == 1); + + cmt_destroy(src); + cmt_destroy(dst); +} + TEST_LIST = { {"cat", test_cat}, {"duplicate_metrics", test_duplicate_metrics}, @@ -676,5 +794,8 @@ TEST_LIST = { {"histogram_empty_to_populated", test_histogram_empty_to_populated}, {"histogram_populated_to_empty", test_histogram_populated_to_empty}, {"exp_histogram_preserves_aggregation_type", test_exp_histogram_preserves_aggregation_type}, + {"summary_concatenation_preserves_series", test_summary_concatenation_preserves_series}, + {"summary_concatenation_rejects_mismatched_label_schema", + test_summary_concatenation_rejects_mismatched_label_schema}, { 0 } }; diff --git a/lib/cmetrics/tests/counter.c b/lib/cmetrics/tests/counter.c index cfe83c6cff0..25faf43f538 100644 --- a/lib/cmetrics/tests/counter.c +++ b/lib/cmetrics/tests/counter.c @@ -21,9 +21,14 @@ #include #include #include +#include #include #include +#if !defined(_WIN32) && !defined(_WIN64) +#include +#endif + #include "cmt_tests.h" static struct cmt *generate_encoder_test_data() @@ -243,6 +248,20 @@ void test_labels() TEST_CHECK(ret == 0); TEST_CHECK(val == 10.55); + /* Label boundaries must be part of identity. These two tuples feed the + * same unframed byte sequence ("abc") into the hash. */ + ret = cmt_counter_add(c, ts, 4.0, 2, (char *[]) {"ab", "c"}); + TEST_CHECK(ret == 0); + ret = cmt_counter_add(c, ts, 7.0, 2, (char *[]) {"a", "bc"}); + TEST_CHECK(ret == 0); + + ret = cmt_counter_get_val(c, 2, (char *[]) {"ab", "c"}, &val); + TEST_CHECK(ret == 0); + TEST_CHECK(val == 4.0); + ret = cmt_counter_get_val(c, 2, (char *[]) {"a", "bc"}, &val); + TEST_CHECK(ret == 0); + TEST_CHECK(val == 7.0); + /* Valid counter set */ ret = cmt_counter_set(c, ts, 12.15, 2, (char *[]) {"localhost", "test"}); TEST_CHECK(ret == 0); @@ -254,11 +273,76 @@ void test_labels() cmt_destroy(cmt); } +#if !defined(_WIN32) && !defined(_WIN64) +#define CONCURRENT_THREAD_COUNT 8 +#define CONCURRENT_UPDATE_COUNT 10000 + +struct concurrent_counter_context { + struct cmt_counter *counter; + int result; +}; + +static void *concurrent_counter_worker(void *data) +{ + int index; + struct concurrent_counter_context *context = data; + + for (index = 0; index < CONCURRENT_UPDATE_COUNT; index++) { + if (cmt_counter_inc(context->counter, (uint64_t) index + 1, 1, + (char *[]) {"shared"}) != 0) { + context->result = -1; + break; + } + } + + return NULL; +} + +void test_concurrent_metric_creation() +{ + int index; + int result; + double value; + pthread_t threads[CONCURRENT_THREAD_COUNT]; + struct concurrent_counter_context contexts[CONCURRENT_THREAD_COUNT]; + struct cmt *cmt; + struct cmt_counter *counter; + + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + counter = cmt_counter_create(cmt, "test", "", "concurrent", "help", + 1, (char *[]) {"series"}); + TEST_ASSERT(counter != NULL); + + for (index = 0; index < CONCURRENT_THREAD_COUNT; index++) { + contexts[index].counter = counter; + contexts[index].result = 0; + result = pthread_create(&threads[index], NULL, + concurrent_counter_worker, &contexts[index]); + TEST_ASSERT(result == 0); + } + for (index = 0; index < CONCURRENT_THREAD_COUNT; index++) { + pthread_join(threads[index], NULL); + TEST_CHECK(contexts[index].result == 0); + } + + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 1); + result = cmt_counter_get_val(counter, 1, (char *[]) {"shared"}, &value); + TEST_CHECK(result == 0); + TEST_CHECK(value == CONCURRENT_THREAD_COUNT * CONCURRENT_UPDATE_COUNT); + + cmt_destroy(cmt); +} +#endif + TEST_LIST = { {"basic", test_counter}, {"labels", test_labels}, {"msgpack", test_msgpack}, {"prometheus", test_prometheus}, {"text", test_text}, +#if !defined(_WIN32) && !defined(_WIN64) + {"concurrent_metric_creation", test_concurrent_metric_creation}, +#endif { 0 } }; diff --git a/lib/cmetrics/tests/encoding.c b/lib/cmetrics/tests/encoding.c index 36c861b444e..24fea1ad12f 100644 --- a/lib/cmetrics/tests/encoding.c +++ b/lib/cmetrics/tests/encoding.c @@ -978,6 +978,201 @@ void test_prometheus_remote_write_with_outdated_timestamps() cmt_destroy(cmt); } +static int remote_write_contains_series(Prometheus__WriteRequest *request, + const char *metric_name, + const char *label_name, + const char *label_value) +{ + size_t series_index; + size_t label_index; + int metric_matches; + int label_matches; + Prometheus__Label *label; + + for (series_index = 0; series_index < request->n_timeseries; series_index++) { + metric_matches = CMT_FALSE; + label_matches = label_name == NULL; + + for (label_index = 0; + label_index < request->timeseries[series_index]->n_labels; + label_index++) { + label = request->timeseries[series_index]->labels[label_index]; + if (label == NULL || label->name == NULL || label->value == NULL) { + continue; + } + + if (strcmp(label->name, "__name__") == 0 && + strcmp(label->value, metric_name) == 0) { + metric_matches = CMT_TRUE; + } + + if (label_name != NULL && strcmp(label->name, label_name) == 0 && + strcmp(label->value, label_value) == 0) { + label_matches = CMT_TRUE; + } + } + + if (metric_matches && label_matches) { + return CMT_TRUE; + } + } + + return CMT_FALSE; +} + +static Prometheus__WriteRequest *decode_remote_write_payload(cfl_sds_t payload) +{ + if (payload == NULL) { + return NULL; + } + + return prometheus__write_request__unpack(NULL, cfl_sds_len(payload), + (uint8_t *) payload); +} + +void test_prometheus_remote_write_skips_only_stale_samples() +{ + uint64_t now; + cfl_sds_t payload; + struct cmt *cmt; + struct cmt_counter *counter; + Prometheus__WriteRequest *request; + + now = cfl_time_now(); + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + + counter = cmt_counter_create(cmt, "test", "remote", "mixed", + "Mixed timestamp samples", 1, + (char *[]) {"state"}); + TEST_ASSERT(counter != NULL); + TEST_ASSERT(cmt_counter_set(counter, + now - CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_THRESHOLD - 1, + 1, 1, (char *[]) {"stale"}) == 0); + TEST_ASSERT(cmt_counter_set(counter, now, 2, 1, + (char *[]) {"fresh"}) == 0); + + payload = cmt_encode_prometheus_remote_write_create(cmt); + request = decode_remote_write_payload(payload); + TEST_ASSERT(request != NULL); + TEST_CHECK(remote_write_contains_series(request, "test_remote_mixed", + "state", "stale") == CMT_FALSE); + TEST_CHECK(remote_write_contains_series(request, "test_remote_mixed", + "state", "fresh") == CMT_TRUE); + + prometheus__write_request__free_unpacked(request, NULL); + cmt_encode_prometheus_remote_write_destroy(payload); + cmt_destroy(cmt); +} + +void test_prometheus_remote_write_continues_after_stale_family() +{ + uint64_t now; + cfl_sds_t payload; + struct cmt *cmt; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + Prometheus__WriteRequest *request; + + now = cfl_time_now(); + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + + counter = cmt_counter_create(cmt, "test", "remote", "stale_counter", + "Stale counter", 0, NULL); + TEST_ASSERT(counter != NULL); + TEST_ASSERT(cmt_counter_set(counter, + now - CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_THRESHOLD - 1, + 1, 0, NULL) == 0); + + gauge = cmt_gauge_create(cmt, "test", "remote", "fresh_gauge", + "Fresh gauge", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, now, 2, 0, NULL) == 0); + + payload = cmt_encode_prometheus_remote_write_create(cmt); + request = decode_remote_write_payload(payload); + TEST_ASSERT(request != NULL); + TEST_CHECK(remote_write_contains_series(request, "test_remote_stale_counter", + NULL, NULL) == CMT_FALSE); + TEST_CHECK(remote_write_contains_series(request, "test_remote_fresh_gauge", + NULL, NULL) == CMT_TRUE); + + prometheus__write_request__free_unpacked(request, NULL); + cmt_encode_prometheus_remote_write_destroy(payload); + cmt_destroy(cmt); +} + +void test_prometheus_remote_write_preserves_future_samples() +{ + uint64_t now; + cfl_sds_t payload; + struct cmt *cmt; + struct cmt_counter *counter; + Prometheus__WriteRequest *request; + + now = cfl_time_now(); + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + + counter = cmt_counter_create(cmt, "test", "remote", "future", + "Future timestamp", 0, NULL); + TEST_ASSERT(counter != NULL); + TEST_ASSERT(cmt_counter_set(counter, now + 1000000000, 1, 0, NULL) == 0); + + payload = cmt_encode_prometheus_remote_write_create(cmt); + request = decode_remote_write_payload(payload); + TEST_ASSERT(request != NULL); + TEST_CHECK(remote_write_contains_series(request, "test_remote_future", + NULL, NULL) == CMT_TRUE); + + prometheus__write_request__free_unpacked(request, NULL); + cmt_encode_prometheus_remote_write_destroy(payload); + cmt_destroy(cmt); +} + +void test_prometheus_remote_write_skips_only_stale_histograms() +{ + uint64_t now; + cfl_sds_t payload; + struct cmt *cmt; + struct cmt_histogram *histogram; + struct cmt_histogram_buckets *buckets; + Prometheus__WriteRequest *request; + + now = cfl_time_now(); + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + + buckets = cmt_histogram_buckets_create(1, 1.0); + TEST_ASSERT(buckets != NULL); + histogram = cmt_histogram_create(cmt, "test", "remote", "mixed_histogram", + "Mixed histogram timestamps", buckets, + 1, (char *[]) {"state"}); + TEST_ASSERT(histogram != NULL); + + TEST_ASSERT(cmt_histogram_observe( + histogram, + now - CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_THRESHOLD - 1, + 0.5, 1, (char *[]) {"stale"}) == 0); + TEST_ASSERT(cmt_histogram_observe(histogram, now, 0.5, 1, + (char *[]) {"fresh"}) == 0); + + payload = cmt_encode_prometheus_remote_write_create(cmt); + request = decode_remote_write_payload(payload); + TEST_ASSERT(request != NULL); + TEST_CHECK(remote_write_contains_series( + request, "test_remote_mixed_histogram_bucket", + "state", "stale") == CMT_FALSE); + TEST_CHECK(remote_write_contains_series( + request, "test_remote_mixed_histogram_bucket", + "state", "fresh") == CMT_TRUE); + + prometheus__write_request__free_unpacked(request, NULL); + cmt_encode_prometheus_remote_write_destroy(payload); + cmt_destroy(cmt); +} + void test_opentelemetry() { cfl_sds_t payload; @@ -1615,6 +1810,10 @@ TEST_LIST = { {"cmt_msgpack_partial_processing", test_cmt_msgpack_partial_processing}, {"prometheus_remote_write", test_prometheus_remote_write}, {"prometheus_remote_write_old_cmt",test_prometheus_remote_write_with_outdated_timestamps}, + {"prometheus_remote_write_skips_only_stale_samples", test_prometheus_remote_write_skips_only_stale_samples}, + {"prometheus_remote_write_continues_after_stale_family", test_prometheus_remote_write_continues_after_stale_family}, + {"prometheus_remote_write_preserves_future_samples", test_prometheus_remote_write_preserves_future_samples}, + {"prometheus_remote_write_skips_only_stale_histograms", test_prometheus_remote_write_skips_only_stale_histograms}, {"cmt_msgpack_stability", test_cmt_to_msgpack_stability}, {"cmt_msgpack_integrity", test_cmt_to_msgpack_integrity}, {"cmt_msgpack_labels", test_cmt_to_msgpack_labels}, diff --git a/lib/cmetrics/tests/expire.c b/lib/cmetrics/tests/expire.c index 94ebefc797f..eaa649dcc62 100644 --- a/lib/cmetrics/tests/expire.c +++ b/lib/cmetrics/tests/expire.c @@ -309,6 +309,79 @@ void test_expire_off_by_one() cmt_destroy(cmt); } +void test_expire_static_metrics() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_counter *counter; + struct cmt_histogram *histogram; + struct cmt_histogram_buckets *buckets; + + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + ts = cfl_time_now(); + + counter = cmt_counter_create(cmt, "test", "expire", "static_counter", + "Static counter expiration", 0, NULL); + TEST_ASSERT(counter != NULL); + TEST_ASSERT(cmt_counter_set(counter, ts - 10, 3, 0, NULL) == 0); + + buckets = cmt_histogram_buckets_create(2, 1.0, 5.0); + TEST_ASSERT(buckets != NULL); + histogram = cmt_histogram_create(cmt, "test", "expire", "static_histogram", + "Static histogram expiration", buckets, + 0, NULL); + TEST_ASSERT(histogram != NULL); + TEST_ASSERT(cmt_histogram_observe(histogram, ts - 10, 2.0, + 0, NULL) == 0); + TEST_CHECK(histogram->map->metric.hist_buckets != NULL); + + cmt_expire(cmt, ts - 1); + + TEST_CHECK(counter->map->metric_static_set == CMT_FALSE); + TEST_CHECK(histogram->map->metric_static_set == CMT_FALSE); + TEST_CHECK(histogram->map->metric.hist_buckets == NULL); + + TEST_ASSERT(cmt_counter_set(counter, ts, 4, 0, NULL) == 0); + TEST_ASSERT(cmt_histogram_observe(histogram, ts, 2.0, 0, NULL) == 0); + TEST_CHECK(counter->map->metric_static_set == CMT_TRUE); + TEST_CHECK(histogram->map->metric_static_set == CMT_TRUE); + TEST_CHECK(histogram->map->metric.hist_buckets != NULL); + TEST_CHECK(cmt_metric_get_timestamp(&counter->map->metric) == ts); + TEST_CHECK(cmt_metric_get_timestamp(&histogram->map->metric) == ts); + + cmt_destroy(cmt); +} + +void test_destroy_unindexed_last_metric() +{ + struct cmt *cmt; + struct cmt_counter *counter; + struct cmt_metric *metric; + + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + counter = cmt_counter_create(cmt, "test", "map", "unindexed", + "Unindexed metric destruction", 1, + (char *[]) {"label"}); + TEST_ASSERT(counter != NULL); + + metric = calloc(1, sizeof(struct cmt_metric)); + TEST_ASSERT(metric != NULL); + cfl_list_init(&metric->labels); + cfl_list_init(&metric->_hash_head); + metric->map = counter->map; + cfl_list_add(&metric->_head, &counter->map->metrics); + counter->map->last_metric = metric; + + cmt_map_metric_destroy(metric); + + TEST_CHECK(counter->map->last_metric == NULL); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 0); + + cmt_destroy(cmt); +} + TEST_LIST = { {"expire_counter" , test_expire_counter}, {"expire_gauge", test_expire_gauge}, @@ -317,5 +390,7 @@ TEST_LIST = { {"expire_untyped", test_expire_untyped}, {"expire_exp_histogram", test_epxire_exp_histogram}, {"expire_off_by_one", test_expire_off_by_one}, + {"expire_static_metrics", test_expire_static_metrics}, + {"destroy_unindexed_last_metric", test_destroy_unindexed_last_metric}, { 0 } }; diff --git a/lib/cmetrics/tests/filter.c b/lib/cmetrics/tests/filter.c index 45248fe5aa5..e4f330b0b66 100644 --- a/lib/cmetrics/tests/filter.c +++ b/lib/cmetrics/tests/filter.c @@ -452,9 +452,60 @@ void test_filter_with_label_key_value_pairs() cmt_destroy(cmt6); } +void test_filter_label_pair_exact_and_all_matches() +{ + int ret; + cfl_sds_t text; + struct cmt *src; + struct cmt *prefix_result; + struct cmt *exact_result; + struct cmt_counter *counter; + + src = cmt_create(); + prefix_result = cmt_create(); + exact_result = cmt_create(); + TEST_ASSERT(src != NULL); + TEST_ASSERT(prefix_result != NULL); + TEST_ASSERT(exact_result != NULL); + + counter = cmt_counter_create(src, "test", "filter", "requests", + "Filter exact label pairs", 2, + (char *[]) {"host", "zone"}); + TEST_ASSERT(counter != NULL); + + TEST_ASSERT(cmt_counter_set(counter, 1, 1, 2, + (char *[]) {"prod", "a"}) == 0); + TEST_ASSERT(cmt_counter_set(counter, 1, 2, 2, + (char *[]) {"prod", "b"}) == 0); + TEST_ASSERT(cmt_counter_set(counter, 1, 3, 2, + (char *[]) {"production", "c"}) == 0); + + ret = cmt_filter_with_label_pair(prefix_result, src, "hostname", "prod"); + TEST_ASSERT(ret == CMT_FILTER_SUCCESS); + text = cmt_encode_text_create(prefix_result); + TEST_ASSERT(text != NULL); + TEST_CHECK(strstr(text, "host=\"prod\",zone=\"a\"") != NULL); + TEST_CHECK(strstr(text, "host=\"prod\",zone=\"b\"") != NULL); + TEST_CHECK(strstr(text, "host=\"production\",zone=\"c\"") != NULL); + cmt_encode_text_destroy(text); + + ret = cmt_filter_with_label_pair(exact_result, src, "host", "prod"); + TEST_ASSERT(ret == CMT_FILTER_SUCCESS); + text = cmt_encode_text_create(exact_result); + TEST_ASSERT(text != NULL); + TEST_CHECK(strstr(text, "host=\"prod\"") == NULL); + TEST_CHECK(strstr(text, "host=\"production\",zone=\"c\"") != NULL); + cmt_encode_text_destroy(text); + + cmt_destroy(exact_result); + cmt_destroy(prefix_result); + cmt_destroy(src); +} + TEST_LIST = { {"filter", test_filter}, {"filter_with_label_pair", test_filter_with_label_key_value_pairs}, + {"filter_label_pair_exact_and_all_matches", test_filter_label_pair_exact_and_all_matches}, { 0 } }; diff --git a/lib/cmetrics/tests/format_conversion.c b/lib/cmetrics/tests/format_conversion.c new file mode 100644 index 00000000000..72e1e695470 --- /dev/null +++ b/lib/cmetrics/tests/format_conversion.c @@ -0,0 +1,440 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2026 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CMT_HAVE_PROMETHEUS_TEXT_DECODER +#include +#endif + +#include "cmt_tests.h" + +static struct cmt *msgpack_round_trip(struct cmt *input) +{ + int result; + char *first_buffer; + char *second_buffer; + size_t first_size; + size_t second_size; + size_t offset; + struct cmt *output; + + first_buffer = NULL; + second_buffer = NULL; + output = NULL; + + result = cmt_encode_msgpack_create(input, &first_buffer, &first_size); + TEST_ASSERT(result == 0); + + offset = 0; + result = cmt_decode_msgpack_create(&output, first_buffer, first_size, + &offset); + TEST_ASSERT(result == CMT_DECODE_MSGPACK_SUCCESS); + TEST_ASSERT(output != NULL); + TEST_CHECK(offset == first_size); + + result = cmt_encode_msgpack_create(output, &second_buffer, &second_size); + TEST_ASSERT(result == 0); + TEST_CHECK(second_size == first_size); + if (second_size == first_size) { + TEST_CHECK(memcmp(first_buffer, second_buffer, first_size) == 0); + } + + cmt_encode_msgpack_destroy(second_buffer); + cmt_encode_msgpack_destroy(first_buffer); + + return output; +} + +static void check_all_encoders(struct cmt *context) +{ + int result; + char *cloudwatch; + size_t cloudwatch_size; + char *msgpack; + size_t msgpack_size; + cfl_sds_t output; + + output = cmt_encode_prometheus_create(context, CMT_TRUE); + TEST_CHECK(output != NULL); + cmt_encode_prometheus_destroy(output); + + output = cmt_encode_text_create(context); + TEST_CHECK(output != NULL); + cmt_encode_text_destroy(output); + + output = cmt_encode_influx_create(context); + TEST_CHECK(output != NULL); + cmt_encode_influx_destroy(output); + + output = cmt_encode_splunk_hec_create(context, "localhost", "metrics", + NULL, NULL); + TEST_CHECK(output != NULL); + cmt_encode_splunk_hec_destroy(output); + + cloudwatch = NULL; + cloudwatch_size = 0; + result = cmt_encode_cloudwatch_emf_create(context, &cloudwatch, + &cloudwatch_size, CMT_TRUE); + TEST_CHECK(result == 0); + TEST_CHECK(cloudwatch != NULL); + if (cloudwatch != NULL) { + cmt_encode_cloudwatch_emf_destroy(cloudwatch); + } + + output = cmt_encode_prometheus_remote_write_create(context); + TEST_CHECK(output != NULL); + cmt_encode_prometheus_remote_write_destroy(output); + + output = cmt_encode_opentelemetry_create(context); + TEST_CHECK(output != NULL); + cmt_encode_opentelemetry_destroy(output); + + msgpack = NULL; + msgpack_size = 0; + result = cmt_encode_msgpack_create(context, &msgpack, &msgpack_size); + TEST_CHECK(result == 0); + TEST_CHECK(msgpack != NULL); + cmt_encode_msgpack_destroy(msgpack); +} + +static struct cmt *create_native_fixture(void) +{ + int result; + uint64_t now; + uint64_t positive_buckets[] = {3, 5}; + uint64_t negative_buckets[] = {2}; + double quantiles[] = {0.5, 0.9}; + double quantile_values[] = {4.0, 8.0}; + struct cmt *cmt; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; + struct cmt_summary *summary; + struct cmt_histogram_buckets *buckets; + struct cmt_metric *metric; + + now = cfl_time_now(); + cmt = cmt_create(); + if (cmt == NULL) { + return NULL; + } + + cmt_label_add(cmt, "environment", "test"); + + counter = cmt_counter_create(cmt, "matrix", "native", "requests_total", + "Requests", 1, (char *[]) {"route"}); + if (counter == NULL || + cmt_counter_set(counter, now, 7, 1, (char *[]) {"/api"}) != 0) { + cmt_destroy(cmt); + return NULL; + } + + gauge = cmt_gauge_create(cmt, "matrix", "native", "large_integer", + "Large integer", 0, NULL); + if (gauge == NULL) { + cmt_destroy(cmt); + return NULL; + } + metric = &gauge->map->metric; + cmt_metric_set_int64(metric, now, 9007199254740993LL); + + untyped = cmt_untyped_create(cmt, "matrix", "native", "temperature", + "Temperature", 1, (char *[]) {"room"}); + if (untyped == NULL || + cmt_untyped_set(untyped, now, 21.5, 1, (char *[]) {"office"}) != 0) { + cmt_destroy(cmt); + return NULL; + } + + buckets = cmt_histogram_buckets_create(2, 1.0, 5.0); + if (buckets == NULL) { + cmt_destroy(cmt); + return NULL; + } + histogram = cmt_histogram_create(cmt, "matrix", "native", "latency", + "Latency", buckets, 1, + (char *[]) {"method"}); + if (histogram == NULL || + cmt_histogram_observe(histogram, now, 2.0, 1, + (char *[]) {"GET"}) != 0) { + cmt_destroy(cmt); + return NULL; + } + histogram->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + + summary = cmt_summary_create(cmt, "matrix", "native", "size", + "Size", 2, quantiles, 1, + (char *[]) {"operation"}); + if (summary == NULL || + cmt_summary_set_default(summary, now, quantile_values, 12.0, 3, + 1, (char *[]) {"write"}) != 0) { + cmt_destroy(cmt); + return NULL; + } + + exp_histogram = cmt_exp_histogram_create(cmt, "matrix", "native", + "payload", "Payload", 1, + (char *[]) {"protocol"}); + if (exp_histogram == NULL) { + cmt_destroy(cmt); + return NULL; + } + result = cmt_exp_histogram_set_default(exp_histogram, now, 1, 2, 0.0, + -1, 2, positive_buckets, + 0, 1, negative_buckets, + CMT_TRUE, 9.5, 10, 1, + (char *[]) {"http"}); + if (result != 0) { + cmt_destroy(cmt); + return NULL; + } + exp_histogram->aggregation_type = CMT_AGGREGATION_TYPE_CUMULATIVE; + + return cmt; +} + +void test_native_msgpack_conversion_matrix(void) +{ + struct cmt *input; + struct cmt *output; + struct cmt_gauge *gauge; + + input = create_native_fixture(); + TEST_ASSERT(input != NULL); + + output = msgpack_round_trip(input); + TEST_ASSERT(output != NULL); + + TEST_CHECK(cfl_list_size(&output->counters) == 1); + TEST_CHECK(cfl_list_size(&output->gauges) == 1); + TEST_CHECK(cfl_list_size(&output->untypeds) == 1); + TEST_CHECK(cfl_list_size(&output->histograms) == 1); + TEST_CHECK(cfl_list_size(&output->summaries) == 1); + TEST_CHECK(cfl_list_size(&output->exp_histograms) == 1); + + gauge = cfl_list_entry_first(&output->gauges, struct cmt_gauge, _head); + TEST_CHECK(cmt_metric_get_value_type(&gauge->map->metric) == + CMT_METRIC_VALUE_INT64); + TEST_CHECK(cmt_metric_get_int64_value(&gauge->map->metric) == + 9007199254740993LL); + + check_all_encoders(output); + + cmt_decode_msgpack_destroy(output); + cmt_destroy(input); +} + +#ifdef CMT_HAVE_PROMETHEUS_TEXT_DECODER +void test_prometheus_msgpack_conversion_matrix(void) +{ + int result; + const char input_text[] = + "# HELP http_requests_total Requests\n" + "# TYPE http_requests_total counter\n" + "http_requests_total{method=\"GET\"} 7\n" + "# HELP queue_depth Queue depth\n" + "# TYPE queue_depth gauge\n" + "queue_depth{queue=\"main\"} 3\n" + "# HELP request_size Request size\n" + "# TYPE request_size histogram\n" + "request_size_bucket{route=\"/\",le=\"1\"} 1\n" + "request_size_bucket{route=\"/\",le=\"5\"} 2\n" + "request_size_bucket{route=\"/\",le=\"+Inf\"} 2\n" + "request_size_sum{route=\"/\"} 4\n" + "request_size_count{route=\"/\"} 2\n"; + struct cmt *decoded; + struct cmt *roundtrip; + cfl_sds_t prometheus; + + decoded = NULL; + result = cmt_decode_prometheus_create(&decoded, input_text, + sizeof(input_text) - 1, NULL); + TEST_ASSERT(result == CMT_DECODE_PROMETHEUS_SUCCESS); + TEST_ASSERT(decoded != NULL); + + roundtrip = msgpack_round_trip(decoded); + TEST_ASSERT(roundtrip != NULL); + + prometheus = cmt_encode_prometheus_create(roundtrip, CMT_FALSE); + TEST_ASSERT(prometheus != NULL); + TEST_CHECK(strstr(prometheus, "http_requests_total{method=\"GET\"} 7") != NULL); + TEST_CHECK(strstr(prometheus, "queue_depth{queue=\"main\"} 3") != NULL); + TEST_CHECK(strstr(prometheus, "request_size_count{route=\"/\"} 2") != NULL); + cmt_encode_prometheus_destroy(prometheus); + + check_all_encoders(roundtrip); + + cmt_decode_msgpack_destroy(roundtrip); + cmt_decode_prometheus_destroy(decoded); +} +#endif + +void test_remote_write_msgpack_conversion_matrix(void) +{ + int result; + cfl_sds_t remote_payload; + struct cmt *source; + struct cmt *decoded; + struct cmt *roundtrip; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + uint64_t now; + + now = cfl_time_now(); + source = cmt_create(); + TEST_ASSERT(source != NULL); + + counter = cmt_counter_create(source, "matrix", "remote", "requests_total", + "Requests", 1, (char *[]) {"method"}); + gauge = cmt_gauge_create(source, "matrix", "remote", "depth", + "Depth", 1, (char *[]) {"queue"}); + TEST_ASSERT(counter != NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_counter_set(counter, now, 5, 1, + (char *[]) {"POST"}) == 0); + TEST_ASSERT(cmt_gauge_set(gauge, now, 2, 1, + (char *[]) {"main"}) == 0); + + remote_payload = cmt_encode_prometheus_remote_write_create(source); + TEST_ASSERT(remote_payload != NULL); + + decoded = NULL; + result = cmt_decode_prometheus_remote_write_create(&decoded, + remote_payload, + cfl_sds_len(remote_payload)); + TEST_ASSERT(result == CMT_DECODE_PROMETHEUS_REMOTE_WRITE_SUCCESS); + TEST_ASSERT(decoded != NULL); + + roundtrip = msgpack_round_trip(decoded); + TEST_ASSERT(roundtrip != NULL); + /* + * The encoder intentionally omits remote-write metadata, so the decoder + * must apply the protocol's safe fallback and represent scalar series as + * gauges. The MessagePack conversion must preserve that decoded model. + */ + TEST_CHECK(cfl_list_size(&roundtrip->counters) == 0); + TEST_CHECK(cfl_list_size(&roundtrip->gauges) == 2); + check_all_encoders(roundtrip); + + cmt_decode_msgpack_destroy(roundtrip); + cmt_decode_prometheus_remote_write_destroy(decoded); + cmt_encode_prometheus_remote_write_destroy(remote_payload); + cmt_destroy(source); +} + +void test_statsd_msgpack_conversion_matrix(void) +{ + int result; + char statsd_payload[] = + "requests:5|c|#environment:test,method:GET\n" + "temperature:21.5|g|#environment:test,room:office\n" + "latency:12|ms|#environment:test,route:/api\n"; + struct cmt *decoded; + struct cmt *roundtrip; + cfl_sds_t prometheus; + + decoded = NULL; + result = cmt_decode_statsd_create(&decoded, statsd_payload, + sizeof(statsd_payload) - 1, + 0); + TEST_ASSERT(result == CMT_DECODE_STATSD_SUCCESS); + TEST_ASSERT(decoded != NULL); + + roundtrip = msgpack_round_trip(decoded); + TEST_ASSERT(roundtrip != NULL); + + prometheus = cmt_encode_prometheus_create(roundtrip, CMT_FALSE); + TEST_ASSERT(prometheus != NULL); + TEST_CHECK(strstr(prometheus, "requests") != NULL); + TEST_CHECK(strstr(prometheus, "temperature") != NULL); + cmt_encode_prometheus_destroy(prometheus); + + check_all_encoders(roundtrip); + + cmt_decode_msgpack_destroy(roundtrip); + cmt_decode_statsd_destroy(decoded); +} + +void test_otlp_msgpack_conversion_matrix(void) +{ + int result; + size_t offset; + cfl_sds_t otlp_payload; + struct cfl_list decoded_list; + struct cmt *source; + struct cmt *decoded; + struct cmt *roundtrip; + + source = create_native_fixture(); + TEST_ASSERT(source != NULL); + + otlp_payload = cmt_encode_opentelemetry_create(source); + TEST_ASSERT(otlp_payload != NULL); + + offset = 0; + result = cmt_decode_opentelemetry_create(&decoded_list, otlp_payload, + cfl_sds_len(otlp_payload), + &offset); + TEST_ASSERT(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + TEST_ASSERT(cfl_list_size(&decoded_list) == 1); + + decoded = cfl_list_entry_first(&decoded_list, struct cmt, _head); + roundtrip = msgpack_round_trip(decoded); + TEST_ASSERT(roundtrip != NULL); + check_all_encoders(roundtrip); + + cmt_decode_msgpack_destroy(roundtrip); + cmt_decode_opentelemetry_destroy(&decoded_list); + cmt_encode_opentelemetry_destroy(otlp_payload); + cmt_destroy(source); +} + +TEST_LIST = { + {"native_msgpack_conversion_matrix", test_native_msgpack_conversion_matrix}, +#ifdef CMT_HAVE_PROMETHEUS_TEXT_DECODER + {"prometheus_msgpack_conversion_matrix", test_prometheus_msgpack_conversion_matrix}, +#endif + {"remote_write_msgpack_conversion_matrix", test_remote_write_msgpack_conversion_matrix}, + {"statsd_msgpack_conversion_matrix", test_statsd_msgpack_conversion_matrix}, + {"otlp_msgpack_conversion_matrix", test_otlp_msgpack_conversion_matrix}, + {NULL, NULL} +}; diff --git a/lib/cmetrics/tests/issues.c b/lib/cmetrics/tests/issues.c index 651a8c405a4..8ac4ee20b9a 100644 --- a/lib/cmetrics/tests/issues.c +++ b/lib/cmetrics/tests/issues.c @@ -24,6 +24,9 @@ #include #include #include +#include +#include +#include #include "cmt_tests.h" @@ -91,6 +94,90 @@ void test_issue_54() cmt_destroy(cmt1); } +static void check_long_label_round_trip(size_t label_length) +{ + char *label_value; + char *label_values[1]; + char *label_keys[1] = {"command"}; + char *msgpack_buffer; + size_t msgpack_size; + size_t offset; + int result; + struct cmt *source; + struct cmt *decoded; + struct cmt_counter *counter; + struct cmt_metric *metric; + struct cmt_map_label *label; + + label_value = malloc(label_length + 1); + TEST_ASSERT(label_value != NULL); + + memset(label_value, 'a', label_length); + label_value[label_length] = '\0'; + label_values[0] = label_value; + + source = cmt_create(); + TEST_ASSERT(source != NULL); + + counter = cmt_counter_create(source, "test", "", "long_label", + "Long label round-trip", 1, label_keys); + TEST_ASSERT(counter != NULL); + TEST_CHECK(cmt_counter_set(counter, 0, 1, 1, label_values) == 0); + + result = cmt_encode_msgpack_create(source, &msgpack_buffer, &msgpack_size); + TEST_ASSERT(result == 0); + + offset = 0; + result = cmt_decode_msgpack_create(&decoded, msgpack_buffer, msgpack_size, + &offset); + TEST_ASSERT(result == 0); + TEST_CHECK(offset == msgpack_size); + + counter = cfl_list_entry_first(&decoded->counters, + struct cmt_counter, _head); + metric = cfl_list_entry_first(&counter->map->metrics, + struct cmt_metric, _head); + label = cfl_list_entry_first(&metric->labels, + struct cmt_map_label, _head); + + TEST_CHECK(cfl_sds_len(label->name) == label_length); + TEST_CHECK(memcmp(label->name, label_value, label_length) == 0); + + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(msgpack_buffer); + cmt_destroy(source); + free(label_value); +} + +void test_long_msgpack_labels() +{ + check_long_label_round_trip(1024); + check_long_label_round_trip(1025); + check_long_label_round_trip(2048); + check_long_label_round_trip(65536); +} + +void test_truncated_msgpack_string() +{ + char *output; + int result; + mpack_error_t error; + mpack_reader_t reader; + const char input[] = { + (char) 0xdb, (char) 0xff, (char) 0xff, (char) 0xff, (char) 0xff + }; + + output = NULL; + mpack_reader_init_data(&reader, input, sizeof(input)); + + result = cmt_mpack_consume_string_tag(&reader, &output); + error = mpack_reader_destroy(&reader); + + TEST_CHECK(result == CMT_MPACK_ENGINE_ERROR); + TEST_CHECK(error != mpack_ok); + TEST_CHECK(output == NULL); +} + #ifdef CMT_HAVE_PROMETHEUS_TEXT_DECODER /* issue: https://github.com/fluent/fluent-bit/issues/10761 */ @@ -123,6 +210,8 @@ void test_prometheus_metric_no_subsystem() TEST_LIST = { {"issue_54", test_issue_54}, + {"long_msgpack_labels", test_long_msgpack_labels}, + {"truncated_msgpack_string", test_truncated_msgpack_string}, #ifdef CMT_HAVE_PROMETHEUS_TEXT_DECODER {"prometheus_metric_no_subsystem", test_prometheus_metric_no_subsystem}, #endif diff --git a/lib/cmetrics/tests/opentelemetry.c b/lib/cmetrics/tests/opentelemetry.c index c24819f0377..918f44253cf 100644 --- a/lib/cmetrics/tests/opentelemetry.c +++ b/lib/cmetrics/tests/opentelemetry.c @@ -992,6 +992,102 @@ void test_opentelemetry_encode_multi_resource_scope_containers() cmt_destroy(cmt); } +void test_opentelemetry_encode_scope_attributes_cleanup() +{ + struct cmt *cmt; + struct cmt_gauge *gauge; + struct cfl_array *resource_metrics_list; + struct cfl_array *scope_metrics_list; + struct cfl_kvlist *resource_entry; + struct cfl_kvlist *scope_entry; + struct cfl_kvlist *scope_root; + struct cfl_kvlist *scope_metadata; + struct cfl_kvlist *scope_attributes; + cfl_sds_t payload; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + if (cmt == NULL) { + return; + } + + gauge = cmt_gauge_create(cmt, "ns", "sub", "scope_attributes", "g", + 0, NULL); + TEST_CHECK(gauge != NULL); + if (gauge == NULL) { + cmt_destroy(cmt); + return; + } + cmt_gauge_set(gauge, 123, 1.5, 0, NULL); + + resource_metrics_list = cfl_array_create(1); + scope_metrics_list = cfl_array_create(2); + TEST_CHECK(resource_metrics_list != NULL && scope_metrics_list != NULL); + if (resource_metrics_list == NULL || scope_metrics_list == NULL) { + if (resource_metrics_list != NULL) { + cfl_array_destroy(resource_metrics_list); + } + if (scope_metrics_list != NULL) { + cfl_array_destroy(scope_metrics_list); + } + cmt_destroy(cmt); + return; + } + + scope_entry = cfl_kvlist_create(); + scope_root = cfl_kvlist_create(); + scope_metadata = cfl_kvlist_create(); + scope_attributes = cfl_kvlist_create(); + cfl_kvlist_insert_string(scope_metadata, "name", "empty-attributes"); + cfl_kvlist_insert_kvlist(scope_root, "metadata", scope_metadata); + cfl_kvlist_insert_kvlist(scope_root, "attributes", scope_attributes); + cfl_kvlist_insert_kvlist(scope_entry, "scope", scope_root); + cfl_array_append_kvlist(scope_metrics_list, scope_entry); + + scope_entry = cfl_kvlist_create(); + scope_root = cfl_kvlist_create(); + scope_metadata = cfl_kvlist_create(); + scope_attributes = cfl_kvlist_create(); + cfl_kvlist_insert_string(scope_metadata, "name", "populated-attributes"); + cfl_kvlist_insert_string(scope_attributes, "language", "c"); + cfl_kvlist_insert_int64(scope_attributes, "generation", 2); + cfl_kvlist_insert_kvlist(scope_root, "metadata", scope_metadata); + cfl_kvlist_insert_kvlist(scope_root, "attributes", scope_attributes); + cfl_kvlist_insert_kvlist(scope_entry, "scope", scope_root); + cfl_array_append_kvlist(scope_metrics_list, scope_entry); + + resource_entry = cfl_kvlist_create(); + cfl_kvlist_insert_array(resource_entry, "scope_metrics_list", + scope_metrics_list); + cfl_array_append_kvlist(resource_metrics_list, resource_entry); + cfl_kvlist_insert_array(cmt->external_metadata, "resource_metrics_list", + resource_metrics_list); + + payload = cmt_encode_opentelemetry_create(cmt); + TEST_CHECK(payload != NULL); + if (payload != NULL) { + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(payload), (uint8_t *) payload); + TEST_CHECK(request != NULL); + if (request != NULL) { + TEST_CHECK(request->n_resource_metrics == 1); + TEST_CHECK(request->resource_metrics[0]->n_scope_metrics == 2); + TEST_CHECK(request->resource_metrics[0]->scope_metrics[0]->scope->n_attributes == 0); + TEST_CHECK(request->resource_metrics[0]->scope_metrics[1]->scope->n_attributes == 2); + TEST_CHECK(strcmp(request->resource_metrics[0]->scope_metrics[1]->scope->attributes[0]->key, + "language") == 0); + TEST_CHECK(strcmp(request->resource_metrics[0]->scope_metrics[1]->scope->attributes[1]->key, + "generation") == 0); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked( + request, NULL); + } + cmt_encode_opentelemetry_destroy(payload); + } + + cmt_destroy(cmt); +} + void test_opentelemetry_api_full_roundtrip_with_msgpack() { int ret; @@ -1900,6 +1996,7 @@ static void test_opentelemetry_omitted_null_key_label_encoded(void) TEST_LIST = { {"opentelemetry_api_full_roundtrip_with_msgpack", test_opentelemetry_api_full_roundtrip_with_msgpack}, {"opentelemetry_encode_multi_resource_scope_containers", test_opentelemetry_encode_multi_resource_scope_containers}, + {"opentelemetry_encode_scope_attributes_cleanup", test_opentelemetry_encode_scope_attributes_cleanup}, {"opentelemetry_exponential_histogram", test_opentelemetry_exponential_histogram}, {"opentelemetry_gauge_int_and_unit_decode", test_opentelemetry_gauge_int_and_unit_decode}, {"opentelemetry_sum_non_monotonic_int_roundtrip", test_opentelemetry_sum_non_monotonic_int_roundtrip}, diff --git a/lib/cmetrics/tests/prometheus_parser.c b/lib/cmetrics/tests/prometheus_parser.c index 2f9c8fafb42..022023b916f 100644 --- a/lib/cmetrics/tests/prometheus_parser.c +++ b/lib/cmetrics/tests/prometheus_parser.c @@ -79,7 +79,7 @@ void test_header_help() TEST_CHECK(f->context.metric.type == 0); cfl_sds_destroy(f->context.metric.name_orig); cfl_sds_destroy(f->context.metric.docstring); - free(f->context.metric.ns); + free(f->context.metric.name_buf); destroy(f); } @@ -97,7 +97,7 @@ void test_header_type() TEST_CHECK(f->context.metric.type == COUNTER); TEST_CHECK(f->context.metric.docstring == NULL); cfl_sds_destroy(f->context.metric.name_orig); - free(f->context.metric.ns); + free(f->context.metric.name_buf); destroy(f); } @@ -118,7 +118,7 @@ void test_header_help_type() TEST_CHECK(f->context.metric.type == SUMMARY); cfl_sds_destroy(f->context.metric.name_orig); cfl_sds_destroy(f->context.metric.docstring); - free(f->context.metric.ns); + free(f->context.metric.name_buf); destroy(f); } @@ -139,11 +139,69 @@ void test_header_type_help() TEST_CHECK(f->context.metric.type == GAUGE); cfl_sds_destroy(f->context.metric.name_orig); cfl_sds_destroy(f->context.metric.docstring); - free(f->context.metric.ns); + free(f->context.metric.name_buf); destroy(f); } +void test_metric_name_ownership() +{ + struct fixture *f; + + f = init(START_HEADER, "# TYPE metric counter\n"); + TEST_ASSERT(parse(f) == 0); + TEST_CHECK(strcmp(f->context.metric.ns, "") == 0); + TEST_CHECK(strcmp(f->context.metric.subsystem, "") == 0); + TEST_CHECK(strcmp(f->context.metric.name, "metric") == 0); + TEST_CHECK(f->context.metric.name_buf != NULL); + cfl_sds_destroy(f->context.metric.name_orig); + free(f->context.metric.name_buf); + destroy(f); + + f = init(START_HEADER, "# TYPE namespace_metric counter\n"); + TEST_ASSERT(parse(f) == 0); + TEST_CHECK(strcmp(f->context.metric.ns, "namespace") == 0); + TEST_CHECK(strcmp(f->context.metric.subsystem, "") == 0); + TEST_CHECK(strcmp(f->context.metric.name, "metric") == 0); + TEST_CHECK(f->context.metric.name_buf == f->context.metric.ns); + cfl_sds_destroy(f->context.metric.name_orig); + free(f->context.metric.name_buf); + destroy(f); +} + +void test_metric_name_ownership_resets() +{ + int status; + struct cmt *cmt = NULL; + const char *input = + "# TYPE metric gauge\n" + "metric 1\n" + "# TYPE namespace_metric gauge\n" + "namespace_metric 2\n" + "# TYPE namespace_subsystem_metric gauge\n" + "namespace_subsystem_metric 3\n"; + + status = cmt_decode_prometheus_create(&cmt, input, 0, NULL); + TEST_ASSERT(status == CMT_DECODE_PROMETHEUS_SUCCESS); + TEST_ASSERT(cmt != NULL); + cmt_decode_prometheus_destroy(cmt); +} + +void test_metric_name_ownership_error_cleanup() +{ + int status; + struct cmt *cmt = NULL; + + status = cmt_decode_prometheus_create(&cmt, + "# TYPE metric counter\nmetric {key=", 0, NULL); + TEST_CHECK(status == CMT_DECODE_PROMETHEUS_SYNTAX_ERROR); + + status = cmt_decode_prometheus_create(&cmt, + "# TYPE namespace_subsystem_metric counter\n" + "namespace_subsystem_metric {key=", 0, NULL); + TEST_CHECK(status == CMT_DECODE_PROMETHEUS_SYNTAX_ERROR); +} + struct cmt_decode_prometheus_context_sample *add_empty_sample(struct fixture *f) { struct cmt_decode_prometheus_context_sample *sample; @@ -1729,6 +1787,9 @@ TEST_LIST = { {"header_type", test_header_type}, {"header_help_type", test_header_help_type}, {"header_type_help", test_header_type_help}, + {"metric_name_ownership", test_metric_name_ownership}, + {"metric_name_ownership_resets", test_metric_name_ownership_resets}, + {"metric_name_ownership_error_cleanup", test_metric_name_ownership_error_cleanup}, {"labels", test_labels}, {"labels_trailing_comma", test_labels_trailing_comma}, {"sample", test_sample}, diff --git a/lib/cmetrics/tests/summary.c b/lib/cmetrics/tests/summary.c index 99ffe579246..299333be3ea 100644 --- a/lib/cmetrics/tests/summary.c +++ b/lib/cmetrics/tests/summary.c @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -94,6 +95,36 @@ void test_set_defaults() cmt_destroy(cmt); } +void test_quantile_bounds() +{ + uint64_t ts; + double quantiles[] = {0.5}; + double values[] = {12.0}; + struct cmt *cmt; + struct cmt_metric *metric; + struct cmt_summary *summary; + + cmt = cmt_create(); + TEST_ASSERT(cmt != NULL); + summary = cmt_summary_create(cmt, "test", "", "summary", "help", + 1, quantiles, 0, NULL); + TEST_ASSERT(summary != NULL); + ts = cfl_time_now(); + TEST_ASSERT(cmt_summary_set_default(summary, ts, values, 12.0, 1, + 0, NULL) == 0); + metric = cmt_map_metric_get(&summary->opts, summary->map, 0, NULL, + CMT_FALSE); + TEST_ASSERT(metric != NULL); + + TEST_CHECK(cmt_summary_quantile_get_value(metric, 0) == 12.0); + TEST_CHECK(cmt_summary_quantile_get_value(metric, -1) == 0.0); + TEST_CHECK(cmt_summary_quantile_get_value(metric, 1) == 0.0); + cmt_summary_quantile_set(metric, ts, 1, 99.0); + TEST_CHECK(cmt_summary_quantile_get_value(metric, 0) == 12.0); + + cmt_destroy(cmt); +} + /* ref: https://github.com/fluent/fluent-bit/issues/5894 */ void fluentbit_bug_5894() { @@ -167,6 +198,7 @@ void fluentbit_bug_5894() TEST_LIST = { {"set_defaults" , test_set_defaults}, + {"quantile_bounds" , test_quantile_bounds}, {"fluentbit_bug_5894", fluentbit_bug_5894}, { 0 } }; From 0b3fdbb8ee4b0fa342056b43793d32c9af0ba060 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sun, 12 Jul 2026 17:10:59 -0600 Subject: [PATCH 2/2] tests: runtime: processor_labels: update sample counts Signed-off-by: Eduardo Silva --- tests/runtime/processor_labels.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/runtime/processor_labels.c b/tests/runtime/processor_labels.c index c58e41c9e47..c4f9d021723 100644 --- a/tests/runtime/processor_labels.c +++ b/tests/runtime/processor_labels.c @@ -61,8 +61,9 @@ static int cb_insert_labels(void *record, size_t size, void *data) text = cmt_encode_text_create(cmt); TEST_CHECK(text != NULL); - TEST_CHECK(count_metrics_matches(text, "static=\"ok\"") == 9); - TEST_CHECK(count_metrics_matches(text, "dynamic=\"test\"") == 9); + /* 3 counters, 1 gauge, 2 summaries, and 2 histograms */ + TEST_CHECK(count_metrics_matches(text, "static=\"ok\"") == 8); + TEST_CHECK(count_metrics_matches(text, "dynamic=\"test\"") == 8); if (record) { flb_free(record); @@ -270,11 +271,11 @@ static int cb_upsert_labels(void *record, size_t size, void *data) text = cmt_encode_text_create(cmt); TEST_CHECK(text != NULL); - /* it should only update the metrics which contains a label with the name 'hostname' */ - TEST_CHECK(count_metrics_matches(text, "hostname=\"updated-2\"") == 8); + /* The unlabeled counter has no existing hostname label to update. */ + TEST_CHECK(count_metrics_matches(text, "hostname=\"updated-2\"") == 7); - /* now check the updated value of 'app' which should include the dynamic tag */ - TEST_CHECK(count_metrics_matches(text, "dynamic-host=\"test\"") == 9); + /* All eight legitimate samples receive the new dynamic label. */ + TEST_CHECK(count_metrics_matches(text, "dynamic-host=\"test\"") == 8); if (record) { flb_free(record); }