diff --git a/Makepkgs b/Makepkgs index 2bcaafd7d5..1d6b72aac1 100755 --- a/Makepkgs +++ b/Makepkgs @@ -41,6 +41,13 @@ fi STRING_DATE_EPOCH=`date +%s`; export STRING_DATE_EPOCH SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$STRING_DATE_EPOCH}"; export SOURCE_DATE_EPOCH +# Optional local patching ... maybe required for vendor code when we're +# waiting for an upstream fix that's otherwise killing the build. +# If the script local.patch exists it is per-host and *NOT* expected +# to be in the git tree. +# +[ -x local.patch ] && ./local.patch + # put VERSION.pcp back, remove temp files and exit # _cleanup() diff --git a/pmlogsynth-phase1-spec.md b/pmlogsynth-phase1-spec.md new file mode 100644 index 0000000000..877b72a74c --- /dev/null +++ b/pmlogsynth-phase1-spec.md @@ -0,0 +1,594 @@ +# pmlogsynth — Phase 1 Specification +## Synthetic PCP Archive Generator: Core Tool + +**Version:** 1.0-draft +**Status:** Proposed + +--- + +## 1. Problem Statement + +Testing PCP tooling — analysis, alerting, dashboards, ML-based anomaly detection — requires +archives with specific, reproducible performance characteristics. Real archives are hard to +curate: you must capture exactly the right event on real hardware, at the right time, with +the right metrics enabled. There is currently no way to say "give me an archive where this +specific failure mode occurs" without either hoping it happens in production or writing +throwaway one-off code against `libpcp_import` from scratch. + +`pmlogsynth` fills this gap: a command-line tool that generates valid, self-consistent PCP +archives from a declarative YAML workload profile. Archives produced by `pmlogsynth` are +indistinguishable in format from those produced by `pmlogger` and are immediately usable +with all PCP tooling. + +--- + +## 2. Naming + +`pmlogsynth` follows the established `pmlog*` tool family convention: + +``` +pmlogger → pmlogrewrite → pmlogextract → pmlogcheck → pmlogdump → pmlogsummary → pmlogsynth +``` + +The name is unambiguous: synthetic archive generation, recognisably part of the PCP archive +toolchain. + +--- + +## 3. Project Overview + +`pmlogsynth` is a standalone Python project distributed via PyPI and developed independently +of PCP. It depends on PCP being installed on the host (for `libpcp_import` and the `pcp` +Python bindings), but is otherwise self-contained. + +The project is intended for eventual contribution back to PCP, but operates as its own +repository to allow faster iteration, independent releases, and contributions from users +who are not PCP committers. + +--- + +## 4. Goals + +- Produce archives indistinguishable in format from `pmlogger` output +- Support real PCP metric namespaces (kernel, disk, network, memory) with correct units + and semantics +- Enforce internal metric consistency (e.g. CPU user + sys + idle = 100%) +- Accept a simple declarative profile format (YAML) +- Require no running `pmcd`, no real hardware, no root access +- Be usable in CI pipelines to regression-test PCP analysis tools +- Ship with a set of named hardware profiles; allow users to define their own + +### Out of Scope (Phase 1) + +The following are explicitly deferred and will not be addressed in this phase: + +| Item | Notes | +|------|-------| +| Multi-host archives | Single host only | +| Event records | Not supported | +| Derived metrics | Not supported | +| Per-process metrics (`proc.*`) | Deferred to future enhancement | +| Hotplug instance domain changes | Instance domains are fixed for archive lifetime | +| GPU / PMDA-specific namespaces | Out of scope | +| Archive v2 format | v3 only; add `--format v2` only if a concrete need arises | +| Per-metric noise granularity | Per-domain override is sufficient | +| Natural language profile generation | Addressed in Phase 2 | + +--- + +## 5. Dependencies + +### Runtime + +| Dependency | How to install | Notes | +|---|---|---| +| **Python 3.8+** | System package manager | `python3` | +| **PCP** | See [PCP installation docs](https://pcp.io/docs/guide.html) | Provides `libpcp_import.so` and the `pcp` Python bindings | +| **`python3-pcp`** | System package manager | RPM: `python3-pcp`; Deb: `python3-pcp`; provides `pcp.pmi`, `pcp.pmapi`, and the `cpmi` C extension | +| **PyYAML** | `pip install pyyaml` | Profile parsing | + +### Optional (Phase 2 — natural language generation) + +| Dependency | How to install | Notes | +|---|---|---| +| **`anthropic>=0.20.0`** | `pip install anthropic` | Anthropic Python SDK; only needed for `--prompt` | + +### Not required + +- No C compiler (pure Python after PCP is installed) +- No `numpy` — Gaussian noise uses `random.gauss` from stdlib +- No running `pmcd` +- No root access +- No database, message queue, or web service +- Phase 3 parallel `--jobs` uses `concurrent.futures` from stdlib + +--- + +## 6. Architecture + +``` +profile.yaml + │ + ▼ + ProfileLoader + │ (parses + validates workload phases, timeline, host config) + ▼ + MetricModel (one per domain) + │ (computes consistent values across related metrics at each tick) + ▼ + ValueSampler + │ (applies Gaussian noise, accumulates counters, coerces types) + ▼ + libpcp_import (via pcp.pmi.pmiLogImport Python bindings) + │ + ▼ +output.{0,index,meta} +``` + +### Implementation Language + +Python 3. Depends only on the `pcp` Python package (installed alongside any PCP +installation that includes Python bindings) and PyYAML. No other third-party dependencies +are required for core archive generation. + +--- + +## 7. Hardware Profile Library + +### 7.1 Concept + +A hardware profile is a named YAML document that describes the physical or virtual host +being simulated: CPU count, RAM, disk devices, and network interfaces. Profiles decouple +the "what hardware" question from the "what workload" question, making profiles reusable +across many workload scenarios. + +### 7.2 Bundled Profiles + +`pmlogsynth` ships with a small set of generic reference host profiles. These are loosely +inspired by common cloud instance tiers but are not tied to any vendor — they serve as +reasonable, recognisable starting points. + +| Profile name | CPUs | RAM | Disk | NIC | Archetype | +|---------------------|------|--------|-----------------|-------------|------------------------| +| `generic-small` | 2 | 8 GB | 1× NVMe | 1× 10 GbE | General purpose, small | +| `generic-medium` | 4 | 16 GB | 1× NVMe | 1× 10 GbE | General purpose, medium| +| `generic-large` | 8 | 32 GB | 2× NVMe | 1× 10 GbE | General purpose, large | +| `generic-xlarge` | 16 | 64 GB | 2× NVMe | 2× 10 GbE | General purpose, xlarge| +| `compute-optimized` | 8 | 16 GB | 1× NVMe | 1× 10 GbE | High CPU, modest RAM | +| `memory-optimized` | 4 | 64 GB | 1× NVMe | 1× 10 GbE | High RAM, modest CPU | +| `storage-optimized` | 4 | 16 GB | 4× HDD | 1× 10 GbE | High disk capacity | + +Bundled profiles are packaged inside the `pmlogsynth/profiles/` directory and installed +as package data alongside the Python source. + +### 7.3 User-Defined Profiles + +Users may define their own profiles — or override bundled ones — by placing YAML files in: + +``` +~/.pcp/pmlogsynth/profiles/ +``` + +**Lookup order:** the user directory is checked first. A user file whose name matches a +bundled profile takes precedence, providing a clean override mechanism without touching +the installation. + +```bash +# List all profiles available (bundled + user-defined, with source shown) +pmlogsynth --list-profiles +``` + +Example user-defined profile: + +```yaml +# ~/.pcp/pmlogsynth/profiles/prod-web-tier.yaml +name: prod-web-tier +cpus: 32 +memory_kb: 134217728 # 128 GB +disks: + - name: nvme0n1 + type: nvme + - name: nvme1n1 + type: nvme +interfaces: + - name: bond0 + speed_mbps: 25000 +``` + +### 7.4 Profile Validation in CI + +The CI pipeline runs a schema validation pass over all bundled profiles in +`pmlogsynth/profiles/`. Any malformed profile fails the test run. Content review +of contributed profiles remains a human responsibility. + +--- + +## 8. Profile Format + +A profile is a YAML file that describes the simulated host and a timeline of workload +**phases**. Each phase has a duration and a set of **stressors** that drive one or more +metric domains. + +### 8.1 Full Example + +```yaml +# cpu-memory-spike.yaml +meta: + hostname: synthetic-host + timezone: UTC + duration: 1800 # total archive length in seconds + interval: 60 # sample interval in seconds [default: 60] + noise: 0.03 # global Gaussian noise factor (3%); overridable per domain + +host: + profile: generic-large # reference a named hardware profile... + # ...or define the host inline (overrides a named profile if both are present): + # name: web-server-01 + # cpus: 16 + # memory_kb: 65536000 + # disks: + # - name: sda + # type: ssd + # interfaces: + # - name: eth0 + # speed_mbps: 10000 + +phases: + - name: baseline + duration: 300 + cpu: + utilization: 0.15 # 15% overall CPU utilisation + user_ratio: 0.70 # fraction of busy time in user space + sys_ratio: 0.20 + iowait_ratio: 0.10 + memory: + used_ratio: 0.40 # fraction of total RAM in use + cache_ratio: 0.30 + disk: + read_mbps: 2.0 + write_mbps: 0.5 + network: + rx_mbps: 10.0 + tx_mbps: 2.0 + + - name: cpu-spike + duration: 300 + cpu: + utilization: 0.92 + user_ratio: 0.85 + sys_ratio: 0.10 + iowait_ratio: 0.05 + memory: + used_ratio: 0.55 + disk: + read_mbps: 8.0 + write_mbps: 3.0 + network: + rx_mbps: 10.0 + tx_mbps: 2.0 + + - name: recovery + duration: 600 + transition: linear # ramp from previous phase's end values over this duration + cpu: + utilization: 0.20 + memory: + used_ratio: 0.42 + disk: + read_mbps: 2.5 + write_mbps: 0.8 + network: + rx_mbps: 10.0 + tx_mbps: 2.0 +``` + +### 8.2 Phase Transitions + +| Value | Behaviour | +|-------|-----------| +| `instant` (default) | Values jump immediately at the phase boundary | +| `linear` | Values interpolate linearly over the full phase duration from prior phase end values | + +### 8.3 Repeating Phases + +A phase may include a `repeat` key to express recurring patterns without copy-pasting. +The timeline sequencer expands repeats before writing begins. + +```yaml +phases: + - name: baseline + duration: 39600 # 11 hours + + - name: noon-peak + duration: 3600 # 1 hour, repeated each day + repeat: daily # valid values: daily, or an integer count + transition: linear + cpu: + utilization: 0.93 + user_ratio: 0.88 + sys_ratio: 0.08 + iowait_ratio: 0.04 + disk: + read_mbps: 25.0 + write_mbps: 12.0 +``` + +When `repeat: daily` is used, the sequencer inserts the baseline phase between each +repetition to fill the 24-hour period. `meta.duration` must accommodate the full +expanded timeline; the validator will reject profiles where this does not hold. + +### 8.4 Noise + +A `noise:` key at domain level overrides `meta.noise` for that domain only: + +```yaml + - name: busy + cpu: + utilization: 0.80 + noise: 0.08 # noisier CPU; other domains use meta.noise + disk: + write_mbps: 5.0 +``` + +### 8.5 Instance Domains + +Disk and NIC instances are derived from the host configuration and remain **fixed** +for the lifetime of the archive. Instance names match the device names in the host +profile (e.g. `nvme0n1`, `eth0`). + +### 8.6 Constraints Enforced at Validation + +- `user_ratio + sys_ratio + iowait_ratio ≤ 1.0` (remainder is steal/other) +- Sum of phase durations == `meta.duration` (when no `repeat` key is present) +- `host.profile` must resolve to a known profile name +- All `noise` values must be in range [0.0, 1.0] +- `interval` must be a positive integer, in seconds. **Default: 60.** + Archive size grows linearly with sample count: a 7-day archive at `interval: 1` + produces ~600,000 samples; at `interval: 60` it produces ~10,000. Use fine-grained + intervals only for short archives where sub-minute resolution is required. + +--- + +## 9. Metric Domains and Consistency Model + +Each domain is a self-contained `MetricModel` subclass that accepts high-level stressor +values and derives all related PCP metrics, enforcing internal constraints at every sample. + +### 9.1 CPU Domain + +**PCP metrics:** `kernel.all.cpu.*`, `kernel.percpu.cpu.*` + +| Input field | Derived PCP metrics | +|-------------|---------------------| +| `utilization` | `kernel.all.cpu.user`, `.sys`, `.idle`, `.wait`, `.steal` | +| `user_ratio`, `sys_ratio`, `iowait_ratio` | Partition the busy fraction accordingly | +| `host.cpus` | `kernel.percpu.cpu.*` distributed across N CPUs with per-CPU ±variance | + +**Constraint enforced:** `user + sys + idle + wait + steal == total_ticks` per CPU per +interval. + +**Metric type:** counter (cumulative milliseconds). The model maintains running totals +across samples so that rate-based tools (`pmval`, `pmrep`) produce correct results when +replaying the archive. + +### 9.2 Memory Domain + +**PCP metrics:** `mem.util.*` + +| Input field | Derived PCP metrics | +|-------------|---------------------| +| `used_ratio` | `mem.util.used`, `.free`, `.cached`, `.bufmem`, `.available` | +| `cache_ratio` | `mem.util.cached` carved from the used portion | +| `host.memory_kb` | All values expressed as absolute KB | + +**Constraint enforced:** `used + free == physmem`. `available ≈ free + cached`. + +### 9.3 Disk Domain + +**PCP metrics:** `disk.all.*`, `disk.dev.*` + +| Input field | Derived PCP metrics | +|-------------|---------------------| +| `read_mbps`, `write_mbps` | `disk.all.read_bytes`, `.write_bytes`, `.read`, `.write` (ops) | +| `host.disks` | `disk.dev.*` split across named device instances | +| `iops_read`, `iops_write` (optional) | If omitted, estimated from throughput at 64 KB mean block size | + +**Metric type:** counter (cumulative bytes and ops). + +### 9.4 Network Domain + +**PCP metrics:** `network.interface.*` + +| Input field | Derived PCP metrics | +|-------------|---------------------| +| `rx_mbps`, `tx_mbps` | `network.interface.in.bytes`, `.out.bytes`, `.in.packets`, `.out.packets` | +| `host.interfaces` | Split across named interface instances | + +Packet counts are estimated from byte totals assuming a 1400-byte mean packet size +(configurable via a top-level `meta.mean_packet_bytes` key). + +### 9.5 Load Average Domain + +**PCP metrics:** `kernel.all.load` + +Derived from CPU utilisation: `load_1min ≈ utilization × num_cpus`. Exponential +smoothing is applied to correctly simulate the 1-minute, 5-minute, and 15-minute +UNIX load average decay constants. + +--- + +## 10. CLI Interface + +``` +pmlogsynth [OPTIONS] PROFILE + +Arguments: + PROFILE Path to YAML profile file + +Options: + -o, --output PATH Output archive base name [default: ./pmlogsynth-out] + --start TIMESTAMP Archive start time [default: now - meta.duration] + --list-metrics Print all PCP metrics this tool can generate, then exit + --list-profiles Print all available hardware profiles (bundled + user), then exit + --validate Validate profile without generating any output + -v, --verbose Print per-sample values to stderr as they are written + -V, --version Show version + -h, --help Show help +``` + +### Examples + +```bash +# Basic generation +pmlogsynth -o ./out cpu-spike.yaml + +# Archive anchored to a specific historical window +pmlogsynth --start "2024-01-15 09:00:00 UTC" -o ./incident-replay spike.yaml + +# Validate a profile without writing any output +pmlogsynth --validate cpu-spike.yaml + +# See all available hardware profiles and where they come from +pmlogsynth --list-profiles + +# See all PCP metrics the tool can produce +pmlogsynth --list-metrics +``` + +--- + +## 11. Output + +`pmlogsynth` produces a standard PCP v3 archive: + +| File | Content | +|------|---------| +| `.0` | Data volume | +| `.index` | Temporal index | +| `.meta` | Metric metadata | + +The archive is immediately usable with all PCP tooling: + +```bash +pmval -a ./out kernel.all.cpu.user +pmrep -a ./out -o csv kernel.all.cpu.user mem.util.used +pmlogcheck ./out +pcp -a ./out atop +``` + +--- + +## 12. Project Layout + +``` +pmlogsynth/ # repository root +├── pyproject.toml # package metadata, dependencies, entry point +├── README.md +├── requirements.txt # pinned dev dependencies +├── pmlogsynth/ # installable Python package +│ ├── __init__.py +│ ├── __main__.py # enables: python -m pmlogsynth +│ ├── cli.py # argument parsing, entry point +│ ├── profile.py # YAML loader and validator +│ ├── timeline.py # phase sequencer, transition interpolation, +│ │ # repeat expansion +│ ├── sampler.py # Gaussian noise, counter accumulation, +│ │ # type coercion +│ ├── writer.py # libpcp_import wrapper (pcp.pmi.pmiLogImport) +│ ├── profiles/ # bundled hardware profiles (package data) +│ │ ├── generic-small.yaml +│ │ ├── generic-medium.yaml +│ │ ├── generic-large.yaml +│ │ ├── generic-xlarge.yaml +│ │ ├── compute-optimized.yaml +│ │ ├── memory-optimized.yaml +│ │ └── storage-optimized.yaml +│ └── domains/ +│ ├── cpu.py +│ ├── memory.py +│ ├── disk.py +│ ├── network.py +│ └── load.py +└── tests/ + ├── test_profile.py # profile loading and validation + ├── test_timeline.py # phase sequencing and repeat expansion + ├── test_sampler.py # noise and counter accumulation + ├── test_domains.py # per-domain metric consistency checks + └── test_writer.py # archive generation (requires PCP installed) +``` + +**User profile directory:** `~/.pcp/pmlogsynth/profiles/` + +### Installation + +```bash +pip install pmlogsynth + +# Or from source: +git clone https://github.com//pmlogsynth +cd pmlogsynth +pip install -e . +``` + +`pyproject.toml` declares the entry point: + +```toml +[project.scripts] +pmlogsynth = "pmlogsynth.cli:main" +``` + +--- + +## 13. Test Requirements + +Tests are written with `pytest` and live in `tests/`. They are split into two tiers: + +### Tier 1 — unit tests (no PCP required) + +Test profile loading, validation, timeline sequencing, phase transitions, repeat +expansion, noise application, and counter accumulation without writing any archive. +All domain consistency constraints are verified at the value-computation level. +These tests run anywhere Python 3.8+ is available. + +### Tier 2 — integration tests (PCP must be installed) + +Generate a real archive from a known profile, then verify it with PCP tooling: + +1. Run `pmlogsynth` against a fixed reference profile +2. Run `pmlogcheck` against the output and assert it passes +3. Run `pmval` against one metric per domain and assert values are within + the expected range (stressor value ± noise tolerance) +4. Assert the archive start and end timestamps match `--start` and `meta.duration` + +Tier 2 tests are skipped automatically (via a pytest fixture) if `pmlogcheck` is not +found on `PATH`. This allows the test suite to run in environments without PCP installed, +with only Tier 1 executing. + +```bash +# Run all tests +pytest + +# Run only unit tests (no PCP needed) +pytest -m "not integration" + +# Run with verbose output +pytest -v +``` + +--- + +## 14. Future Enhancements + +The following items are explicitly deferred from Phase 1: + +| Item | Notes | +|------|-------| +| Per-process metrics (`proc.*`) | Significant additional complexity; high value for profiling workloads | +| Hotplug instance domains | Disks/NICs appearing or disappearing mid-archive | +| Archive v2 format | Add `--format v2` only if a concrete compatibility need arises | +| Per-metric noise granularity | Per-domain override is considered sufficient for now | +| Multi-host archives | Out of scope for a single-tool use case | +| Profile composition (`include:`) | Merging multiple profile files | +| Python scripting API | `from pmlogsynth import Profile, generate` | +| Real-data seeding | Accept a real archive as baseline, overlay synthetic phases | +| Natural language profile generation | Addressed in Phase 2 | diff --git a/pmlogsynth-phase2-spec.md b/pmlogsynth-phase2-spec.md new file mode 100644 index 0000000000..e046c48f60 --- /dev/null +++ b/pmlogsynth-phase2-spec.md @@ -0,0 +1,440 @@ +# pmlogsynth — Phase 2 Specification +## Natural Language Profile Generation via Claude + +**Version:** 1.0-draft +**Status:** Proposed +**Depends on:** pmlogsynth Phase 1 (complete) + +--- + +## 1. Background and Prior Work + +Phase 1 of `pmlogsynth` delivered a command-line tool that generates valid PCP archives +from declarative YAML workload profiles. It ships with a library of named hardware +profiles (bundled in `pmlogsynth/profiles/`, user-extensible via +`~/.pcp/pmlogsynth/profiles/`) and supports multi-phase workload timelines with noise, +linear transitions, and repeating patterns. + +The YAML profile format is expressive but requires the user to know field names, units, +ratio constraints, and phase sequencing rules. For users who are not regular contributors +to PCP — but who need a synthetic archive to test a dashboard, an alerting rule, or an +analysis script — hand-authoring a profile is an unnecessary barrier. + +Phase 2 adds a `--prompt` flag to `pmlogsynth` that accepts a plain English description +of the desired archive and returns a ready-to-use YAML profile, generated by Claude. + +--- + +## 2. Goals + +- Allow a user to generate a valid `pmlogsynth` profile using natural language +- Require no knowledge of the YAML schema, field names, or constraint rules +- Produce a profile that passes `pmlogsynth --validate` without manual correction + in the common case +- Make every assumption Claude makes visible to the user as YAML comments +- Require no persistent service, daemon, or MCP server +- Introduce no new runtime dependencies beyond the Anthropic Python SDK + +### Out of Scope (Phase 2) + +| Item | Notes | +|------|-------| +| Interactive clarification dialogue | See §7 for rationale; deferred | +| MCP server deployment | Overkill for low-frequency use; deferred to future | +| Streaming output | Single-turn request/response is sufficient | +| Conversation history / multi-turn refinement | User iterates by editing YAML or refining prompt | +| Profile validation retry loop | Errors are surfaced to the user; not auto-corrected | + +--- + +## 3. User Experience + +### 3.1 Basic Usage + +```bash +export ANTHROPIC_API_KEY=sk-ant-... + +# Generate a profile and print it to stdout for review +pmlogsynth --prompt \ + "Generate 7 days of data, sampling every 1 minute. Use the generic-large hardware + profile. Baseload CPU at 30%, with daily spikes from 12pm to 1pm reaching 93% + user CPU and heavy disk IO during the same window. Consistent daily pattern." + +# Save the profile to a file +pmlogsynth --prompt "..." -o my-scenario.yaml + +# Generate the profile and immediately produce the archive +pmlogsynth --prompt "..." --run -o ./archive-out + +# Validate a prompt-generated profile before running +pmlogsynth --prompt "..." -o my-scenario.yaml +pmlogsynth --validate my-scenario.yaml +``` + +### 3.2 What the User Sees + +The generated YAML includes a header block documenting every assumption Claude made. +There are no interactive prompts; the result arrives in one step. + +```yaml +# Generated by: pmlogsynth --prompt +# Prompt: "7 days of data, 1 minute sampling, generic-large hardware, baseload CPU +# 30%, daily noon spike to 93% user CPU with heavy disk IO" +# +# Assumptions made: +# - Hardware profile: generic-large (explicitly requested) +# - Spike duration: 12:00–13:00 each day (interpreted literally) +# - Spike transition: linear ramp over 10 minutes either side of peak +# (not specified; avoids unrealistic instantaneous step) +# - Disk IO during spike: read_mbps 22.0, write_mbps 14.0 +# (interpreted "heavy" as ~55% of device throughput ceiling for NVMe) +# - Memory: held at 45% used_ratio throughout (not mentioned; reasonable baseline) +# - Network: held at 8.0 rx_mbps / 2.0 tx_mbps throughout (not mentioned) +# - Noise: 3% global (default) +# +# Review these assumptions. Edit this file and re-run if anything does not match +# your intent, or refine your --prompt and regenerate. + +meta: + hostname: synthetic-host + timezone: UTC + duration: 604800 # 7 days + interval: 60 # 1 minute sampling + noise: 0.03 + +host: + profile: generic-large + +phases: + ... +``` + +If the user disagrees with an assumption, they edit the YAML directly — a targeted, +low-friction change — or refine the prompt and regenerate. + +--- + +## 4. Technical Design + +### 4.1 Overview + +`--prompt` makes a **single API call** to Claude. The system prompt provides Claude with +the complete `pmlogsynth` schema, the hardware profile library, metric domain knowledge, +constraint rules, and workload archetypes. Claude returns a complete YAML profile. +No conversation state, no retry loop, no interactive dialogue. + +``` +User prompt (natural language) + │ + ▼ +┌─────────────────────────────────────────┐ +│ Claude API (single-turn) │ +│ System prompt: │ +│ - pmlogsynth YAML schema │ +│ - Hardware profile summaries │ +│ - Metric domain knowledge │ +│ - Constraint rules │ +│ - Workload archetypes │ +│ - Output instructions │ +└─────────────────────────────────────────┘ + │ + ▼ + Complete YAML profile (with assumption comments) + │ + ├── stdout / -o file + └── optional --run → pmlogsynth generates archive +``` + +### 4.2 Implementation + +The Phase 2 addition to `pmlogsynth` is small. The core client is straightforward: + +```python +# pmlogsynth/agent/client.py + +import anthropic +import sys +from pathlib import Path + + +MODEL = "claude-opus-4-6" +MAX_TOKENS = 4096 + + +def generate_profile(prompt: str) -> str: + """ + Send prompt to Claude and return a pmlogsynth YAML profile string. + Raises anthropic.APIError on failure. + """ + client = anthropic.Anthropic() # reads ANTHROPIC_API_KEY from environment + + response = client.messages.create( + model=MODEL, + max_tokens=MAX_TOKENS, + system=_load_system_prompt(), + messages=[{"role": "user", "content": prompt}], + ) + + return response.content[0].text + + +def _load_system_prompt() -> str: + prompt_path = Path(__file__).parent / "system_prompt.md" + return prompt_path.read_text() +``` + +The CLI change in `pmlogsynth` is equally minimal: + +```python +# In pmlogsynth/cli.py: + +if args.prompt: + _check_api_key() + from pmlogsynth.agent.client import generate_profile + yaml_text = generate_profile(args.prompt) + + if args.output and args.output.endswith(".yaml"): + Path(args.output).write_text(yaml_text) + print(f"Profile written to {args.output}", file=sys.stderr) + else: + print(yaml_text) + + if args.run: + profile = ProfileLoader.from_string(yaml_text) + generate_archive(profile, args.output or "pmlogsynth-out") + + +def _check_api_key(): + if not os.environ.get("ANTHROPIC_API_KEY"): + print( + "pmlogsynth: --prompt requires ANTHROPIC_API_KEY to be set.\n" + "Obtain a key at https://console.anthropic.com", + file=sys.stderr, + ) + sys.exit(1) +``` + +There is no conversation loop, no JSON parsing, no structured output format negotiation. +Claude returns YAML text directly; the system prompt is responsible for ensuring it does. + +### 4.3 Error Handling + +| Condition | Behaviour | +|-----------|-----------| +| `ANTHROPIC_API_KEY` not set | Clear error message with URL; exit non-zero | +| `anthropic` package not installed | Clear error message with `pip install pmlogsynth[ai]`; exit non-zero | +| API call fails (network, rate limit, etc.) | Surface the API error message; exit non-zero | +| Generated YAML fails `--validate` | Inform the user; suggest editing the file or refining the prompt; exit non-zero | +| `--run` with invalid profile | Same as above; no archive is written | + +There is no automatic retry or self-correction loop. If the generated profile is invalid, +the user sees the validation error and decides how to proceed. + +--- + +## 5. The System Prompt + +The system prompt is the core of Phase 2. It is stored as a versioned file +(`pmlogsynth/agent/system_prompt.md`) and maintained alongside the YAML schema — +if the schema changes, the system prompt must be updated in the same commit. + +### 5.1 Contents + +The system prompt embeds four things: + +**a) The complete YAML schema** +Every field, its type, valid range, and the constraints that must hold. This is the +authoritative source for what Claude must produce. + +Key constraints stated explicitly: +- `user_ratio + sys_ratio + iowait_ratio ≤ 1.0` +- Sum of phase durations must equal `meta.duration` when no `repeat` key is used +- `noise` values must be in [0.0, 1.0] +- `host.profile` must be one of the known profile names + +**b) Hardware profile summaries** +Names and key specs of all bundled profiles, so Claude can reference them by name +in the output YAML. At invocation time, the CLI also enumerates any profiles in the +user's `~/.pcp/pmlogsynth/profiles/` directory and appends them to the system prompt, +making user-defined profiles available to Claude. + +**c) Workload archetypes** +Named stressor patterns Claude can compose. These give Claude a vocabulary for common +scenarios so it does not have to derive reasonable values from first principles every time. + +| Archetype name | Description | +|----------------|-------------| +| `web-noon-peak` | Predictable daily request surge; CPU and network rise together | +| `batch-job` | High CPU and disk IO in a fixed time window; low network | +| `memory-pressure` | `used_ratio` rises steadily; iowait increases as swapping begins | +| `disk-saturation` | Write throughput at device ceiling; iowait rises proportionally | +| `network-storm` | `rx_mbps` spikes to near interface ceiling; CPU stays low | +| `idle-baseline` | Low utilisation across all domains; useful named starting phase | + +**d) Output instructions** +The final section of the system prompt instructs Claude on how to format its response: + +``` +Always produce a complete, valid pmlogsynth YAML profile. Never ask clarifying +questions. When the user's prompt does not specify a value, choose a reasonable +default and document it as a YAML comment immediately above the relevant field +using the format: + + # Assumed: + +Begin the file with a header block listing all assumptions made, as shown in the +example below. The header must appear before the first YAML key. + +Do not include any text outside the YAML document. Do not wrap the output in +a code fence. +``` + +--- + +## 6. New CLI Options + +The following options are added to `pmlogsynth` in Phase 2: + +``` + --prompt TEXT Describe the desired archive in natural language. + Requires ANTHROPIC_API_KEY. Outputs a pmlogsynth + YAML profile to stdout (or -o FILE if FILE ends + in .yaml). Combine with --run to also generate + the archive immediately. + --run When used with --prompt, generate the archive after + producing the profile. Output archive base name is + taken from -o (minus .yaml suffix if present) or + defaults to ./pmlogsynth-out. +``` + +The existing `-o / --output` flag is extended to serve double duty: if it ends in +`.yaml`, the profile is written there; if it does not, it is used as the archive +base name (for `--run`). + +--- + +## 7. Why No Interactive Clarification Mode + +An earlier draft of this specification included an interactive clarification dialogue: +Claude would return clarifying questions when the prompt was underspecified, the CLI +would print them and read answers from stdin, and a second API call would be made with +the answers appended. + +This was dropped for the following reasons: + +**Technical complexity is disproportionate to the benefit.** Detecting whether Claude's +response is a finished YAML profile or a block of questions requires either fragile text +heuristics or a structured JSON output contract with its own parsing layer. Either +approach introduces a second code path, second API call, conversation state management, +and additional failure modes — all for a tool invoked infrequently. + +**The iteration model is already good without it.** Claude documents every assumption +it makes as a YAML comment. The user reads the profile, spots a disagreement, edits +one line, and reruns. This is faster and less error-prone than answering terminal +questions whose impact on the output is not immediately visible. + +**Prompts can be refined.** If the first output is substantially wrong, the user +sharpens the prompt and regenerates. This is the natural interaction model for LLM +tooling and requires no special infrastructure. + +Interactive mode remains listed as a future enhancement (§10) if a clear use case +emerges. + +--- + +## 8. API Key and Access + +`--prompt` calls the Anthropic API and requires an `ANTHROPIC_API_KEY` environment +variable. + +**Important:** An `ANTHROPIC_API_KEY` is a separate credential from a Claude.ai +subscription (including Pro and Max plans). A Claude.ai subscription does not grant +access to a standalone API key usable by third-party tools. API access is provisioned +and billed independently at [console.anthropic.com](https://console.anthropic.com). + +Usage costs for profile generation are very low. A single `--prompt` invocation is a +small, single-turn request — typically well under 2,000 output tokens. + +--- + +## 9. Installation + +Phase 2 is shipped as an optional extras group so the `anthropic` SDK is not installed +for users who do not need `--prompt`: + +```bash +# Core install (no AI features) +pip install pmlogsynth + +# With natural language profile generation +pip install "pmlogsynth[ai]" +``` + +`pyproject.toml`: + +```toml +[project.optional-dependencies] +ai = ["anthropic>=0.20.0"] +``` + +If `--prompt` is invoked without the `ai` extras installed, `pmlogsynth` exits with a +clear error: + +``` +pmlogsynth: --prompt requires the 'anthropic' package. +Install it with: pip install "pmlogsynth[ai]" +``` + +--- + +## 10. Project Layout Changes + +Phase 2 adds the following to the Phase 1 layout: + +``` +pmlogsynth/ +└── agent/ + ├── client.py # Anthropic API wrapper; generate_profile() + └── system_prompt.md # Schema + constraints + archetypes + output instructions + # Versioned alongside the YAML schema + +tests/ +└── test_agent.py # Phase 2 tests (all mocked; no live API calls) +``` + +--- + +## 11. Test Requirements + +Tests live in `tests/test_agent.py` and use `unittest.mock` (stdlib) to mock the +Anthropic API. No live API calls are made; no API key is required to run the test suite. + +The test suite must cover: + +1. Mock the `anthropic.Anthropic().messages.create()` call to return a known, fixed + YAML profile; assert the profile is written correctly to the output path +2. Run `pmlogsynth --validate` on the mocked output and assert it passes +3. Assert that the header assumption block is present in the generated output +4. Assert that a missing `ANTHROPIC_API_KEY` produces a clear error message and + non-zero exit code +5. Assert that `--prompt` without the `anthropic` package installed produces a clear + install hint and non-zero exit code + +```bash +# Run all tests including Phase 2 +pytest + +# Run only Phase 2 tests +pytest tests/test_agent.py +``` + +--- + +## 12. Future Enhancements + +| Item | Notes | +|------|-------| +| Interactive clarification dialogue | Requires structured JSON output contract + multi-turn conversation state; see §7 for full rationale | +| MCP server (`pmlogsynth-mcp`) | Expose `generate_profile(prompt)` as an MCP tool for use within Claude Code / Claude.ai agentic sessions; useful but overkill given invocation frequency | +| Profile composition from multiple prompts | Generate sub-profiles and merge them | +| `--refine FILE` flag | Feed an existing profile + a change description back to Claude for targeted edits | diff --git a/pmlogsynth-phase3-spec.md b/pmlogsynth-phase3-spec.md new file mode 100644 index 0000000000..f270dfcfa8 --- /dev/null +++ b/pmlogsynth-phase3-spec.md @@ -0,0 +1,412 @@ +# pmlogsynth — Phase 3 Specification +## Fleet Archive Generation: Bulk Synthetic Server Farm + +**Version:** 1.0-draft +**Status:** Proposed +**Depends on:** pmlogsynth Phase 1 (complete), Phase 2 (optional but recommended) + +--- + +## 1. Background and Prior Work + +Phase 1 of `pmlogsynth` produces a single PCP archive from a declarative YAML workload +profile, with a hardware profile describing the simulated host. Phase 2 adds natural +language profile generation via Claude. + +A single archive is sufficient for testing tools that analyse one host in isolation. +However, many real PCP use cases involve fleets: pmlogger archives arriving from dozens +or hundreds of hosts, analysed together to find outliers, detect coordinated failures, +or benchmark fleet-wide health. Testing these use cases requires a realistic set of +archives — mostly well-behaved "background" hosts, with a small number exhibiting +specific fault conditions. + +Phase 3 adds fleet generation: the ability to produce a named, structured set of +archives from a single fleet profile, representing a heterogeneous server farm with +controllable fault injection. + +--- + +## 2. Goals + +- Generate a set of PCP archives in a single command, one per simulated host +- Support a majority of "healthy" hosts sharing a common baseline workload profile +- Support a minority of "anomalous" hosts with fault conditions overlaid on top of + the baseline +- Introduce per-host variation among healthy hosts so archives are not identical +- Name output archives consistently so downstream tools can discover them +- Reuse Phase 1 profile and hardware profile formats without modification +- Support Phase 2 `--prompt` for fleet profile generation + +### Out of Scope (Phase 3) + +| Item | Notes | +|------|-------| +| Cross-host metric correlation | Each host archive is independent | +| Network topology simulation | No inter-host traffic modelling | +| Rolling failures / cascades | Faults are per-host, not triggered by other hosts | +| Live replay to pmcd | Out of scope; archives only | + +--- + +## 3. Concepts + +### 3.1 Fleet Profile + +A fleet profile is a new YAML document type (distinct from a workload profile) that +describes a collection of host groups. Each group specifies a count of hosts, a +hardware profile, a base workload profile, and optionally one or more anomaly overlays. + +### 3.2 Host Groups + +A host group is a set of hosts that share the same hardware profile and base workload. +Groups are named; host archives are named `-NN.{0,index,meta}`. + +### 3.3 Anomaly Overlays + +An anomaly overlay is a partial workload profile — it specifies only the fields that +deviate from the base workload, for a defined time window within the archive. The +overlay is merged on top of the base workload for the affected hosts only. + +Overlays use the same stressor syntax as Phase 1 workload profiles. Any field not +present in the overlay retains its base workload value. + +### 3.4 Per-Host Variation (Jitter) + +Among hosts in the same group, values are not identical. A `jitter` factor (default 5%) +applies a small, per-host random offset to all stressor values at profile load time. +This simulates the natural variation seen in a real fleet without requiring separate +profiles per host. + +Jitter is seeded per host name, so the same fleet profile with the same `--seed` +produces the same archives. + +--- + +## 4. Fleet Profile Format + +```yaml +# fleet: web-cluster.yaml + +meta: + name: web-cluster + duration: 86400 # 24 hours — applies to all hosts in this fleet + interval: 60 # 1 minute sampling — applies to all hosts + timezone: UTC + output_dir: ./fleet-archives # base output directory + +groups: + + - name: web-frontend + count: 16 + host_prefix: web # archives named web-01, web-02, ... web-16 + hardware: generic-large + workload: profiles/normal-web.yaml + jitter: 0.05 # ±5% per-host variation on all stressor values + + - name: web-degraded + count: 2 + host_prefix: web-degraded + hardware: generic-large + workload: profiles/normal-web.yaml + jitter: 0.02 + anomalies: + - name: cpu-saturation + start_offset: 14400 # fault begins 4 hours in + duration: 7200 # lasts 2 hours + transition: linear # ramp into and out of fault condition + cpu: + utilization: 0.96 + user_ratio: 0.90 + iowait_ratio: 0.06 + disk: + read_mbps: 18.0 + write_mbps: 9.0 + + - name: db-primary + count: 1 + host_prefix: db-primary + hardware: memory-optimized + workload: profiles/normal-db.yaml + + - name: db-replica + count: 3 + host_prefix: db-replica + hardware: memory-optimized + workload: profiles/normal-db.yaml + jitter: 0.03 + anomalies: + - name: memory-pressure + start_offset: 28800 # 8 hours in + duration: 14400 # 4 hours + transition: linear + memory: + used_ratio: 0.91 + cache_ratio: 0.04 +``` + +### 4.1 Anomaly Overlay Rules + +- Only fields explicitly set in the anomaly override the base workload; all others + are inherited unchanged +- Multiple anomalies can be listed per group; they are applied in order and may + overlap in time +- `start_offset` is in seconds from the archive start (`meta.start` or `--start`) +- `transition: linear` ramps the anomaly values in over the first 10% of the + anomaly duration and out over the last 10% (configurable via `transition_ramp`) +- An anomaly with no `duration` runs to the end of the archive + +### 4.2 Per-Host Variation Detail + +Jitter is applied as a multiplicative factor drawn from a Gaussian distribution with +mean 1.0 and standard deviation equal to `jitter`. It is applied independently per +stressor field, per host, at profile load time. The same host name always produces the +same jitter offsets for a given `--seed`. + +``` +effective_value = base_value × Normal(mean=1.0, stddev=jitter) +``` + +Values are clamped to valid ranges after jitter is applied (e.g. ratios to [0.0, 1.0]). + +--- + +## 5. Output Layout + +Archives are written into `meta.output_dir`, organised by group: + +``` +./fleet-archives/ +├── web-01.{0,index,meta} +├── web-02.{0,index,meta} +│ ... +├── web-16.{0,index,meta} +├── web-degraded-01.{0,index,meta} +├── web-degraded-02.{0,index,meta} +├── db-primary-01.{0,index,meta} +├── db-replica-01.{0,index,meta} +├── db-replica-02.{0,index,meta} +├── db-replica-03.{0,index,meta} +└── fleet.manifest +``` + +### 5.1 Fleet Manifest + +`fleet.manifest` is a machine-readable YAML file listing every archive in the fleet, +its group, hardware profile, and whether it carries any anomalies. This allows +downstream tooling to know which archives should be flagged as anomalous during +test assertions. + +```yaml +# fleet.manifest — generated by pmlogsynth fleet +meta: + name: web-cluster + generated: "2024-01-15T09:00:00Z" + pmlogsynth_version: "1.0" + duration: 86400 + interval: 60 + +archives: + - host: web-01 + path: web-01 + group: web-frontend + hardware: generic-large + anomalies: [] + + - host: web-degraded-01 + path: web-degraded-01 + group: web-degraded + hardware: generic-large + anomalies: + - name: cpu-saturation + start_offset: 14400 + duration: 7200 + + ... +``` + +--- + +## 6. CLI Interface + +Fleet generation is a subcommand of `pmlogsynth`: + +``` +pmlogsynth fleet [OPTIONS] FLEET_PROFILE + +Arguments: + FLEET_PROFILE Path to fleet YAML profile + +Options: + -o, --output-dir PATH Override meta.output_dir from profile + --start TIMESTAMP Archive start time for all hosts [default: now - duration] + --seed INT PRNG seed for reproducible jitter and noise + --jobs INT Parallel archive generation workers [default: CPU count] + --dry-run Print what would be generated without writing any output + --validate Validate fleet profile without generating output + -v, --verbose Show per-host progress + -h, --help Show help +``` + +### Examples + +```bash +# Generate a 22-host fleet +pmlogsynth fleet -o ./cluster web-cluster.yaml + +# Reproducible fleet (same seed = same jitter offsets and noise) +pmlogsynth fleet --seed 42 -o ./cluster web-cluster.yaml + +# See what would be generated without writing anything +pmlogsynth fleet --dry-run web-cluster.yaml + +# Generate fleet anchored to a historical window +pmlogsynth fleet --start "2024-01-15 00:00:00 UTC" -o ./cluster web-cluster.yaml + +# Use Phase 2 natural language to generate the fleet profile first +pmlogsynth --prompt \ + "A web cluster: 16 healthy frontend servers on generic-large hardware, \ + 2 with CPU saturation faults starting 4 hours in, and 1 database server \ + on memory-optimized hardware." \ + -o web-cluster.yaml +pmlogsynth fleet -o ./cluster web-cluster.yaml +``` + +### Parallel Generation + +Each host archive is independent and can be generated concurrently. `--jobs` defaults +to the number of available CPU cores, implemented via `concurrent.futures.ProcessPoolExecutor` +from the Python standard library — no additional dependency is required. For large fleets +this makes generation time scale with hardware rather than host count. + +--- + +## 7. Phase 2 Integration: Natural Language Fleet Profiles + +The Phase 2 `--prompt` flag works for fleet profiles as well as single-host profiles. +The system prompt (§5 of the Phase 2 specification) is extended with: + +- The fleet profile schema and its additional fields (`groups`, `anomalies`, `jitter`) +- The anomaly overlay merge rules +- Common fault scenario archetypes (see below) + +Claude infers from context whether the user wants a single-host profile or a fleet +profile and generates accordingly. + +### Fleet Fault Archetypes + +Named fault scenarios that Claude can reference when generating anomaly overlays: + +| Archetype | Description | +|---|---| +| `cpu-saturation` | CPU utilisation at ceiling; user space dominant; iowait elevated | +| `memory-pressure` | RAM nearly exhausted; cache evicted; iowait rising | +| `disk-saturation` | Write throughput at device limit; iowait dominates CPU time | +| `network-degraded` | rx/tx throughput well below interface capacity | +| `noisy-neighbour` | CPU steal elevated (virtualisation contention) | +| `slow-drain` | Gradual linear degradation across all domains over archive lifetime | + +--- + +## 8. Referencing Fleet Archives from Tests + +The fleet manifest enables straightforward test assertions in any test framework: + +```bash +# Shell example: verify a fleet-aware analysis tool flags the right hosts +pmlogsynth fleet --seed 42 -o ./cluster web-cluster.yaml + +# Run the tool under test against the fleet +pcp-fleet-analyser --archives ./cluster --manifest ./cluster/fleet.manifest \ + --flag-threshold cpu:0.90 > ./results.txt + +# Assert that exactly the anomalous hosts were flagged +expected=$(grep -l 'cpu-saturation' ./cluster/fleet.manifest | wc -l) +got=$(wc -l < ./results.txt) +[ "$expected" -eq "$got" ] || fail "Expected $expected flagged hosts, got $got" +``` + +In `pytest`: + +```python +# tests/test_fleet_integration.py + +def test_fleet_anomalous_hosts_are_identifiable(tmp_path): + subprocess.run( + ["pmlogsynth", "fleet", "--seed", "42", "-o", str(tmp_path), "tests/fixtures/web-cluster.yaml"], + check=True, + ) + manifest = yaml.safe_load((tmp_path / "fleet.manifest").read_text()) + anomalous = [a for a in manifest["archives"] if a["anomalies"]] + assert len(anomalous) == 2 + assert all(a["group"] == "web-degraded" for a in anomalous) +``` + +--- + +## 9. Project Layout Changes + +Phase 3 adds the following to the Phase 1 + Phase 2 layout: + +``` +pmlogsynth/ +├── fleet.py # fleet profile loader, group expander, manifest writer +├── overlay.py # anomaly overlay merge logic +├── jitter.py # per-host deterministic value variation +└── profiles/ + └── fleet/ # example fleet profiles (package data) + ├── small-web-cluster.yaml + └── mixed-db-web.yaml + +tests/ +├── test_fleet.py # fleet profile loading, overlay merging, jitter +├── test_fleet_integration.py # end-to-end fleet generation (requires PCP installed) +└── fixtures/ + └── web-cluster.yaml # reference fleet profile used in tests +``` + +The example fleet profiles in `pmlogsynth/profiles/fleet/` are installed as package +data alongside the bundled hardware profiles. + +--- + +## 10. Test Requirements + +### Tier 1 — unit tests (no PCP required) + +- Fleet profile loading and validation +- Anomaly overlay merge logic (field precedence, time window application) +- Jitter reproducibility: same host name + same seed → same offsets +- Jitter clamping: ratio fields stay in [0.0, 1.0] after jitter +- `--dry-run` output matches expected host list and group assignments + +### Tier 2 — integration tests (PCP must be installed) + +- Generate a small fleet (3–5 hosts) from the reference fixture +- Assert `fleet.manifest` is written and well-formed YAML +- Assert each archive passes `pmlogcheck` +- Assert anomalous hosts in the manifest match the fleet profile definition +- Assert `--seed` reproducibility: two runs with the same seed produce byte-identical archives + +Tier 2 tests are skipped automatically if `pmlogcheck` is not found on `PATH`. + +```bash +# Run all tests +pytest + +# Run only fleet tests +pytest tests/test_fleet.py tests/test_fleet_integration.py + +# Run only unit tests (no PCP needed) +pytest -m "not integration" +``` + +--- + +## 11. Future Enhancements + +| Item | Notes | +|------|-------| +| Rolling / cascading faults | Fault on host A triggers fault onset on host B after a delay | +| Fleet profile via Phase 2 `--prompt` refinement (`--refine`) | Adjust specific groups after initial generation | +| Heterogeneous archive durations per group | Different groups covering different time windows | +| Archive merging | Combine fleet archives into a single multi-host archive (if PCP adds multi-host support) | diff --git a/qa/1352 b/qa/1352 index c64c1ed137..8bd22c894e 100755 --- a/qa/1352 +++ b/qa/1352 @@ -1,5 +1,5 @@ #!/bin/sh -# PCP QA Test No. 1346 +# PCP QA Test No. 1352 # dpkg packaging botches for old conffiles # # Copyright (c) 2021 Ken McDonell. All Rights Reserved. diff --git a/qa/1882 b/qa/1882 index a2068c80d5..7c51dfed8b 100755 --- a/qa/1882 +++ b/qa/1882 @@ -25,7 +25,7 @@ _cleanup() } status=1 # failure is the default! -$sudo rm -rf $tmp $tmp.* $seq.full +$sudo rm -rf $tmp $tmp.* $seq_full trap "_cleanup; exit \$status" 0 1 2 3 15 _filter_valkey() @@ -56,7 +56,7 @@ $sudo sh -c "echo 'host=127.0.0.1' > $pmda_path/valkey.conf" $sudo sh -c "echo 'port=$valkey_port' >> $pmda_path/valkey.conf" echo "== Installing valkey PMDA ==" -_prepare_pmda_install valkey +_prepare_pmda valkey cd $PCP_PMDAS_DIR/valkey $sudo ./Install $tmp.out 2>&1 cat $tmp.out >>$seq_full @@ -89,7 +89,7 @@ pmprobe -v valkey.stats.total_commands 2>&1 | _filter_valkey echo echo "== Restoring valkey PMDA ==" -_restore_pmda_install valkey +# done via _cleanup_pmda() in _cleanup() # success, all done status=0 diff --git a/qa/1883 b/qa/1883 index 3510c0783d..93aae7913e 100755 --- a/qa/1883 +++ b/qa/1883 @@ -62,7 +62,7 @@ $sudo sh -c "echo 'host=127.0.0.1' > $pmda_path/valkey.conf" $sudo sh -c "echo 'port=$valkey_port' >> $pmda_path/valkey.conf" echo "== Installing valkey PMDA ==" -_prepare_pmda_install valkey +_prepare_pmda valkey cd $PCP_PMDAS_DIR/valkey $sudo ./Install $tmp.out 2>&1 cat $tmp.out >>$seq_full @@ -132,7 +132,7 @@ fi echo echo "== Restoring valkey PMDA ==" -_restore_pmda_install valkey +# done via _cleanup_pmda() in _cleanup() # success, all done status=0 diff --git a/qa/1998 b/qa/1998 old mode 100644 new mode 100755 index 1a4f6461db..11a94dcf80 --- a/qa/1998 +++ b/qa/1998 @@ -45,18 +45,18 @@ pmdards_remove() { cd $pmda_path echo - echo "=== Removing RDS agent ===" - $sudo ./Remove >$tmp.out 2>&1 + echo "=== Removing RDS agent ===" | tee -a $seq_full + $sudo ./Remove >>$seq_full 2>&1 } pmdards_install() { cd $pmda_path - $sudo ./Remove >/dev/null 2>&1 + $sudo ./Remove >>$seq_full 2>&1 echo - echo "=== Installing RDS agent ===" - $sudo ./Install $tmp.out 2>&1 + echo "=== Installing RDS agent ===" | tee -a $seq_full + $sudo ./Install >$seq_full 2>&1 cd $here } @@ -65,14 +65,25 @@ _prepare_pmda rds pmdards_install +if ! pminfo -f pmcd.agent.status 2>&1 | grep -q '"rds"' +then + echo "Arrgh! PMDA install failed ... see $seq.full" + _exit 1 +fi + echo "=== Report metric values ===" metrics=`pminfo rds | LC_COLLATE=POSIX sort` - -pminfo -dfmtT $metrics 2>&1 +echo "metric=$metrics" >>$seq_full +if [ -z "$metrics" ] +then + echo "Botch: no rds metrics in the PMNS" +else + pminfo -dfmtT $metrics 2>&1 +fi pmdards_remove # Success, all done status=0 -exit \ No newline at end of file +exit diff --git a/qa/662 b/qa/662 index 03e87e4999..c22bdee110 100755 --- a/qa/662 +++ b/qa/662 @@ -38,7 +38,7 @@ _cleanup() _sighup_pmcd fi _restore_auto_restart pmproxy - $pmproxy_was_running && _service pmproxy restart >$seq_full 2>&1 + $pmproxy_was_running && _service pmproxy restart >>$seq_full 2>&1 $sudo rm -f $tmp.* } @@ -66,7 +66,25 @@ PCP_DERIVED_CONFIG="$PCP_VAR_DIR/config/derived"; export PCP_DERIVED_CONFIG if ! _service pmproxy restart >/dev/null 2>&1; then _exit 1; fi +echo "+++ pmproxy before test_webapi.py ..." >>$seq_full +_ps_full_by_name pmproxy >>$seq_full $python $here/src/test_webapi.py | _webapi_response_filter | _filter_labels +echo "+++ pmproxy after test_webapi.py ..." >>$seq_full +_ps_full_by_name pmproxy >>$seq_full + +# metrics from src/test_webapi.py are enumerated in these files +# webapi-host and webapi-pmcd-host, and +# webapi-host-kernel and webapi-pmcd-host-kernel +# +# help triage by checking lists of metrics ... +# +LC_COLLATE=POSIX; export LC_COLLATE +echo "+++ webapi-host vs webapi-pmcd-host diffs ..." >>$seq_full +sed -e 's/ \[.*//' $tmp.tmp +sort webapi-host | diff $tmp.tmp - >>$seq_full +echo "+++ webapi-host-kernel vs webapi-pmcd-host-kernel diffs ..." >>$seq_full +sed -e 's/ \[.*//' $tmp.tmp +sort webapi-host-kernel | diff $tmp.tmp - >>$seq_full echo >>$seq_full echo "=== pmproxy log ===" >>$seq_full diff --git a/qa/admin/list-packages b/qa/admin/list-packages index ca2bfbffbf..897c8d734e 100755 --- a/qa/admin/list-packages +++ b/qa/admin/list-packages @@ -663,7 +663,7 @@ fi if $missing then - # special handling for cpan() packages + # special handling for cpan() and pip3() packages # $very_verbose && cp $tmp.list $tmp.list.before rm -f $tmp.sed @@ -676,13 +676,35 @@ then echo "/cpan($module)/d" >>$tmp.sed fi done + grep '^pip3(' $tmp.list \ + | sed -e 's/pip3(//' -e 's/)//' \ + | while read module + do + if which python3 >/dev/null 2>&1 + then + python=python3 + elif which pmpython >/dev/null 2>&1 + then + python=pmpython + else + python='' + $very_verbose && echo >&2 "Warning cannot file python" + fi + if [ -n "$python" ] + then + if $python -c "import $module" >/dev/null 2>&1 + then + echo "/pip3($module)/d" >>$tmp.sed + fi + fi + done if [ -f $tmp.sed ] then sed -f $tmp.sed <$tmp.list >$tmp.tmp mv $tmp.tmp $tmp.list if $very_verbose then - echo >&2 "Diffs after cpan() mangling ..." + echo >&2 "Diffs after cpan() and pip3() mangling ..." diff $tmp.list.before $tmp.list fi fi diff --git a/qa/admin/other-packages/check-manifest b/qa/admin/other-packages/check-manifest new file mode 100755 index 0000000000..b13ebcbc48 --- /dev/null +++ b/qa/admin/other-packages/check-manifest @@ -0,0 +1,56 @@ +#!/bin/sh +# +# Look for incomplete blocks in the manifest +# + +tmp=/var/tmp/whack-manifest-$$ +trap "rm -f $tmp.*; exit 0" 0 1 2 3 15 + +# pass 1 .. build block start and end index +# +awk $tmp.map ' +BEGIN { start = 0 } +/^# -- / { if (start) + print start,NR-1,tag + start = NR + tag = $3 + for (i = 4; i <= NF; i++) + tag = tag " " $i + next + } +NF == 0 { if (start) + print start,NR-1,tag + start = 0 + } +END { if (start) + print start,NR-1,tag + }' + +# pass 2 ... find blocks that do not have lines for all platforms +# or multiple lines for the same platform +# +cat $tmp.map \ +| while read start end tag +do + awk 1) + printf "%d,%d (%s): extra %s\n",'"$start"','"$end"',"'"$tag"'",p + } + }' +done diff --git a/qa/admin/other-packages/manifest b/qa/admin/other-packages/manifest index 37451fb6f6..a0508d6f4f 100644 --- a/qa/admin/other-packages/manifest +++ b/qa/admin/other-packages/manifest @@ -46,7 +46,7 @@ # # common executables # -# --bash(1) +# -- bash(1) dpkg? bash [bash] rpm? bash [bash] emerge? bash [bash] @@ -57,7 +57,7 @@ S_pkg? bash [shell/bash] slackpkg? bash [bash] pacman? bash [core/bash] brew? bash [bash] -# --sed(1) +# -- sed(1) dpkg? sed [sed] rpm? sed [sed] emerge? sed [sed] @@ -68,7 +68,7 @@ S_pkg? sed [text/gnu-sed] slackpkg? sed [sed] pacman? sed [core/sed] brew? sed [sed] -# --grep(1) +# -- grep(1) dpkg? grep [grep] rpm? grep [grep] emerge? grep [grep] @@ -79,7 +79,7 @@ S_pkg? grep [text/gnu-grep] slackpkg? grep [grep] pacman? grep [core/grep] brew? grep [grep] -# --ed(1) +# -- ed(1) dpkg? ed [ed] rpm? ed [ed] emerge? ed [ed] @@ -90,7 +90,7 @@ S_pkg? ed [SUNWcs] slackpkg? ed [ed] pacman? ed [extra/ed] brew? ed [ed] -# --make(1) +# -- make(1) dpkg? make [make] rpm? make [make] emerge? make [make] @@ -101,7 +101,7 @@ S_pkg? gmake [developer/build/gnu-make] slackpkg? make [make] pacman? make [core/make] brew? make [make] -# --flex(1) +# -- flex(1) dpkg? flex [flex] rpm? flex [flex] emerge? flex [flex] @@ -199,7 +199,7 @@ pkg_add? man [base OpenBSD install (QA optional)] F_pkg? man [base FreeBSD install (QA optional)] S_pkg? man [system/man (QA optional)] slackpkg? man [man-db (QA optional)] -pacman? man [? (QA optional)] +pacman? man [man-db (QA optional)] brew? man [? (QA optional)] # NetBSD/OpenBSD/FreeBSD specials @@ -272,7 +272,7 @@ pkg_add? python3 [python] F_pkg? python3.9|python3 [python39 or python3] S_pkg? python3 [runtime/python-39] slackpkg? python3 [python3] -pacman? python3 [?] +pacman? python3 [python] brew? python3 [python] # -- pylint dpkg? pylint [pylint (build optional)] @@ -294,7 +294,7 @@ pkg_add? mandoc [mandoc or base OpenBSD install (build optional)] F_pkg? mandoc [mandoc or base FreeBSD install (build optional)] S_pkg? mandoc [N/A (build optional)] slackpkg? mandoc [mandoc (build optional)] -pacman? mandoc [N/A (build optional)] +pacman? mandoc [extra/mandoc (build optional)] brew? mandoc [mandoc (build optional)] # -- cppcheck for "make check" dpkg? cppcheck [cppcheck (build optional)] @@ -358,7 +358,7 @@ pkg_add? umad.h [N/A (build optional)] F_pkg? umad.h [N/A (build optional)] S_pkg? /usr/include/sys/ib/clients/of/sol_umad/sol_umad.h [system/header (build optional)] slackpkg? umad.h [N/A (build optional)] -pacman? umad.h [N/A (build optional)] +pacman? /usr/include/infiniband/umad.h [extra/rdma-core (build optional)] brew? umad.h [N/A (build optional)] dpkg? /usr/include/infiniband/mad.h [libibmad-dev (build optional)] rpm? /usr/include/infiniband/mad.h [infiniband-diags-devel or libibmad-devel or rdma-core-devel (build optional)] @@ -368,7 +368,7 @@ pkg_add? /usr/local/include/mad.h [libmad (build optional)] F_pkg? mad.h [N/A (build optional)] S_pkg? /usr/include/sys/ib/clients/of/rdma/ib_user_mad.h [system/header (build optional)] slackpkg? mad.h [N/A (build optional)] -pacman? /usr/include/mad.h [extra/libmad (build optional)] +pacman? /usr/include/infiniband/mad.h [extra/libmad (build optional)] brew? mad.h [N/A (build optional)] dpkg? /usr/include/infiniband/verbs.h [libibverbs-dev (build optional)] rpm? /usr/include/infiniband/verbs.h [rdma-core-devel or libibverbs-devel (build optional)] @@ -378,7 +378,7 @@ pkg_add? verbs.h [N/A (build optional)] F_pkg? verbs.h [N/A (build optional)] S_pkg? verbs.h [N/A (build optional)] slackpkg? verbs.h [N/A (build optional)] -pacman? verbs.h [N/A (build optional)] +pacman? /usr/include/infiniband/verbs.h [rdma-core (build optional)] brew? verbs.h [N/A (build optional)] # -- avahi API dpkg? /usr/include/avahi-common/defs.h [libavahi-common-dev (build optional)] @@ -398,7 +398,7 @@ emerge? /usr/include/perfmon/pfmlib_perf_event.h [dev-libs/libpfm (build optiona F_pkg? pfmlib_perf_event.h [N/A (build optional)] S_pkg? pfmlib_perf_event.h [N/A (build optional)] slackpkg? pfmlib_perf_event.h [N/A (build optional)] -pacman? /usr/include/perfmon/pfmlib_perf_event.h [N/A (build optional)] +pacman? /usr/include/perfmon/pfmlib_perf_event.h [extra/libpfm (build optional)] brew? pfmlib_perf_event.h [N/A (build optional)] # -- gcc C++ compiler dpkg? g++ [g++] @@ -507,6 +507,7 @@ brew? /usr/local/lib/python3.*/*-packages/openpyxl/conftest.py [pip3(openpyxl) ( # -- bpftrace dpkg? bpftrace [bpftrace (build optional)] rpm? bpftrace [bpftrace (build optional)] +pacman? bpftrace [extra/bpftrace (build optional)] # -- python pillow dpkg? /usr/lib/python3/*-packages/PIL/Image.py [python3-pil (QA optional)] rpm? /usr/lib*/python3.*/*-packages/PIL/Image.py [python3[0-9]*-pillow or python3[0-9]*-Pillow (QA optional)] @@ -531,9 +532,18 @@ brew? /usr/local/lib/python3.*/*-packages/psycopg2 [pip3(psycopg2)] # -- python pyodbc (mssql PMDA) rpm? /usr/lib*/python3.*/*-packages/pyodbc* [python3-pyodbc] dpkg? /usr/lib/python3/dist-packages/pyodbc.pyi [python3-pyodbc] +pacman? /usr/lib*/python3.*/*-packages/pyodbc [N/A] # -- python pymongo (mongodb PMDA) -rpm? /usr/lib*/python3.*/*-packages/pymongo* [python3[0-9]*-pymongo] dpkg? /usr/lib/python3/dist-packages/pymongo [python3-pymongo] +rpm? /usr/lib*/python3.*/*-packages/pymongo* [python3[0-9]*-pymongo] +emerge? ? [? {Gentoo}] +pkgin? ? [? {NetBSD}] +pkg_add? ? [? {OpenBSD}] +F_pkg? ? [? {FreeBSD}] +S_pkg? ? [? {Solaris}] +slackpkg? ? [? {Slackware}] +pacman? /usr/lib*/python3.*/*-packages/pymongo [extra/python-pymongo] +brew? ? [? {MacOS?}] # perl modules # -- Time::HiRes dpkg? Time::HiRes [libperl[0-9][0-9.]* or libperl[0-9][0-9.]*t64] @@ -638,7 +648,6 @@ S_pkg? File::Slurp [library/perl-5/file-slurp (QA optional)] slackpkg? File::Slurp [cpan(File::Slurp) (QA optional)] pacman? File::Slurp [extra/perl-file-slurp or cpan(File::Slurp) (QA optional)] brew? File::Slurp [base Darwin install (QA optional)] -# -- Slurm::Hostlist ... skip this one, it comes in the pcp-testsuite package # -- List::MoreUtils dpkg? List::MoreUtils [liblist-moreutils-perl (QA optional)] rpm? List::MoreUtils [perl-List-MoreUtils or cpan(List::MoreUtils) (QA optional)] @@ -762,7 +771,16 @@ slackpkg? Net::SNMP [cpan(Net::SNMP)] pacman? Net::SNMP [extra/perl-net-snmp] brew? Net::SNMP [base Darwin install] # -- Archive::Zip -F_pkg? Archive::Zip [p5-Archive-Zip] +dpkg? ? [? {dpkg-based, Debian, Ubuntu, LinuxMint, MX, ...}] +rpm? ? [? {rpm-based, RHEL, Fedora, OpenSuSE, ...}] +emerge? ? [? {Gentoo}] +pkgin? ? [? {NetBSD}] +pkg_add? ? [? {OpenBSD}] +F_pkg? Archive::Zip [p5-Archive-Zip] +S_pkg? ? [? {Solaris}] +slackpkg? ? [? {Slackware}] +pacman? Archive::Zip [extra/perl-archive-zip] +brew? ? [? {MacOS?}] # # other run-time # @@ -897,7 +915,7 @@ pkg_add? kubectl [N/A (build optional)] F_pkg? kubectl [kubectl (build optional)] S_pkg? kubectl [N/A (build optional)] slackpkg? kubectl [N/A (build optional)] -pacman? kubectl [N/A (build optional)] +pacman? kubectl [extra/kubectl (build optional)] brew? kubectl [N/A (build optional)] # -- kubelet dpkg? kubelet [N/A (build optional)] @@ -908,7 +926,7 @@ pkg_add? kubelet [N/A (build optional)] F_pkg? kubelet [N/A (build optional)] S_pkg? kubelet [N/A (build optional)] slackpkg? kubelet [N/A (build optional)] -pacman? kubelet [N/A (build optional)] +pacman? kubelet [extra/kubelet (build optional)] brew? kubelet [N/A (build optional)] # -- crontab dpkg? crontab [cron] @@ -985,7 +1003,7 @@ pkg_add? /usr/local/lib/python3.*/*-packages/pandas [py3-pandas (QA optional)] F_pkg? /usr/local/lib/python3.*/*-packages/pandas [py3[0-9]*-pandas (QA optional)] S_pkg? ? [N/A] slackpkg? ? [?] -pacman? ? [?] +pacman? /usr/lib/python3.*/*-packages/pandas [extra/python-pandas] brew? ? [?] # -- python-pyarrow dpkg? ? [N/A] @@ -996,7 +1014,7 @@ pkg_add? ? [N/A] F_pkg? /usr/local/lib/python3.*/*-packages/pyarrow [py3[0-9]*-pyarrow (QA optional)] S_pkg? ? [N/A] slackpkg? ? [?] -pacman? ? [?] +pacman? /usr/lib/python3.*/*-packages/pyarrow [extra/python-pyarrow] brew? ? [?] # -- logconf for ds389 PMDA rpm? /usr/bin/logconv.pl|/usr/lib/389-ds/bin/logconv.pl [389-ds-base or 389-ds (QA optional)] @@ -1075,7 +1093,7 @@ pkg_add? zabbix_agentd [zabbix-agent (QA optional)] F_pkg? zabbix_agentd [zabbix[0-9][0-9]*-agent (QA optional)] S_pkg? zabbix_agentd [N/A (QA optional)] slackpkg? zabbix_agentd [N/A (QA optional)] -pacman? zabbix_agentd [N/A (QA optional)] +pacman? zabbix_agentd [extra/zabbix-agent (QA optional)] brew? zabbix_agentd [N/A (QA optional)] # -- mysql dpkg? mysql [mariadb-client-core or mariadb-client-core-[0-9][0-9.]* or mysql-client-core or mysql-client-core-[0-9][0-9.]* (QA optional)] @@ -1338,10 +1356,19 @@ pkg_add? valkey-server [N/A (QA optional)] F_pkg? valkey-server [N/A (QA optional)] S_pkg? valkey-server [N/A (QA optional)] slackpkg? valkey-server [N/A (QA optional)] -pacman? valkey-server [community/valkey (QA optional)] +pacman? valkey-server [extra/valkey (QA optional)] brew? valkey-server [valkey (QA optional)] # -- valkeysearch (pmproxy, pmsearch QA) +dpkg? valkeysearch.so [? {dpkg-based, Debian, Ubuntu, LinuxMint, MX, ...}] rpm? /usr/lib*/valkey/modules/valkeysearch.so [ValkeySearch (QA optional)] +emerge? valkeysearch.so [? {Gentoo}] +pkgin? valkeysearch.so [? {NetBSD}] +pkg_add? valkeysearch.so [? {OpenBSD}] +F_pkg? valkeysearch.so [? {FreeBSD}] +S_pkg? valkeysearch.so [? {Solaris}] +slackpkg? valkeysearch.so [? {Slackware}] +pacman? valkeysearch.so [? {Arch Linux}] +brew? valkeysearch.so [? {MacOS?}] # -- redis (pmproxy, pmseries QA) dpkg? redis-server [redis-server (QA optional)] dpkg? redis-cli [redis-cli or redis-tools (QA optional)] @@ -1352,7 +1379,7 @@ pkg_add? redis-server [N/A (QA optional)] F_pkg? redis-server [N/A (QA optional)] S_pkg? redis-server [database/redis (QA optional)] slackpkg? redis-server [N/A (QA optional)] -pacman? redis-server [extra/redis (QA optional)] +pacman? redis-server [N/A (QA optional)] brew? redis-server [redis (QA optional)] # -- redisearch (pmproxy, pmsearch QA) dpkg? /usr/lib*/redis/modules/redisearch.so [redis-redisearch (QA optional)] @@ -1467,7 +1494,7 @@ pkg_add? libbpf.so|bpf.h [N/A (build optional)] F_pkg? libbpf.so [N/A (build optional)] S_pkg? ? [N/A (build optional)] slackpkg? ? [N/A (build optional)] -pacman? ? [? (build optional)] +pacman? /usr/lib/libbpf.so [core/libbpf (build optional)] brew? ? [? (QA optional)] # -- expect (for bpf PMDA QA) @@ -1480,7 +1507,7 @@ pkg_add? expect [N/A (QA optional)] # no bpf PMDA here F_pkg? expect [expect (QA optional)] S_pkg? expect [shell/expect (QA optional)] slackpkg? expect [expect (QA optional)] -pacman? expect [? (QA optional)] +pacman? expect [extra/expect (QA optional)] brew? expect [? (QA optional)] # -- llvm-strip (for bpf PMDA) @@ -1493,7 +1520,7 @@ pkg_add? llvm-strip [N/A (build optional)] # no bpf PMDA here F_pkg? llvm-strip [N/A (build optional)] S_pkg? llvm-strip [N/A (build optional)] slackpkg? llvm-strip [N/A (build optional)] -pacman? llvm-strip [? (build optional)] +pacman? llvm-strip [extra/llvm (build optional)] brew? llvm-strip [? (build optional)] # -- clang (for bpf PMDA) @@ -1506,7 +1533,7 @@ pkg_add? clang [N/A (build optional)] # no bpf PMDA here F_pkg? clang [N/A (build optional)] # no bpf PMDA here S_pkg? clang [N/A (build optional)] # no bpf PMDA here slackpkg? clang [N/A (build optional)] -pacman? clang [? (build optional)] +pacman? clang [extra/clang (build optional)] brew? clang [? (build optional)] # -- libcmocka (for url encode/decode QA) @@ -1519,7 +1546,7 @@ pkg_add? /usr/local/lib/libcmocka.so.* [cmocka (QA optional)] F_pkg? /usr/local/lib/libcmocka.so [cmocka (QA optional)] S_pkg? ? [N/A (QA optional)] slackpkg? ? [N/A (QA optional)] -pacman? ? [? (QA optional)] +pacman? /usr/lib/libcmocka.so [extra/cmocka (QA optional)] brew? ? [? (QA optional)] # -- libdrm (for amdgpu PMDA) @@ -1532,7 +1559,7 @@ F_pkg? libdrm.so [N/A (QA optional)] S_pkg? /usr/lib/xorg/*/libdrm.so.* [x11/library/libdrm (QA optional)] S_pkg? /usr/include/drm/drm.h [system/header/header-drm] slackpkg? ? [N/A (QA optional)] -pacman? ? [? (QA optional)] +pacman? /usr/lib/libdrm.so [extra/libdrm (QA optional)] brew? ? [? (QA optional)] # -- libinih (optional for libpcp_web) @@ -1558,7 +1585,7 @@ pkg_add? jq [jq (QA optional)] F_pkg? jq [jq (QA optional)] S_pkg? jq [text/jq (QA optional)] slackpkg? jq [N/A (QA optional)] -pacman? jq [? (QA optional)] +pacman? jq [extra/jq (QA optional)] brew? jq [? (QA optional)] # -- bpftrace-dbgsym (for bpftrace PMDA QA?) @@ -1588,7 +1615,7 @@ pkg_add? sudo [sudo {OpenBSD}] F_pkg? sudo [sudo {FreeBSD}] S_pkg? sudo [security/sudo {Solaris}] slackpkg? sudo [? {Slackware}] -pacman? sudo [? {Arch Linux}] +pacman? sudo [core/sudo] brew? sudo [? {MacOS?}] # -- targetcli (for lio PMDA QA) @@ -1601,7 +1628,7 @@ pkg_add? targetcli [N/A] F_pkg? targetcli [N/A] S_pkg? targetcli [N/A] slackpkg? targetcli [? {Slackware}] -pacman? targetcli [? {Arch Linux}] +pacman? targetcli [N/A] brew? targetcli [N/A] # -- ethtool (for rocestat PMDA) @@ -1614,7 +1641,7 @@ pkg_add? ethtool [N/A] F_pkg? ethtool [N/A] S_pkg? ethtool [N/A] slackpkg? ethtool [? {Slackware}] -pacman? ethtool [? {Arch Linux}] +pacman? ethtool [extra/ethtool] brew? ethtool [? {MacOS?}] # -- ibdev2netdev (for rocestat PMDA) @@ -1627,9 +1654,22 @@ pkg_add? ibdev2netdev [N/A] F_pkg? ibdev2netdev [N/A] S_pkg? ibdev2netdev [N/A] slackpkg? ibdev2netdev [? {Slackware}] -pacman? ibdev2netdev [? {Arch Linux}] +pacman? ibdev2netdev [N/A] brew? ibdev2netdev [? {MacOS?}] +# -- qshape (for postfix PMDA) +dpkg? ? [? {dpkg-based, Debian, Ubuntu, LinuxMint, MX, ...}] +rpm? /usr/sbin/qshape [postfix-perl-scripts] +suse? /usr/share/doc/packages/postfix-doc/auxiliary/qshape/qshape.pl [postfix-doc] +emerge? ? [? {Gentoo}] +pkgin? ? [? {NetBSD}] +pkg_add? ? [? {OpenBSD}] +F_pkg? /usr/local/bin/qshape [postfix] +S_pkg? ? [? {Solaris}] +slackpkg? ? [? {Slackware}] +pacman? /usr/bin/qshape [extra/postfix] +brew? ? [? {MacOS?}] + # -- /usr/share/pkgconfig/systemd.pc needed for configure testing to # set systemd paths # @@ -1638,11 +1678,6 @@ rpm? /usr/share/pkgconfig/systemd.pc [systemd] # misc # -rpm? /usr/sbin/qshape [postfix-perl-scripts] -mandriva? /usr/sbin/qshape [postfix] -suse? /usr/share/doc/packages/postfix-doc/auxiliary/qshape/qshape.pl [postfix-doc] -arch? /usr/bin/qshape [N/A (build optional)] -F_pkg? /usr/local/bin/qshape [postfix] F_pkg? gdiff [diffutils] netbsd? gdiff [diffutils] pkg_add? gdiff [gdiff] diff --git a/qa/admin/package-lists/ArchLinux++x86_64 b/qa/admin/package-lists/ArchLinux++x86_64 index f4300ce7b8..bd9c4a4ba7 100644 --- a/qa/admin/package-lists/ArchLinux++x86_64 +++ b/qa/admin/package-lists/ArchLinux++x86_64 @@ -15,6 +15,9 @@ core/bash extra/bc extra/bind core/bison +extra/bpftrace +extra/clang +extra/cmocka core/coreutils extra/cppcheck extra/cronie @@ -22,6 +25,8 @@ core/curl core/device-mapper extra/docker extra/ed +extra/ethtool +extra/expect core/flex core/gawk core/gcc @@ -30,14 +35,22 @@ extra/git core/grep core/icu core/iproute2 +extra/jq +extra/kubectl +extra/kubelet +core/libbpf +extra/libdrm core/libinih extra/libmad +extra/libpfm core/libsasl extra/libuv extra/libvirt extra/libvirt-python +extra/llvm extra/lm_sensors core/make +extra/mandoc extra/mariadb-clients extra/memcached extra/mesa @@ -45,6 +58,7 @@ core/ncurses core/net-tools extra/nmap core/perl +extra/perl-archive-zip extra/perl-cpanplus extra/perl-dbd-mysql extra/perl-file-slurp @@ -55,14 +69,18 @@ extra/perl-timedate extra/perl-xml-libxml extra/perl-yaml-libyaml core/pkgconf +extra/postfix extra/postgresql extra/postgresql-libs core/psmisc core/python extra/python-defusedxml extra/python-elasticsearch +extra/python-pymongo extra/python-openpyxl extra/python-pillow +extra/python-pandas +extra/python-pyarrow extra/python-prometheus_client extra/python-psycopg2 extra/python-pylint @@ -71,19 +89,22 @@ extra/python-setuptools extra/qt5-base extra/qt5-svg core/readline -extra/redis +extra/rdma-core extra/rrdtool core/sed extra/smartmontools core/s-nail extra/socat core/sqlite +core/sudo extra/sysstat core/systemd extra/time extra/unbound extra/valgrind +extra/valkey core/xfsprogs extra/xkeyboard-config core/xz +extra/zabbix-agent core/zstd diff --git a/qa/admin/package-lists/Ubuntu+24.04+x86_64 b/qa/admin/package-lists/Ubuntu+24.04+x86_64 index d8de591b38..d35fd54fd1 100644 --- a/qa/admin/package-lists/Ubuntu+24.04+x86_64 +++ b/qa/admin/package-lists/Ubuntu+24.04+x86_64 @@ -106,6 +106,7 @@ libperl-dev pkg-config postgresql-client-common psmisc +pyarrow pip3 pylint python3-all python3-all-dev diff --git a/qa/common.check b/qa/common.check index 9866582ab6..6b7ca5cbf3 100644 --- a/qa/common.check +++ b/qa/common.check @@ -421,11 +421,21 @@ _prepare_pmda() __agent=$1 __names=$2 + if [ -n "$__install_on_cleanup" ] + then + echo "Botch: _prepare_pmda($__agent,$__iopts) called twice" + _exit 1 + fi + iam=$__agent [ "X$__names" = "X" ] && __names=$iam __done_clean=false - __install_on_cleanup=false - pminfo $__names >/dev/null 2>&1 && __install_on_cleanup=true + if grep -q "^$iam[ ]" $PCP_PMCDCONF_PATH + then + __install_on_cleanup=true + else + __install_on_cleanup=false + fi echo "_prepare_pmda(agent=$__agent, names=$__names) __install_on_cleanup=$__install_on_cleanup" >>$seq_full # copy the pmcd config file to restore state later. _save_config $PCP_PMCDCONF_PATH @@ -437,9 +447,14 @@ _cleanup_pmda() __agent=$1 __iopts=$2 - if $__done_clean + if [ -z "$__done_clean" ] + then + echo "Botch: _cleanup_pmda($__agent,$__iopts) called with no prior _prepare_pmda() call" + _exit 1 + elif "$__done_clean" then - echo "_cleanup_pmda($__agent,$__iopts) called twice?" >>$seq_full + echo "Botch: _cleanup_pmda($__agent,$__iopts) called twice" + _exit 1 else _restore_config $PCP_PMCDCONF_PATH _service pmcd restart 2>&1 | _filter_pcp_restart @@ -447,7 +462,7 @@ _cleanup_pmda() _restore_auto_restart pmcd _service pmlogger restart 2>&1 | _filter_pcp_restart _wait_for_pmlogger - if $__install_on_cleanup + if "$__install_on_cleanup" then [ "X$__iopts" = "X" ] && __iopts=/dev/null ( cd $PCP_PMDAS_DIR/$__agent; $sudo ./Install <$__iopts >/dev/null 2>&1 ) diff --git a/scripts/rebuild b/scripts/rebuild index 34f5920f36..ce664ff61c 100755 --- a/scripts/rebuild +++ b/scripts/rebuild @@ -23,6 +23,13 @@ then exit fi +# Optional local patching ... maybe required for vendor code when we're +# waiting for an upstream fix that's otherwise killing the build. +# If the script local.patch exists it is per-host and *NOT* expected +# to be in the git tree. +# +[ -x local.patch ] && ./local.patch + quick=false while [ $# -gt 0 ] do diff --git a/src/GNUmakefile b/src/GNUmakefile index 27af504658..16a7179263 100644 --- a/src/GNUmakefile +++ b/src/GNUmakefile @@ -31,6 +31,7 @@ LIBS_SUBDIRS = \ libpcp_qed \ libpcp_qmc \ libpcp_qwt \ + libvalkey \ libpcp_web \ libpcp_fault \ # diff --git a/src/libpcp/src/auxconnect.c b/src/libpcp/src/auxconnect.c index e42cd7f23b..4cac85c147 100644 --- a/src/libpcp/src/auxconnect.c +++ b/src/libpcp/src/auxconnect.c @@ -321,8 +321,8 @@ __pmStringToSockAddr(const char *cp) else #endif if (strchr(cp, ':') != NULL) { - char *cp1; - char *scope; + char *cp1; + const char *scope; /* * inet_pton(3) does not support the "%" extension for specifying the * scope of a link-local address. If one is present, then strip it out and diff --git a/src/libpcp/src/context.c b/src/libpcp/src/context.c index ac2d79c0ef..a7197b1ca4 100644 --- a/src/libpcp/src/context.c +++ b/src/libpcp/src/context.c @@ -886,7 +886,7 @@ initarchive(__pmContext *ctxp, const char *name) int i; int sts; char *namelist = NULL; - const char *current; + char *current; char *end; __pmArchCtl *acp; __pmMultiLogCtl *mlcp = NULL; diff --git a/src/libpcp/src/discovery.c b/src/libpcp/src/discovery.c index b79510c639..561457fb85 100644 --- a/src/libpcp/src/discovery.c +++ b/src/libpcp/src/discovery.c @@ -63,11 +63,12 @@ __pmServerUnadvertisePresence(__pmServerPresence *s) /* * Service discovery API entry points. */ -char * +const char * __pmServiceDiscoveryParseTimeout(const char *s, struct timeval *timeout) { double seconds; - char *end, *result; + char *end; + const char *result; /* * The string is a floating point number representing the number of seconds @@ -84,13 +85,13 @@ __pmServiceDiscoveryParseTimeout(const char *s, struct timeval *timeout) /* Set the specified timeout. */ pmtimevalFromReal(seconds, timeout); - return end; + return (const char *)end; } static int parseOptions(const char *optionsString, __pmServiceDiscoveryOptions *options) { - char *result; + const char *result; if (optionsString == NULL) return 0; /* no options to parse */ diff --git a/src/libpcp/src/getopt.c b/src/libpcp/src/getopt.c index 5b5b0b074f..b335780a22 100644 --- a/src/libpcp/src/getopt.c +++ b/src/libpcp/src/getopt.c @@ -439,7 +439,7 @@ __pmSetGuiPort(pmOptions *opts, char *arg) } static char * -comma_or_end(const char *start) +comma_or_end(char *start) { char *end; @@ -1651,7 +1651,7 @@ pmgetopt_r(int argc, char *const *argv, pmOptions *d) { char c = *d->__nextchar++; - char *temp = strchr(optstring, c); + const char *temp = strchr(optstring, c); /* Increment `optind' when we start to process its last character. */ if (*d->__nextchar == '\0') diff --git a/src/libpcp/src/internal.h b/src/libpcp/src/internal.h index a71f90b831..4a647205ea 100644 --- a/src/libpcp/src/internal.h +++ b/src/libpcp/src/internal.h @@ -317,7 +317,7 @@ typedef struct { extern int __pmAddDiscoveredService(__pmServiceInfo *, const __pmServiceDiscoveryOptions *, int, char ***) _PCP_HIDDEN; -extern char *__pmServiceDiscoveryParseTimeout(const char *s, +extern const char *__pmServiceDiscoveryParseTimeout(const char *s, struct timeval *timeout) _PCP_HIDDEN; extern int __pmServiceAddPorts(const char *, int **, int) _PCP_HIDDEN; diff --git a/src/libpcp/src/util.c b/src/libpcp/src/util.c index 4ff5db9670..4c51cf8eed 100644 --- a/src/libpcp/src/util.c +++ b/src/libpcp/src/util.c @@ -1621,7 +1621,7 @@ static int debug(const char *spec, int action) { const char *p; - char *pend; + const char *pend; int i; int sts = 0; diff --git a/src/libpcp3/src/auxconnect.c b/src/libpcp3/src/auxconnect.c index 088490573d..9d82ae539b 100644 --- a/src/libpcp3/src/auxconnect.c +++ b/src/libpcp3/src/auxconnect.c @@ -322,8 +322,8 @@ __pmStringToSockAddr(const char *cp) else #endif if (strchr(cp, ':') != NULL) { - char *cp1; - char *scope; + char *cp1; + const char *scope; /* * inet_pton(3) does not support the "%" extension for specifying the * scope of a link-local address. If one is present, then strip it out and diff --git a/src/libpcp3/src/context.c b/src/libpcp3/src/context.c index 5c05d12a71..7a05d45f0d 100644 --- a/src/libpcp3/src/context.c +++ b/src/libpcp3/src/context.c @@ -862,7 +862,7 @@ initarchive(__pmContext *ctxp, const char *name) int i; int sts; char *namelist = NULL; - const char *current; + char *current; char *end; __pmArchCtl *acp; __pmMultiLogCtl *mlcp = NULL; diff --git a/src/libpcp3/src/discovery.c b/src/libpcp3/src/discovery.c index b79510c639..561457fb85 100644 --- a/src/libpcp3/src/discovery.c +++ b/src/libpcp3/src/discovery.c @@ -63,11 +63,12 @@ __pmServerUnadvertisePresence(__pmServerPresence *s) /* * Service discovery API entry points. */ -char * +const char * __pmServiceDiscoveryParseTimeout(const char *s, struct timeval *timeout) { double seconds; - char *end, *result; + char *end; + const char *result; /* * The string is a floating point number representing the number of seconds @@ -84,13 +85,13 @@ __pmServiceDiscoveryParseTimeout(const char *s, struct timeval *timeout) /* Set the specified timeout. */ pmtimevalFromReal(seconds, timeout); - return end; + return (const char *)end; } static int parseOptions(const char *optionsString, __pmServiceDiscoveryOptions *options) { - char *result; + const char *result; if (optionsString == NULL) return 0; /* no options to parse */ diff --git a/src/libpcp3/src/getopt.c b/src/libpcp3/src/getopt.c index a2a0345779..0777618f5a 100644 --- a/src/libpcp3/src/getopt.c +++ b/src/libpcp3/src/getopt.c @@ -399,7 +399,7 @@ __pmAddOptArchive(pmOptions *opts, char *arg) } static char * -comma_or_end(const char *start) +comma_or_end(char *start) { char *end; @@ -1586,7 +1586,7 @@ pmgetopt_r(int argc, char *const *argv, pmOptions *d) { char c = *d->__nextchar++; - char *temp = strchr(optstring, c); + const char *temp = strchr(optstring, c); /* Increment `optind' when we start to process its last character. */ if (*d->__nextchar == '\0') diff --git a/src/libpcp3/src/internal.h b/src/libpcp3/src/internal.h index daf999ebef..8fe152173a 100644 --- a/src/libpcp3/src/internal.h +++ b/src/libpcp3/src/internal.h @@ -315,7 +315,7 @@ typedef struct { extern int __pmAddDiscoveredService(__pmServiceInfo *, const __pmServiceDiscoveryOptions *, int, char ***) _PCP_HIDDEN; -extern char *__pmServiceDiscoveryParseTimeout(const char *s, +extern const char *__pmServiceDiscoveryParseTimeout(const char *s, struct timeval *timeout) _PCP_HIDDEN; extern int __pmServiceAddPorts(const char *, int **, int) _PCP_HIDDEN; diff --git a/src/libpcp3/src/util.c b/src/libpcp3/src/util.c index dca3dce476..f118f3326d 100644 --- a/src/libpcp3/src/util.c +++ b/src/libpcp3/src/util.c @@ -1623,7 +1623,7 @@ debug(const char *spec, int action, int style) break; } - pend = strchr(p, ','); + pend = (char *)strchr(p, ','); if (pend == NULL) pend = (char *)&p[strlen(p)]; diff --git a/src/libpcp_web/src/GNUmakefile b/src/libpcp_web/src/GNUmakefile index 3d4ec1ee19..da787444c7 100644 --- a/src/libpcp_web/src/GNUmakefile +++ b/src/libpcp_web/src/GNUmakefile @@ -26,28 +26,17 @@ else INIH_XFILES = endif -LIBVALKEY_HFILES = $(addprefix deps/libvalkey/, \ - include/valkey/alloc.h include/valkey/async.h src/async_private.h src/fmacros.h include/valkey/valkey.h include/valkey/net.h \ - include/valkey/read.h include/valkey/visibility.h src/sds.h src/sdsalloc.h include/valkey/sockcompat.h src/win32.h include/valkey/adapters/libuv.h \ - src/adlist.h src/command.h src/vkutil.h include/valkey/cluster.h src/valkey_private.h src/cmddef.h src/dict.h) -LIBVALKEY_CFILES = $(addprefix deps/libvalkey/, \ - src/alloc.c src/async.c src/valkey.c src/net.c src/sds.c src/sockcompat.c src/read.c \ - src/adlist.c src/command.c src/crc16.c src/vkutil.c src/cluster.c src/conn.c src/dict.c) -LIBVALKEY_XFILES = $(LIBVALKEY_HFILES) $(LIBVALKEY_CFILES) - CFILES = jsmn.c http_client.c http_parser.c siphash.c \ query.c schema.c load.c sha1.c util.c slots.c \ keys.c maps.c batons.c encoding.c \ - search.c json_helpers.c config.c \ - $(LIBVALKEY_CFILES) + search.c json_helpers.c config.c ifneq "$(HAVE_LIBINIH)" "true" CFILES += $(INIH_CFILES) endif HFILES = jsmn.h http_client.h http_parser.h zmalloc.h \ query.h schema.h load.h sha1.h util.h slots.h \ keys.h maps.h batons.h encoding.h \ - search.h discover.h private.h \ - $(LIBVALKEY_HFILES) + search.h discover.h private.h ifneq "$(HAVE_LIBINIH)" "true" HFILES += $(INIH_HFILES) endif @@ -55,7 +44,8 @@ YFILES = query_parser.y XFILES = jsmn.c jsmn.h http_parser.c http_parser.h \ sha1.c sha1.h siphash.c -LLDLIBS = $(PCPWEBLIB_EXTRAS) $(LIB_FOR_MATH) $(LIB_FOR_REGEX) +LLDLIBS = $(PCPWEBLIB_EXTRAS) $(LIB_FOR_MATH) $(LIB_FOR_REGEX) \ + $(TOPDIR)/src/libvalkey/src/libvalkey.a ifeq "$(TARGET_OS)" "mingw" LLDLIBS += -lws2_32 CFILES += fnmatch.c @@ -67,7 +57,9 @@ ifneq "$(HAVE_LIBINIH)" "true" LCFLAGS += -Iinih endif -LCFLAGS += $(C99_CFLAGS) -DJSMN_PARENT_LINKS=1 -DJSMN_STRICT=1 -DHTTP_PARSER_STRICT=0 -Ideps -Ideps/libvalkey/include/valkey -Ideps/libvalkey/include -Ideps/libvalkey/src +LCFLAGS += $(C99_CFLAGS) -DJSMN_PARENT_LINKS=1 -DJSMN_STRICT=1 -DHTTP_PARSER_STRICT=0 -Ideps \ + -I$(TOPDIR)/vendor/github.com/valkey-io/libvalkey/include \ + -I$(TOPDIR)/vendor/github.com/valkey-io/libvalkey/src ifeq "$(HAVE_LIBUV)" "true" CFILES += discover.c loggroup.c webgroup.c timer.c @@ -108,13 +100,13 @@ SYMTARGET = endif VERSION_SCRIPT = exports -LDIRT = $(XFILES) $(YFILES) $(LIBVALKEY_XFILES) $(SYMTARGET) \ +LDIRT = $(XFILES) $(YFILES) $(SYMTARGET) \ $(YFILES:%.y=%.tab.?) exports ifneq "$(HAVE_LIBINIH)" "true" LDIRT += $(INIH_XFILES) endif -default: $(XFILES) $(LIBVALKEY_XFILES) $(INIH_XFILES) $(LIBTARGET) $(SYMTARGET) $(STATICLIBTARGET) +default: $(XFILES) $(INIH_XFILES) $(LIBTARGET) $(SYMTARGET) $(STATICLIBTARGET) include $(BUILDRULES) @@ -144,10 +136,6 @@ $(INIH_XFILES): mkdir -p deps/inih $(LN_S) -f $(realpath $(TOPDIR))/vendor/github.com/benhoyt/$(@:deps/%=%) $@ endif -$(LIBVALKEY_XFILES): - mkdir -p deps/libvalkey/include/valkey/adapters - mkdir -p deps/libvalkey/src - $(LN_S) -f $(realpath $(TOPDIR))/vendor/github.com/valkey-io/$(@:deps/%=%) $@ .NOTPARALLEL: query_parser.tab.h query_parser.tab.c: query_parser.y query.h @@ -161,10 +149,10 @@ default_pcp: default install_pcp: install ifneq ($(LIBTARGET),) -$(LIBTARGET): $(VERSION_SCRIPT) $(XFILES) $(LIBVALKEY_XFILES) $(INIH_XFILES) +$(LIBTARGET): $(VERSION_SCRIPT) $(XFILES) $(INIH_XFILES) endif -%.o: $(LIBVALKEY_XFILES) $(INIH_XFILES) +%.o: $(INIH_XFILES) jsmn.o: jsmn.c jsmn.h discover.o: discover.h discover.c diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 1a398d08e7..6dfd78db98 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -114,7 +114,7 @@ keys_search_text_prep(sds s, int min_length, char *prefix, char *suffix) int token_count, non_digit_found; for (i = 0; i < len; i++) { - char *is_found = strchr(delimiters, s[i]); + const char *is_found = strchr(delimiters, s[i]); if (is_found != NULL) { result = sdscat(result, " "); } else { diff --git a/src/libvalkey/GNUmakefile b/src/libvalkey/GNUmakefile new file mode 100644 index 0000000000..e17507a23a --- /dev/null +++ b/src/libvalkey/GNUmakefile @@ -0,0 +1,30 @@ +# +# Copyright (c) 2026 Red Hat. +# +# This library is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +# License for more details. +# + +# libvalkey.a - vendored Valkey (Redis-compatible) C client library + +TOPDIR = ../.. + +include $(TOPDIR)/src/include/builddefs + +SUBDIRS = src + +default install: $(SUBDIRS) + $(SUBDIRS_MAKERULE) + +include $(BUILDRULES) + +default_pcp: default + +install_pcp: install diff --git a/src/libvalkey/src/GNUmakefile b/src/libvalkey/src/GNUmakefile new file mode 100644 index 0000000000..c1608bb90a --- /dev/null +++ b/src/libvalkey/src/GNUmakefile @@ -0,0 +1,58 @@ +# +# Copyright (c) 2026 Red Hat. +# +# This library is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +# License for more details. +# + +TOPDIR = ../../.. +include $(TOPDIR)/src/include/builddefs + +LIBVALKEY_HFILES = $(addprefix deps/libvalkey/, \ + include/valkey/alloc.h include/valkey/async.h include/valkey/cluster.h \ + include/valkey/valkey.h include/valkey/net.h include/valkey/read.h \ + include/valkey/visibility.h include/valkey/sockcompat.h \ + include/valkey/adapters/libuv.h \ + src/adlist.h src/async_private.h src/cmddef.h src/command.h src/dict.h \ + src/fmacros.h src/sds.h src/sdsalloc.h src/valkey_private.h \ + src/vkutil.h src/win32.h) +LIBVALKEY_CFILES = $(addprefix deps/libvalkey/, \ + src/adlist.c src/alloc.c src/async.c src/cluster.c src/command.c \ + src/conn.c src/crc16.c src/dict.c src/net.c src/read.c src/sds.c \ + src/sockcompat.c src/valkey.c src/vkutil.c) +LIBVALKEY_XFILES = $(LIBVALKEY_HFILES) $(LIBVALKEY_CFILES) + +CFILES = $(LIBVALKEY_CFILES) + +STATICLIBTARGET = libvalkey.a + +LCFLAGS = -U_GNU_SOURCE -Ideps -Ideps/libvalkey/include/valkey \ + -Ideps/libvalkey/include -Ideps/libvalkey/src + +LDIRT = $(LIBVALKEY_XFILES) + +default: $(LIBVALKEY_XFILES) $(STATICLIBTARGET) + +include $(BUILDRULES) + +deps/libvalkey/include/valkey/adapters deps/libvalkey/include/valkey deps/libvalkey/src: + mkdir -p $@ + +$(LIBVALKEY_XFILES): | deps/libvalkey/include/valkey/adapters deps/libvalkey/include/valkey deps/libvalkey/src + $(LN_S) -f $(realpath $(TOPDIR))/vendor/github.com/valkey-io/$(@:deps/%=%) $@ + +%.o: $(LIBVALKEY_XFILES) + +install: default + $(INSTALL) -m 755 $(STATICLIBTARGET) $(PCP_LIB_DIR)/$(STATICLIBTARGET) + +default_pcp: default + +install_pcp: install diff --git a/src/pmdas/valkey/GNUmakefile b/src/pmdas/valkey/GNUmakefile index 06d67127a5..c5d2c82384 100644 --- a/src/pmdas/valkey/GNUmakefile +++ b/src/pmdas/valkey/GNUmakefile @@ -21,39 +21,26 @@ CMDTARGET = pmdavalkey$(EXECSUFFIX) DFILES = README help CONFIG = valkey.conf -LIBVALKEY_HFILES = $(addprefix deps/libvalkey/, \ - include/valkey/alloc.h include/valkey/async.h include/valkey/cluster.h \ - include/valkey/valkey.h include/valkey/net.h include/valkey/read.h \ - include/valkey/visibility.h include/valkey/sockcompat.h src/adlist.h \ - src/async_private.h src/cmddef.h src/command.h src/dict.h src/fmacros.h \ - src/sds.h src/sdsalloc.h src/valkey_private.h src/vkutil.h src/win32.h) -LIBVALKEY_CFILES = $(addprefix deps/libvalkey/, \ - src/adlist.c src/alloc.c src/async.c src/cluster.c src/command.c \ - src/conn.c src/crc16.c src/dict.c src/net.c src/read.c src/sds.c \ - src/sockcompat.c src/valkey.c src/vkutil.c) -LIBVALKEY_XFILES = $(LIBVALKEY_HFILES) $(LIBVALKEY_CFILES) +CFILES = pmdavalkey.c CFILES = pmdavalkey.c $(LIBVALKEY_CFILES) -LLDLIBS = $(PCP_PMDALIB) -LCFLAGS = -I. -Ideps/libvalkey/include/valkey -Ideps/libvalkey/include -Ideps/libvalkey/src +# deps/libvalkey/src/read.c calls isfinite(), so -lm needed +# +LLDLIBS = $(PCP_PMDALIB) $(TOPDIR)/src/libvalkey/src/libvalkey.a -lm +LCFLAGS = -I. -I$(TOPDIR)/vendor/github.com/valkey-io/libvalkey/include \ + -I$(TOPDIR)/vendor/github.com/valkey-io/libvalkey/src PMDATMPDIR = $(PCP_PMDAS_DIR)/$(IAM) PMDACONFIG = $(PCP_SYSCONF_DIR)/$(IAM) PMDAADMDIR = $(PCP_PMDASADM_DIR)/$(IAM) -LDIRT = domain.h *.o $(IAM).log pmda$(IAM) pmda_$(IAM).$(DSOSUFFIX) $(LIBVALKEY_XFILES) +LDIRT = domain.h *.o $(IAM).log pmda$(IAM) pmda_$(IAM).$(DSOSUFFIX) -default_pcp default: $(LIBVALKEY_XFILES) $(CMDTARGET) +default_pcp default: $(CMDTARGET) include $(BUILDRULES) -deps/libvalkey/include/valkey deps/libvalkey/src: - mkdir -p $@ - -$(LIBVALKEY_XFILES): | deps/libvalkey/include/valkey deps/libvalkey/src - $(LN_S) -f $(realpath $(TOPDIR))/vendor/github.com/valkey-io/$(@:deps/%=%) $@ - install_pcp install: default $(INSTALL) -m 755 -d $(PMDAADMDIR) $(INSTALL) -m 755 -d $(PMDATMPDIR) @@ -69,8 +56,6 @@ $(OBJECTS): domain.h domain.h: ../../pmns/stdpmid $(DOMAIN_MAKERULE) -%.o: $(LIBVALKEY_XFILES) - pmdavalkey.o: pmdavalkey.c domain.h default_pcp : default diff --git a/src/pmlogconf/pmlogconf.c b/src/pmlogconf/pmlogconf.c index e9c16850c1..bfda73a936 100644 --- a/src/pmlogconf/pmlogconf.c +++ b/src/pmlogconf/pmlogconf.c @@ -1735,7 +1735,7 @@ void group_dircheck(const char *path) { size_t length; - char *end; + const char *end; if ((end = strrchr(path, '\n')) == NULL) return;