Add (categorized) stats

common/src/api/v1/mod.rs

@@ -665,3 +665,38 @@ impl WorkerRestApi for Client {
         Ok(())
     }
 }
+
+#[async_trait]
+pub trait StatsRestApi {
+    async fn collect_stats(&self, request: StatsCollectRequest) -> Result<Vec<StatsSnapshot>>;
+    async fn get_stats(&self, filter: Option<&StatsFilter>) -> Result<Vec<StatsSnapshot>>;
+}
+
+#[async_trait]
+impl StatsRestApi for Client {
+    async fn collect_stats(&self, request: StatsCollectRequest) -> Result<Vec<StatsSnapshot>> {
+        let snapshots = self
+            .post(Cow::Borrowed("api/v1/stats"))
+            .json(&request)
+            .send_encoded()
+            .await?
+            .error_for_status()?
+            .json()
+            .await?;
+
+        Ok(snapshots)
+    }
+
+    async fn get_stats(&self, filter: Option<&StatsFilter>) -> Result<Vec<StatsSnapshot>> {
+        let snapshots = self
+            .get(Cow::Borrowed("api/v1/stats"))
+            .query(&filter)
+            .send()
+            .await?
+            .error_for_status()?
+            .json()
+            .await?;
+
+        Ok(snapshots)
+    }
+}

common/src/api/v1/models/mod.rs

@@ -3,6 +3,7 @@ mod dashboard;
 mod meta;
 mod package;
 mod queue;
+mod stats;
 mod worker;
 
 pub use build::*;
@@ -11,6 +12,7 @@ pub use meta::*;
 pub use package::*;
 pub use queue::*;
 use serde::{Deserialize, Serialize};
+pub use stats::*;
 pub use worker::*;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

common/src/api/v1/models/stats.rs

@@ -0,0 +1,46 @@
+use chrono::NaiveDateTime;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct StatsCategoryCount {
+    pub category: String,
+    pub count: i32,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct StatsSnapshot {
+    pub id: i32,
+    pub captured_at: NaiveDateTime,
+    pub distribution: Option<String>,
+    pub release: Option<String>,
+    pub architecture: Option<String>,
+    pub good: i32,
+    pub bad: i32,
+    pub fail: i32,
+    pub unknown: i32,
+    /// Per-category breakdown of bad/failed packages. Empty when no stats config
+    /// backend matched or when categories have not been configured.
+    pub categories: Vec<StatsCategoryCount>,
+}
+
+/// Sent by `rebuildctl stats collect` to trigger a snapshot on the daemon.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct StatsCollectRequest {
+    /// Backend name used to look up error categories in rebuilderd-stats.conf
+    /// (e.g. "debian"). Leave empty to skip error categorization.
+    pub backend: Option<String>,
+    pub distribution: Option<String>,
+    pub release: Option<String>,
+    pub architecture: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct StatsFilter {
+    pub distribution: Option<String>,
+    pub release: Option<String>,
+    pub architecture: Option<String>,
+    /// Only return snapshots captured at or after this timestamp.
+    pub since: Option<NaiveDateTime>,
+    /// Maximum number of snapshots to return (default: 100).
+    pub limit: Option<i64>,
+}

common/src/config.rs

@@ -90,6 +90,7 @@ pub struct HttpConfig {
     pub post_body_size_limit: Option<usize>,
     pub transparently_sign_attestations: Option<bool>,
     pub endpoint: Option<String>,
+    pub permissive_cors: Option<bool>,
 }
 
 impl HttpConfig {

contrib/confs/rebuilderd-stats.conf

@@ -0,0 +1,102 @@
+## rebuilderd-stats.conf
+##
+## Error category definitions used by `rebuildctl stats collect --backend <name>`.
+## Categories are evaluated in order; the first match wins for each package.
+##
+## Each category requires exactly one matcher field:
+##   log_has      = "string"        # build log contains this string
+##   log_has_re   = "regex"         # build log matches regex; '.' always matches newlines so
+##                                  # patterns can span multiple lines without (?s)
+##   log_has_any  = ["s1", "s2"]    # build log contains any of these strings
+##   log_has_all  = ["s1", "s2"]    # build log contains all of these strings
+##   diff_has     = "string"        # diffoscope log contains this string
+##   diff_has_re  = "regex"         # diffoscope log matches regex; same DOTALL behaviour as log_has_re
+##   diff_has_any = ["s1", "s2"]    # diffoscope log contains any of these strings
+##   has_diff     = true            # a non-empty diffoscope log exists
+##   catch_all    = true            # always matches (use as last entry)
+##
+## Optional:
+##   exclude_architectures = ["all"]  # skip this category for these architecture prefixes
+
+## Categories listed here are appended to every backend's category list.
+[backend.all]
+categories = [
+    { name = "timeout",          log_has   = "TRUNCATED DUE TO TIMEOUT: " },
+    { name = "size limit",       log_has   = "TRUNCATED DUE TO SIZE LIMIT: " },
+    { name = "rebuilderd error", log_has   = "rebuilderd: unexpected error while rebuilding package:" },
+    { name = "other errors",     catch_all = true },
+]
+
+[backend.archlinux]
+categories = [
+    ## Infrastructure / rebuilderd-level failures
+    { name = "out of disk space",                                        log_has     = "No space left on device" },
+    { name = "out of memory",                                            log_has     = "Cannot allocate memory" },
+
+    ## Chroot / environment setup failures
+    ## PGP check comes first: it is the root cause of "Failed to install" errors
+    { name = "cached package has invalid PGP signature (infra issue)",   log_has     = "invalid or corrupted package (PGP signature)" },
+    { name = "failed to install packages to chroot",                     log_has     = "==> ERROR: Failed to install packages to new root" },
+    { name = "failed to retrieve packages",                              log_has     = "failed to retrieve" },
+    { name = "fakeroot error",                                           log_has     = "libfakeroot internal error: payload not recognized!" },
+
+    ## Source acquisition failures
+    { name = "source download failed",                                   log_has     = "ERROR: Failure while downloading" },
+    { name = "source checksum mismatch",                                 log_has     = "ERROR: One or more files did not pass the validity check!" },
+
+    ## Build failures (most specific first)
+    { name = "build failed: linker error",                               log_has_any = ["collect2: error: ld returned 1 exit status", "ld: error:"] },
+    { name = "build failed: ninja",                                      log_has     = "ninja: build stopped" },
+    { name = "build failed",                                             log_has_any = ["==> ERROR: A failure occurred in build().", "==> ERROR: A failure occurred in package()."] },
+]
+
+# Initial set of error categorizations for Debian are taken from
+# https://salsa.debian.org/qa/jenkins.debian.net/-/blob/master/bin/rebuilderd_stats.py
+#
+# Copyright © 2024-2025 Jochen Sprickerhof <jspricke@debian.org>
+# Copyright © 2024-2026 Holger Levsen <holger@layer-acht.org>
+# Copyright © 2024-2025 Gioele Barabucci <gioele@svario.it>
+#
+# Licensed under GPL-2
+[backend.debian]
+categories = [
+    { name = "package no longer in the archive",                                       log_has     = "E: Unable to find a source package for" },
+    { name = "rebuilderd failed, out of memory (temporary)",                           log_has_re  = 'dpkg \(subprocess\):.*Cannot allocate memory' },
+    { name = "rebuilderd failed, no diskspace (temporary)",                            log_has_re  = 'fatal error: error writing to /.*: No space left on device' },
+    { name = "rebuilderd failed, repository not signed (temporary)",                   log_has_re  = "E: The repository 'https://deb.debian.org/debian .* InRelease' is not signed." },
+    { name = "buildinfo file 404 (maybe temporary)",                                   log_has     = "rebuilderd: unexpected error while rebuilding package: Failed to download build input from" },
+    { name = "package file 404 (temporary)",                                           log_has     = "rebuilderd: unexpected error while rebuilding package: Failed to download original package from" },
+    { name = "packages missing on snapshot.d.o (maybe temporary, #1096129)",           log_has_re  = 'IndexError: list index out of range.*debootsnap failed at /usr/bin/debrebuild line 4' },
+    { name = "packages missing on metasnap (maybe temporary, #1096129)",               log_has_re  = 'cannot find:.*debootsnap failed at /usr/bin/debrebuild line 4' },
+    { name = "multiarch-support missing (upload needed)",                              log_has_all = ["multiarch-support but it is not installable", "Build Type: all"] },
+    { name = "multiarch-support missing (binNMU needed)",                              log_has     = "multiarch-support but it is not installable" },
+    { name = "mmdebstrap failed (#1094165)",                                           log_has_all = ["fatal: `/bin/chfn -f Debian OWFS system account Debian-ow' returned error code 1. Exiting.", "debootsnap failed at /usr/bin/debrebuild line 4"] },
+    { name = "debootsnap failed (temporary)",                                          log_has_all = ["Failed to fetch http://snapshot.debian.org/archive/debian", "debootsnap failed at /usr/bin/debrebuild line 4"] },
+    { name = "debootsnap failed, no diskspace (temporary)",                            log_has_any = ["W: creating tarball failed: E: cannot copy to /tmp/debrebuild", "dpkg-new': failed to write (No space left on device)"] },
+    { name = "debootsnap failed (maybe temporary)",                                    log_has     = "debootsnap failed at /usr/bin/debrebuild line 4" },
+    { name = "dpkg-source failed (maybe temporary)",                                   log_has     = "E: FAILED [dpkg-source died]" },
+    { name = "network error (temporary)",                                              log_has_any = ["Unable to connect to 127.0.0.1:3128", "E: You must put some 'deb-src' URIs in your sources.list", "E: Failed to fetch https://deb.debian.org/debian/dists/"] },
+    { name = "download failed (temporary)",                                            log_has_re  = '400 URL must be absolute.E: Could not download.*sbuild failed' },
+    { name = "sbuild chroot failed (temporary)",                                       log_has     = "E: Error creating chroot session: skipping" },
+    { name = "sbuild failed due to insufficient disk space",                           log_has     = "E: Disk space is probably not sufficient for building." },
+    { name = "sbuild failed due to insufficient memory",                               log_has_re  = 'fork\(\) failed: Cannot allocate memory at /usr/libexec/sbuild-usernsexec line' },
+    { name = "timeout: freedict (#998683)",                                            log_has_all = ["TRUNCATED DUE TO TIMEOUT: ", "inputs/freedict_20"] },
+    { name = "old dpkg (<1.19.0, upload needed)",                                      log_has_re  = 'dpkg is already the newest version \(1\.1[0-8].*fakeroot not found, either install the fakeroot.*Build Type: all' },
+    { name = "old dpkg (<1.19.0, binNMU needed)",                                      log_has_re  = 'dpkg is already the newest version \(1\.1[0-8].*fakeroot not found, either install the fakeroot' },
+    { name = "fakeroot not found (https://deb.li/3P46G)",                              log_has     = "fakeroot not found, either install the fakeroot" },
+    { name = "dpkg-buildpackage failed to allocate memory for gcc (maybe temporary)",  log_has_all = ["E: Build failure (dpkg-buildpackage died", "powerpc64le-linux-gnu-gcc: builderMainLoop: posix_spawnp: resource exhausted (Cannot allocate memory)"] },
+    { name = "dpkg-buildpackage failed to allocate memory for make (maybe temporary)", log_has_all = ["E: Build failure (dpkg-buildpackage died", "/usr/bin/make: Cannot allocate memory"] },
+    { name = "dpkg-buildpackage failed due to linker failure (maybe temporary)",       log_has_all = ["E: Build failure (dpkg-buildpackage died", "collect2: error: ld returned 1 exit status"] },
+    { name = "dpkg-buildpackage failed: dh-r (#1089197)",                              log_has_re  = '(Source|Binary): r-(cran|bioc|other)-.*dpkg-buildpackage died' },
+    { name = "dpkg-buildpackage failed",                                               log_has     = "E: Build failure (dpkg-buildpackage died" },
+    { name = "failed to reproduce: diffoscope timeout (not fatal)",                    diff_has    = "TRUNCATED DUE TO TIMEOUT: " },
+    { name = "failed to reproduce: dh-buildinfo (#1068809)",                           diff_has_any = ["buildinfo_{arch}.gz", "buildinfo_all.gz"] },
+    { name = "failed to reproduce: NT_GNU_BUILD_ID",                                   diff_has_re = ' -(  GNU                  0x00000014\tNT_GNU_BUILD_ID \(unique build ID bitstring\)\t    Build ID: ).* \+\1' },
+    { name = "failed to reproduce: chmod +s",                                          diff_has_re = ' -([drwx-]{3,6})s([drwx-]{3,6} [0-9a-z() ]{,60} [0-9]+ [0-9-]{10} [012][0-9]:).* \+\1x\2' },
+    { name = "failed to reproduce: fakeroot in LD path",                               diff_has_re = 'linux-gnu[^/]*/libfakeroot' },
+    { name = "failed to reproduce: PackageVerificationCode",                           diff_has    = "-PackageVerificationCode" },
+    { name = "failed to reproduce: 1-second offset (#1089088)",                        diff_has_re = ' -([drwx-]{10} [0-9a-z() ]{,60} [0-9]+ [0-9-]{10} [012][0-9]:).* \+\1', exclude_architectures = ["all"] },
+    { name = "failed to reproduce: dh-r (#1089197)",                                   log_has_re  = '(Source|Binary): r-(cran|bioc|other)-' },
+    { name = "failed to reproduce",                                                    has_diff    = true },
+    { name = "failed to reproduce (no diffoscope)",                                    log_has_re  = 'checking [^ ]*: (size(\.\.\.)? ?(value of [^ ]*)? (differs for [^ ]*)?)? ?$' },
+]

contrib/confs/rebuilderd.conf

@@ -11,6 +11,9 @@
 #transparently_sign_attestations = true
 ## Set a default endpoint for rebuildctl. This is especially useful for the sync timer.
 #endpoint = "http://127.0.0.1:8484"
+## Allow cross-origin requests from any origin (e.g. for local dashboard development).
+## Do not enable this in production unless behind a trusted reverse proxy.
+#permissive_cors = false
 
 ## A random cookie for administration is generated at startup and written to /var/lib/rebuilderd/auth-cookie
 ## You can set this to a fixed value here. Use `pwgen -1s 32` to generate one.