Ingestion throughput parallellization

.github/workflows/docker.yml

@@ -1,49 +1,83 @@
-name: Docker Image CI
+name: Publish Containers
 
 on:
   push:
-    branches: ["main"]
+    branches:
+      - main
+    tags:
+      - "v*"
+  workflow_dispatch:
+
+concurrency:
+  group: docker-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: idlab-discover/rustiflow
 
 jobs:
+  docker:
+    name: Publish ${{ matrix.variant.name }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        variant:
+          - name: default
+            dockerfile: Dockerfile
+            latest_tag: latest
+            flavor_suffix: ""
+          - name: slim
+            dockerfile: Dockerfile-slim
+            latest_tag: slim
+            flavor_suffix: -slim
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract image metadata
+        id: meta
+        uses: docker/metadata-action@v6
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=${{ matrix.variant.latest_tag }},enable={{is_default_branch}}
+            type=ref,event=branch,suffix=${{ matrix.variant.flavor_suffix }}
+            type=ref,event=tag,suffix=${{ matrix.variant.flavor_suffix }}
+            type=sha,prefix=sha-,suffix=${{ matrix.variant.flavor_suffix }}
+          labels: |
+            org.opencontainers.image.title=RustiFlow
+            org.opencontainers.image.description=Network flow extractor with offline and realtime capture modes
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
 
-    docker:
-
-        name: Publish Docker image
-        runs-on: ubuntu-latest
-
-        permissions:
-          contents: read
-          packages: write
-
-        steps:
-        - name: Checkout repository
-          uses: actions/checkout@v4
-
-        - name: Set up QEMU
-          uses: docker/setup-qemu-action@v3
-
-        - name: Set up Docker Buildx
-          uses: docker/setup-buildx-action@v3
-
-        - name: Log in to the GitHub Container Registry
-          uses: docker/login-action@v3
-          with:
-            registry: ghcr.io
-            username: ${{ github.actor }}
-            password: ${{ secrets.GITHUB_TOKEN }}
-
-        - name: Build and push the Docker image
-          uses: docker/build-push-action@v6
-          with:
-            context: .
-            file: Dockerfile
-            push: true
-            tags: ghcr.io/idlab-discover/rustiflow:ubuntu-20
-
-        - name: Build and push the slim Docker image
-          uses: docker/build-push-action@v6
-          with:
-            context: .
-            file: Dockerfile-slim
-            push: true
-            tags: ghcr.io/idlab-discover/rustiflow:slim
+      - name: Build and push image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ${{ matrix.variant.dockerfile }}
+          push: true
+          provenance: false
+          sbom: false
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

AGENTS.md

@@ -13,6 +13,9 @@ This repository is a Rust workspace for a network flow extractor. The main crate
 - Linux is the source of truth for build, runtime, and performance validation.
 - Do not assume that successful non-Linux builds imply realtime correctness.
 - When touching `aya`/eBPF/realtime code, prefer validating on Linux.
+- On the local Arch `rustiflow-t0` veth harness, legacy netlink tc attach is
+  currently more reliable than `aya`'s automatic TCX attach path for realtime
+  validation.
 
 ## Local Test Network
 
@@ -21,7 +24,8 @@ This repository is a Rust workspace for a network flow extractor. The main crate
   - host namespace capture side: `rustiflow-t0`
   - peer namespace side: `rustiflow-p0`
   - peer namespace: `rustiflow-peer`
-  - addressing: `10.203.0.1/30` on `rustiflow-t0`, `10.203.0.2/30` on `rustiflow-p0`
+  - IPv4 addressing: `10.203.0.1/30` on `rustiflow-t0`, `10.203.0.2/30` on `rustiflow-p0`
+  - IPv6 addressing: `fd42:203::1/64` on `rustiflow-t0`, `fd42:203::2/64` on `rustiflow-p0`
 - This setup is intended to stress the RustiFlow software path without depending on the physical LAN.
 - Treat it as a high-throughput local test harness, not as a substitute for true physical wire-rate validation.
 
@@ -117,22 +121,18 @@ in `docs/engineering-notes.md`.
 
 ### Current Focus
 
-- [ ] Stabilize and measure before expanding the eBPF event payload further.
-- [x] Finish the remaining TCP quality signals that current metadata already supports:
-  duplicate ACKs, zero-window events, and close style.
-- [x] Add the next IP and path signals once they can be trusted in both offline
-  and realtime modes.
+None currently. See `docs/engineering-notes.md` for completed experiments and
+decision history.
 
 Primary files:
 
-- `rustiflow/src/packet_features.rs`
-- `rustiflow/src/pcap.rs`
-- `rustiflow/src/realtime.rs`
-- `common/src/lib.rs`
-- `ebpf-ipv4/src/main.rs`
-- `ebpf-ipv6/src/main.rs`
+- `rustiflow/src/output.rs`
 - `rustiflow/src/flows/basic_flow.rs`
+- `rustiflow/src/flows/rusti_flow.rs`
 - `rustiflow/src/flows/features/`
+- `rustiflow/src/flow_table.rs`
+- `rustiflow/src/realtime.rs`
+- `docs/engineering-notes.md`
 
 ### Later Work
 

README.md

@@ -259,7 +259,7 @@ Options:
           [default: 60]
 
       --threads <THREADS>
-          The numbers of threads to use for processing packets (optional) (default: 5, maximum number of logical CPUs)
+          The numbers of threads to use for processing packets (optional) (default: realtime uses 12, capped at the number of logical CPUs; pcap uses 5; maximum number of logical CPUs)
 
       -o, --output <OUTPUT>
               Output method (required if no config file is provided)

common/src/lib.rs

@@ -2,6 +2,9 @@
 
 pub use network_types::{icmp::IcmpHdr, tcp::TcpHdr, udp::UdpHdr};
 
+pub const REALTIME_EVENT_QUEUE_COUNT: usize = 8;
+pub const REALTIME_EVENT_RINGBUF_BYTES: u32 = 1024 * 1024 * 64;
+
 /// BasicFeaturesIpv4 is a struct collection all ipv4 traffic data.
 #[repr(C, packed)]
 #[derive(Copy, Clone)]