Entlein/adaptive write perf

.arclint

@@ -20,6 +20,7 @@
     "(^private\/credentials\/.*\\.yaml)",
     "(^src/operator/client/versioned/)",
     "(^src/operator/apis/px.dev/v1alpha1/zz_generated.deepcopy.go)",
+    "(^src/e2e_test/perf_tool/pkg/suites/k8s/sovereign-soc/helm-rendered/)",
     "(^src/stirling/bpf_tools/bcc_bpf/system-headers)",
     "(^src/stirling/mysql/testing/.*\\.json$)",
     "(^src/stirling/obj_tools/testdata/go/test_go_binary.go)",
@@ -107,6 +108,9 @@
       ],
       "include": [
         "(\\.py$)"
+      ],
+      "exclude": [
+        "(^\\.local/)"
       ]
     },
     "flake8-pxl": {
@@ -121,8 +125,10 @@
     "mypy": {
       "type": "script-and-regex",
       "include": [
-        "(\\.py$)",
-        "(\\.pxl$)"
+        "(\\.py$)"
+      ],
+      "exclude": [
+        "(^\\.local/)"
       ],
       "script-and-regex.script": "mypy --config-file=mypy.ini",
       "script-and-regex.regex": "/^(?P<file>.*):(?P<line>\\d+): (?P<severity>error|warning): (?P<message>.*)$/m"
@@ -142,7 +148,7 @@
         "(\\.go$)"
       ],
       "flags": [
-        "--timeout=5m0s",
+        "--timeout=15m0s",
         "--output.checkstyle.path=stdout"
       ]
     },
@@ -201,6 +207,9 @@
       "type": "shellcheck",
       "include": [
         "(.*\\.sh$)"
+      ],
+      "exclude": [
+        "(^\\.local/)"
       ]
     },
     "spelling": {

.flake8rc

@@ -4,3 +4,9 @@ max-line-length = 120
 # N802: Function names have to be lower case. This is for GRPC service.
 # E999: Mistaken error see https://github.com/PyCQA/pycodestyle/issues/584
 ignore = N802,E999,W503
+
+# .local/ holds working artifacts (sweep render scripts, runbook tooling)
+# that evolve quickly and don't justify production-grade style
+# enforcement. The scripts already pass mypy; flake8's aesthetic rules
+# would just generate churn in every PR that touches them.
+exclude = .local/

.github/workflows/adaptive_export_image.yaml

@@ -0,0 +1,100 @@
+---
+# Build and push the adaptive_export operator image to
+# ghcr.io/k8sstormcenter/vizier-adaptive_export_image. Modelled on
+# vizier_release.yaml + ci/image_utils.sh::push_images_for_arch — the
+# Pixie-idiomatic flow that does `bazel run :<container_push>` with
+# --//k8s:image_repository / --//k8s:image_version overrides. The
+# scope here is intentionally just the adaptive_export image (a
+# dedicated :adaptive_export_image_push bundle in
+# src/vizier/services/adaptive_export/BUILD.bazel) so the SBOB PoC
+# can rebuild this one component without rebuilding kelvin / pem /
+# metadata.
+#
+# Triggers:
+#   - workflow_dispatch (manual rebuild for any commit)
+#   - push to entlein/adaptive-write-perf (the PoC branch)
+#
+# Tag scheme matches the existing manually-pushed tags on
+# ghcr.io/k8sstormcenter/vizier-adaptive_export_image:
+#   - <YYYY-MM-DD_HH-MM-SS.mmm_UTC>  (timestamp, primary tag)
+#   - <short-sha>                    (commit pin, secondary tag)
+# `latest` is intentionally NOT updated so we don't shift what an
+# `:latest` puller resolves to without an explicit ack.
+name: adaptive-export-image
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Branch, tag or SHA to build (defaults to the workflow ref)'
+        required: false
+        type: string
+  push:
+    branches:
+    - entlein/adaptive-write-perf
+    paths:
+    - 'src/vizier/services/adaptive_export/**'
+    - '.github/workflows/adaptive_export_image.yaml'
+permissions:
+  contents: read
+  packages: write
+jobs:
+  get-dev-image-with-extras:
+    uses: ./.github/workflows/get_image.yaml
+    with:
+      image-base-name: "dev_image_with_extras"
+      ref: ${{ inputs.ref }}
+
+  build-and-push:
+    name: Build and push adaptive_export image
+    needs: get-dev-image-with-extras
+    runs-on: oracle-vm-16cpu-64gb-x86-64
+    container:
+      image: ${{ needs.get-dev-image-with-extras.outputs.image-with-tag }}
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+      with:
+        ref: ${{ inputs.ref }}
+        fetch-depth: 0
+    - name: Add pwd to git safe dir
+      run: git config --global --add safe.directory `pwd`
+    - id: tags
+      run: |
+        TS="$(date -u +%Y-%m-%d_%H-%M-%S.%3N_UTC)"
+        SHA="$(git rev-parse --short HEAD)"
+        echo "ts=${TS}" >> "$GITHUB_OUTPUT"
+        echo "sha=${SHA}" >> "$GITHUB_OUTPUT"
+    - name: Use github bazel config
+      uses: ./.github/actions/bazelrc
+      with:
+        download_toplevel: 'true'
+        BB_API_KEY: ${{ secrets.BB_IO_API_KEY }}
+    - name: Log in to GHCR
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: echo "${GH_TOKEN}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+    - name: Build and push image
+      shell: bash
+      env:
+        IMAGE_REPO: ghcr.io/k8sstormcenter
+        TS: ${{ steps.tags.outputs.ts }}
+        SHA: ${{ steps.tags.outputs.sha }}
+      run: |
+        # Same shape as ci/image_utils.sh::push_images_for_arch — bazel
+        # run on the container_push target with the standard
+        # --//k8s:image_repository / --//k8s:image_version flags. Run
+        # twice with two image_versions so we publish both <ts> and
+        # <sha> tags; analysis is cached after the first run so only
+        # the push action actually re-executes.
+        for TAG in "${TS}" "${SHA}"; do
+          echo "::group::push ${IMAGE_REPO}/vizier-adaptive_export_image:${TAG}"
+          bazel run -c opt \
+            --config=stamp \
+            --config=x86_64_sysroot \
+            --//k8s:image_repository="${IMAGE_REPO}" \
+            --//k8s:image_version="${TAG}" \
+            //src/vizier/services/adaptive_export:adaptive_export_image_push
+          echo "::endgroup::"
+        done
+        echo "Pushed:"
+        echo "  ${IMAGE_REPO}/vizier-adaptive_export_image:${TS}"
+        echo "  ${IMAGE_REPO}/vizier-adaptive_export_image:${SHA}"

.github/workflows/perf_clickhouse.yaml

@@ -74,7 +74,9 @@ jobs:
       run: |
         tailscale status
         tailscale netcheck
-        api_host="$(kubectl --kubeconfig="$KUBECONFIG" config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed -E 's|https?://||; s|/.*||')"
+        api_host="$(kubectl --kubeconfig="$KUBECONFIG" config view --minify \
+          -o jsonpath='{.clusters[0].cluster.server}' \
+          | sed -E 's|https?://||; s|/.*||')"
         api_ip="${api_host%%:*}"
         api_port="${api_host##*:}"
         echo "--- tailscale ping ${api_ip} ---"
@@ -118,7 +120,7 @@ jobs:
 
     - name: Build and install px CLI
       run: |
-        bazel build //src/pixie_cli:px
+        bazel build --config=x86_64_sysroot //src/pixie_cli:px
         install -m 0755 bazel-bin/src/pixie_cli/px_/px /usr/local/bin/px
         px version
 
@@ -130,7 +132,7 @@ jobs:
       run: |
         bazel run //src/e2e_test/perf_tool:perf_tool -- run \
           --api_key="${PX_API_KEY}" \
-          --cloud_addr=${{ vars.PERF_CLOUD_ADDR }}
+          --cloud_addr=pixie.austrianopencloudcommunity.org:443 \
           --commit_sha="${{ steps.get-commit-sha.outputs.commit-sha }}" \
           --experiment_name=clickhouse-export \
           --suite=clickhouse-exec \
@@ -141,6 +143,14 @@ jobs:
           --prom_recorder_override 'clickhouse-operator=:k8ss-forensic' \
           --tags "${{ inputs.tags }}"
 
+    - name: Upload skaffold stderr log
+      if: always()
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+      with:
+        name: skaffold-stderr-${{ github.run_id }}-${{ github.run_attempt }}
+        path: ${{ runner.temp }}/skaffold-stderr.log
+        if-no-files-found: ignore
+
     - name: Deactivate gcloud service account
       if: always()
       run: gcloud auth revoke || true

.github/workflows/perf_soc_attack.yaml

@@ -74,7 +74,9 @@ jobs:
       run: |
         tailscale status
         tailscale netcheck
-        api_host="$(kubectl --kubeconfig="$KUBECONFIG" config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed -E 's|https?://||; s|/.*||')"
+        api_host="$(kubectl --kubeconfig="$KUBECONFIG" config view --minify \
+          -o jsonpath='{.clusters[0].cluster.server}' \
+          | sed -E 's|https?://||; s|/.*||')"
         api_ip="${api_host%%:*}"
         api_port="${api_host##*:}"
         echo "--- tailscale ping ${api_ip} ---"
@@ -138,18 +140,20 @@ jobs:
         PX_API_KEY: ${{ secrets.PX_API_KEY }}
         GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
         KUBECONFIG: ${{ runner.temp }}/kubeconfig
+        SOC_VIZIER_EXISTING: "1"
       run: |
         bazel run //src/e2e_test/perf_tool:perf_tool -- run \
           --api_key="${PX_API_KEY}" \
           --cloud_addr=pixie.austrianopencloudcommunity.org:443 \
           --commit_sha="${{ steps.get-commit-sha.outputs.commit-sha }}" \
-          --experiment_name=redis-attack \
+          --experiment_name=redis-attack-4x \
           --suite=sovereign-soc \
           --use_local_cluster \
           --export_backend=parquet-gcs \
           --gcs_bucket=k8sstormcenter-soc-perf \
           --container_repo=ghcr.io/k8sstormcenter \
           --prom_recorder_override 'clickhouse-operator=:k8ss-forensic' \
+          --max_retries=1 \
           --tags "${{ inputs.tags }}"
 
     - name: Tailscale logout