diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml new file mode 100644 index 000000000..1cd539b3d --- /dev/null +++ b/.github/workflows/check-dependencies.yml @@ -0,0 +1,19 @@ +name: PR Dependency Check + +on: + pull_request: + types: [opened, edited, closed, reopened] + +permissions: + issues: read + pull-requests: read + checks: write + +jobs: + check_dependencies: + runs-on: ubuntu-latest + name: Check Dependencies + steps: + - uses: astubbs/dependencies-action@a09974c14e84fb3e4c0df10c04f17bdc6ccc8878 # feat/auto-unblock-children-on-merge + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 000000000..0d3503722 --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,39 @@ +name: Claude Code Review + +on: + pull_request: + types: [opened, synchronize, ready_for_review, reopened] + # Optional: Only run on specific file changes + # paths: + # - "src/**/*.ts" + # - "src/**/*.tsx" + # - "src/**/*.js" + # - "src/**/*.jsx" + +jobs: + claude-review: + if: github.event.sender.type != 'Bot' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Code Review + id: claude-review + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' + plugins: 'code-review@claude-code-plugins' + prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://code.claude.com/docs/en/cli-reference for available options + diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 000000000..d300267f1 --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,50 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + actions: read # Required for Claude to read CI results on PRs + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + + # This is an optional setting that allows Claude to read CI results on PRs + additional_permissions: | + actions: read + + # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. + # prompt: 'Update the pull request description to include a summary of changes.' + + # Optional: Add claude_args to customize behavior and configuration + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://code.claude.com/docs/en/cli-reference for available options + # claude_args: '--allowed-tools Bash(gh pr:*)' + diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0a4cde6a1..28abdafe1 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -1,113 +1,471 @@ -# This workflow will build a Java project with Maven -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven +# PR builds: +# Default Kafka version (3.9.1) split into unit, integration, and performance +# suites in parallel for fast feedback, plus an experimental Kafka 4.x +# compatibility check. +# +# Push builds (master): +# Single full build on default Kafka version (3.9.1) to gate SNAPSHOT publishing. +# +# Caching strategy: +# All jobs use explicit cache/restore with restore-keys fallback to the +# prepare-deps rotating cache. Do NOT use setup-java's built-in `cache: maven` +# anywhere - its immutable keys can freeze an incomplete cache permanently. +# See docs/solutions/build-errors/maven-central-timeout-azure-west-regions-2026-04-21.md -# Tests disabled due to flakiness with under resourced github test machines. Confluent Jira works fine. Will fix later. -name: Unit tests only +name: Build and Test on: push: branches: [ master ] pull_request: - branches: [ master ] + +permissions: + contents: read + pull-requests: write + checks: write + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true jobs: - build: + + # ── Shared ──────────────────────────────────────────────────────────── + + # Pre-warm the Maven dependency cache so all downstream jobs start with + # everything local. Without this, jobs on Azure West US regions hit + # Maven Central CDN timeouts (see docs/solutions/build-errors/). + prepare-deps: + name: "Prepare Maven Cache" + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + # No `cache: maven` - see caching strategy note at top of file + - name: Restore Maven cache + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + key: setup-java-Linux-x64-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + setup-java-Linux-x64-maven- + - name: Download all dependencies + run: ./mvnw --batch-mode -Pci dependency:go-offline -Dlicense.skip -DincludeScope=test -U + - name: Save Maven cache (rotating key) + if: success() + uses: actions/cache/save@v4 + with: + path: ~/.m2/repository + key: setup-java-Linux-x64-maven-${{ hashFiles('**/pom.xml') }}-${{ github.run_id }} + + # ── PR Builds ────────────────────────────────────────────────────────── + + # Default Kafka version: split suites for fast, parallel feedback. + # Uses the same scripts as local development (ci-unit-test.sh, etc.) + test: + if: github.event_name == 'pull_request' + needs: prepare-deps strategy: fail-fast: false matrix: - # Why not? because we can. - # 2.0.1, 2.1.1, 2.2.2, 2.3.1, 2.4.1 don't work - needs zstd and some kafka client libs. - # Doesn't mean it couldn't be modified slightly to work... - #ak: [ 2.5.1, 2.6.1, 2.7.0, 2.8.1, 3.0.1, 3.1.0 ] - # 25 and 26 include a dep with a vulnerability which ossindex fails the build for - ak: [ 2.7.0, 2.8.1, 3.0.1, 3.1.0 ] - #ak: [ 2.7.0 ] - #jdk: [ '-P jvm8-release -Djvm8.location=/opt/hostedtoolcache/Java_Zulu_jdk/8.0.332-9/x64', '' ] - # TG currently targets 11, so can't run the tests on 8 https://github.com/astubbs/truth-generator/issues/114 - jdk: [ '' ] - experimental: [ false ] - name: [ "Stable AK version" ] include: - # AK 2.4 not supported - # - ak: "'[2.4.1,2.5)'" # currently failing - # experimental: true - # name: "Oldest AK breaking version 2.4.1+ (below 2.5.0) expected to fail" - - ak: "'[2.7.0,4)'" # currently failing - experimental: true - name: "Newest AK version 2.7.0+?" - - continue-on-error: ${{ matrix.experimental }} - name: "AK: ${{ matrix.ak }} JDK: ${{ matrix.jdk }}" + - suite: unit + name: "Unit Tests" + cmd: "bin/ci-unit-test.sh" + timeout: 15 + - suite: integration + name: "Integration Tests" + cmd: "bin/ci-integration-test.sh" + timeout: 60 + - suite: performance + name: "Performance Tests" + cmd: "bin/performance-test.sh" + timeout: 60 + name: "${{ matrix.name }}" runs-on: ubuntu-latest - + timeout-minutes: ${{ matrix.timeout }} steps: - - uses: actions/checkout@v3 - - - name: Setup JDK 1.8 - uses: actions/setup-java@v3 + - uses: actions/checkout@v6 + - uses: actions/setup-java@v5 with: - java-version: '8' - distribution: 'zulu' - cache: 'maven' - - # the patch version will be upgraded silently causing the build to eventually start failing - need to store this as a var - possible? - - name: Show java 1.8 home - # /opt/hostedtoolcache/Java_Zulu_jdk/8.0.332-9/x64/bin/java - run: which java - - # - name: Setup JDK 1.9 - # uses: actions/setup-java@v1 - # with: - # java-version: 1.9 - - # - name: Show java 1.9 home - # /opt/hostedtoolcache/jdk/9.0.7/x64 - # run: which java + distribution: 'temurin' + java-version: '17' + - name: Restore Maven cache + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + key: setup-java-Linux-x64-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + setup-java-Linux-x64-maven- + - name: ${{ matrix.name }} + run: ${{ matrix.cmd }} + - name: Upload coverage to Codecov + if: always() + uses: codecov/codecov-action@v5 + with: + files: '**/target/site/jacoco/jacoco.xml,**/target/site/jacoco-it/jacoco.xml' + flags: ${{ matrix.suite }} + token: ${{ secrets.CODECOV_TOKEN }} - - name: Setup JDK 17 - uses: actions/setup-java@v3 + # Experimental Kafka 4.x compatibility check. Non-blocking. + test-kafka-compat: + if: github.event_name == 'pull_request' + needs: prepare-deps + name: "Kafka Compat (experimental 4.x)" + runs-on: ubuntu-latest + timeout-minutes: 45 + continue-on-error: true + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-java@v5 with: - distribution: 'zulu' + distribution: 'temurin' java-version: '17' - cache: 'maven' + - name: Restore Maven cache + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + key: setup-java-Linux-x64-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + setup-java-Linux-x64-maven- + - name: Build and Test (Kafka 4.x) + run: bin/ci-build.sh '[3.9.1,5)' + - name: Upload coverage to Codecov + if: always() + uses: codecov/codecov-action@v5 + with: + files: '**/target/site/jacoco/jacoco.xml,**/target/site/jacoco-it/jacoco.xml' + flags: ak-experimental + token: ${{ secrets.CODECOV_TOKEN }} - - name: Show java 17 home - # /opt/hostedtoolcache/jdk/13.0.2/x64/bin/java - run: which java + # Duplicate code detection using both PMD CPD and jscpd engines. + # See https://github.com/astubbs/duplicate-code-cross-check + duplicate-detection: + if: github.event_name == 'pull_request' + name: "Duplicate Code Check" + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: astubbs/duplicate-code-cross-check@d3140ef6e9a4adf68c9f729a789e8b7ec8d058f7 # v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + directories: 'parallel-consumer-core/src parallel-consumer-vertx/src parallel-consumer-reactor/src parallel-consumer-mutiny/src' + cpd-max-duplication: '5' + jscpd-max-duplication: '4' - # - name: Show java version - # run: java -version + # File similarity detection - finds files that are semantically similar overall + file-similarity: + if: github.event_name == 'pull_request' + name: "File Similarity Check" + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: astubbs/duplicate-code-detection-tool@4e302e720f86c4e95f709196bae21333a9696937 # feat/base-vs-pr-comparison + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + directories: 'parallel-consumer-core/src,parallel-consumer-vertx/src,parallel-consumer-reactor/src,parallel-consumer-mutiny/src' + file_extensions: 'java' + ignore_below: 30 + fail_above: 80 + warn_above: 50 + one_comment: true + compare_with_base: true + max_increase: 10 - # - name: Show mvn version - # run: mvn -version + # SpotBugs static analysis - finds null derefs, concurrency issues, resource leaks. + # Uses a baseline from the base branch so only NEW bugs introduced by the PR are + # reported. The baseline is generated by the push build and cached. + spotbugs: + if: github.event_name == 'pull_request' + name: "SpotBugs" + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: prepare-deps + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + - name: Restore Maven cache + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + key: setup-java-Linux-x64-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + setup-java-Linux-x64-maven- + - name: Restore SpotBugs baseline + id: spotbugs-baseline + uses: actions/cache/restore@v4 + with: + path: .spotbugs-baseline.xml + key: spotbugs-baseline-${{ github.base_ref }}-${{ hashFiles('**/src/main/**/*.java') }} + restore-keys: | + spotbugs-baseline-${{ github.base_ref }}- + spotbugs-baseline- + - name: Compile and run SpotBugs + run: | + EXCLUDE_ARG="" + if [ -f .spotbugs-baseline.xml ]; then + EXCLUDE_ARG="-Dspotbugs.excludeFilterFile=.spotbugs-baseline.xml" + echo "Using baseline — only new bugs will be reported" + else + echo "No baseline found — reporting all bugs (first run on this base branch)" + fi + ./mvnw --batch-mode -Pci compile spotbugs:spotbugs -Dlicense.skip $EXCLUDE_ARG -pl parallel-consumer-core -am + - name: Post SpotBugs annotations on PR + if: always() + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + with: + path: '**/target/spotbugsXml.xml' + token: ${{ secrets.GITHUB_TOKEN }} + - name: Post SpotBugs summary to PR + if: always() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const { execSync } = require('child_process'); + const hasBaseline = fs.existsSync('.spotbugs-baseline.xml'); + const files = execSync('find . -path "*/target/spotbugsXml.xml" -type f').toString().trim().split('\n').filter(f => f); + let body; + if (files.length === 0) { + body = `## SpotBugs Report\n\nNo SpotBugs XML report found. The compile or analysis step may have failed.\n`; + } else { + let totalBugs = 0; + for (const f of files) { + const content = fs.readFileSync(f, 'utf8'); + const matches = content.match(/ c.body.includes('SpotBugs Report')); + if (existing) { + await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, body }); + } - # - name: Build with Maven on Java 13 - # run: mvn -B package --file pom.xml + # Dependency vulnerability scanning - GitHub's own dependency review + dependency-scan: + if: github.event_name == 'pull_request' + name: "Dependency Vulnerabilities" + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + - uses: actions/dependency-review-action@v4 + with: + fail-on-severity: high + comment-summary-in-pr: always + # Mutation testing (PIT) - verifies test assertions are meaningful. + # Non-blocking: posts a PR comment with mutation score but doesn't + # gate merge. TODO: move to self-hosted performance runner for speed. + mutation-testing: + if: github.event_name == 'pull_request' + name: "Mutation Testing (PIT)" + runs-on: ubuntu-latest + continue-on-error: true + timeout-minutes: 300 + needs: prepare-deps + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + - name: Restore Maven cache + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + key: setup-java-Linux-x64-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + setup-java-Linux-x64-maven- + # Restore PIT history for incremental analysis — only re-mutates + # code that changed since last run. PIT writes its history to /tmp/ + # with a filename derived from groupId.artifactId.version, ignoring + # explicit -DhistoryInputLocation CLI flags when -DwithHistory is set. + - name: Restore PIT history cache + uses: actions/cache@v4 + with: + path: /tmp/*_pitest_history.bin + key: pit-history-${{ github.base_ref }}-${{ hashFiles('**/src/main/**/*.java') }} + restore-keys: | + pit-history-${{ github.base_ref }}- + pit-history- + # -Djacoco.skip=true prevents Jacoco from injecting its agent into PIT + # minions (PIT has its own coverage; Jacoco's agent + # causes UNKNOWN_ERROR crashes). + # -DjvmArgs=-Xmx1g gives minion JVMs 1GB heap (default too small). + # -DtimeoutConstant/Factor generous per-mutation timeout for CI hardware + # -Dthreads=1 I/O-bound tests; parallelism adds contention not speed + # -DwithHistory incremental: skips re-mutating unchanged code. + # PIT writes history to /tmp/{groupId}.{artifactId}... + # which we cache above via actions/cache. + # targetClasses: internal.* only — the core engine where mutations matter most. + # Broadening to io.confluent.parallelconsumer.* hit GitHub's 6h job cap (#41). + - name: Run PIT mutation testing + run: | + ./mvnw --batch-mode -Pci test-compile org.pitest:pitest-maven:mutationCoverage \ + -Dlicense.skip \ + -Djacoco.skip=true \ + -DtargetClasses="io.confluent.parallelconsumer.internal.*" \ + -DtargetTests="io.confluent.parallelconsumer.*" \ + -DjvmArgs=-Xmx1g \ + -DtimeoutConstant=30000 -DtimeoutFactor=3.0 \ + -Dthreads=1 \ + -DwithHistory \ + -pl parallel-consumer-core -am + - name: Upload PIT report + if: always() + uses: actions/upload-artifact@v4 + with: + name: pit-report + path: '**/target/pit-reports/**' + - name: Post PIT summary to PR + if: always() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const { execSync } = require('child_process'); + const files = execSync('find . -path "*/pit-reports/*/mutations.csv" -type f').toString().trim().split('\n').filter(f => f); + let body; + if (files.length === 0) { + body = `## :x: Mutation Testing (PIT) Report\n\n`; + body += `**PIT did not produce a report.** Most commonly this means a test failed in the baseline (PIT runs all tests unmodified first to establish green) and PIT aborted before mutating. See the "Run PIT mutation testing" step logs for the failing test, then either fix it or add it to \`-DexcludedTestClasses\` in the workflow.\n`; + } else { + let killed = 0, survived = 0, noCov = 0, total = 0; + for (const f of files) { + const lines = fs.readFileSync(f, 'utf8').split('\n').filter(l => l.trim()); + for (const line of lines) { + total++; + if (line.includes('KILLED')) killed++; + else if (line.includes('SURVIVED')) survived++; + else if (line.includes('NO_COVERAGE')) noCov++; + } + } + const score = total > 0 ? ((killed / total) * 100).toFixed(1) : '0'; + body = `## Mutation Testing (PIT) Report\n\n`; + body += `| Metric | Value |\n|--------|-------|\n`; + body += `| Mutations generated | ${total} |\n`; + body += `| Killed (detected) | ${killed} |\n`; + body += `| Survived (missed) | ${survived} |\n`; + body += `| No coverage | ${noCov} |\n`; + body += `| **Mutation score** | **${score}%** |\n\n`; + body += `Full HTML report available as artifact: \`pit-report\`\n`; + } + const comments = await github.rest.issues.listComments({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number }); + const existing = comments.data.find(c => c.body.includes('Mutation Testing (PIT) Report')); + if (existing) { + await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, body }); + } - # done automatically now - # - name: Cache Maven packages - # uses: actions/cache@v2.1.7 - # with: - # path: ~/.m2/repository - # key: ${{ runner.os }}-m2 - # restore-keys: ${{ runner.os }}-m2 + # ── Push Builds (master) ─────────────────────────────────────────────── - - name: Test with Maven - run: mvn -Pci -B package ${{ matrix.jdk }} -Dkafka.version=${{ matrix.ak }} -Dlicense.skip + # Generate SpotBugs baseline for PR comparisons. Runs on every push to + # master so PR builds can exclude known bugs and only report new ones. + spotbugs-baseline: + if: github.event_name == 'push' + name: "SpotBugs Baseline" + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: prepare-deps + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + - name: Restore Maven cache + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + key: setup-java-Linux-x64-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + setup-java-Linux-x64-maven- + - name: Compile and generate SpotBugs XML + run: ./mvnw --batch-mode -Pci compile spotbugs:spotbugs -Dlicense.skip -pl parallel-consumer-core -am + - name: Convert SpotBugs results to exclude filter + run: | + # Transform the SpotBugs XML into an exclude filter that can be + # passed to future runs via -Dspotbugs.excludeFilterFile. Each + # BugInstance becomes a Match element keyed on class + method + bug type. + python3 -c " + import xml.etree.ElementTree as ET, sys + tree = ET.parse('parallel-consumer-core/target/spotbugsXml.xml') + root = tree.getroot() + out = ET.Element('FindBugsFilter') + for bug in root.findall('.//BugInstance'): + match = ET.SubElement(out, 'Match') + cls = bug.find('Class') + method = bug.find('Method') + if cls is not None: + ET.SubElement(match, 'Class', name=cls.get('classname', '')) + if method is not None: + ET.SubElement(match, 'Method', name=method.get('name', '')) + ET.SubElement(match, 'Bug', pattern=bug.get('type', '')) + ET.ElementTree(out).write('.spotbugs-baseline.xml', xml_declaration=True, encoding='utf-8') + print(f'Baseline: {len(root.findall(\".//BugInstance\"))} bugs excluded') + " + - name: Cache SpotBugs baseline + uses: actions/cache/save@v4 + with: + path: .spotbugs-baseline.xml + key: spotbugs-baseline-${{ github.ref_name }}-${{ hashFiles('**/src/main/**/*.java') }} -# - name: Archive test results -# if: ${{ always() }} -# uses: actions/upload-artifact@v2 -# with: -# name: test-reports -# path: target/**-reports/* -# retention-days: 14 -# -# - name: Archive surefire test results -# if: ${{ always() }} -# uses: actions/upload-artifact@v2 -# with: -# name: test-reports -# path: target/surefire-reports/* -# retention-days: 14 + # Single full build on default Kafka version to gate SNAPSHOT publishing. + # Uses the same ci-build.sh script that PRs use for Kafka compat testing. + build: + if: github.event_name == 'push' + needs: prepare-deps + name: "Build and Test" + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + - name: Restore Maven cache + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + key: setup-java-Linux-x64-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + setup-java-Linux-x64-maven- + - name: Build and Test + run: bin/ci-build.sh + - name: Upload coverage to Codecov + if: always() + uses: codecov/codecov-action@v5 + with: + files: '**/target/site/jacoco/jacoco.xml,**/target/site/jacoco-it/jacoco.xml' + flags: default + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 000000000..b33de3a14 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,82 @@ +# Performance test suite, run on a self-hosted Windows runner with Docker Desktop. +# +# These tests are tagged @Tag("performance") and excluded from the regular CI +# build because they need substantial hardware (CPU, memory, disk). They run +# on dedicated machines where the user has labelled their runner with the +# "performance" custom label. +# +# Triggers: +# - workflow_dispatch (manual) - primary trigger +# - schedule (weekly) - automated regression check +# - NOT on PRs from forks - self-hosted runners + untrusted code = bad +# +# See docs/SELF_HOSTED_RUNNER.md for one-time runner setup instructions. + +name: Performance Tests + +on: + workflow_dispatch: + inputs: + kafka_version: + description: 'Kafka version to test against (default: project default)' + required: false + type: string + default: '' + schedule: + # Weekly on Sunday at 02:00 UTC + - cron: '0 2 * * 0' + +concurrency: + # Only run one performance test at a time per branch - they're slow and resource-heavy + group: performance-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + performance: + name: "Performance suite (self-hosted)" + # Targets a self-hosted runner labelled "performance" running Windows. + # The "self-hosted" label is automatic; "windows" and "performance" are + # added when the runner is registered. See docs/SELF_HOSTED_RUNNER.md. + runs-on: [self-hosted, windows, performance] + timeout-minutes: 180 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup JDK 17 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + # Don't cache here - self-hosted runners persist .m2 across runs already + cache: '' + + - name: Show environment + shell: cmd + run: | + java -version + docker --version + docker info + + - name: Run performance tests + shell: cmd + env: + KAFKA_VERSION: ${{ inputs.kafka_version }} + run: | + if defined KAFKA_VERSION ( + call bin\performance-test.cmd -Dkafka.version=%KAFKA_VERSION% + ) else ( + call bin\performance-test.cmd + ) + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: performance-reports-${{ github.run_number }} + path: '**/target/*-reports/*.xml' + retention-days: 30 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..84b55385b --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,119 @@ +# Publishes to Maven Central on every push to master. +# +# The pom.xml version is the source of truth: +# - Versions ending in -SNAPSHOT -> publish snapshot +# - Versions without -SNAPSHOT -> publish release + create git tag + GitHub release +# +# To cut a release: open a PR that strips -SNAPSHOT, merge it. CI will publish. +# Then open another PR bumping to the next -SNAPSHOT. +# +# Required GitHub secrets: +# MAVEN_CENTRAL_USERNAME - Sonatype Central Portal token username +# MAVEN_CENTRAL_PASSWORD - Sonatype Central Portal token password +# MAVEN_GPG_PRIVATE_KEY - Armored GPG private key (gpg --export-secret-keys --armor KEYID) +# MAVEN_GPG_PASSPHRASE - Passphrase for the GPG key + +name: Publish to Maven Central + +on: + # Only publish after CI passes - workflow_run triggers when "Build and Test" completes + workflow_run: + workflows: ["Build and Test"] + branches: [ master ] + types: [completed] + workflow_dispatch: # allow manual re-runs + +concurrency: + group: publish-${{ github.ref }} + cancel-in-progress: false # never cancel a publish mid-flight + +permissions: + contents: write # needed to create tags and releases + +jobs: + publish: + name: Publish + # Skip if CI failed (workflow_run fires on both success and failure) + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 # need full history for tag creation + + - name: Setup JDK 17 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + # Configure Maven settings.xml with Sonatype credentials and GPG + server-id: central + server-username: MAVEN_CENTRAL_USERNAME + server-password: MAVEN_CENTRAL_PASSWORD + gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Read project version + id: version + run: | + VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout) + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + if [[ "$VERSION" == *-SNAPSHOT ]]; then + echo "is_snapshot=true" >> "$GITHUB_OUTPUT" + echo "Detected SNAPSHOT version: $VERSION" + else + echo "is_snapshot=false" >> "$GITHUB_OUTPUT" + echo "Detected RELEASE version: $VERSION" + fi + + - name: Check tag does not exist (releases only) + if: steps.version.outputs.is_snapshot == 'false' + run: | + TAG="v${{ steps.version.outputs.version }}" + if git rev-parse "$TAG" >/dev/null 2>&1; then + echo "Tag $TAG already exists - aborting to prevent re-release" + exit 1 + fi + + - name: Deploy to Maven Central + env: + MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }} + MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + run: | + # Unified publish path: central-publishing-maven-plugin handles both + # snapshots and releases. For snapshots it PUTs directly to the Central + # Portal snapshot endpoint; for releases it bundles, validates, and + # auto-publishes via the Portal API. Requires SNAPSHOT publishing to be + # enabled on the namespace in central.sonatype.com/publishing/namespaces. + # + # -pl exclusion: examples are sample code, not library artifacts; skip them + # from the deploy reactor. + ./mvnw --batch-mode \ + -Pmaven-central \ + -Pci \ + -pl '!:parallel-consumer-examples,!:parallel-consumer-example-core,!:parallel-consumer-example-metrics,!:parallel-consumer-example-streams,!:parallel-consumer-example-vertx,!:parallel-consumer-example-reactor' \ + clean deploy \ + -DskipTests \ + -Dlicense.skip + + - name: Create git tag (releases only) + if: steps.version.outputs.is_snapshot == 'false' + run: | + TAG="v${{ steps.version.outputs.version }}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag -a "$TAG" -m "Release $TAG" + git push origin "$TAG" + + - name: Create GitHub release (releases only) + if: steps.version.outputs.is_snapshot == 'false' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG="v${{ steps.version.outputs.version }}" + gh release create "$TAG" \ + --title "$TAG" \ + --generate-notes diff --git a/.gitignore b/.gitignore index e7ef34e89..4bf56e9af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /mk-include /.mk-include-timestamp .DS_Store +CLAUDE.md # Compiled class file *.class diff --git a/.mvn/maven.config b/.mvn/maven.config new file mode 100644 index 000000000..bfe087449 --- /dev/null +++ b/.mvn/maven.config @@ -0,0 +1,6 @@ +-Dmaven.wagon.http.connectionTimeout=10000 +-Dmaven.wagon.http.readTimeout=120000 +-Dmaven.wagon.httpconnectionManager.ttlSeconds=120 +-Dmaven.wagon.http.retryHandler.count=3 +-Daether.connector.connectTimeout=10000 +-Daether.connector.requestTimeout=120000 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..f079a569e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,102 @@ +# Parallel Consumer - Agent Context + +Project context for AI coding agents (Claude Code, Copilot, Cursor, etc.). + +## Overview + +Parallel Consumer is a Java library that enables concurrent message processing from Apache Kafka with a single consumer, avoiding the need to increase partition counts. It maintains ordering guarantees (by partition or key) while processing messages in parallel. + +This is a community-maintained fork of `confluentinc/parallel-consumer` (the upstream is no longer actively maintained), published to Maven Central as `io.github.astubbs.parallelconsumer`. + +## Build Requirements + +- **JDK 17** (required - the project uses Jabel to compile Java 17 source to Java 8 bytecode) +- **Docker** (required for integration tests - TestContainers spins up Kafka brokers) +- **Maven** via wrapper (`./mvnw`) - do not use system Maven + +## How to Build + +```bash +# Quick local build (compile + unit tests) +bin/build.sh + +# Unit tests only (no Docker needed) +bin/ci-unit-test.sh + +# Integration tests only (requires Docker for TestContainers) +bin/ci-integration-test.sh + +# Full CI build with all tests against a Kafka version matrix (used by push-to-master CI) +bin/ci-build.sh + +# Full CI build against a specific Kafka version +bin/ci-build.sh 3.9.1 + +# Performance tests only (requires substantial hardware) +bin/performance-test.sh +``` + +## Module Structure + +| Module | Purpose | +|--------|---------| +| `parallel-consumer-core` | Core library - consumer, producer, offset management, sharding | +| `parallel-consumer-vertx` | Vert.x integration for async HTTP | +| `parallel-consumer-reactor` | Project Reactor integration | +| `parallel-consumer-mutiny` | SmallRye Mutiny integration (Quarkus) | +| `parallel-consumer-examples` | Example implementations for each module | + +## Key Architecture Decisions + +- **Jabel cross-compilation**: Source is Java 17, bytecode targets Java 8 via Jabel annotation processor. This means `--release 8` is set in the compiler plugin, which restricts available APIs to Java 8 surface. The Mutiny module overrides this to `--release 9` because Mutiny uses `java.util.concurrent.Flow` (Java 9+). +- **Offset encoding**: Custom offset map encoding (run-length, bitset) stored in Kafka commit metadata for tracking in-flight messages. +- **Sharding**: Messages are distributed to processing shards by key or partition for ordering guarantees. + +## Testing + +- **Unit tests**: `mvn test` / surefire plugin. Source in `src/test/java/`. +- **Integration tests**: `mvn verify` / failsafe plugin. Source in `src/test-integration/java/`. Uses TestContainers with `confluentinc/cp-kafka` Docker image. +- **Test exclusion patterns**: `**/integrationTest*/**/*.java` and `**/*IT.java` are excluded from surefire, included in failsafe. +- **Kafka version matrix**: CI tests against multiple Kafka versions via `-Dkafka.version=X.Y.Z`. + +## Known Issues + +- **Mutiny module**: Has a `release.target=9` override in its pom.xml because Mutiny's `Multi` implements `java.util.concurrent.Flow.Publisher` which is not available with `--release 8`. + +## Code Style + +- **Lombok**: Used extensively (builders, getters, logging). IntelliJ Lombok plugin required. +- **EditorConfig**: Enforced via `.editorconfig` - 4-space indent for Java, 120 char line length. +- **License headers**: Managed by `license-maven-plugin` (Mycila). Use `-Dlicense.skip` locally to skip checks. +- **Copyright rules for this fork**: + - Do not change copyright headers on existing files unless the file has substantive code changes in the same commit + - Do not bump copyright years as an incidental or standalone change + - The `NOTICE` file at repo root contains the legal attribution structure for the fork + - New files written entirely for the fork should not claim Confluent copyright + - Always pass `-Dlicense.skip` to Maven to prevent the license plugin from auto-bumping years +- **Google Truth**: Used for test assertions alongside JUnit 5 and Mockito. + +## CI + +- **`.github/workflows/maven.yml`** — Build and test on every push/PR. PRs run two tiers in parallel: (1) split suites on default Kafka 3.9.1 for fast feedback (`bin/ci-unit-test.sh`, `bin/ci-integration-test.sh`, `bin/performance-test.sh`), and (2) an experimental Kafka 4.x compatibility check (`bin/ci-build.sh`). Push to master runs a single full build on default Kafka version via `bin/ci-build.sh` to gate SNAPSHOT publishing. All jobs use explicit `cache/restore` with rotating keys from the `prepare-deps` job - never `setup-java cache: 'maven'`. Includes SpotBugs, duplicate detection, mutation testing (PIT), and dependency vulnerability scanning on PRs. +- **`.github/workflows/publish.yml`** — Publishes to Maven Central on every push to `master`. The pom.xml version is the source of truth: `-SNAPSHOT` versions deploy as snapshots, non-snapshot versions deploy as full releases (and create a git tag + GitHub release). +- **`.semaphore/`** — Legacy Confluent internal CI/release pipelines, retained but inactive on the fork. + +## Releasing + +The pom.xml version drives publishing — there is no `maven-release-plugin` dance. + +**Cut a release:** +1. Open a PR removing `-SNAPSHOT` from `` in the parent pom (e.g. `0.6.0.0-SNAPSHOT` → `0.6.0.0`) +2. Merge it to master → CI publishes to Maven Central, tags `v0.6.0.0`, creates a GitHub release +3. Open another PR bumping to the next snapshot (e.g. `0.6.0.1-SNAPSHOT`) and merge + +**Required GitHub repo secrets** for `publish.yml`: +- `MAVEN_CENTRAL_USERNAME` — Sonatype Central Portal token username +- `MAVEN_CENTRAL_PASSWORD` — Sonatype Central Portal token password +- `MAVEN_GPG_PRIVATE_KEY` — Armored GPG private key for signing artifacts +- `MAVEN_GPG_PASSPHRASE` — Passphrase for the GPG key + +## Documented Solutions + +`docs/solutions/` - documented solutions to past problems and workflow patterns, organized by category with YAML frontmatter (`module`, `tags`, `problem_type`). Relevant when implementing or debugging in documented areas. diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 543a21f07..605f2b0cb 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -14,6 +14,23 @@ ifndef::github_name[] toc::[] endif::[] +== 0.6.0.0 + +=== Breaking + +* Maven coordinates rebranded from `io.confluent.parallelconsumer` to `io.github.astubbs.parallelconsumer`. Update your dependency declarations. + +=== Fixes + +* fix: null epoch race condition in EpochAndRecordsMap - poll() returning records before onPartitionsAssigned() no longer causes NPE; records are safely skipped and re-delivered +* fix: test pollution from leaked threads and Awaitility global state across test classes + +=== Improvements + +* ci: modernized GitHub Actions CI with parallel test suites (unit, integration, performance), SpotBugs, duplicate detection, mutation testing (PIT), and dependency vulnerability scanning +* ci: added publish workflow for automated Maven Central deployment +* docs: updated README and added upstream PR analysis report + == 0.5.3.4 === Fixes diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..964fb9171 --- /dev/null +++ b/NOTICE @@ -0,0 +1,10 @@ +Parallel Consumer +Copyright 2020-2026 Confluent, Inc. +Copyright 2026 Antony Stubbs and contributors + +This product includes software originally developed by Confluent, Inc. +(https://github.com/confluentinc/parallel-consumer), licensed under the +Apache License, Version 2.0. + +This fork is maintained independently at: + https://github.com/astubbs/parallel-consumer diff --git a/README.adoc b/README.adoc index 39f4db1be..9c9b8b247 100644 --- a/README.adoc +++ b/README.adoc @@ -22,7 +22,7 @@ TIP:: Editing template file endif::[] -= Confluent Parallel Consumer += Kafka Parallel Consumer :icons: :toc: macro :toclevels: 3 @@ -31,7 +31,8 @@ endif::[] :sectanchors: true :github_name: parallel-consumer -:base_url: https://github.com/confluentinc/{github_name} +:base_confluent_url: https://github.com/confluentinc/{github_name} +:base_url: https://github.com/astubbs/{github_name} :issues_link: {base_url}/issues @@ -43,26 +44,40 @@ ifdef::env-github[] :warning-caption: :warning: endif::[] -image:https://maven-badges.herokuapp.com/maven-central/io.confluent.parallelconsumer/parallel-consumer-parent/badge.svg?style=flat[link=https://mvnrepository.com/artifact/io.confluent.parallelconsumer/parallel-consumer-parent,Latest Parallel Consumer on Maven Central] +image:https://maven-badges.herokuapp.com/maven-central/io.github.astubbs.parallelconsumer/parallel-consumer-parent/badge.svg?style=flat[link=https://mvnrepository.com/artifact/io.github.astubbs.parallelconsumer/parallel-consumer-parent,Latest Parallel Consumer on Maven Central] -// Github actions disabled since codecov -//image:https://github.com/confluentinc/parallel-consumer/actions/workflows/maven.yml/badge.svg[Java 8 Unit Test GitHub] + -//^(^^full^ ^test^ ^suite^ ^currently^ ^running^ ^only^ ^on^ ^Confluent^ ^internal^ ^CI^ ^server^^)^ +image:https://github.com/astubbs/parallel-consumer/actions/workflows/maven.yml/badge.svg?branch=master[link=https://github.com/astubbs/parallel-consumer/actions/workflows/maven.yml,Build and Test] // travis badges temporarily disabled as travis isn't running CI currently //image:https://travis-ci.com/astubbs/parallel-consumer.svg?branch=master["Build Status", link="https://travis-ci.com/astubbs/parallel-consumer"] image:https://codecov.io/gh/astubbs/parallel-consumer/branch/master/graph/badge.svg["Coverage",https://codecov.io/gh/astubbs/parallel-consumer] Parallel Apache Kafka client wrapper with client side queueing, a simpler consumer/producer API with *key concurrency* and *extendable non-blocking IO* processing. -Confluent's https://www.confluent.io/confluent-accelerators/#parallel-consumer[product page for the project is here]. +IMPORTANT: This is a community-maintained fork of https://github.com/confluentinc/parallel-consumer[confluentinc/parallel-consumer], published under different Maven coordinates (`io.github.astubbs.parallelconsumer`). The original upstream project is no longer actively maintained. + +Confluent's https://www.confluent.io/confluent-accelerators/#parallel-consumer[product page for the original project is here]. TIP: If you like this project, please ⭐ Star it in GitHub to show your appreciation, help us gauge popularity of the project and allocate resources. -NOTE: This is not a part of the Confluent commercial support offering, except through consulting engagements. +NOTE: This is a community-maintained project with no commercial support. See the <> section for more information. -IMPORTANT: This project has been stable and reached its initial target feature set in Q1 2021. -It is actively maintained by the CSID team at Confluent. +[[when-to-use]] +== When to use this library (vs KIP-932 Share Groups) + +The Kafka landscape has shifted since this library's 2021 stable release. *KIP-932 Share Groups* is now GA on Confluent Cloud and ships with Confluent Platform 8.2 / Apache Kafka 4.2. It covers a large part of what people historically reached for Parallel Consumer to do, at the broker level rather than in a client wrapper. + +*Share Groups (broker-native):* many-to-many consumer↔partition mapping, per-message ack, broker-side delivery counts with poison-message protection, elastic scaling decoupled from partition count. Unordered queue semantics -- "RabbitMQ on Kafka". Already wrapped by Spring Kafka via `ShareConsumerFactory`. + +*Parallel Consumer (client-side):* keeps the partition model and adds *per-key parallelism* on top. Messages within a key stay ordered; different keys run concurrently; concurrency is independent of partition count. + +The two are not strict alternatives -- they solve different problems. + +[TIP] +==== +* If you want unordered queue semantics on Kafka 4.2+, reach for *Share Groups*. The "partitions are fixed, I need more consumers" motivation is now solved at the broker. +* If you need *key-level ordering with concurrency beyond partition count*, reach for *Parallel Consumer*. Nothing else does that cleanly today. +==== [[intro]] This library lets you process messages in parallel via a single Kafka Consumer meaning you can increase consumer parallelism without increasing the number of partitions in the topic you intend to process. @@ -283,7 +298,7 @@ The user just has to provide a function to extract from the message the HTTP cal === Illustrative Performance Example -.(see link:./parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VolumeTests.java[VolumeTests.java]) +.(see link:./parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VeryLargeMessageVolumeTest.java[VeryLargeMessageVolumeTest.java]) These performance comparison results below, even though are based on real performance measurement results, are for illustrative purposes. To see how the performance of the tool is related to instance counts, partition counts, key distribution and how it would relate to the vanilla client. Actual results will vary wildly depending upon the setup being deployed into. @@ -374,7 +389,7 @@ As an illustrative example of relative performance, given: == Support and Issues -If you encounter any issues, or have any suggestions or future requests, please create issues in the {issues_link}[github issue tracker]. +If you encounter any issues, or have any suggestions or future requests, please create issues in the {issues_link}[fork issue tracker]. Issues will be dealt with on a good faith, best efforts basis, by the small team maintaining this library. We also encourage participation, so if you have any feature ideas etc, please get in touch, and we will help you work on submitting a PR! @@ -382,30 +397,30 @@ We also encourage participation, so if you have any feature ideas etc, please ge NOTE: We are very interested to hear about your experiences! And please vote on your favourite issues! -If you have questions, head over to the https://launchpass.com/confluentcommunity[Confluent Slack community], or raise an https://github.com/confluentinc/parallel-consumer/issues[issue] on GitHub. +If you have questions or find a bug, raise an {issues_link}[issue] on GitHub. == License -This library is copyright Confluent Inc, and licensed under the Apache License Version 2.0. +This library is copyright Confluent, Inc. and contributors, and licensed under the Apache License Version 2.0. == Usage === Maven -This project is available in maven central, https://repo1.maven.org/maven2/io/confluent/parallelconsumer/[repo1], along with SNAPSHOT builds (starting with 0.5-SNAPSHOT) in https://oss.sonatype.org/content/repositories/snapshots/io/confluent/parallelconsumer/[repo1's SNAPSHOTS repo]. +This project is available in Maven Central, https://repo1.maven.org/maven2/io/github/astubbs/parallelconsumer/[repo1]. -Latest version can be seen https://search.maven.org/artifact/io.confluent.parallelconsumer/parallel-consumer-core[here]. +Latest version can be seen https://search.maven.org/artifact/io.github.astubbs.parallelconsumer/parallel-consumer-core[here]. Where `${project.version}` is the version to be used: -* group ID: `io.confluent.parallelconsumer` +* group ID: `io.github.astubbs.parallelconsumer` * artifact ID: `parallel-consumer-core` -* version: image:https://maven-badges.herokuapp.com/maven-central/io.confluent.parallelconsumer/parallel-consumer-parent/badge.svg?style=flat[link=https://mvnrepository.com/artifact/io.confluent.parallelconsumer/parallel-consumer-parent,Latest Parallel Consumer on Maven Central] +* version: image:https://maven-badges.herokuapp.com/maven-central/io.github.astubbs.parallelconsumer/parallel-consumer-parent/badge.svg?style=flat[link=https://mvnrepository.com/artifact/io.github.astubbs.parallelconsumer/parallel-consumer-parent,Latest Parallel Consumer on Maven Central] .Core Module Dependency [source,xml,indent=0] - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} @@ -413,7 +428,7 @@ Where `${project.version}` is the version to be used: .Reactor Module Dependency [source,xml,indent=0] - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-reactor ${project.version} @@ -421,7 +436,7 @@ Where `${project.version}` is the version to be used: .Vert.x Module Dependency [source,xml,indent=0] - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-vertx ${project.version} @@ -468,7 +483,7 @@ After this setup, one then has the choice of interfaces: * `JStreamVertxParallelStreamProcessor` There is another interface: `ParallelConsumer` which is integrated, however there is currently no immediate implementation. -See {issues_link}/12[issue #12], and the `ParallelConsumer` JavaDoc: +See {base_confluent_url}/issues/12[issue #12], and the `ParallelConsumer` JavaDoc: [source,java] ---- @@ -651,21 +666,6 @@ image::https://lucid.app/publicSegments/view/43f2740c-2a7f-4b7f-909e-434a5bbe3fb See the link:{project_root}/parallel-consumer-examples/parallel-consumer-example-streams/src/main/java/io/confluent/parallelconsumer/examples/streams/StreamsApp.java[Kafka Streams example] project, and it's test. -[[confluent-cloud]] -=== Confluent Cloud - -. Provision your fully managed Kafka cluster in Confluent Cloud -.. Sign up for https://www.confluent.io/confluent-cloud/tryfree/[Confluent Cloud], a fully-managed Apache Kafka service. -.. After you log in to Confluent Cloud, click on `Add cloud environment` and name the environment `learn-kafka`. -Using a new environment keeps your learning resources separate from your other Confluent Cloud resources. -.. Click on https://confluent.cloud/learn[LEARN] and follow the instructions to launch a Kafka cluster and to enable Schema Registry. -. Access the client configuration settings -.. From the Confluent Cloud Console, navigate to your Kafka cluster. -From the `Clients` view, get the connection information customized to your cluster (select `Java`). -.. Create new credentials for your Kafka cluster, and then Confluent Cloud will show a configuration block with your new credentials automatically populated (make sure `show API keys` is checked). -.. Use these settings presented to https://docs.confluent.io/clients-kafka-java/current/overview.html[configure your clients]. -. Use these clients for steps outlined in the <> section. - [[upgrading]] == Upgrading @@ -794,7 +794,7 @@ You can access the retry count of a record through it's wrapped `WorkContainer` .Example retry delay function implementing exponential backoff [source,java,indent=0] ---- - final double multiplier = 2.0; + final double multiplier = 0.5; final int baseDelaySecond = 1; ParallelConsumerOptions.builder() @@ -1341,6 +1341,50 @@ Note:: See https://github.com/confluentinc/parallel-consumer/issues/162[issue #162] and this https://stackoverflow.com/questions/4786881/why-is-test-jar-dependency-required-for-mvn-compile[Stack Overflow question]. +=== Build Scripts + +Helper scripts are in the `bin/` directory: + +[qanda] +Quick local build (compile + unit tests):: +`bin/build.sh` + +Unit tests only (no Docker needed):: +`bin/ci-unit-test.sh` + +Integration tests only (requires Docker for TestContainers):: +`bin/ci-integration-test.sh` + +Full CI build with all tests (unit + integration):: +`bin/ci-build.sh` + +CI build against a specific Kafka version:: +`bin/ci-build.sh 3.9.1` + +All `ci-*` scripts use the `-Pci` Maven profile which enables license checking and disables parallel test execution. The GitHub Actions CI workflow uses these scripts, so running them locally reproduces the CI environment. + +=== Releasing + +The `pom.xml` version is the source of truth for publishing — there is no `maven-release-plugin` step. + +On every push to `master`, `.github/workflows/publish.yml` deploys to Maven Central: + +* If the version ends in `-SNAPSHOT` → publishes a snapshot +* If the version does not end in `-SNAPSHOT` → publishes a full release, creates a `v` git tag, and creates a GitHub release + +To cut a release: + +. Open a PR removing `-SNAPSHOT` from `` in the parent `pom.xml` (e.g. `0.6.0.0-SNAPSHOT` → `0.6.0.0`) +. Merge it to master → CI publishes the release +. Open another PR bumping to the next snapshot (e.g. `0.6.0.1-SNAPSHOT`) and merge + +Required GitHub repository secrets: + +* `MAVEN_CENTRAL_USERNAME` — Sonatype Central Portal token username +* `MAVEN_CENTRAL_PASSWORD` — Sonatype Central Portal token password +* `MAVEN_GPG_PRIVATE_KEY` — Armored GPG private key for signing artifacts +* `MAVEN_GPG_PASSPHRASE` — Passphrase for the GPG key + === Testing The project has good automated test coverage, of all features. @@ -1534,6 +1578,23 @@ ifndef::github_name[] toc::[] endif::[] +== 0.6.0.0 + +=== Breaking + +* Maven coordinates rebranded from `io.confluent.parallelconsumer` to `io.github.astubbs.parallelconsumer`. Update your dependency declarations. + +=== Fixes + +* fix: null epoch race condition in EpochAndRecordsMap - poll() returning records before onPartitionsAssigned() no longer causes NPE; records are safely skipped and re-delivered +* fix: test pollution from leaked threads and Awaitility global state across test classes + +=== Improvements + +* ci: modernized GitHub Actions CI with parallel test suites (unit, integration, performance), SpotBugs, duplicate detection, mutation testing (PIT), and dependency vulnerability scanning +* ci: added publish workflow for automated Maven Central deployment +* docs: updated README and added upstream PR analysis report + == 0.5.3.4 === Fixes @@ -1976,3 +2037,4 @@ without operational burden or harming the clusters performance ** Clean draining shutdown cycle //:leveloffset: -1 - Duplicate key leveloffset (attempted merging values +1 and -1): https://github.com/whelk-io/asciidoc-template-maven-plugin/issues/118 + diff --git a/RELEASE.adoc b/RELEASE.adoc deleted file mode 100644 index e6c5c6838..000000000 --- a/RELEASE.adoc +++ /dev/null @@ -1,43 +0,0 @@ -= Releasing - -Releases are automated through a Semaphore Task and are always built from latest commit of the master branch. - - -- Verify that Changelog.md and Readme.md are up-to-date with all the changes in the release. If not update them through normal PR process. -- Go to Tasks in Parallel Consumer project on Semaphore CI - https://semaphore.ci.confluent.io/projects/parallel-consumer/schedulers -- Click on Run Now for the Release task -- Enter Release Version (e.g. 0.5.3.3) -- Enter Next Development Version (e.g. 0.5.3.4-SNAPSHOT) -- Click Run the Task -- The task will create a Tag branch and corresponding Semaphore Job will start automatically -- Once the build job for the Tag is complete - go to the Semaphore CI page for that job and click Publish to Maven Central in the Promotions box. -- Wait for the promotion to complete - note that it may show as failed on Semaphore - but actually succeed - verify logs on Semaphore -- Wait for Sonatype to publish from it's staging area (~15 minutes) https://repo1.maven.org/maven2/io/confluent/parallelconsumer/parallel-consumer-parent/[repo1 link] -- Verify the release is available on Maven Central https://repo1.maven.org/maven2/io/confluent/parallelconsumer/parallel-consumer-parent/[repo1 link] -- Create the release on GH from the tag -- Paste in the details from the changelog, save, share as discussion -- Announce on slack (community #parallel-consumer and internal channels), mailing list, twitter - - -== Troubleshooting - -There seems to be a bug somewhere in Maven plugins or just timeout getting caught when deploying to sonatype - after actual deployment is done - logs like this: -``` -[INFO] * Upload of locally staged artifacts finished.04:49 -[INFO] * Closing staging repository with ID "ioconfluent".04:49 -Waiting for operation to complete...04:49 -..................................06:30 -[INFO] Remote staged 1 repositories, finished with success.06:30 -[INFO] Remote staging repositories are being released...06:30 -Waiting for operation to complete...06:30 -.......Oct 21, 2024 1:04:39 PM com.sun.jersey.api.client.ClientResponse getEntity07:22 -SEVERE: A message body reader for Java class com.sonatype.nexus.staging.api.dto.StagingProfileRepositoryDTO, and Java type class com.sonatype.nexus.staging.api.dto.StagingProfileRepositoryDTO, and MIME media type text/html was not found07:22 -Oct 21, 2024 1:04:39 PM com.sun.jersey.api.client.ClientResponse getEntity -``` -Shows up as promotion failed - but as you can see the actual deployment was successful. - -In case of actual upload failure / build failure of the tagged branch - the tag might need to be reverted - which is locked by devprod and github task has to be ran to revert the Tag. - -`delete-github-tag` in https://semaphore.ci.confluent.io/projects/github-tasks/schedulers. - -This will delete the tag (actually it will rename the tag to TAG-DELETED - i.e. 0.5.3.3-deleted) so that whatever build issue there is can be remedied (with revert / new PRs to Master branch) and when ready to do a release again - you can follow this process again. diff --git a/bin/build.sh b/bin/build.sh new file mode 100755 index 000000000..4f45a4056 --- /dev/null +++ b/bin/build.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2026 Antony Stubbs and contributors +# + +# Local development build - compile and run unit tests +# Usage: bin/build.sh [extra-maven-args...] +# Example: bin/build.sh -pl parallel-consumer-core + +set -euo pipefail + +./mvnw --batch-mode clean package -Dlicense.skip "$@" diff --git a/bin/ci-build.sh b/bin/ci-build.sh new file mode 100755 index 000000000..acf353f6a --- /dev/null +++ b/bin/ci-build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2026 Antony Stubbs and contributors +# + +# CI build script - run the full build and test suite +# Usage: bin/ci-build.sh [kafka-version] +# Example: bin/ci-build.sh 3.9.1 +# If no version is specified, uses the default from pom.xml + +set -euo pipefail + +KAFKA_VERSION_ARG="" +if [ $# -ge 1 ]; then + KAFKA_VERSION_ARG="-Dkafka.version=$1" + echo "Building with Kafka version: $1" +else + echo "Building with default Kafka version from pom.xml" +fi + +./mvnw --batch-mode \ + -Pci \ + clean verify \ + ${KAFKA_VERSION_ARG:+"$KAFKA_VERSION_ARG"} \ + -Dlicense.skip \ + -Dexcluded.groups=performance \ + -Dsurefire.rerunFailingTestsCount=2 diff --git a/bin/ci-integration-test.sh b/bin/ci-integration-test.sh new file mode 100755 index 000000000..24a34bffc --- /dev/null +++ b/bin/ci-integration-test.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2026 Antony Stubbs and contributors +# + +# Run integration tests only (failsafe, requires Docker for TestContainers). +# Skips unit tests to avoid duplicate work. +# Usage: bin/ci-integration-test.sh + +set -euo pipefail + +./mvnw --batch-mode \ + -Pci \ + clean verify \ + -DskipUTs=true \ + -Dlicense.skip \ + -Dexcluded.groups=performance \ + -Dsurefire.rerunFailingTestsCount=2 diff --git a/bin/ci-unit-test.sh b/bin/ci-unit-test.sh new file mode 100755 index 000000000..3d1e9afc6 --- /dev/null +++ b/bin/ci-unit-test.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2026 Antony Stubbs and contributors +# + +# Run unit tests only (surefire, no Docker/TestContainers needed). +# Usage: bin/ci-unit-test.sh + +set -euo pipefail + +./mvnw --batch-mode \ + -Pci \ + clean test \ + -Dlicense.skip \ + -Dexcluded.groups=performance \ + -Dsurefire.rerunFailingTestsCount=2 diff --git a/bin/performance-test.cmd b/bin/performance-test.cmd new file mode 100644 index 000000000..b086f05d7 --- /dev/null +++ b/bin/performance-test.cmd @@ -0,0 +1,18 @@ +@REM Copyright (C) 2026 Antony Stubbs and contributors +@REM +@REM Run only the performance test suite (tests tagged @Tag("performance")). +@REM These are excluded from the regular CI build because they take a long time +@REM and need substantial hardware. Used by the self-hosted Windows runner. +@REM +@REM Usage: bin\performance-test.cmd [extra-maven-args...] + +@echo off +setlocal + +call mvnw.cmd --batch-mode ^ + -Pci ^ + clean verify ^ + -Dincluded.groups=performance ^ + -Dexcluded.groups= ^ + -Dlicense.skip ^ + %* diff --git a/bin/performance-test.sh b/bin/performance-test.sh new file mode 100755 index 000000000..596121c0f --- /dev/null +++ b/bin/performance-test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2026 Antony Stubbs and contributors +# + +# Run only the performance test suite (tests tagged @Tag("performance")). +# These are excluded from the regular CI build because they take a long time +# and need substantial hardware. The self-hosted runner workflow +# (.github/workflows/performance.yml) calls this script. +# +# Usage: bin/performance-test.sh [extra-maven-args...] + +set -euo pipefail + +./mvnw --batch-mode \ + -Pci \ + clean verify \ + -DskipUTs=true \ + -Dincluded.groups=performance \ + -Dexcluded.groups= \ + -Dlicense.skip \ + "$@" diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..0b0bba041 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,11 @@ +coverage: + status: + project: + default: + # Fail if overall coverage drops from the base branch + target: auto + threshold: 1% + patch: + default: + # Don't enforce a minimum on new code — just track it + informational: true diff --git a/docs/solutions/build-errors/maven-central-timeout-azure-west-regions-2026-04-21.md b/docs/solutions/build-errors/maven-central-timeout-azure-west-regions-2026-04-21.md new file mode 100644 index 000000000..3719147a8 --- /dev/null +++ b/docs/solutions/build-errors/maven-central-timeout-azure-west-regions-2026-04-21.md @@ -0,0 +1,122 @@ +--- +title: Maven Central download timeouts on GitHub Actions runners in Azure West US regions +date: 2026-04-21 +category: build-errors +module: build-system +problem_type: build_error +component: development_workflow +symptoms: + - "Could not transfer artifact io.vertx:vertx-web-client:pom:4.5.7 from/to central: Read timed out" + - Build fails consistently on vertx module after core module succeeds + - Exactly 240-second (4-minute) hang per artifact download attempt from Maven Central + - AK 3.9.1 matrix entry passes while AK 3.7.0 and 3.1.0 fail in the same workflow run +root_cause: config_error +resolution_type: config_change +severity: high +tags: + - maven-central + - github-actions + - azure-region + - vertx + - timeout + - cache-warming + - ci +--- + +# Maven Central download timeouts on GitHub Actions runners in Azure West US regions + +## Problem + +GitHub Actions CI builds consistently fail downloading Maven dependencies (appearing as vertx timeouts), but only on some matrix entries while others pass in the same workflow run. The failures are not random - they correlate with which Azure data center the runner is assigned to. + +## Symptoms + +- `Could not transfer artifact io.vertx:vertx-web-client:pom:4.5.7 from/to central (https://repo1.maven.org/maven2/): Read timed out` +- Each download attempt hangs for exactly 240 seconds before falling through to the next repository +- The vertx module always fails; core module always passes (because core's deps are already cached) +- Re-running the failed jobs produces the same failure +- The same artifact downloads in under 200ms locally and in under 200ms from East US runners + +## What Didn't Work + +- **Re-running failed jobs** - same runners in the same regions, same timeout +- **`retryHandler.count=3` in `.mvn/maven.config`** - retries don't help when the CDN route itself is broken. Each retry adds another 120s timeout, totalling 240s+ per artifact per repository +- **Assuming it was a vertx-specific issue** - vertx appeared to be the problem because it's the first module with uncached dependencies. The real issue is the network route from certain Azure regions to Maven Central's CDN + +## Solution + +The root cause is that Maven Central's CDN has degraded connectivity from Azure's western US data centers (westcentralus, westus3). Since you can't control which region GitHub assigns your runner to, the fix is to pre-warm the Maven cache so no module needs to download from Central during the actual build. + +The `prepare-deps` cache warming job downloads all dependencies once (including vertx), then the matrix jobs restore from that cache: + +```yaml +# .github/workflows/maven.yml + +prepare-deps: + # Removed: if: github.event_name == 'pull_request' + # Now runs on BOTH PR and push builds + name: "Prepare Maven Cache" + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + - name: Restore Maven cache + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + key: setup-java-Linux-x64-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + setup-java-Linux-x64-maven- + - name: Download all dependencies + run: ./mvnw --batch-mode -Pci dependency:go-offline -Dlicense.skip -DincludeScope=test -U + - name: Save Maven cache (rotating key) + if: success() + uses: actions/cache/save@v4 + with: + path: ~/.m2/repository + key: setup-java-Linux-x64-maven-${{ hashFiles('**/pom.xml') }}-${{ github.run_id }} + +build: + if: github.event_name == 'push' + needs: prepare-deps # <-- added: wait for cache warming + # ... matrix config ... +``` + +The rotating `...-${{ github.run_id }}` save key ensures every successful cache-warming run can update the cache, unlike `setup-java`'s built-in caching which uses `actions/cache` (won't save when the primary key hits). + +## Why This Works + +**The root cause is Azure region routing, not Maven Central or vertx.** + +Evidence from the same workflow run (24700111718): + +| Job | Azure Region | vertx-web-client download | Result | +|-----|-------------|--------------------------|--------| +| AK 3.9.1 | **eastus** | 160ms | Passed | +| AK 3.7.0 | **westcentralus** | 240s timeout | Failed | +| AK 3.1.0 | **westus3** | 30min timeout | Failed | + +All three jobs started from the same cache, ran the same code, and tried to download the same 2.9KB POM file from the same Maven Central URL. The only difference was which Azure data center the runner was in. + +Vertx appeared to be the culprit because: +1. The reactor build order puts vertx as the **first module after core** that needs non-core dependencies +2. Core module dependencies (kafka-clients, slf4j, lombok) are either already cached or hosted on faster CDN paths +3. When vertx fails, reactor and mutiny are SKIPPED - so vertx always appears to be the only failing module + +With cache warming, all artifacts (including vertx) are downloaded by a single `prepare-deps` job. If that job lands on a bad region, it may take longer but will eventually succeed within its 15-minute timeout and 3 retries. The subsequent matrix jobs then read everything from the local cache regardless of their runner's region. + +## Prevention + +- **Always run `prepare-deps` before jobs that need Maven dependencies** - don't rely on `setup-java`'s built-in `cache: 'maven'` alone for push builds. The built-in cache uses `actions/cache` which won't overwrite an existing (possibly incomplete) cache key. +- **Use rotating cache keys** (`...-${{ github.run_id }}`) so each successful run can update the cache. Static keys based only on `hashFiles('**/pom.xml')` get stuck if the first cache entry was incomplete. +- **Don't chase the symptom** - when a specific dependency consistently times out in CI, check the Azure region of passing vs failing runners before assuming the dependency or its repository is the problem. + +## Related Issues + +- PR #48 (`fix/prepare-deps-push-builds`) - the fix that extends cache warming to push builds +- `docs/solutions/security-issues/ci-hardening-pull-request-target-mutable-refs-publish-gate-2026-04-21.md` - companion CI hardening doc +- `.mvn/maven.config` - connection timeout settings (10s connect, 120s read, 3 retries) diff --git a/docs/solutions/security-issues/ci-hardening-pull-request-target-mutable-refs-publish-gate-2026-04-21.md b/docs/solutions/security-issues/ci-hardening-pull-request-target-mutable-refs-publish-gate-2026-04-21.md new file mode 100644 index 000000000..d2d499704 --- /dev/null +++ b/docs/solutions/security-issues/ci-hardening-pull-request-target-mutable-refs-publish-gate-2026-04-21.md @@ -0,0 +1,186 @@ +--- +title: "CI Security Hardening: pull_request_target, Mutable Action Refs, Missing Test Gates, and Fragile Static Parsers" +date: 2026-04-21 +category: security-issues +module: build-system +problem_type: security_issue +component: development_workflow +root_cause: config_error +resolution_type: config_change +severity: high +applies_when: + - Setting up GitHub Actions CI for a fork that accepts external PRs + - Using custom or third-party actions with write permissions + - Publishing artifacts to Maven Central from CI + - Parsing version strings in test infrastructure static initializers +tags: + - ci + - github-actions + - supply-chain + - action-pinning + - publish-gate + - static-initializer + - pull-request-target +--- + +# CI Security Hardening: pull_request_target, Mutable Action Refs, Missing Test Gates, and Fragile Static Parsers + +## Context + +During code review of the `dev/rebrand-fork` branch (parallel-consumer fork rebranding), four CI security and robustness issues were identified. These ranged from a critical supply-chain vulnerability (`pull_request_target` + mutable action ref) to a test-killing `ExceptionInInitializerError` from unparseable version strings. None had been exploited, but all represented real attack surface or reliability gaps. + +The fixes establish patterns applicable to any project using GitHub Actions with fork PRs, reusable actions, and Maven publishing. + +## Guidance + +### 1. Never use `pull_request_target` with mutable action refs + +`pull_request_target` runs workflows in the context of the *base* repository, with access to base-repo secrets and elevated permissions. When combined with a mutable action ref (branch name), an attacker controlling that action branch can execute arbitrary code with write permissions on every fork PR. + +**Rule**: If you need `pull_request_target`, pin every action to a commit SHA. Prefer switching to `pull_request` where the elevated capability is not actually needed. + +### 2. Pin all action refs to commit SHAs + +Branch and tag refs are mutable pointers. A force-push upstream silently changes what code runs in your CI, with whatever permissions `GITHUB_TOKEN` grants. SHA pins are immutable. + +**Rule**: Every `uses:` entry in a workflow that has any write permission must reference a commit SHA. Keep the original ref as a trailing comment for human readability. + +### 3. Gate publishing on a successful test run + +Publishing on raw `push` with `-DskipTests` means any test-failing commit produces a published SNAPSHOT artifact. Downstream consumers inherit broken code. + +**Rule**: Use `workflow_run` to trigger publishing only after the build-and-test workflow succeeds. Add an explicit `if:` guard on the conclusion. Keep `workflow_dispatch` so maintainers can manually re-run. + +### 4. Harden version string parsing for pre-release suffixes + +When parsing version strings at runtime, `Integer.parseInt()` on a string like `"4.0.0-SNAPSHOT"` throws `NumberFormatException`. If this code runs in a static initializer, the resulting `ExceptionInInitializerError` kills every test in every class that inherits the initializer - a confusing failure mode that appears unrelated to the actual cause. + +**Rule**: Strip pre-release suffixes before parsing. Wrap in a try/catch with a safe fallback. Never let version detection run in a static initializer without a fallback. + +## Why This Matters + +**Supply chain risk (Issues 1 & 2).** `pull_request_target` with mutable action refs is a well-documented GitHub Actions attack vector. An attacker who can push to the referenced action branch - including via a compromised maintainer account on a third-party repo - silently executes code with `GITHUB_TOKEN` write access on every incoming PR. This can exfiltrate secrets, poison releases, or modify repository state. + +**Artifact integrity (Issue 3).** Publishing broken SNAPSHOTs trains downstream developers to distrust the artifact feed. In a library project, a bad SNAPSHOT propagates into dependent projects before anyone notices. + +**Test infrastructure fragility (Issue 4).** Static initializer failures produce `ExceptionInInitializerError` which JUnit reports as a class-level error, not a test failure. The root cause is buried in the stack trace with no obvious connection to the test being run. + +## When to Apply + +- **SHA pinning**: Any GitHub Actions workflow with write permissions, secrets access, or `pull_request_target` trigger +- **`pull_request` vs `pull_request_target`**: Use `pull_request_target` only when you explicitly need base-repo secrets for fork PRs (e.g., posting a comment after a permission check). For read-only checks, use `pull_request` +- **Publish gating**: Whenever publishing is triggered by a push and the build/test workflow lives in a separate workflow file. If tests and publishing are in the same file, a `needs:` dependency is simpler +- **Version string hardening**: Whenever parsing version strings from dependency metadata, environment variables, or external configuration - especially in static initializers or `@BeforeAll` + +## Examples + +### check-dependencies.yml (pull_request_target + mutable ref) + +Before (vulnerable): +```yaml +on: + pull_request_target: + types: [opened, edited, closed, reopened] +permissions: + checks: write +jobs: + check_dependencies: + steps: + - uses: astubbs/dependencies-action@feat/auto-unblock-children-on-merge +``` + +After (hardened): +```yaml +on: + pull_request: + types: [opened, edited, closed, reopened] +permissions: + checks: write +jobs: + check_dependencies: + steps: + - uses: astubbs/dependencies-action@a09974c # feat/auto-unblock-children-on-merge +``` + +### maven.yml (mutable action refs) + +Before: +```yaml +- uses: astubbs/duplicate-code-cross-check@v1 +- uses: astubbs/duplicate-code-detection-tool@feat/base-vs-pr-comparison +``` + +After: +```yaml +- uses: astubbs/duplicate-code-cross-check@d3140ef # v1 +- uses: astubbs/duplicate-code-detection-tool@4e302e7 # feat/base-vs-pr-comparison +``` + +### publish.yml (missing test gate) + +Before: +```yaml +on: + push: + branches: [ master ] +jobs: + publish: + steps: + - run: ./mvnw deploy -DskipTests +``` + +After: +```yaml +on: + workflow_run: + workflows: ["Build and Test"] + branches: [ master ] + types: [completed] + workflow_dispatch: +jobs: + publish: + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + steps: + - run: ./mvnw deploy -DskipTests +``` + +### BrokerIntegrationTest.java (fragile version parsing) + +Before: +```java +static String deriveCpKafkaImage() { + String akVersion = AppInfoParser.getVersion(); + String[] parts = akVersion.split("\\."); + int akMajor = Integer.parseInt(parts[0]); // throws on "4.0.0-SNAPSHOT" + int akMinor = Integer.parseInt(parts[1]); + return "confluentinc/cp-kafka:" + (akMajor + 4) + "." + akMinor + ".0"; +} +``` + +After: +```java +private static final String FALLBACK_CP_IMAGE = "confluentinc/cp-kafka:7.9.0"; + +static String deriveCpKafkaImage() { + String akVersion = AppInfoParser.getVersion(); + try { + String cleanVersion = akVersion.split("-")[0]; // strip -SNAPSHOT, -rc1 + String[] parts = cleanVersion.split("\\."); + int akMajor = Integer.parseInt(parts[0]); + int akMinor = Integer.parseInt(parts[1]); + return "confluentinc/cp-kafka:" + (akMajor + 4) + "." + akMinor + ".0"; + } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) { + log.warn("Could not parse Kafka version '{}', falling back to {}", + akVersion, FALLBACK_CP_IMAGE, e); + return FALLBACK_CP_IMAGE; + } +} +``` + +## Related + +- `docs/solutions/workflow-issues/copyright-header-rules-for-fork-2026-04-21.md` - companion fork workflow guidance (copyright headers) +- `.github/workflows/check-dependencies.yml` - primary subject of Issue 1 +- `.github/workflows/maven.yml` - primary subject of Issue 2 +- `.github/workflows/publish.yml` - primary subject of Issue 3 +- `parallel-consumer-core/src/test-integration/.../BrokerIntegrationTest.java` - primary subject of Issue 4 diff --git a/docs/solutions/workflow-issues/copyright-header-rules-for-fork-2026-04-21.md b/docs/solutions/workflow-issues/copyright-header-rules-for-fork-2026-04-21.md new file mode 100644 index 000000000..f139f5658 --- /dev/null +++ b/docs/solutions/workflow-issues/copyright-header-rules-for-fork-2026-04-21.md @@ -0,0 +1,132 @@ +--- +title: Copyright header management rules for Apache 2.0 fork +date: 2026-04-21 +category: workflow-issues +module: build-system +problem_type: workflow_issue +component: development_workflow +severity: medium +applies_when: + - Rebranding an Apache 2.0 fork under new Maven coordinates + - Creating new files in a forked repository + - Running Maven builds that trigger the license plugin + - Code review of changes to files with copyright headers +tags: + - copyright + - license-headers + - fork + - maven + - mycila + - apache-2 + - agents-md +--- + +# Copyright header management rules for Apache 2.0 fork + +## Context + +The `astubbs/parallel-consumer` repository is an Apache 2.0 fork of `confluentinc/parallel-consumer`, rebranded under `io.github.astubbs.parallelconsumer` Maven coordinates. During the `dev/rebrand-fork` branch review, three copyright header problems surfaced: + +1. **License plugin auto-bumps years**: The `license-maven-plugin` (Mycila) auto-bumps copyright year ranges when run without `-Dlicense.skip`. A routine pom.xml property addition caused `parallel-consumer-mutiny/pom.xml` to silently gain a year bump from `2020-2025` to `2020-2026` as an unintended side effect. + +2. **New fork files wrongly attributed**: Six new `bin/` scripts (build.sh, ci-build.sh, ci-integration-test.sh, ci-unit-test.sh, performance-test.cmd, performance-test.sh) were written entirely for the fork but carried `Copyright (C) 2020-2026 Confluent, Inc.` headers. Confluent didn't write them and they didn't exist before 2026. + +3. **Review fix changed copyright incorrectly**: During code review autofix, the reviewer changed `EpochAndRecordsMap.java`'s copyright from `2020-2022 Confluent, Inc.` to `2020-2026 Confluent, Inc. and contributors`. The original PR commit only added code (a null-epoch guard), not a copyright change - the reviewer introduced a spurious copyright modification that had to be reverted. + +None of this was caught early because AGENTS.md had no guidance on copyright header management. + +## Guidance + +Rules added to AGENTS.md under "Code Style": + +``` +- **Copyright rules for this fork**: + - Do not change copyright headers on existing files unless the file has + substantive code changes in the same commit + - Do not bump copyright years as an incidental or standalone change + - The `NOTICE` file at repo root contains the legal attribution structure + - New files written entirely for the fork should not claim Confluent copyright + - Always pass `-Dlicense.skip` to Maven to prevent the license plugin from + auto-bumping years +``` + +The NOTICE file at the repository root is the authoritative attribution record: + +``` +Parallel Consumer +Copyright 2020-2026 Confluent, Inc. +Copyright 2026 Antony Stubbs and contributors +``` + +The pom.xml license template header was changed from: + +``` +Copyright (C) ${license.git.copyrightYears} ${project.organization.name} +``` + +to: + +``` +Copyright (C) 2020-${license.git.copyrightYears} Confluent, Inc. and contributors +``` + +## Why This Matters + +**Legal accuracy.** Copyright headers are legal statements. Attributing work to Confluent that Confluent did not write, or claiming a file dates from 2020 when it was created in 2026, produces inaccurate legal records. + +**Diff noise.** Spurious year bumps pollute commits and PRs with meaningless diffs. A reviewer looking at a pom.xml change to add `9` should not have to mentally filter out an unrelated copyright line change. This also makes git blame less useful. + +**Review trust.** When an automated reviewer or agent changes copyright headers as a "fix," it undermines confidence in the review. The `EpochAndRecordsMap.java` incident changed a historically accurate range (`2020-2022`) to a wrong one (`2020-2026`). + +**Plugin footgun.** The Mycila license plugin runs as part of the normal build cycle. Without `-Dlicense.skip`, any Maven command that triggers `license:check` or `license:format` will rewrite headers silently. The plugin also breaks in git worktrees. + +## When to Apply + +- Any time you run a Maven command: always include `-Dlicense.skip` +- Any time you create a new file that did not exist in upstream: do not add Confluent copyright +- Any time a code review suggests changing a copyright header: only accept if the same commit has substantive code changes +- Any time you see a standalone "bump copyright year" commit: reject it +- Any time you add a property, dependency, or config-only change to a pom.xml: the copyright header must not change + +## Examples + +**Bad - license plugin auto-bump as side effect:** + +```xml + + + +``` + +Fix: run Maven with `-Dlicense.skip` and revert the header change. + +**Bad - new fork-only file claiming upstream copyright:** + +```bash +#!/bin/bash +# Copyright (C) 2020-2026 Confluent, Inc. <-- WRONG +# bin/build.sh - created entirely for the astubbs fork in 2026 +``` + +**Bad - reviewer adding spurious copyright change alongside a code fix:** + +```java +// EpochAndRecordsMap.java +// Before (historically accurate): Copyright (C) 2020-2022 Confluent, Inc. +// After reviewer autofix (wrong): Copyright (C) 2020-2026 Confluent, Inc. and contributors +``` + +The commit only added a null-epoch guard. The range `2020-2022` was correct. The reviewer's change was reverted. + +**Correct - substantive code change warrants a header update:** + +```java +// Before: Copyright (C) 2020-2022 Confluent, Inc. +// After a real 2026 code change: Copyright (C) 2020-2026 Confluent, Inc. and contributors +``` + +## Related + +- `AGENTS.md` lines 70-76 - the codified rules (authoritative source) +- `NOTICE` file at repo root - legal attribution structure +- `pom.xml` line ~613 - license template `inlineHeader` configuration diff --git a/parallel-consumer-core/pom.xml b/parallel-consumer-core/pom.xml index 5970f43b4..e984a10bf 100644 --- a/parallel-consumer-core/pom.xml +++ b/parallel-consumer-core/pom.xml @@ -6,15 +6,15 @@ --> - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-parent - 0.5.3.4-SNAPSHOT + 0.6.0.0-SNAPSHOT 4.0.0 parallel-consumer-core - Confluent Parallel Consumer Core + Kafka Parallel Consumer Core diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/EpochAndRecordsMap.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/EpochAndRecordsMap.java index 1ad4f6aa0..0b482ce8b 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/EpochAndRecordsMap.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/internal/EpochAndRecordsMap.java @@ -7,6 +7,7 @@ import io.confluent.parallelconsumer.state.PartitionStateManager; import lombok.NonNull; import lombok.Value; +import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.common.TopicPartition; @@ -18,6 +19,7 @@ * * @see BrokerPollSystem#partitionAssignmentEpoch */ +@Slf4j @Value public class EpochAndRecordsMap { @@ -27,6 +29,15 @@ public EpochAndRecordsMap(ConsumerRecords poll, PartitionStateManager { var records = poll.records(partition); Long epochOfPartition = pm.getEpochOfPartition(partition); + if (epochOfPartition == null) { + // Race: poll() returned records for a partition before onPartitionsAssigned() + // has fired. This is more likely with Kafka 2.x's eager rebalance protocol. + // Safe to skip — these records haven't been committed, so Kafka will re-deliver + // them on the next poll after the assignment callback completes. + log.warn("Skipping {} records for partition {} — no epoch assigned yet. " + + "Records will be re-delivered on next poll after assignment completes.", records.size(), partition); + return; + } RecordsAndEpoch entry = new RecordsAndEpoch(partition, epochOfPartition, records); recordMap.put(partition, entry); }); diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/PartitionStateManager.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/PartitionStateManager.java index fd47dd552..c2c55cb5d 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/PartitionStateManager.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/PartitionStateManager.java @@ -248,12 +248,13 @@ private void resetOffsetMapAndRemoveWork(Collection allRemovedPa } /** - * @return the current epoch of the partition + * @return the current epoch of the partition, or null if not yet assigned */ public Long getEpochOfPartition(TopicPartition partition) { return partitionsAssignmentEpochs.get(partition); } + private void incrementPartitionAssignmentEpoch(final Collection partitions) { for (final TopicPartition partition : partitions) { Long epoch = partitionsAssignmentEpochs.getOrDefault(partition, PartitionState.KAFKA_OFFSET_ABSENCE); diff --git a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/ProcessingShard.java b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/ProcessingShard.java index 6dc4d6890..f2ec527a7 100644 --- a/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/ProcessingShard.java +++ b/parallel-consumer-core/src/main/java/io/confluent/parallelconsumer/state/ProcessingShard.java @@ -59,8 +59,17 @@ public class ProcessingShard { public void addWorkContainer(WorkContainer wc) { long key = wc.offset(); - if (entries.containsKey(key)) { - log.debug("Entry for {} already exists in shard queue, dropping record", wc); + WorkContainer existing = entries.get(key); + if (existing != null) { + // Check if the existing entry is stale and should be replaced + if (isWorkContainerStale(existing)) { + log.debug("Replacing stale entry (epoch {}) for offset {} with fresh one (epoch {})", + existing.getEpoch(), key, wc.getEpoch()); + entries.put(key, wc); + // availableWorkContainerCnt stays the same since we're replacing, not adding + } else { + log.debug("Entry for {} already exists in shard queue, dropping record", wc); + } } else { entries.put(key, wc); availableWorkContainerCnt.incrementAndGet(); diff --git a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/BrokerIntegrationTest.java b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/BrokerIntegrationTest.java index 57275a8b3..c04b2e11a 100644 --- a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/BrokerIntegrationTest.java +++ b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/BrokerIntegrationTest.java @@ -55,8 +55,41 @@ public abstract class BrokerIntegrationTest { */ public static KafkaContainer kafkaContainer = createKafkaContainer(null); + /** + * Derives the Confluent Platform version from the Apache Kafka client version so that + * the broker under test matches the client. The CI matrix overrides {@code kafka.version} + * via {@code -Dkafka.version=X.Y.Z}, so we read it at runtime from the client jar. + *

+ * Mapping: CP major = AK major + 4 (e.g., AK 3.1 → CP 7.1, AK 3.9 → CP 7.9). + */ + private static final String FALLBACK_CP_IMAGE = "confluentinc/cp-kafka:7.9.0"; + + static String deriveCpKafkaImage() { + String akVersion = org.apache.kafka.common.utils.AppInfoParser.getVersion(); + log.info("Kafka client version detected: {}", akVersion); + + try { + // Strip pre-release suffixes (e.g. "4.0.0-SNAPSHOT" -> "4.0.0") + String cleanVersion = akVersion.split("-")[0]; + String[] parts = cleanVersion.split("\\."); + int akMajor = Integer.parseInt(parts[0]); + int akMinor = Integer.parseInt(parts[1]); + + // CP major = AK major + 4, CP minor = AK minor + int cpMajor = akMajor + 4; + int cpMinor = akMinor; + + String cpImage = "confluentinc/cp-kafka:" + cpMajor + "." + cpMinor + ".0"; + log.info("Using CP Kafka image: {} (derived from AK {})", cpImage, akVersion); + return cpImage; + } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) { + log.warn("Could not parse Kafka version '{}', falling back to {}", akVersion, FALLBACK_CP_IMAGE, e); + return FALLBACK_CP_IMAGE; + } + } + public static KafkaContainer createKafkaContainer(String logSegmentSize) { - KafkaContainer base = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:7.6.0")) + KafkaContainer base = new KafkaContainer(DockerImageName.parse(deriveCpKafkaImage())) .withEnv("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1") //transaction.state.log.replication.factor .withEnv("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", "1") //transaction.state.log.min.isr .withEnv("KAFKA_TRANSACTION_STATE_LOG_NUM_PARTITIONS", "1") //transaction.state.log.num.partitions diff --git a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/LargeVolumeInMemoryTests.java b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/LargeVolumeInMemoryTests.java index 2e507eb6d..877434a04 100644 --- a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/LargeVolumeInMemoryTests.java +++ b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/LargeVolumeInMemoryTests.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.TopicPartition; import org.assertj.core.util.Lists; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import pl.tlinkowski.unij.api.UniLists; @@ -44,6 +45,7 @@ * Mocked out comparative volume tests */ @Slf4j +@Tag("performance") class LargeVolumeInMemoryTests extends ParallelEoSStreamProcessorTestBase { @SneakyThrows diff --git a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/MultiInstanceHighVolumeTest.java b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/MultiInstanceHighVolumeTest.java index 56efc1e4f..5db19804e 100644 --- a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/MultiInstanceHighVolumeTest.java +++ b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/MultiInstanceHighVolumeTest.java @@ -17,6 +17,7 @@ import org.assertj.core.api.Assertions; import org.assertj.core.api.SoftAssertions; import org.awaitility.core.ConditionTimeoutException; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -34,6 +35,7 @@ import static pl.tlinkowski.unij.api.UniLists.of; @Slf4j +@Tag("performance") class MultiInstanceHighVolumeTest extends BrokerIntegrationTest { public List consumedKeys = Collections.synchronizedList(new ArrayList<>()); diff --git a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/PartitionOrderProcessingTest.java b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/PartitionOrderProcessingTest.java index 91a5d8706..9a9660f2d 100644 --- a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/PartitionOrderProcessingTest.java +++ b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/PartitionOrderProcessingTest.java @@ -17,6 +17,7 @@ import org.junit.jupiter.api.Test; import pl.tlinkowski.unij.api.UniSets; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -96,8 +97,18 @@ void allPartitionsAreProcessedInParallel() { partitionCounts.get(recordContexts.getSingleConsumerRecord().partition()).getAndIncrement(); ThreadUtils.sleepQuietly(10); // introduce a bit of processing delay - to make sure polling backpressure kicks in. }); - await().until(() -> partitionCounts.values().stream().mapToInt(AtomicInteger::get).sum() > 500); // wait until we process some messages to get the counts in. - Assertions.assertTrue(partitionCounts.values().stream().allMatch(v -> v.get() > 0), "Expect all partitions to have some messages processed, actual partitionCounts:" + partitionCounts); + // Wait for BOTH conditions: enough total messages AND all partitions represented. + // Previously the await only checked total > 500, then the assertion checked all + // partitions — a race, because Kafka may deliver from one partition first. + // Moving the partition check into the await lets Awaitility retry until + // all partitions have been reached. + await().atMost(Duration.ofSeconds(120)).untilAsserted(() -> { + int total = partitionCounts.values().stream().mapToInt(AtomicInteger::get).sum(); + Assertions.assertTrue(total > 500, + "Expect > 500 total messages processed, actual: " + total); + Assertions.assertTrue(partitionCounts.values().stream().allMatch(v -> v.get() > 0), + "Expect all partitions to have some messages processed, actual partitionCounts:" + partitionCounts); + }); } @@ -129,7 +140,8 @@ void allPartitionsAreNotProcessedInParallel() { partitionCounts.get(recordContexts.getSingleConsumerRecord().partition()).getAndIncrement(); ThreadUtils.sleepQuietly(10); // introduce a bit of processing delay - to make sure polling backpressure kicks in. }); - await().until(() -> partitionCounts.values().stream().mapToInt(AtomicInteger::get).sum() > 500); // wait until we process some messages to get the counts in. + // 120s explicit timeout — bare await() used shaded Awaitility's 10s default, too tight for CI. + await().atMost(Duration.ofSeconds(120)).until(() -> partitionCounts.values().stream().mapToInt(AtomicInteger::get).sum() > 500); Assertions.assertFalse(partitionCounts.values().stream().allMatch(v -> v.get() > 0), "Expect some processing thread starving and not all partition counts to have some messages processed, actual partitionCounts:" + partitionCounts); } diff --git a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VeryLargeMessageVolumeTest.java b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VeryLargeMessageVolumeTest.java index 6c32c8c09..f27922288 100644 --- a/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VeryLargeMessageVolumeTest.java +++ b/parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VeryLargeMessageVolumeTest.java @@ -25,6 +25,7 @@ import org.assertj.core.api.Assertions; import org.assertj.core.api.SoftAssertions; import org.awaitility.core.ConditionTimeoutException; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.util.*; @@ -51,6 +52,7 @@ * RuntimeException when running with very high options in 0.2.0.0 (Bitset too long to encode) #35 */ @Slf4j +@Tag("performance") public class VeryLargeMessageVolumeTest extends BrokerIntegrationTest { int HIGH_MAX_POLL_RECORDS_CONFIG = 10_000; diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/AbstractParallelEoSStreamProcessorTestBase.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/AbstractParallelEoSStreamProcessorTestBase.java index 918eb0657..652cf88ec 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/AbstractParallelEoSStreamProcessorTestBase.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/AbstractParallelEoSStreamProcessorTestBase.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.producer.MockProducer; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.Serdes; +import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import pl.tlinkowski.unij.api.UniLists; @@ -144,6 +145,12 @@ protected ParallelConsumerOptions.ParallelConsumerOptionsBuilder @AfterEach public void close() { + // Reset Awaitility's global thread-local timeout state so per-test overrides + // (e.g. setDefaultTimeout) don't leak into other tests under non-deterministic + // test order (PIT baseline/mutations surface this; surefire's default ordering + // happens to mask it). Runs even if the test body threw. + Awaitility.reset(); + // don't try to close if error'd (at least one test purposefully creates an error to tests error handling) - we // don't want to bubble up an error here that we expect from here. if (!parentParallelConsumer.isClosedOrFailed()) { diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/MockConsumerTestWithCommitTimeoutException.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/MockConsumerTestWithCommitTimeoutException.java index 7593375b7..2c443eae8 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/MockConsumerTestWithCommitTimeoutException.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/MockConsumerTestWithCommitTimeoutException.java @@ -87,39 +87,47 @@ public synchronized void commitSync(Map offse parallelConsumer.onPartitionsAssigned(of(tp)); mockConsumer.updateBeginningOffsets(startOffsets); - // - new Thread() { - public void run() { - addRecords(mockConsumer); - } - }.start(); - - // - ConcurrentLinkedQueue> records = new ConcurrentLinkedQueue<>(); - parallelConsumer.poll(recordContexts -> { - recordContexts.forEach(recordContext -> { - log.warn("Processing: {}", recordContext); - records.add(recordContext); + // Daemon thread: must NOT survive past this test method, or when it wakes + // from sleep it'll addRecord() on a closed mockConsumer and throw an + // uncaught exception that PIT attributes to whatever test is running next + // in the same minion JVM. We also interrupt it and close PC in the finally + // block. + Thread recordAdder = new Thread(() -> addRecords(mockConsumer), "commit-timeout-record-adder"); + recordAdder.setDaemon(true); + recordAdder.start(); + + try { + // + ConcurrentLinkedQueue> records = new ConcurrentLinkedQueue<>(); + parallelConsumer.poll(recordContexts -> { + recordContexts.forEach(recordContext -> { + log.warn("Processing: {}", recordContext); + records.add(recordContext); + }); }); - }); - // temporarily set the wait timeout - Awaitility.setDefaultTimeout(Duration.ofSeconds(50)); - // - Awaitility.await().untilAsserted(() -> { - assertThat(records).hasSize(10); - }); - - Awaitility.reset(); + // Scope the timeout locally (don't mutate Awaitility's global default — + // that was leaking across tests if the assertion below throws before reset()). + Awaitility.await().atMost(Duration.ofSeconds(50)).untilAsserted(() -> { + assertThat(records).hasSize(10); + }); + } finally { + recordAdder.interrupt(); + parallelConsumer.close(); + } } private void addRecords(MockConsumer mockConsumer) { - for(int i = 0; i < 10; i++) { - mockConsumer.addRecord(new org.apache.kafka.clients.consumer.ConsumerRecord<>(topic, 0, i, "key", "value")); + for (int i = 0; i < 10; i++) { try { + mockConsumer.addRecord(new org.apache.kafka.clients.consumer.ConsumerRecord<>(topic, 0, i, "key", "value")); Thread.sleep(1000L); + } catch (IllegalStateException e) { + // mockConsumer was closed - test has ended, stop quietly + return; } catch (InterruptedException e) { - throw new RuntimeException(e); + Thread.currentThread().interrupt(); + return; } } } diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/MockConsumerTestWithEarlyClose.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/MockConsumerTestWithEarlyClose.java index e12480913..9813296df 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/MockConsumerTestWithEarlyClose.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/MockConsumerTestWithEarlyClose.java @@ -85,39 +85,49 @@ public synchronized void commitSync(Map offse parallelConsumer.onPartitionsAssigned(of(tp)); mockConsumer.updateBeginningOffsets(startOffsets); - // - new Thread() { - public void run() { - addRecords(mockConsumer); - } - }.start(); + // Daemon thread: must NOT survive past this test method, or when it wakes + // from sleep it'll addRecord() on a closed mockConsumer and throw an + // uncaught exception that PIT attributes to whatever test is running next + // in the same minion JVM. We also interrupt it explicitly in the finally + // block to stop the loop promptly. + Thread recordAdder = new Thread(() -> addRecords(mockConsumer), "early-close-record-adder"); + recordAdder.setDaemon(true); + recordAdder.start(); - // - ConcurrentLinkedQueue> records = new ConcurrentLinkedQueue<>(); - parallelConsumer.poll(recordContexts -> { - recordContexts.forEach(recordContext -> { - log.warn("Processing: {}", recordContext); - records.add(recordContext); - }); - }); try { - Thread.sleep(5000L); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + // + ConcurrentLinkedQueue> records = new ConcurrentLinkedQueue<>(); + parallelConsumer.poll(recordContexts -> { + recordContexts.forEach(recordContext -> { + log.warn("Processing: {}", recordContext); + records.add(recordContext); + }); + }); + try { + Thread.sleep(5000L); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } - log.info("Trying to close..."); - parallelConsumer.close(); // request close after 5 seconds - log.info("Close successful!"); + log.info("Trying to close..."); + parallelConsumer.close(); // request close after 5 seconds + log.info("Close successful!"); + } finally { + recordAdder.interrupt(); + } } private void addRecords(MockConsumer mockConsumer) { - for(int i = 0; i < 100000; i++) { - mockConsumer.addRecord(new org.apache.kafka.clients.consumer.ConsumerRecord<>(topic, 0, i, "key", "value")); + for (int i = 0; i < 100000; i++) { try { + mockConsumer.addRecord(new org.apache.kafka.clients.consumer.ConsumerRecord<>(topic, 0, i, "key", "value")); Thread.sleep(1000L); + } catch (IllegalStateException e) { + // mockConsumer was closed - test has ended, stop quietly + return; } catch (InterruptedException e) { - throw new RuntimeException(e); + Thread.currentThread().interrupt(); + return; } } } diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/MockConsumerTestWithSaslAuthenticationException.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/MockConsumerTestWithSaslAuthenticationException.java index c324119f1..5825bab89 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/MockConsumerTestWithSaslAuthenticationException.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/MockConsumerTestWithSaslAuthenticationException.java @@ -12,6 +12,7 @@ import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.SaslAuthenticationException; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.testcontainers.shaded.org.awaitility.Awaitility; @@ -27,14 +28,16 @@ import static pl.tlinkowski.unij.api.UniLists.of; /** - * Test that PC can survive for a temporary SaslAuthenticationException. + * Test that PC can survive a temporary SaslAuthenticationException. * - * In this test, MockConsumer starts to throw SaslAuthenticationException from the beginning until 20 seconds later. + * In this test, MockConsumer throws SaslAuthenticationException from the beginning for 8 seconds, then + * goes back to normal. * - * After that MockConsumer will back to normal. - * - * The saslAuthenticationRetryTimeout is set to 25 seconds. It is expected to resume normal after 20 seconds and will - * be able to consume all produced messages. + * The saslAuthenticationRetryTimeout is set to 30 seconds (generous margin over the 8s outage window) so + * PC has room to recover. The whole test fits comfortably inside PIT's per-test baseline coverage budget + * (which caps around ~80s). The earlier 20s outage + 25s retry version intermittently failed PIT's baseline + * because the total runtime scraped that cap. Test still verifies the same property: PC recovers if the + * retry budget exceeds the outage window. * @author Shilin Wu */ @Slf4j @@ -43,12 +46,25 @@ class MockConsumerTestWithSaslAuthenticationException { private final String topic = MockConsumerTestWithSaslAuthenticationException.class.getSimpleName(); + // Field so @AfterEach can close it. This class doesn't extend + // AbstractParallelEoSStreamProcessorTestBase, so no base-class cleanup runs. + private ParallelEoSStreamProcessor parallelConsumer; + + @AfterEach + void close() { + if (parallelConsumer != null && !parallelConsumer.isClosedOrFailed()) { + parallelConsumer.close(); + } + } + /** * Test that the mock consumer works as expected */ @Test void mockConsumer() { - final AtomicLong failUntil = new AtomicLong(System.currentTimeMillis() + 20000L); + // 8s mock-failure window (was 20s) — keeps total test runtime well within PIT's baseline + // per-test budget while still triggering PC's SASL retry path meaningfully. + final AtomicLong failUntil = new AtomicLong(System.currentTimeMillis() + 8000L); var mockConsumer = new MockConsumer(OffsetResetStrategy.EARLIEST) { @Override public synchronized ConsumerRecords poll(Duration timeout) { @@ -74,9 +90,11 @@ public synchronized void commitSync(Map offse // var options = ParallelConsumerOptions.builder() .consumer(mockConsumer) - .saslAuthenticationRetryTimeout(Duration.ofSeconds(25L)) // set retry to 25 seconds. + // 30s retry budget over an 8s mock-failure window — generous margin (22s) for + // PC's recovery poll even under PIT's slower JVM. + .saslAuthenticationRetryTimeout(Duration.ofSeconds(30L)) .build(); - var parallelConsumer = new ParallelEoSStreamProcessor(options); + parallelConsumer = new ParallelEoSStreamProcessor<>(options); parallelConsumer.subscribe(of(topic)); // MockConsumer is not a correct implementation of the Consumer contract - must manually rebalance++ - or use LongPollingMockConsumer @@ -96,14 +114,13 @@ public synchronized void commitSync(Map offse }); }); - // temporarily set the wait timeout - Awaitility.setDefaultTimeout(Duration.ofSeconds(50)); - // - Awaitility.await().untilAsserted(() -> { + // Scope the timeout locally (don't mutate Awaitility's global default — that was leaking + // across tests under PIT's different ordering, since this class doesn't have base-class + // Awaitility.reset() cleanup). + // 45s: 8s mock-failure window + retry + PIT's JVM slowdown, with headroom. + Awaitility.await().atMost(Duration.ofSeconds(45)).untilAsserted(() -> { assertThat(records).hasSize(3); }); - - Awaitility.reset(); } private void addRecords(MockConsumer mockConsumer) { diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/PCMetricsTest.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/PCMetricsTest.java index c85cafcef..a93d0c5ff 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/PCMetricsTest.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/PCMetricsTest.java @@ -91,7 +91,9 @@ void metricsRegisterBinding() { }); // metrics show processing is complete - await().untilAsserted(() -> { + // 120s budget (was default 10s) - matches the atMost budgets elsewhere in this method, + // and gives headroom under PIT's instrumented JVM processing 1500 records. + await().atMost(Duration.ofSeconds(120)).untilAsserted(() -> { log.info("counterP0: {}, counterP1: {}", counterP0.get(), counterP1.get()); log.info(registry.getMetersAsString()); assertThat(registeredGaugeValueFor(PCMetricsDef.NUM_PAUSED_PARTITIONS)).isEqualTo(2); @@ -177,7 +179,7 @@ void metricsRegisterBinding() { numberToBlockAt.set(5000); latchPartition0.countDown(); latchPartition1.countDown(); - await().untilAsserted(() -> { + await().atMost(Duration.ofSeconds(120)).untilAsserted(() -> { assertThat(counterP0.get()).isEqualTo(quantityP0); }); diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSSStreamProcessorRebalancedTest.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSSStreamProcessorRebalancedTest.java index 224549f12..2c358e078 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSSStreamProcessorRebalancedTest.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/ParallelEoSSStreamProcessorRebalancedTest.java @@ -9,7 +9,6 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.TopicPartition; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; @@ -30,10 +29,6 @@ public void setupAsyncConsumerTestBase() { setupClients(); } - @AfterEach() - public void close() { - } - @ParameterizedTest @EnumSource(CommitMode.class) @SneakyThrows diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/internal/EpochAndRecordsMapRaceTest.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/internal/EpochAndRecordsMapRaceTest.java new file mode 100644 index 000000000..703f10321 --- /dev/null +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/internal/EpochAndRecordsMapRaceTest.java @@ -0,0 +1,171 @@ +package io.confluent.parallelconsumer.internal; + +/*- + * Copyright (C) 2026 Antony Stubbs and contributors + */ + +import io.confluent.parallelconsumer.state.ModelUtils; +import io.confluent.parallelconsumer.state.PartitionStateManager; +import io.confluent.parallelconsumer.state.ShardManager; +import io.confluent.parallelconsumer.state.WorkContainer; +import io.confluent.parallelconsumer.state.WorkManager; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.common.TopicPartition; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import pl.tlinkowski.unij.api.UniLists; +import pl.tlinkowski.unij.api.UniMaps; + +import java.util.List; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; + +/** + * Verifies that the epoch initialization race is handled safely: + *

    + *
  1. poll() returns records for a partition before onPartitionsAssigned() fires
    2. + *
  2. Records are safely skipped (no NPE crash)
    3. + *
  3. onPartitionsAssigned() fires, establishing the epoch and partition state
    4. + *
  4. Next poll creates valid work at the correct epoch
    5. + *
+ * This race is more likely with Kafka 2.x's eager rebalance protocol. + */ +@Slf4j +class EpochAndRecordsMapRaceTest { + + ModelUtils mu = new ModelUtils(); + WorkManager wm; + ShardManager sm; + PartitionStateManager pm; + + String topic = "topic"; + TopicPartition tp = new TopicPartition(topic, 0); + + @BeforeEach + void setup() { + PCModuleTestEnv module = mu.getModule(); + wm = module.workManager(); + sm = wm.getSm(); + pm = wm.getPm(); + // Deliberately NOT calling onPartitionsAssigned — simulating the race + } + + /** + * Core race scenario: poll returns records before onPartitionsAssigned fires. + * Records should be safely skipped (no NPE), and the map should be empty. + */ + @Test + void pollBeforeAssignmentShouldSkipRecordsNotCrash() { + // No onPartitionsAssigned called — epoch map is empty + assertThat(pm.getEpochOfPartition(tp)).isNull(); + + // poll() returns records for the unassigned partition + ConsumerRecords poll = new ConsumerRecords<>(UniMaps.of(tp, UniLists.of( + new ConsumerRecord<>(topic, 0, 0, "key", "value"), + new ConsumerRecord<>(topic, 0, 1, "key", "value") + ))); + + // This should NOT throw NPE — records are skipped + EpochAndRecordsMap recordsMap = new EpochAndRecordsMap<>(poll, pm); + + // Records should NOT be in the map (skipped due to missing epoch) + assertThat(recordsMap.count()).isEqualTo(0); + assertThat(recordsMap.partitions()).isEmpty(); + } + + /** + * Full lifecycle: poll before assignment (skipped) → assignment fires → re-poll succeeds. + * Proves records are recovered after the assignment callback completes. + */ + @Test + void fullLifecycleRecordsRecoveredAfterAssignment() { + // Step 1: poll returns records before onPartitionsAssigned — safely skipped + ConsumerRecords firstPoll = new ConsumerRecords<>(UniMaps.of(tp, UniLists.of( + new ConsumerRecord<>(topic, 0, 0, "key-0", "value"), + new ConsumerRecord<>(topic, 0, 1, "key-1", "value") + ))); + EpochAndRecordsMap firstRecords = new EpochAndRecordsMap<>(firstPoll, pm); + + // Records were skipped — nothing to register + assertThat(firstRecords.count()).isEqualTo(0); + + // Step 2: onPartitionsAssigned fires (late) — epoch and partition state established + wm.onPartitionsAssigned(UniLists.of(tp)); + long epoch = pm.getEpochOfPartition(tp); + assertThat(epoch).isEqualTo(0L); + + // Step 3: Re-poll — Kafka re-delivers the same records (they were never committed) + ConsumerRecords secondPoll = new ConsumerRecords<>(UniMaps.of(tp, UniLists.of( + new ConsumerRecord<>(topic, 0, 0, "key-0", "value"), + new ConsumerRecord<>(topic, 0, 1, "key-1", "value") + ))); + EpochAndRecordsMap secondRecords = new EpochAndRecordsMap<>(secondPoll, pm); + + // Records should now be accepted with the correct epoch + assertThat(secondRecords.count()).isEqualTo(2); + assertThat(secondRecords.records(tp).getEpochOfPartitionAtPoll()).isEqualTo(0L); + + // Step 4: Register and verify work is created + wm.registerWork(secondRecords); + List> work = sm.getWorkIfAvailable(10); + assertWithMessage("Work should be available after assignment + re-poll") + .that(work).hasSize(2); + for (var wc : work) { + assertThat(wc.getEpoch()).isEqualTo(0L); + } + } + + /** + * Multi-partition poll where only some partitions have an epoch assigned. + * The assigned partition's records should be accepted while the unassigned + * partition's records are skipped in the same map construction call. + */ + @Test + void mixedPartitionPollSkipsOnlyUnassignedPartitions() { + TopicPartition tp1 = new TopicPartition(topic, 1); + + // Assign only tp (partition 0), NOT tp1 (partition 1) + wm.onPartitionsAssigned(UniLists.of(tp)); + assertThat(pm.getEpochOfPartition(tp)).isEqualTo(0L); + assertThat(pm.getEpochOfPartition(tp1)).isNull(); + + // poll returns records for both partitions + ConsumerRecords poll = new ConsumerRecords<>(UniMaps.of( + tp, UniLists.of( + new ConsumerRecord<>(topic, 0, 0, "key-0", "value") + ), + tp1, UniLists.of( + new ConsumerRecord<>(topic, 1, 0, "key-1", "value"), + new ConsumerRecord<>(topic, 1, 1, "key-1b", "value") + ) + )); + EpochAndRecordsMap recordsMap = new EpochAndRecordsMap<>(poll, pm); + + // Only tp (assigned) should be in the map; tp1 (unassigned) should be skipped + assertThat(recordsMap.partitions()).containsExactly(tp); + assertThat(recordsMap.count()).isEqualTo(1); + assertThat(recordsMap.records(tp).getEpochOfPartitionAtPoll()).isEqualTo(0L); + } + + /** + * When epoch is already present (normal case), records are processed normally. + */ + @Test + void normalCaseWithPreExistingEpochIsUnaffected() { + // Normal flow: onPartitionsAssigned first + wm.onPartitionsAssigned(UniLists.of(tp)); + assertThat(pm.getEpochOfPartition(tp)).isEqualTo(0L); + + // poll returns records — should use existing epoch + ConsumerRecords poll = new ConsumerRecords<>(UniMaps.of(tp, UniLists.of( + new ConsumerRecord<>(topic, 0, 0, "key", "value") + ))); + EpochAndRecordsMap recordsMap = new EpochAndRecordsMap<>(poll, pm); + + assertThat(recordsMap.count()).isEqualTo(1); + assertThat(recordsMap.records(tp).getEpochOfPartitionAtPoll()).isEqualTo(0L); + } +} diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/internal/ProducerManagerTest.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/internal/ProducerManagerTest.java index f716f151a..120167277 100644 --- a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/internal/ProducerManagerTest.java +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/internal/ProducerManagerTest.java @@ -19,6 +19,8 @@ import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -76,7 +78,17 @@ class ProducerManagerTest { void setup() { setup(ParallelConsumerOptions.builder() .commitMode(PERIODIC_TRANSACTIONAL_PRODUCER) - .commitLockAcquisitionTimeout(ofSeconds(2))); + // 10s (was 2s): 2s is too tight on a CI JVM under PIT instrumentation. + .commitLockAcquisitionTimeout(ofSeconds(10))); + } + + // This class doesn't extend AbstractParallelEoSStreamProcessorTestBase, so + // nothing else resets Awaitility between tests. Not closing pc here on purpose: + // buildModule() overrides close() as a no-op so each test manages its own pc + // lifecycle explicitly (by design, to inspect mid-commit state). + @AfterEach + void tearDown() { + Awaitility.reset(); } private void setup(ParallelConsumerOptions.ParallelConsumerOptionsBuilder optionsBuilder) { @@ -318,7 +330,9 @@ void producedRecordsCantBeInTransactionWithoutItsOffsetDirect() { { var msg = "wait for first record to finish"; log.debug(msg); - await(msg).untilAsserted(() -> assertThat(pc.getWorkMailBox()).hasSize(1)); + // 20s (was default 10s): tight under PIT's instrumented JVM + await(msg).atMost(ofSeconds(20)) + .untilAsserted(() -> assertThat(pc.getWorkMailBox()).hasSize(1)); } // send another record, register the work @@ -338,7 +352,7 @@ void producedRecordsCantBeInTransactionWithoutItsOffsetDirect() { // blocks, as offset 1 is blocked sending and so cannot acquire commit lock var msg = "Ensure expected produce lock is now held by blocked worker thread"; log.debug(msg); - await(msg).untilTrue(blockedOn1); + await(msg).atMost(ofSeconds(20)).untilTrue(blockedOn1); var commitBlocks = new BlockedThreadAsserter(); @@ -354,12 +368,13 @@ void producedRecordsCantBeInTransactionWithoutItsOffsetDirect() { }, () -> { log.debug("Unblocking offset processing offset1Mutex..."); offset1Mutex.countDown(); - }, ofSeconds(10)); + }, ofSeconds(20)); // was 10s; too tight under PIT // - await().untilAsserted(() -> Truth.assertWithMessage("commit should now have unlocked and returned") - .that(commitBlocks.functionHasCompleted()) - .isTrue()); + await().atMost(ofSeconds(20)) + .untilAsserted(() -> Truth.assertWithMessage("commit should now have unlocked and returned") + .that(commitBlocks.functionHasCompleted()) + .isTrue()); final int nextExpectedOffset = 2; // as only first of two work completed diff --git a/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/state/ProcessingShardStaleReplacementTest909.java b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/state/ProcessingShardStaleReplacementTest909.java new file mode 100644 index 000000000..45b976dd3 --- /dev/null +++ b/parallel-consumer-core/src/test/java/io/confluent/parallelconsumer/state/ProcessingShardStaleReplacementTest909.java @@ -0,0 +1,122 @@ +package io.confluent.parallelconsumer.state; + +/*- + * Copyright (C) 2020-2026 Confluent, Inc. and contributors + */ + +import io.confluent.parallelconsumer.internal.PCModuleTestEnv; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.TopicPartition; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import pl.tlinkowski.unij.api.UniLists; + +import java.util.List; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; + +/** + * Regression test for upstream PR #909: a stale {@link WorkContainer} at the same offset + * blocks fresh work from being added after rebalance. + *

+ * The race condition: during rebalance, {@code removeStaleContainers()} runs but can't clean + * shards that don't exist yet. When the control thread resumes processing the old batch, it + * adds a stale container to a newly-created shard. The next poll's fresh container for the + * same offset is then dropped because {@code addWorkContainer} sees "entry already exists." + * + * @see confluentinc/parallel-consumer#909 + * @see ProcessingShard#addWorkContainer(WorkContainer) + */ +@Slf4j +class ProcessingShardStaleReplacementTest909 { + + ModelUtils mu = new ModelUtils(); + WorkManager wm; + ShardManager sm; + PartitionStateManager pm; + + String topic = "topic"; + TopicPartition tp = new TopicPartition(topic, 0); + + @BeforeEach + void setup() { + PCModuleTestEnv module = mu.getModule(); + wm = module.workManager(); + sm = wm.getSm(); + pm = wm.getPm(); + + // initial assignment at epoch 0 + wm.onPartitionsAssigned(UniLists.of(tp)); + } + + /** + * Exact reproduction of the PR #909 race condition timeline: + *

    + *
  1. Control thread adds work at offset 200 for key "K_B" at epoch N
    2. + *
  2. Rebalance happens (epoch advances to N+2)
    3. + *
  3. Control thread (still processing old batch) adds STALE work at offset 300 + * for key "K_B" at OLD epoch N — this creates shard K_B with a stale entry
    4. + *
  4. New poll adds FRESH work at offset 300 for key "K_B" at NEW epoch N+2
    5. + *
  5. BUG (before fix): fresh work is DROPPED because offset 300 already exists
    6. + *
  6. FIX: stale entry is replaced with fresh one
    7. + *
+ */ + @Test + void staleContainerAtSameOffsetShouldBeReplacedByFreshOne() { + long epoch0 = pm.getEpochOfPartition(tp); + + // Step 1: add initial work at epoch 0 + addWork(epoch0, 100, "K_A"); + addWork(epoch0, 200, "K_B"); + + // Step 2: rebalance (epoch 0 → epoch 2: revoke increments, assign increments again) + wm.onPartitionsRevoked(UniLists.of(tp)); + wm.onPartitionsAssigned(UniLists.of(tp)); + long epoch2 = pm.getEpochOfPartition(tp); + assertThat(epoch2).isGreaterThan(epoch0); + + // Step 3: late-arriving stale work at offset 300 from old epoch + // (control thread still processing old batch after rebalance) + addWork(epoch0, 300, "K_B"); + + // Step 4: new poll adds fresh work at the SAME offset 300, new epoch + addWork(epoch2, 300, "K_B"); + + // Step 5: verify the fresh work replaced the stale entry + List> work = sm.getWorkIfAvailable(100); + var offset300 = work.stream().filter(wc -> wc.offset() == 300).findFirst(); + + assertWithMessage("Fresh work at offset 300 should be available (not blocked by stale entry). " + + "See https://github.com/confluentinc/parallel-consumer/pull/909") + .that(offset300.isPresent()).isTrue(); + assertWithMessage("Work at offset 300 should be from the new epoch, not the stale one") + .that(offset300.get().getEpoch()).isEqualTo(epoch2); + } + + /** + * Verify that a non-stale duplicate at the same offset is still correctly dropped + * (preserving original behavior). + */ + @Test + void nonStaleDuplicateAtSameOffsetShouldStillBeDropped() { + long epoch0 = pm.getEpochOfPartition(tp); + + // Add work at offset 100 + addWork(epoch0, 100, "K_A"); + + // Try to add duplicate at same offset, same epoch — should be dropped + addWork(epoch0, 100, "K_A"); + + // Should only have one entry + List> work = sm.getWorkIfAvailable(100); + long count = work.stream().filter(wc -> wc.offset() == 100).count(); + assertThat(count).isEqualTo(1); + } + + private void addWork(long epoch, long offset, String key) { + var record = new ConsumerRecord<>(topic, 0, offset, key, "value"); + sm.addWorkContainer(epoch, record); + } +} diff --git a/parallel-consumer-examples/parallel-consumer-example-core/pom.xml b/parallel-consumer-examples/parallel-consumer-example-core/pom.xml index d9d03103b..e0ec57295 100644 --- a/parallel-consumer-examples/parallel-consumer-example-core/pom.xml +++ b/parallel-consumer-examples/parallel-consumer-example-core/pom.xml @@ -8,24 +8,24 @@ 4.0.0 - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-examples - 0.5.3.4-SNAPSHOT + 0.6.0.0-SNAPSHOT parallel-consumer-example-core - Confluent Parallel Consumer Example - Core + Kafka Parallel Consumer Example - Core - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} tests diff --git a/parallel-consumer-examples/parallel-consumer-example-metrics/pom.xml b/parallel-consumer-examples/parallel-consumer-example-metrics/pom.xml index 8ecba9dac..95072e93d 100644 --- a/parallel-consumer-examples/parallel-consumer-example-metrics/pom.xml +++ b/parallel-consumer-examples/parallel-consumer-example-metrics/pom.xml @@ -8,18 +8,18 @@ 4.0.0 - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-examples - 0.5.3.4-SNAPSHOT + 0.6.0.0-SNAPSHOT parallel-consumer-example-metrics - Confluent Parallel Consumer Example - Metrics + Kafka Parallel Consumer Example - Metrics - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} @@ -30,7 +30,7 @@ - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} tests diff --git a/parallel-consumer-examples/parallel-consumer-example-reactor/pom.xml b/parallel-consumer-examples/parallel-consumer-example-reactor/pom.xml index b26dde0f3..f5b22546b 100644 --- a/parallel-consumer-examples/parallel-consumer-example-reactor/pom.xml +++ b/parallel-consumer-examples/parallel-consumer-example-reactor/pom.xml @@ -8,31 +8,31 @@ 4.0.0 - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-examples - 0.5.3.4-SNAPSHOT + 0.6.0.0-SNAPSHOT parallel-consumer-example-reactor - Confluent Parallel Consumer Example - Project Reactor.io + Kafka Parallel Consumer Example - Project Reactor.io - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-reactor ${project.version} - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} tests test - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-reactor ${project.version} tests diff --git a/parallel-consumer-examples/parallel-consumer-example-streams/pom.xml b/parallel-consumer-examples/parallel-consumer-example-streams/pom.xml index fc31adfad..5d0e4a41c 100644 --- a/parallel-consumer-examples/parallel-consumer-example-streams/pom.xml +++ b/parallel-consumer-examples/parallel-consumer-example-streams/pom.xml @@ -8,17 +8,17 @@ 4.0.0 - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-examples - 0.5.3.4-SNAPSHOT + 0.6.0.0-SNAPSHOT parallel-consumer-example-streams - Confluent Parallel Consumer Example - Streams + Kafka Parallel Consumer Example - Streams - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} @@ -41,7 +41,7 @@ test - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} tests diff --git a/parallel-consumer-examples/parallel-consumer-example-vertx/pom.xml b/parallel-consumer-examples/parallel-consumer-example-vertx/pom.xml index 9296f0f1c..73761b45e 100644 --- a/parallel-consumer-examples/parallel-consumer-example-vertx/pom.xml +++ b/parallel-consumer-examples/parallel-consumer-example-vertx/pom.xml @@ -8,31 +8,31 @@ 4.0.0 - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-examples - 0.5.3.4-SNAPSHOT + 0.6.0.0-SNAPSHOT parallel-consumer-example-vertx - Confluent Parallel Consumer Example - Vert.x + Kafka Parallel Consumer Example - Vert.x - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-vertx ${project.version} - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} tests test - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-vertx ${project.version} tests diff --git a/parallel-consumer-examples/pom.xml b/parallel-consumer-examples/pom.xml index 7ac9e4918..1902f4f44 100644 --- a/parallel-consumer-examples/pom.xml +++ b/parallel-consumer-examples/pom.xml @@ -9,12 +9,12 @@ parallel-consumer-parent - io.confluent.parallelconsumer - 0.5.3.4-SNAPSHOT + io.github.astubbs.parallelconsumer + 0.6.0.0-SNAPSHOT parallel-consumer-examples - Confluent Parallel Consumer Examples + Kafka Parallel Consumer Examples pom @@ -25,4 +25,27 @@ parallel-consumer-example-reactor + + + true + true + true + + + + + + org.sonatype.central + central-publishing-maven-plugin + + true + + + + + diff --git a/parallel-consumer-mutiny/pom.xml b/parallel-consumer-mutiny/pom.xml index 08fc730e0..06d684f38 100644 --- a/parallel-consumer-mutiny/pom.xml +++ b/parallel-consumer-mutiny/pom.xml @@ -1,28 +1,33 @@ parallel-consumer-parent - io.confluent.parallelconsumer - 0.5.3.4-SNAPSHOT + io.github.astubbs.parallelconsumer + 0.6.0.0-SNAPSHOT 4.0.0 - Confluent Parallel Consumer SmallRye Mutiny + Kafka Parallel Consumer SmallRye Mutiny parallel-consumer-mutiny + + + 9 + + - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} tests diff --git a/parallel-consumer-reactor/pom.xml b/parallel-consumer-reactor/pom.xml index 638e23192..7f63d1ea7 100644 --- a/parallel-consumer-reactor/pom.xml +++ b/parallel-consumer-reactor/pom.xml @@ -7,22 +7,22 @@ parallel-consumer-parent - io.confluent.parallelconsumer - 0.5.3.4-SNAPSHOT + io.github.astubbs.parallelconsumer + 0.6.0.0-SNAPSHOT 4.0.0 - Confluent Parallel Consumer Project Reactor.io + Kafka Parallel Consumer Project Reactor.io parallel-consumer-reactor - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} tests diff --git a/parallel-consumer-vertx/pom.xml b/parallel-consumer-vertx/pom.xml index f6f90545f..d37fa8f8f 100644 --- a/parallel-consumer-vertx/pom.xml +++ b/parallel-consumer-vertx/pom.xml @@ -6,13 +6,13 @@ --> - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-parent - 0.5.3.4-SNAPSHOT + 0.6.0.0-SNAPSHOT parallel-consumer-vertx - Confluent Parallel Consumer Vert.x + Kafka Parallel Consumer Vert.x 4.0.0 @@ -22,12 +22,12 @@ - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-core ${project.version} tests diff --git a/parallel-consumer-vertx/src/test-integration/java/io/confluent/parallelconsumer/vertx/integrationTests/VertxConcurrencyIT.java b/parallel-consumer-vertx/src/test-integration/java/io/confluent/parallelconsumer/vertx/integrationTests/VertxConcurrencyIT.java index 31838f9b8..836e4f64f 100644 --- a/parallel-consumer-vertx/src/test-integration/java/io/confluent/parallelconsumer/vertx/integrationTests/VertxConcurrencyIT.java +++ b/parallel-consumer-vertx/src/test-integration/java/io/confluent/parallelconsumer/vertx/integrationTests/VertxConcurrencyIT.java @@ -32,6 +32,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.parallel.Isolated; import org.testcontainers.junit.jupiter.Testcontainers; import pl.tlinkowski.unij.api.UniMaps; @@ -40,6 +41,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static com.github.tomakehurst.wiremock.client.WireMock.*; @@ -134,6 +136,7 @@ static void close() { * would expect. */ @Test + @Timeout(value = 5, unit = TimeUnit.MINUTES) @SneakyThrows void testVertxConcurrency() { var commitMode = PERIODIC_CONSUMER_ASYNCHRONOUS; @@ -208,7 +211,7 @@ void testVertxConcurrency() { var failureMessage = msg("Mock server receives {} requests in parallel from vertx engine", expectedMessageCount / 2); try { - waitAtMost(ofSeconds(20)) + waitAtMost(ofSeconds(120)) .pollInterval(ofMillis(200)) .alias(failureMessage) .untilAsserted(() -> { @@ -217,11 +220,12 @@ void testVertxConcurrency() { }); } catch (ConditionTimeoutException e) { fail(failureMessage + "\n" + e.getMessage()); + } finally { + // Always release the latch — if the test fails, this prevents WireMock threads + // from hanging for 30s each on the unreleased latch + log.info("{} requests received by server, releasing server response lock.", requestsReceivedOnServer.size()); + LatchTestUtils.release(responseLock); } - log.info("{} requests received in parallel by server, releasing server response lock.", requestsReceivedOnServer.size()); - - // all requests were received in parallel, so unlock the server to respond to all of them - LatchTestUtils.release(responseLock); // assertNumberOfThreads(); diff --git a/pom.xml b/pom.xml index 78a81326a..3b9512737 100644 --- a/pom.xml +++ b/pom.xml @@ -1,24 +1,24 @@ 4.0.0 - io.confluent.parallelconsumer + io.github.astubbs.parallelconsumer parallel-consumer-parent - Confluent Parallel Consumer - 0.5.3.4-SNAPSHOT + Kafka Parallel Consumer + 0.6.0.0-SNAPSHOT Parallel Apache Kafka client wrapper with client side queueing, a simpler consumer/producer API with key concurrency and extendable non-blocking IO processing. - https://confluent.io + https://github.com/astubbs/parallel-consumer 2020 - Confluent, Inc. - https://confluent.io + Antony Stubbs + https://stub.bz @@ -42,24 +42,23 @@ - antony@confluent.io + astubbs Antony Stubbs - antony@confluent.io - https://confluent.io - Confluent - https://confluent.io + antony.stubbs@gmail.com + https://stub.bz + Antony Stubbs + https://stub.bz Europe/London - scm:git:git://github.com:confluentinc/parallel-consumer.git - scm:git:git@github.com:confluentinc/parallel-consumer.git - https://github.com/confluentinc/parallel-consumer.git - 0.5.3.2 + scm:git:git://github.com:astubbs/parallel-consumer.git + scm:git:git@github.com:astubbs/parallel-consumer.git + https://github.com/astubbs/parallel-consumer.git + HEAD - 17 8 @@ -81,12 +80,23 @@ ${skipTests} ${skipTests} + + + performance + 5.0.0 1.18.28 1.1.1 3.2.5 + 4.8.6 + 4.8.6.6 + 1.17.4 + 1.2.2 2.0.13 @@ -201,7 +211,7 @@ false false - io.confluent.parallelconsumer: + io.github.astubbs.parallelconsumer: @@ -234,13 +244,20 @@ sign + + + + --pinentry-mode + loopback + + org.sonatype.central central-publishing-maven-plugin - 0.8.0 + 0.10.0 true central @@ -592,7 +609,7 @@ true - Copyright (C) ${license.git.copyrightYears} ${project.organization.name} + Copyright (C) 2020-${license.git.copyrightYears} Confluent, Inc. and contributors **/release-pom.xml @@ -706,6 +723,11 @@ ${skipTests} ${skipITs} methods + + ${included.groups} + ${excluded.groups} @@ -725,8 +747,50 @@ report + + prepare-agent-integration + + prepare-agent-integration + + + + report-integration + post-integration-test + + report-integration + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + Max + Medium + true + + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + + + + + org.pitest + pitest-maven + ${pitest.version} + + + org.pitest + pitest-junit5-plugin + ${pitest-junit5.version} + + + org.apache.maven.plugins maven-enforcer-plugin @@ -779,6 +843,12 @@ test-jar + + + true + @@ -950,18 +1020,21 @@ + - - confluent - https://packages.confluent.io/maven/ - central https://repo1.maven.org/maven2/ - jitpack.io - https://jitpack.io + confluent + https://packages.confluent.io/maven/ astubbs-truth-generator diff --git a/src/docs/README_TEMPLATE.adoc b/src/docs/README_TEMPLATE.adoc index b7805c2ef..b29d833e8 100644 --- a/src/docs/README_TEMPLATE.adoc +++ b/src/docs/README_TEMPLATE.adoc @@ -22,7 +22,7 @@ TIP:: Editing template file endif::[] -= Confluent Parallel Consumer += Kafka Parallel Consumer :icons: :toc: macro :toclevels: 3 @@ -31,7 +31,8 @@ endif::[] :sectanchors: true :github_name: parallel-consumer -:base_url: https://github.com/confluentinc/{github_name} +:base_confluent_url: https://github.com/confluentinc/{github_name} +:base_url: https://github.com/astubbs/{github_name} :issues_link: {base_url}/issues @@ -43,26 +44,40 @@ ifdef::env-github[] :warning-caption: :warning: endif::[] -image:https://maven-badges.herokuapp.com/maven-central/io.confluent.parallelconsumer/parallel-consumer-parent/badge.svg?style=flat[link=https://mvnrepository.com/artifact/io.confluent.parallelconsumer/parallel-consumer-parent,Latest Parallel Consumer on Maven Central] +image:https://maven-badges.herokuapp.com/maven-central/io.github.astubbs.parallelconsumer/parallel-consumer-parent/badge.svg?style=flat[link=https://mvnrepository.com/artifact/io.github.astubbs.parallelconsumer/parallel-consumer-parent,Latest Parallel Consumer on Maven Central] -// Github actions disabled since codecov -//image:https://github.com/confluentinc/parallel-consumer/actions/workflows/maven.yml/badge.svg[Java 8 Unit Test GitHub] + -//^(^^full^ ^test^ ^suite^ ^currently^ ^running^ ^only^ ^on^ ^Confluent^ ^internal^ ^CI^ ^server^^)^ +image:https://github.com/astubbs/parallel-consumer/actions/workflows/maven.yml/badge.svg?branch=master[link=https://github.com/astubbs/parallel-consumer/actions/workflows/maven.yml,Build and Test] // travis badges temporarily disabled as travis isn't running CI currently //image:https://travis-ci.com/astubbs/parallel-consumer.svg?branch=master["Build Status", link="https://travis-ci.com/astubbs/parallel-consumer"] image:https://codecov.io/gh/astubbs/parallel-consumer/branch/master/graph/badge.svg["Coverage",https://codecov.io/gh/astubbs/parallel-consumer] Parallel Apache Kafka client wrapper with client side queueing, a simpler consumer/producer API with *key concurrency* and *extendable non-blocking IO* processing. -Confluent's https://www.confluent.io/confluent-accelerators/#parallel-consumer[product page for the project is here]. +IMPORTANT: This is a community-maintained fork of https://github.com/confluentinc/parallel-consumer[confluentinc/parallel-consumer], published under different Maven coordinates (`io.github.astubbs.parallelconsumer`). The original upstream project is no longer actively maintained. + +Confluent's https://www.confluent.io/confluent-accelerators/#parallel-consumer[product page for the original project is here]. TIP: If you like this project, please ⭐ Star it in GitHub to show your appreciation, help us gauge popularity of the project and allocate resources. -NOTE: This is not a part of the Confluent commercial support offering, except through consulting engagements. +NOTE: This is a community-maintained project with no commercial support. See the <> section for more information. -IMPORTANT: This project has been stable and reached its initial target feature set in Q1 2021. -It is actively maintained by the CSID team at Confluent. +[[when-to-use]] +== When to use this library (vs KIP-932 Share Groups) + +The Kafka landscape has shifted since this library's 2021 stable release. *KIP-932 Share Groups* is now GA on Confluent Cloud and ships with Confluent Platform 8.2 / Apache Kafka 4.2. It covers a large part of what people historically reached for Parallel Consumer to do, at the broker level rather than in a client wrapper. + +*Share Groups (broker-native):* many-to-many consumer↔partition mapping, per-message ack, broker-side delivery counts with poison-message protection, elastic scaling decoupled from partition count. Unordered queue semantics -- "RabbitMQ on Kafka". Already wrapped by Spring Kafka via `ShareConsumerFactory`. + +*Parallel Consumer (client-side):* keeps the partition model and adds *per-key parallelism* on top. Messages within a key stay ordered; different keys run concurrently; concurrency is independent of partition count. + +The two are not strict alternatives -- they solve different problems. + +[TIP] +==== +* If you want unordered queue semantics on Kafka 4.2+, reach for *Share Groups*. The "partitions are fixed, I need more consumers" motivation is now solved at the broker. +* If you need *key-level ordering with concurrency beyond partition count*, reach for *Parallel Consumer*. Nothing else does that cleanly today. +==== [[intro]] This library lets you process messages in parallel via a single Kafka Consumer meaning you can increase consumer parallelism without increasing the number of partitions in the topic you intend to process. @@ -281,7 +296,7 @@ The user just has to provide a function to extract from the message the HTTP cal === Illustrative Performance Example -.(see link:./parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VolumeTests.java[VolumeTests.java]) +.(see link:./parallel-consumer-core/src/test-integration/java/io/confluent/parallelconsumer/integrationTests/VeryLargeMessageVolumeTest.java[VeryLargeMessageVolumeTest.java]) These performance comparison results below, even though are based on real performance measurement results, are for illustrative purposes. To see how the performance of the tool is related to instance counts, partition counts, key distribution and how it would relate to the vanilla client. Actual results will vary wildly depending upon the setup being deployed into. @@ -372,7 +387,7 @@ As an illustrative example of relative performance, given: == Support and Issues -If you encounter any issues, or have any suggestions or future requests, please create issues in the {issues_link}[github issue tracker]. +If you encounter any issues, or have any suggestions or future requests, please create issues in the {issues_link}[fork issue tracker]. Issues will be dealt with on a good faith, best efforts basis, by the small team maintaining this library. We also encourage participation, so if you have any feature ideas etc, please get in touch, and we will help you work on submitting a PR! @@ -380,25 +395,25 @@ We also encourage participation, so if you have any feature ideas etc, please ge NOTE: We are very interested to hear about your experiences! And please vote on your favourite issues! -If you have questions, head over to the https://launchpass.com/confluentcommunity[Confluent Slack community], or raise an https://github.com/confluentinc/parallel-consumer/issues[issue] on GitHub. +If you have questions or find a bug, raise an {issues_link}[issue] on GitHub. == License -This library is copyright Confluent Inc, and licensed under the Apache License Version 2.0. +This library is copyright Confluent, Inc. and contributors, and licensed under the Apache License Version 2.0. == Usage === Maven -This project is available in maven central, https://repo1.maven.org/maven2/io/confluent/parallelconsumer/[repo1], along with SNAPSHOT builds (starting with 0.5-SNAPSHOT) in https://oss.sonatype.org/content/repositories/snapshots/io/confluent/parallelconsumer/[repo1's SNAPSHOTS repo]. +This project is available in Maven Central, https://repo1.maven.org/maven2/io/github/astubbs/parallelconsumer/[repo1]. -Latest version can be seen https://search.maven.org/artifact/io.confluent.parallelconsumer/parallel-consumer-core[here]. +Latest version can be seen https://search.maven.org/artifact/io.github.astubbs.parallelconsumer/parallel-consumer-core[here]. Where `${project.version}` is the version to be used: -* group ID: `io.confluent.parallelconsumer` +* group ID: `io.github.astubbs.parallelconsumer` * artifact ID: `parallel-consumer-core` -* version: image:https://maven-badges.herokuapp.com/maven-central/io.confluent.parallelconsumer/parallel-consumer-parent/badge.svg?style=flat[link=https://mvnrepository.com/artifact/io.confluent.parallelconsumer/parallel-consumer-parent,Latest Parallel Consumer on Maven Central] +* version: image:https://maven-badges.herokuapp.com/maven-central/io.github.astubbs.parallelconsumer/parallel-consumer-parent/badge.svg?style=flat[link=https://mvnrepository.com/artifact/io.github.astubbs.parallelconsumer/parallel-consumer-parent,Latest Parallel Consumer on Maven Central] .Core Module Dependency [source,xml,indent=0] @@ -439,7 +454,7 @@ After this setup, one then has the choice of interfaces: * `JStreamVertxParallelStreamProcessor` There is another interface: `ParallelConsumer` which is integrated, however there is currently no immediate implementation. -See {issues_link}/12[issue #12], and the `ParallelConsumer` JavaDoc: +See {base_confluent_url}/issues/12[issue #12], and the `ParallelConsumer` JavaDoc: [source,java] ---- @@ -550,21 +565,6 @@ include::{project_root}/parallel-consumer-examples/parallel-consumer-example-str See the link:{project_root}/parallel-consumer-examples/parallel-consumer-example-streams/src/main/java/io/confluent/parallelconsumer/examples/streams/StreamsApp.java[Kafka Streams example] project, and it's test. -[[confluent-cloud]] -=== Confluent Cloud - -. Provision your fully managed Kafka cluster in Confluent Cloud -.. Sign up for https://www.confluent.io/confluent-cloud/tryfree/[Confluent Cloud], a fully-managed Apache Kafka service. -.. After you log in to Confluent Cloud, click on `Add cloud environment` and name the environment `learn-kafka`. -Using a new environment keeps your learning resources separate from your other Confluent Cloud resources. -.. Click on https://confluent.cloud/learn[LEARN] and follow the instructions to launch a Kafka cluster and to enable Schema Registry. -. Access the client configuration settings -.. From the Confluent Cloud Console, navigate to your Kafka cluster. -From the `Clients` view, get the connection information customized to your cluster (select `Java`). -.. Create new credentials for your Kafka cluster, and then Confluent Cloud will show a configuration block with your new credentials automatically populated (make sure `show API keys` is checked). -.. Use these settings presented to https://docs.confluent.io/clients-kafka-java/current/overview.html[configure your clients]. -. Use these clients for steps outlined in the <> section. - [[upgrading]] == Upgrading @@ -1092,6 +1092,50 @@ Note:: See https://github.com/confluentinc/parallel-consumer/issues/162[issue #162] and this https://stackoverflow.com/questions/4786881/why-is-test-jar-dependency-required-for-mvn-compile[Stack Overflow question]. +=== Build Scripts + +Helper scripts are in the `bin/` directory: + +[qanda] +Quick local build (compile + unit tests):: +`bin/build.sh` + +Unit tests only (no Docker needed):: +`bin/ci-unit-test.sh` + +Integration tests only (requires Docker for TestContainers):: +`bin/ci-integration-test.sh` + +Full CI build with all tests (unit + integration):: +`bin/ci-build.sh` + +CI build against a specific Kafka version:: +`bin/ci-build.sh 3.9.1` + +All `ci-*` scripts use the `-Pci` Maven profile which enables license checking and disables parallel test execution. The GitHub Actions CI workflow uses these scripts, so running them locally reproduces the CI environment. + +=== Releasing + +The `pom.xml` version is the source of truth for publishing — there is no `maven-release-plugin` step. + +On every push to `master`, `.github/workflows/publish.yml` deploys to Maven Central: + +* If the version ends in `-SNAPSHOT` → publishes a snapshot +* If the version does not end in `-SNAPSHOT` → publishes a full release, creates a `v` git tag, and creates a GitHub release + +To cut a release: + +. Open a PR removing `-SNAPSHOT` from `` in the parent `pom.xml` (e.g. `0.6.0.0-SNAPSHOT` → `0.6.0.0`) +. Merge it to master → CI publishes the release +. Open another PR bumping to the next snapshot (e.g. `0.6.0.1-SNAPSHOT`) and merge + +Required GitHub repository secrets: + +* `MAVEN_CENTRAL_USERNAME` — Sonatype Central Portal token username +* `MAVEN_CENTRAL_PASSWORD` — Sonatype Central Portal token password +* `MAVEN_GPG_PRIVATE_KEY` — Armored GPG private key for signing artifacts +* `MAVEN_GPG_PASSPHRASE` — Passphrase for the GPG key + === Testing The project has good automated test coverage, of all features. @@ -1272,3 +1316,4 @@ https://www.google.com/url?q=https://www.yourkit.com/.net/profiler/&source=gmail include::{project_root}/CHANGELOG.adoc[] //:leveloffset: -1 - Duplicate key leveloffset (attempted merging values +1 and -1): https://github.com/whelk-io/asciidoc-template-maven-plugin/issues/118 + diff --git a/src/docs/development/upstream-pr-analysis.adoc b/src/docs/development/upstream-pr-analysis.adoc new file mode 100644 index 000000000..95b45c13c --- /dev/null +++ b/src/docs/development/upstream-pr-analysis.adoc @@ -0,0 +1,542 @@ += Upstream PR & Issue Analysis for parallel-consumer +:toc: macro +:toclevels: 3 +:numbered: 1 + +toc::[] + +== Part 1: Open Upstream PRs (confluentinc/parallel-consumer) + +The upstream repository has accumulated *19 open PRs* (565 closed). The repo appears sparsely maintained -- some PRs from early 2025 remain unreviewed. The local fork (`astubbs/parallel-consumer`) is pinned to Kafka 3.1.0, JUnit 5.8.2, with a Java 17 WIP branch in progress. + +=== Summary Table (19 PRs, most to least important) + +[cols="1,1,4,2,3", options="header"] +|=== +|Rank |PR |Title |Group |Why it matters + +|1 |#893 |Fix offset reset on partition reassignment |Correctness bug |Data-loss / reset risk during rebalance +|2 |#909 |Replace stale container when adding work container |Correctness bug |Record drops during rebalance +|3 |#915 |Feature: select batch construction strategy |Feature (closes #266) |Unlocks throughput & ordering flexibility +|4 |#908 |Feature: Support Virtual Threads (JDK 21+) |Feature (closes #896) |Major platform modernisation +|5 |#905 |Metric: max queued records per shard |Observability |Diagnose hot-key bottlenecks +|6 |#866 |Update Kafka to v7 (BREAKING) |Major dep bump |Keeps library on supported Kafka +|7 |#867 |Update vertx to v5 (BREAKING) |Major dep bump |Keeps vertx module alive +|8 |#901 |Fix failing licence check + gitignore |Build/tooling |Unblocks CI +|9 |#877 |chore: update repo by service bot |Build/tooling |Bot-maintained config +|10 |#851 |postgres 42.7.2 (security) |Security CVE |Fork should not ship vulnerable deps +|11 |#913 |assertj 3.27.7 (security) |Security CVE |Test-only, low severity +|12 |#914 |logback 1.5.25 (security, DRAFT) |Security CVE |Logging stack CVE +|13 |#855 |wiremock-jre8 to v3 (BREAKING) |Test dep |Test-only, unblocks other updates +|14 |#899 |JUnit platform 1.10.2 to 6.0.0 |Test dep |Huge jump, needs care +|15 |#900 |testcontainers 1.19.8 to 1.21.3 |Test dep |Routine bump +|16 |#897 |vertx 4.5.7 to 5.0.5 |Dep (overlaps #867) |Duplicate of #867 +|17 |#869 |threeten-extra 1.7.2 to 1.8.0 |Dep |Trivial +|18 |#898 |maven-gpg-plugin 3.1.0 to 3.2.8 |Build plugin |Trivial +|19 |#854 |renovate minor+patch batch |Dep batch |Routine +|=== + +=== Group A -- Correctness fixes (merge first) + +These address real bugs observed in production and should be the highest priority to cherry-pick. + +==== #893 -- Accurate committed offset on partition assignment + +* *Author:* Martyn Ye (sangreal), opened *Oct 2025* +* *Problem:* Race condition in `PartitionState.getOffsetToCommit()`. Between offset calculation and commit, `incompletes` may drain, so a higher offset gets committed than intended. After rebalance, the consumer tries to fetch a non-existent offset and *triggers `auto.offset.reset`* (data loss or replay). +* *Fix:* Call `getOffsetToCommit()` once per commit cycle, track `lastProcessedOffset`, add synchronisation. +* *Status:* Has unit tests, approved by Roman Kolesnev, run privately for >1 week with zero recurrence. Awaiting codeowner merge. +* *Verdict:* Should merge immediately. High-impact, low-risk. + +==== #909 -- Replace stale work container on rebalance + +* *Author:* cserspring, opened *Jan 2026* +* *Problem:* Broker-poll thread and control thread race during rebalance. Control thread adds a container at an outdated epoch _after_ `removeStaleContainers()` runs. The stale entry then blocks subsequent valid additions, *dropping records*. +* *Fix:* `addWorkContainer` replaces stale entries rather than rejecting new ones. +* *Status:* No visible tests yet, no review comments. Needs more scrutiny than #893 but the root-cause analysis is compelling. +* *Verdict:* High priority, but request tests before merging. + +=== Group B -- Features (real user value) + +==== #915 -- Batch construction strategy (closes #266) + +* *Author:* Devingryu, opened *Mar 2026* (most recent PR) +* Adds `batchStrategy` enum with three modes: +** `SEQUENTIAL` (default, current behaviour) -- one record per shard per cycle +** `BATCH_MULTIPLEX` -- multiple records from a shard, batches span shards +** `BATCH_BY_SHARD` -- multi-record batches, each batch from a single shard +* Closes a long-standing feature request (#266). Touches `ProcessingShard` work-selection and batch construction. +* Author explicitly flags open design questions (UNORDERED semantics, stage separation) and notes the repo looks unmaintained. +* *Verdict:* Most valuable user-facing feature in the queue. Needs an architectural review before merge. + +==== #908 -- Virtual Threads support (closes #896) + +* *Author:* Devingryu, opened *Jan 2026* +* Adds `useVirtualThreads` option for JDK 21+, migrates `synchronized` to `ReentrantLock` to avoid carrier-thread pinning, generalises `setupWorkerPool` return to `ExecutorService`. +* Tests skip on JDK 17 via `Assumptions`. +* *Caveats:* Backwards-compat risk for subclasses overriding executor type; queue-size querying uses `WorkerManager.numberRecordsOutForProcessing` as a proxy. +* *Verdict:* Strategically important modernisation. Sequence behind Java 17 branch. + +==== #905 -- Max-queued-records-per-shard metric + +* *Author:* flashmouse, opened *Jan 2026* +* Adds monitoring metric; useful with `orderType.KEY` to detect *hot-key shards*. +* Small, self-contained (2 commits). No review comments. +* *Verdict:* Easy merge, good observability win. + +=== Group C -- Major (breaking) dependency bumps + +[cols="1,3,4", options="header"] +|=== +|PR |Bump |Note + +|#866 |Kafka to v7 |Upstream Confluent Kafka major bump. Essential for staying current but will cascade test/API changes. +|#867 |vertx to v5 |Breaking for the `parallel-consumer-vertx` module. +|#897 |vertx 4.5.7 to 5.0.5 |Duplicate of #867; close one. +|#855 |wiremock-jre8 to v3 |Test-only, blocks other test-dep updates. +|#899 |junit platform 1.10.2 to *6.0.0* |Dependabot misread -- 6.0.0 likely incorrect. Close or pin. +|=== + +*Verdict:* Tackle #866 (Kafka v7) first. Consolidate #867/#897. Close #899. + +=== Group D -- Security CVEs + +[cols="1,2,2,2", options="header"] +|=== +|PR |Dep |Scope |Priority + +|#851 |postgresql 42.7.2 |Runtime (JDBC sample) |Medium +|#913 |assertj 3.27.7 |Test only |Low +|#914 |logback 1.5.25 (DRAFT) |Runtime logging |Medium +|=== + +*Verdict:* All renovate-generated and low-effort. Merge in a single batch once CI is green. + +=== Group E -- Routine / housekeeping + +* *#854* -- renovate minor+patch dependency batch (trivial) +* *#869* -- threeten-extra 1.7.2 to 1.8.0 (trivial) +* *#898* -- maven-gpg-plugin 3.1.0 to 3.2.8 (trivial) +* *#900* -- testcontainers 1.19.8 to 1.21.3 (routine test dep) +* *#877* -- service-bot repo config update +* *#901* -- fix failing license check + gitignore (unblocks CI -- promote to Group A priority) + +=== Recommended merge order for the local fork + +. *#901* -- unbreak the licence-check build (prerequisite) +. *#893* -- offset reset fix (correctness, approved upstream) +. *#909* -- stale container fix (correctness, request tests first) +. *#905* -- hot-key observability metric (cheap win) +. *Security batch:* #851, #913, #914 +. *Routine deps batch:* #854, #869, #898, #900, #877 +. *#915* -- batch strategy feature (needs architectural review) +. *#866* -- Kafka v7 (scheduled breaking upgrade) +. *#908* -- Virtual Threads (after Java 17 branch lands; requires JDK 21+ CI) +. *#867* (and close duplicate #897) -- vertx v5 +. *#855* -- wiremock v3 (after other test deps stabilise) +. *#899* -- close as incorrect; re-open with a sensible target version + +== Part 2: astubbs Closed-Unmerged PRs (Revival Candidates) + +A GitHub search of `confluentinc/parallel-consumer` for `author:astubbs is:closed is:unmerged` returns *53 PRs*. A striking pattern: *~40 of them were closed on the same day, 2023-06-15*, which looks like an administrative sweep (coinciding with astubbs leaving Confluent) rather than a substantive rejection of the ideas. The PRs represent a trove of unfinished but intentional design work. + +=== Group A -- Major features (top revival candidates) + +[cols="1,1,4,1,3", options="header"] +|=== +|Rank |PR |Title |Closes |Notes + +|A1 |#464 |Feature: Health check and metrics |#71 |Observability gap; still needed today. Consolidate with upstream #905. +|A2 |#366 |Dead Letter Queue (DLQ) as option for retry expiration |-- |Most-requested missing feature; no clean upstream alternative. +|A3 |#441 |Queue priority |#50 |Closes long-standing issue; enables priority-aware shards. +|A4 |#316 |Combine queues from different partitions/topics with KEY ordering |#314 |Powerful composition primitive. +|A5 |#300 |Loom POC (Virtual Threads) |#299 |Predates upstream #908. Compare approaches before redoing. +|A6 |#473 |Producer sends to least-loaded broker |-- |Throughput optimisation. +|A7 |#390 |Streams integration |-- |Expands reach to Kafka Streams users. +|A8 |#346 |Thread-safe user API exposure of ALL Consumer APIs |#520 |Safety-critical API surface expansion. +|A9 |#353 |Broker status informer |#185 |Surface broker health to user code. +|A10 |#291 |Explicit terminal and retry exceptions + poison pills |#242 |Pairs with #268; cleaner error semantics. +|A11 |#197 |Retry count available in header |-- |Small, widely useful. +|A12 |#22 |Dynamic concurrency control (WIP) |#21 |Adaptive parallelism. +|=== + +=== Group B -- Performance / correctness work + +[cols="1,1,4,3", options="header"] +|=== +|Rank |PR |Title |Notes + +|B1 |#530 |Caching of shard management counts (shard scanning O(n) to cached) |Concrete hot-path perf fix. High ROI. +|B2 |#237 |Shard starvation fix |#236 -- real user-visible bug. +|B3 |#356 |Faster record producing: remove Future blocks, async result processing |#29 -- throughput-critical. +|B4 |#408 |Runlength v3 encoding using Longs instead of Integers |Extends offset encoder capacity. +|B5 |#46 |Continuous encoding with per-partition realtime offset-space backpressure (WIP!) |Aspirational; pairs with #408. +|B6 |#66 |Retry system improvements |Iterative correctness work. +|B7 |#140 |Don't allow use of core methods from vert.x or reactor module |#99 -- prevents footguns. +|=== + +=== Group C -- Architectural refactors (keep ideas, not code) + +Use as design references. Branches are far too stale to apply directly. + +[cols="1,1,4,3", options="header"] +|=== +|Rank |PR |Title |Notes + +|C1 |#488 |Refactor God class to components |Long-standing maintainability pain. +|C2 |#270 |Shared-nothing architecture -- Partition Events |#200 -- scalability rearchitecture. +|C3 |#524 |Use Actor for commit commands |Pairs with C4; cleaner concurrency. +|C4 |#325 |New IPC system using lightweight Lambda actor queue |Underpins many other improvements. +|C5 |#271 |Major package restructure |Blast radius too large on its own; fold into C1/C2. +|C6 |#303 |Extend Consumer and Function for more cohesive API |Public-API evolution. +|C7 |#45 |Direct work loading, direct result processing (WIP!) |POC; informs C2/C4. +|C8 |#405 |Remove static state |Ties into B-group testability work. +|=== + +=== Group D -- Testing & chaos infrastructure + +Reviving these pays dividends on every other revival in Groups A-C. + +[cols="1,1,4,3", options="header"] +|=== +|Rank |PR |Title |Notes + +|D1 |#345 |Broker disconnect testing + ChaosBroker |#203 -- essential for validating correctness fixes. +|D2 |#126 |Remove static state manipulation that tests use |Unlocks parallel test execution. +|D3 |#143 |Parallel tests in CI |Dependent on D2. +|D4 |#106 |Client factory config: prevent client reuse, safer config validation |Test hygiene. +|D5 |#492 |Fix incorrect assumption in main PC unit test |Small, easy. +|D6 |#494 |Re-enable disabled legacy tests in ParallelEoSStreamProcessorTest |Coverage recovery. +|D7 |#496 |Implement the empty tests |Fill test placeholders. +|D8 |#73 |Enable integration tests on GH Workflow CI |Verify if merged-equivalent elsewhere. +|D9 |#75, #74 |Run CI on Java 8 / Java 9 |Obsolete; close permanently -- covered by Java 17 effort. +|=== + +=== Group E -- UX, logging, docs + +[cols="1,1,4,3", options="header"] +|=== +|Rank |PR |Title |Notes + +|E1 |#268 |Explicit retriable exception for cleaner ERROR logging |#242 -- superseded by #291 (A10); pick one. +|E2 |#139 |Warning when no SLF4J logger detected |Small UX polish. +|E3 |#508 |'Back Pressure' notes and Javadoc |Valuable docs addition. +|E4 |#81 |Parallel join technique documentation |Design docs. +|E5 |#506 |Fix chart links (docs) |Trivial. +|=== + +=== Group F -- Vertx / Reactor module work + +[cols="1,1,4,3", options="header"] +|=== +|Rank |PR |Title |Notes + +|F1 |#204 |Run user functions on a Vert.x vertical instead of Java ThreadPool |Re-evaluate in light of upstream #867 / Virtual Threads. +|F2 |#181 |Convenience methods for Vertx calls |Small API addition. +|F3 |#179 |Scatter-gather parallel webservice requests (POC) |Useful pattern for docs/examples. +|=== + +=== Group G -- Build, tooling, plumbing (mostly obsolete) + +[cols="1,3,2", options="header"] +|=== +|PR |Title |Status + +|#442 |Setup mvnw |Consider re-doing -- mvnw is still useful. +|#91 |Upgrade Apache Kafka version |Obsolete (see upstream #866). +|#220 |Docker naming, fixes and improvements |Niche. +|#10 |Extract Interface from implementation |Likely already done. +|#23 |Playing around with interface naming (WIP!) |Skip. +|#6 |Fix plugin versions / surefire bug |Obsolete. +|#2 |Fix groupid in poms |Obsolete. +|#7 |Store offset map of incomplete messages (Draft) |Likely superseded by current offset encoder. +|=== + +=== Top 10 revival candidates (flat ranking) + +. *#530* -- shard-count caching perf fix (small, high-impact, hot path) +. *#464* -- health check and metrics (still-missing observability) +. *#366* -- dead letter queue option +. *#237* -- shard starvation bug fix +. *#345* -- ChaosBroker testing infrastructure (amplifies every other fix) +. *#441* -- queue priority +. *#316* -- combine queues across partitions with KEY ordering +. *#291* -- terminal/retry exception types + poison pill handling +. *#300* -- Loom / Virtual Threads POC (compare directly to upstream #908) +. *#197* -- retry count header + +=== Cross-references to Part 1 + +* *#300 (Loom POC)* <-> upstream *#908 (Virtual Threads)* -- astubbs POC may offer earlier-design insight. +* *#464 (Health check + metrics)* <-> upstream *#905 (max-queued-per-shard metric)* -- merge philosophies. +* *#91 (Kafka upgrade)* <-> upstream *#866 (Kafka v7)* -- close #91 as obsolete. +* *#204 (Vertx vertical)* <-> upstream *#867 (vertx v5)* -- re-scope against v5 APIs. +* *#237 / #345* <-> upstream *#893 / #909* (rebalance correctness) -- astubbs chaos testing would help validate upstream fixes. + +== Part 3: Upstream Issues Analysis + +`confluentinc/parallel-consumer` has *~61 open issues* (dating back to Nov 2020) and *8 issues formally closed as "Not Planned"*. The issues were NOT bulk-closed in the Jun 2023 sweep -- only the PRs were. Most feature requests and bug reports are still sitting open with no maintainer triage since mid-2023. + +Issue #907 ("Is the project still actively maintained?") filed Jan 2026 summarises the community frustration. + +=== Category 1: Stability & timeout bugs (work on FIRST) + +Production-blocking issues reported by multiple users. Highest urgency. + +[cols="1,4,2,1,1,2", options="header"] +|=== +|Issue |Title |Author |Date |Labels |Cross-ref PRs + +|#803 |Transactional Producer timeout getting commit lock |PatrickChauveau |2024-06 |verified bug |-- +|#809 |Sporadic timeouts from ConsumerOffsetCommitter.CommitRequest |tedcaozoom |2024-06 |-- |PR #893 may help +|#833 |PC runs then exits due to InternalRuntimeException(Timeout) |dumontxiong |2024-09 |-- |PR #893 may help +|#597 |PC doesn't close Kafka consumer if commit fails during close |BartoszSta |2023 |-- |-- +|#402 |Max loading factor steps reached: 100/100 |bartman64 |2022-08 |-- |-- +|#857 |Paused consumption across multiple consumers |rbokade-rbk |2025-03 |-- |-- +|#825 |checkAutoCommitIsDisabled fails with kafka-clients < 3.7.0 |ddqof |2024-08 |-- |-- +|=== + +*Verdict:* #803 is the only "verified bug" in the entire tracker. Start here. #809 and #833 are likely related -- investigate together. + +=== Category 2: Rebalance / offset correctness (data loss risk) + +[cols="1,4,2,1,2", options="header"] +|=== +|Issue |Title |Author |Date |Cross-ref PRs + +|#777 |Partition revocation leads to duplicate event processing |ajax-levashov-m |2024-05 |PR #893 (offset reset), PR #909 (stale container) +|#843 |Record picked up by multiple threads simultaneously |singhsaurabh2409 |2024-12 |PR #909 likely root cause +|#326 |Error in onPartitionsAssigned |milansanjeev |2022-07 |labeled not-a-bug, but related to rebalance +|=== + +*Verdict:* Merging PR #893 and PR #909 likely resolves #777 and #843. Verify after cherry-pick. + +=== Category 3: Observability & health (most requested feature gap) + +[cols="1,4,2,1,1,2", options="header"] +|=== +|Issue |Title |Author |Date |Labels |Cross-ref PRs + +|#27 |Micrometer metrics |astubbs |2020-11 |enhancement, medium |-- +|#71 |Health-checks |JorgenRingen |2021-01 |medium |PR #464 (astubbs, closed) +|#859 |Memory leak in PCMetrics class |CMExile |2025-04 |-- |PR #905 (metric addition) +|#484 |Question: PC state to read from? |Ehud-Lev |2022-11 |question |-- +|=== + +*Verdict:* #27 and #71 have been open for 5+ years. PR #464 was the intended fix; redo it. PR #905 adds one metric but doesn't close these. + +=== Category 4: Error handling / DLQ / retry (most demanded feature) + +[cols="1,4,2,1,2", options="header"] +|=== +|Issue |Title |Author |Date |Cross-ref PRs + +|#310 |DLQ implementation |astubbs |2022-05 |PR #366 (astubbs, closed) +|#196 |Max retries with DLQ callback |astubbs |2022-02 |PR #366, PR #291 +|#304 |Handle deserialization exceptions |astubbs |2022-05 |-- +|#391 |Serialization error handling / flexibility |astubbs |2022-08 |PR #291 (terminal/retry exceptions) +|=== + +*Verdict:* DLQ (#310) is the single most impactful missing feature. Revive PR #366 and PR #291 as a pair. + +=== Category 5: Thread safety & modern JDK support + +[cols="1,4,2,1,1,2", options="header"] +|=== +|Issue |Title |Author |Date |Labels |Cross-ref PRs + +|#186 |Ensure all PC APIs are thread safe |fowlerp-qlik |2022-02 |*blocker, ver:1.0* |PR #346 (closed) +|#78 |Allow customization of ThreadPoolExecutor |mauricioszabo |2021-02 |-- |PR #908 generalises this +|#192 |Unique thread names for PC instances |nioertel |2022-02 |-- |-- +|#299 |Loom integration POC |astubbs |2022-05 |-- |PR #300 (closed), PR #908 (open) +|#520 |Safe user API exposure of ALL Consumer APIs |astubbs |2022-12 |-- |PR #346 (closed) +|#862 |PC cannot run on Java 24 |jvissers |2025-04 |-- |PR #908 may help +|=== + +*Verdict:* #186 is labeled *blocker for 1.0*. #862 (Java 24 compat) is time-critical. PR #908 ReentrantLock migration may resolve #862. + +=== Category 6: Batch processing + +[cols="1,4,2,1,1,2", options="header"] +|=== +|Issue |Title |Author |Date |Labels |Cross-ref PRs + +|#266 |Batch option: same key only |astubbs |2022-04 |good first issue |*PR #915* (implements this!) +|#551 |Batching not working as expected |dixitsingla |2023 |-- |PR #915 may clarify +|#560 |Min batch size + batch max wait time |Ehud-Lev-Forter |2023 |-- |Complementary to #266 +|=== + +*Verdict:* PR #915 directly closes #266. #560 could be a follow-up enhancement. + +=== Category 7: Advanced features + +[cols="1,4,2,1,2", options="header"] +|=== +|Issue |Title |Author |Date |Cross-ref PRs + +|#314 |Combine queues across partitions/topics with KEY ordering |astubbs |2022-05 |PR #316 (closed) +|#321 |Transparent large message chunking |astubbs |2022-06 |-- +|#322 |Disk-backed produce queue |astubbs |2022-06 |-- +|#394 |Producer sends to least loaded broker |astubbs |2022-08 |PR #473 (closed) +|#718 |Missing feature to terminate processing |Zordid |2024 |-- +|#782 |Seeking to a specific offset for a partition |ebrockman1 |2024-05 |-- +|=== + +*Verdict:* #718 (terminate processing) and #782 (seek to offset) are practical, self-contained features worth implementing. + +=== Category 8: Logging / UX polish + +[cols="1,4,2,1,2", options="header"] +|=== +|Issue |Title |Author |Date |Cross-ref PRs + +|#629 |Missing topic/offset info in ConsumerOffsetCommitter error log |bmaggi |2023 |-- +|#631 |Warning log too verbose in RemovedPartitionState |bmaggi |2023 |-- +|#640 |Error log too verbose in AbstractParallelEoSStreamProcessor |bmaggi |2023 |PR #291 +|#622 |Wrong multiplier value in retry delay function example |zerda |2023 |-- +|=== + +*Verdict:* All four are low-effort, high-ROI. Batch them in a single "logging cleanup" PR. + +=== Category 9: Documentation gaps + +[cols="1,4,2,1,2", options="header"] +|=== +|Issue |Title |Author |Date |Cross-ref PRs + +|#115 |Clarify tombstone message handling (javadoc) |astubbs |2021-05 |-- +|#171 |Spring boot example |astubbs |2021-10 |-- +|#178 |Distribute single message to HTTP endpoints |astubbs |2021-11 |-- +|#642 |Add explanation of close modes |rkolesnev |2023 |-- +|=== + +*Verdict:* #171 (Spring Boot example) is highest-impact doc. #642 (close modes) addresses real operational confusion. + +=== Category 10: Vertx / Reactor module + +[cols="1,4,2,1,2", options="header"] +|=== +|Issue |Title |Author |Date |Cross-ref PRs + +|#170 |CompletableFuture API support + docs |astubbs |2021-10 |-- +|#180 |vertxHttpReqInfo only supports GET |OpenSourceTycoon |2021-11 |-- +|#480 |Question: Produce events using reactor? |Ehud-Lev |2022-11 |-- +|#860 |Accept instance params for managedExecutorService |dougcavalheiro |2025-04 |-- +|#912 |Memory Leak from JStreamVertxParallelStreamProcessor |sharifahmad2061 |2026-01 |-- +|=== + +*Verdict:* #912 (memory leak) is a production bug and highest priority in this group. + +=== Category 11: Build / CI / tooling + +[cols="1,4,2,1,2", options="header"] +|=== +|Issue |Title |Author |Date |Cross-ref PRs + +|#103 |Matrix test against multiple AK/JDK versions |astubbs |2021-03 |-- +|#130 |Remove static state in tests |astubbs |2021-07 |PR #126, PR #405 (both closed) +|#162 |mvn compile fails without test-jar |rkolesnev |2021-09 |-- +|#259 |Adopt error-prone and checker |astubbs |2022-04 |-- +|#290 |Refactor test base |astubbs |2022-05 |-- +|#526 |Move LongPollingMockConsumer to main artefact |astubbs |2022-12 |-- +|#861 |Error running tests: ManagedTruth.assertThat not found |gihong-park |2025-04 |PR #901 +|#906 |pom.xml version mismatch |lutzh |2026-01 |-- +|=== + +*Verdict:* #162 and #861 are contributor-friction issues. Fix these early to unblock external contributors. + +=== Category 12: Architecture / refactoring (internal) + +[cols="1,4,2,1,2", options="header"] +|=== +|Issue |Title |Author |Date |Cross-ref PRs + +|#200 |Shared-nothing architecture |astubbs |2022-02 |PR #270 (closed) +|#233 |Refactor OffsetMapCodecManager |astubbs |2022-03 |-- +|#241 |Refactor WC type from String to Enum |astubbs |2022-03 |-- +|#109 |Review serialisation versioning strategy |astubbs |2021-04 |-- +|=== + +*Verdict:* Internal maintainability improvements. Tackle when touching adjacent code. + +=== Category 13: Meta / project governance + +[cols="1,4,2,1,1", options="header"] +|=== +|Issue |Title |Author |Date |Labels + +|#172 |Release train for 1.0 |astubbs |2021-10 |*high* +|#177 |Investigate Release Drafter |astubbs |2021-11 |-- +|#907 |Is the project still actively maintained? |amrynsky |2026-01 |-- +|=== + +*Verdict:* #907 is the community distress signal. Answering it with a new release would address the maintenance concern directly. + +=== Closed as "Not Planned" (8 issues) + +[cols="1,4,3", options="header"] +|=== +|Issue |Title |Reason + +|#847 |Python support? |Out of scope (Java only) +|#462 |How to set starting offset? |Question, answered +|#446 |Poller behavior question |Question, answered +|#433 |Different results with different concurrency configs |Cannot reproduce +|#395 |API to send records through wrapped producer |Closed by astubbs +|#379 |consumeAndProduce with read-one write-many |Not a bug / by design +|#377 |Multiple instances/nodes support |Not a bug / by design +|#373 |batchSize(100) but only getting 1-3 messages |Not a bug / by design +|=== + +*Verdict:* Only #395 might be worth reconsidering. The rest are correctly closed. + +=== Issue priority ranking (top 15 across all categories) + +[cols="1,1,4,2,3", options="header"] +|=== +|Rank |Issue |Title |Category |Why first + +|1 |#803 |Transactional Producer timeout (verified bug) |Stability |Only verified bug; production-blocking +|2 |#71 |Health-checks |Observability |5+ years open; most basic operational need +|3 |#27 |Micrometer metrics |Observability |5+ years open; pairs with #71 +|4 |#310 |DLQ implementation |Error handling |Most demanded feature; PR #366 exists as design ref +|5 |#186 |Thread-safe PC APIs (blocker) |Safety |Labeled *blocker for 1.0* +|6 |#777 |Partition revocation leads to duplicates |Correctness |Fixed by merging PR #893 + #909 +|7 |#862 |PC cannot run on Java 24 |Compat |Time-critical; JDK moves fast +|8 |#266 |Batch same-key-only option |Batch |PR #915 already implements this +|9 |#809 |Sporadic commit timeouts |Stability |Multiple users report; likely related to #803 +|10 |#859 |Memory leak in PCMetrics |Stability |Production leak +|11 |#912 |Memory leak in Vertx module |Stability |Production leak +|12 |#162 |mvn compile fails without test-jar |Build |Blocks contributors +|13 |#718 |Terminate processing feature |Feature |Practical operational gap +|14 |#196 |Max retries + DLQ callback |Error handling |Companion to #310 +|15 |#629+#631+#640 |Logging verbosity fixes |UX |Low-effort batch fix +|=== + +=== Issue to PR cross-reference matrix + +[cols="2,2,2,3", options="header"] +|=== +|Issue |Open upstream PR |Closed astubbs PR |Action + +|#71 (health checks) |#905 (partial -- one metric only) |#464 (full health check impl) |Revive #464, merge #905 +|#266 (batch same-key) |*#915* (full implementation) |-- |Merge #915 +|#299 (Loom POC) |*#908* (Virtual Threads) |#300 (Loom POC) |Compare #908 vs #300, merge #908 +|#310 (DLQ) |-- |#366 (DLQ impl) |Revive #366 +|#314 (combine queues) |-- |#316 (impl) |Revive #316 +|#520 (safe user API) |-- |#346 (thread-safe API) |Revive #346 +|#200 (shared-nothing) |-- |#270 (impl) |Keep as design ref +|#130 (static state tests) |-- |#126, #405 |Revive #126 +|#394 (least-loaded broker) |-- |#473 (impl) |Revive #473 +|#196 (max retries) |-- |#291 (terminal exceptions) |Revive #291 +|#777 (rebalance dupes) |*#893* (offset fix) |-- |Merge #893 +|#843 (multi-thread pickup) |*#909* (stale container) |-- |Merge #909 +|#862 (Java 24 compat) |*#908* (ReentrantLock migration) |-- |Merge #908 +|=== + +== Part 4: How to use this document + +. *Part 1 (open PRs)* = short-term backlog -- fixes and features already written, just needing review / merge / cherry-pick. +. *Part 2 (astubbs closed PRs)* = medium-term idea bank -- intentional design work that was never rejected on technical grounds. Use each entry as a *spec*, not a branch to resurrect (branches will have bitrot beyond usefulness). +. *Part 3 (issues)* = the requirements backlog -- user-reported needs and bugs. Use the issue priority ranking and the cross-reference matrix to decide what to tackle next. Issues with linked PRs (open or closed) can be resolved by merging/reviving those PRs. +. Before picking up a Part 2 item, check Part 1 for overlap (see cross-references) to avoid duplicating effort.