diff --git a/.asf.yaml b/.asf.yaml index 1b02ecc60c0d..ea0b72024c43 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -39,7 +39,7 @@ github: - CosmosNi - fcb-xiaobo - LeonYoah - - silenceland + - heye1005 - SEZ9 - boy-xiaozhang - nzw921rx @@ -48,10 +48,15 @@ github: squash: true merge: false rebase: false + pull_requests: + allow_update_branch: true protected_branches: dev: required_status_checks: - strict: true + strict: false + checks: + - context: Build + app_id: 15368 required_pull_request_reviews: dismiss_stale_reviews: true required_approving_review_count: 1 diff --git a/.dlc.json b/.dlc.json index b25f2512f20f..a43af666416d 100644 --- a/.dlc.json +++ b/.dlc.json @@ -42,6 +42,7 @@ 0, 200, 401, - 403 + 403, + 406 ] } diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index e2938d3959c9..91e713b95ccd 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -31,6 +31,9 @@ concurrency: env: TEST_IN_PR: ${{ inputs.TEST_IN_PR }} + # GitHub-hosted runners are ephemeral, so CI does not need Ryuk to reap containers after tests. + # Disabling it avoids transient Docker Hub pulls for testcontainers/ryuk that can fail unrelated PRs. + TESTCONTAINERS_RYUK_DISABLED: 'true' jobs: license-header: @@ -38,7 +41,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Check license header @@ -49,7 +52,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Check code style @@ -58,6 +61,8 @@ jobs: run: ./mvnw -B -T 1 clean test -D"license.skipAddThirdParty"=true -pl seatunnel-ci-tools -am --no-snapshot-updates env: MAVEN_OPTS: -Xmx512m + - name: Check CI scope rules + run: python3 -m unittest discover -s tools/update_modules_check -p 'test_*.py' - name: Check for .class files in git run: | echo "Checking for .class files tracked by git..." @@ -94,8 +99,11 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Helm - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 - id: install + run: | + if ! command -v helm >/dev/null 2>&1; then + curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + fi + helm version - name: Lint Chart run: helm lint deploy/kubernetes/seatunnel @@ -106,7 +114,7 @@ jobs: # Temporarily ignore this job to avoid blocking PRs continue-on-error: true steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - run: sudo npm install -g markdown-link-check@3.8.7 - run: | for file in $(find . -name "*.md"); do @@ -131,6 +139,9 @@ jobs: api: ${{ steps.filter.outputs.api }} engine: ${{ steps.filter.outputs.engine }} engine-e2e: ${{ steps.filter.outputs.engine-e2e }} + edge-agent: ${{ steps.filter.outputs.edge-agent }} + edge-agent-e2e: ${{ steps.filter.outputs.edge-agent-e2e }} + benchmarks: ${{ steps.filter.outputs.benchmarks }} docs: ${{ steps.filter.outputs.docs }} ut-modules: ${{ steps.ut-modules.outputs.modules }} it-modules: ${{ steps.it-modules.outputs.modules }} @@ -146,7 +157,7 @@ jobs: /usr/bin/git checkout apache/dev /usr/bin/git checkout '${{ github.ref }}' echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.11.0' - name: Check for file changes by python @@ -185,6 +196,12 @@ jobs: file_list=${doc_files#*$'\n'} echo "docs=$true_or_false" >> $GITHUB_OUTPUT echo "docs_files=$file_list" >> $GITHUB_OUTPUT + + benchmark_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-benchmarks/**" "tools/benchmarks/**" ".github/workflows/benchmarks.yml"` + true_or_false=${benchmark_files%%$'\n'*} + file_list=${benchmark_files#*$'\n'} + echo "benchmarks=$true_or_false" >> $GITHUB_OUTPUT + echo "benchmarks_files=$file_list" >> $GITHUB_OUTPUT engine_e2e_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-e2e/seatunnel-engine-e2e/**"` true_or_false=${engine_e2e_files%%$'\n'*} @@ -192,10 +209,37 @@ jobs: echo "engine-e2e=$true_or_false" >> $GITHUB_OUTPUT echo "engine-e2e_files=$file_list" >> $GITHUB_OUTPUT + edge_agent_e2e_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-e2e/seatunnel-edge-agent-e2e/**"` + true_or_false=${edge_agent_e2e_files%%$'\n'*} + file_list=${edge_agent_e2e_files#*$'\n'} + echo "edge-agent-e2e=$true_or_false" >> $GITHUB_OUTPUT + echo "edge-agent-e2e_files=$file_list" >> $GITHUB_OUTPUT + + edge_agent_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-edge-agent/**"` + true_or_false=${edge_agent_files%%$'\n'*} + file_list=${edge_agent_files#*$'\n'} + echo "edge-agent=$true_or_false" >> $GITHUB_OUTPUT + echo "edge-agent_files=$file_list" >> $GITHUB_OUTPUT + + dist_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-dist/**" "bin/install-plugin.sh"` + true_or_false=${dist_files%%$'\n'*} + file_list=${dist_files#*$'\n'} + echo "dist=$true_or_false" >> $GITHUB_OUTPUT + echo "dist_files=$file_list" >> $GITHUB_OUTPUT + api_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-api/**" "seatunnel-common/**" "seatunnel-config/**" "seatunnel-core/**" "seatunnel-e2e/seatunnel-e2e-common/**" "seatunnel-formats/**" "seatunnel-plugin-discovery/**" "seatunnel-transforms-v2/**" "seatunnel-translation/**" "seatunnel-e2e/seatunnel-transforms-v2-e2e/**" "pom.xml" "**/workflows/**" "tools/**" "seatunnel-dist/**"` true_or_false=${api_files%%$'\n'*} file_list=${api_files#*$'\n'} - if [[ $repository_owner == 'apache' ]];then + # Apache dev, main, master, and numeric *-release pushes force full API coverage. + # On fork pushes, workflow files, tools/update_modules_check, seatunnel-dist, and + # bin/install-plugin.sh do not trigger the full connector matrix by themselves. + ci_scope_ref="${GITHUB_BASE_REF:-$GITHUB_REF}" + if ! true_or_false=$(python tools/update_modules_check/ci_scope.py "$repository_owner" "$ci_scope_ref" --api-changed "$true_or_false" --api-files-json "$file_list"); then + echo "::warning::CI scope helper failed; forcing full API coverage" + true_or_false='true' + fi + if [[ $true_or_false != 'true' && $true_or_false != 'false' ]]; then + echo "::warning::CI scope helper returned an invalid value; forcing full API coverage" true_or_false='true' fi echo "api=$true_or_false" >> $GITHUB_OUTPUT @@ -249,9 +293,13 @@ jobs: - name: Make unit test modules id: ut-modules timeout-minutes: 60 - if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '') }} + if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '' || steps.filter.outputs.dist == 'true') }} run: | - modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}' + dist_modules='' + if [[ '${{ steps.filter.outputs.dist }}' == 'true' ]]; then + dist_modules=',seatunnel-dist' + fi + modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'$dist_modules modules=${modules: 1} pl_modules=`python tools/update_modules_check/update_modules_check.py replace "$modules"` # remove deleted modules @@ -266,6 +314,11 @@ jobs: engine_modules='${{ steps.engine-modules.outputs.modules }}' connector_modules='${{ steps.cv2-modules.outputs.modules }}' + if [[ "zz${engine_modules}${connector_modules}" == "zz" && "zz"$dist_modules != "zz" ]];then + echo $pl_modules + echo "modules=$pl_modules" >> $GITHUB_OUTPUT + exit 0 + fi if [[ "zz"$connector_modules == "zz" && "zz"$engine_modules != "zz" ]];then # Engine changes already route the full downstream engine test set through engine=true. # Keep the fixed smoke modules as the output without resolving the Maven dependency tree. @@ -289,9 +342,9 @@ jobs: - name: Make integration test modules id: it-modules timeout-minutes: 60 - if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '' || steps.cv2-e2e-modules.outputs.modules != '' || steps.cv2-flink-e2e-modules.outputs.modules != '' || steps.cv2-spark-e2e-modules.outputs.modules != '') }} + if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '' || steps.cv2-e2e-modules.outputs.modules != '' || steps.engine-e2e-modules.outputs.modules != '') }} run: | - modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.cv2-flink-e2e-modules.outputs.modules }}${{ steps.cv2-spark-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}' + modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}' modules=${modules: 1} pl_modules=`python tools/update_modules_check/update_modules_check.py replace "$modules"` # remove deleted modules @@ -307,8 +360,6 @@ jobs: engine_modules='${{ steps.engine-modules.outputs.modules }}' connector_modules='${{ steps.cv2-modules.outputs.modules }}' connector_modules="$connector_modules"'${{ steps.cv2-e2e-modules.outputs.modules }}' - connector_modules="$connector_modules"'${{ steps.cv2-flink-e2e-modules.outputs.modules }}' - connector_modules="$connector_modules"'${{ steps.cv2-spark-e2e-modules.outputs.modules }}' engine_e2e_modules='${{ steps.engine-e2e-modules.outputs.modules }}' if [[ "zz${connector_modules}${engine_e2e_modules}" == "zz" && "zz"$engine_modules != "zz" ]];then # Engine changes already trigger the broad downstream integration jobs through engine=true. @@ -334,16 +385,16 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '8' cache: 'maven' - name: Install - uses: nick-fields/retry@v2 + uses: nick-fields/retry@v3 with: timeout_minutes: 40 max_attempts: 3 @@ -361,21 +412,24 @@ jobs: timeout-minutes: 90 steps: - name: Checkout PR - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: seatunnel-pr - name: Checkout website repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: apache/seatunnel-website path: seatunnel-website - name: Sync PR changes to website run: | bash seatunnel-pr/tools/documents/sync.sh seatunnel-pr seatunnel-website - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v4 with: node-version: 18.20.7 - name: Run docusaurus build + env: + GITHUB_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ github.token }} run: | cd seatunnel-website npm set strict-ssl false @@ -390,8 +444,8 @@ jobs: timeout-minutes: 60 steps: - name: Checkout PR - uses: actions/checkout@v3 - - uses: actions/setup-node@v2 + uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: node-version: 20.x - name: Install Dependencies and Check Code Style @@ -418,18 +472,61 @@ jobs: os: [ 'ubuntu-latest', 'windows-latest' ] timeout-minutes: 90 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' cache: 'maven' - name: run all modules unit test - run: | - ./mvnw -B -T 1 clean verify -DskipUT=false -DskipIT=true -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates + run: ./mvnw -B -T 1 clean verify -DskipUT=false -DskipIT=true -D"license.skipAddThirdParty"=true -D"skip.ui"=true "-Dsurefire.jvm.args=$SUREFIRE_JVM_ARGS" --no-snapshot-updates env: - MAVEN_OPTS: -Xmx4096m + MAVEN_OPTS: -Xmx1536m -Dfile.encoding=UTF-8 + SUREFIRE_JVM_ARGS: -Xmx3072m -XX:MaxMetaspaceSize=512m -Dfile.encoding=UTF-8 + if: runner.os != 'Windows' + - name: run all modules unit test (Windows) + run: ./mvnw -B -T 1 clean verify -DskipUT=false -DskipIT=true -D"license.skipAddThirdParty"=true -D"skip.ui"=true "-Dsurefire.jvm.args=$env:SUREFIRE_JVM_ARGS" --no-snapshot-updates + env: + MAVEN_OPTS: -Xmx1536m -Dfile.encoding=UTF-8 + SUREFIRE_JVM_ARGS: -Xmx3072m -XX:MaxMetaspaceSize=512m -Dfile.encoding=UTF-8 + if: runner.os == 'Windows' + + benchmark-test: + needs: [ changes, sanity-check ] + if: needs.changes.outputs.benchmarks == 'true' + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + java: [ '8', '11' ] + os: [ 'ubuntu-latest' ] + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java }} + distribution: 'temurin' + cache: 'maven' + - name: Build benchmark dependencies + run: | + ./mvnw -B -Pbenchmark -pl seatunnel-benchmarks -am -DskipTests install \ + -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates + - name: Run benchmark unit tests + run: | + ./mvnw -B -Pbenchmark -pl seatunnel-benchmarks test \ + -D"license.skipAddThirdParty"=true --no-snapshot-updates + - name: Run benchmark validation + run: | + result_dir="${RUNNER_TEMP}/seatunnel-benchmark-results" + mkdir -p "${result_dir}" + java -Dseatunnel.benchmark.result.dir="${result_dir}" \ + -jar seatunnel-benchmarks/target/benchmarks.jar \ + '.*' \ + -f 1 -wi 0 -i 1 -r 1s -foe true + find "${result_dir}" -name '*.json' -print -quit | grep -q . updated-modules-integration-test-part-1: needs: [ changes, sanity-check ] @@ -441,9 +538,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -460,7 +557,7 @@ jobs: echo "sub modules is empty, skipping" fi env: - MAVEN_OPTS: -Xmx2048m + MAVEN_OPTS: -Xmx4096m updated-modules-integration-test-part-2: needs: [ changes, sanity-check ] @@ -472,9 +569,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -501,11 +598,11 @@ jobs: matrix: java: [ '8', '11' ] os: [ 'ubuntu-latest' ] - timeout-minutes: 210 + timeout-minutes: 300 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -522,7 +619,7 @@ jobs: echo "sub modules is empty, skipping" fi env: - MAVEN_OPTS: -Xmx2048m + MAVEN_OPTS: -Xmx4096m updated-modules-integration-test-part-4: needs: [ changes, sanity-check ] @@ -534,9 +631,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 200 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -564,9 +661,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -583,7 +680,7 @@ jobs: echo "sub modules is empty, skipping" fi env: - MAVEN_OPTS: -Xmx2048m + MAVEN_OPTS: -Xmx4096m updated-modules-integration-test-part-6: needs: [ changes, sanity-check ] if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '' @@ -594,9 +691,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -613,7 +710,7 @@ jobs: echo "sub modules is empty, skipping" fi env: - MAVEN_OPTS: -Xmx2048m + MAVEN_OPTS: -Xmx4096m updated-modules-integration-test-part-7: needs: [ changes, sanity-check ] if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '' @@ -622,11 +719,11 @@ jobs: matrix: java: [ '8', '11' ] os: [ 'ubuntu-latest' ] - timeout-minutes: 120 + timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -643,7 +740,7 @@ jobs: echo "sub modules is empty, skipping" fi env: - MAVEN_OPTS: -Xmx2048m + MAVEN_OPTS: -Xmx4096m updated-modules-integration-test-part-8: needs: [ changes, sanity-check ] @@ -653,11 +750,11 @@ jobs: matrix: java: [ '8', '11' ] os: [ 'ubuntu-latest' ] - timeout-minutes: 120 + timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -674,7 +771,7 @@ jobs: echo "sub modules is empty, skipping" fi env: - MAVEN_OPTS: -Xmx2048m + MAVEN_OPTS: -Xmx4096m engine-v2-it: needs: [ changes, sanity-check ] @@ -686,9 +783,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 150 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -701,6 +798,31 @@ jobs: env: MAVEN_OPTS: -Xmx4096m + edge-agent-it: + needs: [ changes, sanity-check ] + if: needs.changes.outputs.edge-agent == 'true' || needs.changes.outputs.edge-agent-e2e == 'true' + runs-on: ${{ matrix.os }} + strategy: + matrix: + java: [ '8', '11' ] + os: [ 'ubuntu-latest' ] + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java }} + distribution: 'temurin' + cache: 'maven' + - name: free disk space + run: tools/github/free_disk_space.sh + - name: run seatunnel edge agent integration test + run: | + ./mvnw -T 1 -B verify -DskipUT=false -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :seatunnel-edge-agent-e2e -am -Pci + env: + MAVEN_OPTS: -Xmx4096m + engine-k8s-it: needs: [ changes, sanity-check ] if: needs.changes.outputs.api == 'true' || contains(needs.changes.outputs.it-modules, 'seatunnel-engine-k8s-e2e') @@ -719,11 +841,11 @@ jobs: cp /etc/rancher/k3s/k3s.yaml ~/.kube/config env: KUBECONFIG: /etc/rancher/k3s/k3s.yaml - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: free disk space run: tools/github/free_disk_space.sh - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -748,9 +870,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -776,9 +898,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 150 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -804,9 +926,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -835,9 +957,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -849,6 +971,13 @@ jobs: ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 1` + # Keep the longest Iceberg and HBase suites in a dedicated shard so this hosted runner + # does not lose heartbeat before the rest of the part-2 modules finish. + # Handle both first-position (no leading comma) and mid/last-position (leading comma). + run_it_modules=${run_it_modules//:connector-iceberg-e2e,/} + run_it_modules=${run_it_modules//,:connector-iceberg-e2e/} + run_it_modules=${run_it_modules//:connector-hbase-e2e,/} + run_it_modules=${run_it_modules//,:connector-hbase-e2e/} ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci env: MAVEN_OPTS: -Xmx4096m @@ -864,11 +993,11 @@ jobs: matrix: java: [ '8', '11' ] os: [ 'ubuntu-latest' ] - timeout-minutes: 210 + timeout-minutes: 240 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -897,9 +1026,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 270 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -928,9 +1057,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 270 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -959,9 +1088,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -990,9 +1119,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1008,6 +1137,35 @@ jobs: env: MAVEN_OPTS: -Xmx4096m + all-connectors-it-8: + needs: [ changes, sanity-check ] + if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' + runs-on: ${{ matrix.os }} + env: + RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} + RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} + strategy: + matrix: + java: [ '8', '11' ] + os: [ 'ubuntu-latest' ] + timeout-minutes: 210 + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java }} + distribution: 'temurin' + cache: 'maven' + - name: free disk space + run: tools/github/free_disk_space.sh + - name: run connector-v2 integration test (part-8) + run: | + # These two suites dominated part-2 and triggered hosted-runner heartbeat loss. + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-iceberg-e2e,:connector-hbase-e2e -am -Pci + env: + MAVEN_OPTS: -Xmx4096m + jdbc-connectors-it-part-1: needs: [ changes, sanity-check ] if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' @@ -1021,9 +1179,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1049,9 +1207,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1077,9 +1235,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1105,9 +1263,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1133,9 +1291,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1161,9 +1319,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1189,9 +1347,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1217,9 +1375,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1243,9 +1401,9 @@ jobs: # Kudu E2E expands each @TestTemplate case across several PR test containers. timeout-minutes: 90 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1268,9 +1426,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1293,9 +1451,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1318,9 +1476,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1333,6 +1491,56 @@ jobs: env: MAVEN_OPTS: -Xmx4096m + elasticsearch-connector-it: + needs: [ changes, sanity-check ] + if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-elasticsearch-e2e') + runs-on: ${{ matrix.os }} + strategy: + matrix: + java: [ '8', '11' ] + os: [ 'ubuntu-latest' ] + timeout-minutes: 180 + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java }} + distribution: 'temurin' + cache: 'maven' + - name: free disk space + run: tools/github/free_disk_space.sh + - name: run elasticsearch connector integration test + run: | + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-elasticsearch-e2e -am -Pci + env: + MAVEN_OPTS: -Xmx4096m + + mysql-cdc-connector-it: + needs: [ changes, sanity-check ] + if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-cdc-mysql-e2e') + runs-on: ${{ matrix.os }} + strategy: + matrix: + java: [ '8', '11' ] + os: [ 'ubuntu-latest' ] + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java }} + distribution: 'temurin' + cache: 'maven' + - name: free disk space + run: tools/github/free_disk_space.sh + - name: run mysql cdc connector integration test + run: | + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-cdc-mysql-e2e -am -Pci + env: + MAVEN_OPTS: -Xmx4096m + doris-connector-it: needs: [ changes, sanity-check ] @@ -1344,9 +1552,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1369,9 +1577,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1395,9 +1603,9 @@ jobs: timeout-minutes: 210 steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1424,9 +1632,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1449,9 +1657,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1474,9 +1682,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 210 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' @@ -1499,9 +1707,9 @@ jobs: os: [ 'ubuntu-latest' ] timeout-minutes: 180 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: 'temurin' diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 000000000000..25f74cb37cb8 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,127 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the 'License'); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Benchmarks + +on: + schedule: + # Avoid the high-load top of the hour for scheduled workflows. + - cron: '17 0 * * *' + workflow_dispatch: + inputs: + seatunnel_ref: + description: 'SeaTunnel branch, tag, or commit SHA to benchmark.' + required: false + type: string + default: 'dev' + benchmarks: + description: 'Common benchmark selector.' + required: true + type: choice + options: + - '.*' + - 'SeaTunnelRowBenchmark' + - 'SeaTunnelPipelineBenchmark' + - 'sourceSink$' + - 'sourceTransformSink$' + - 'sourceTransformSinkWithObservability$' + - 'sourceTransformSinkWithTrace$' + - 'sourceTransformSinkWithObservabilityAndTrace$' + default: '.*' + custom_benchmarks: + description: 'Optional JMH selector that overrides the choice above.' + required: false + type: string + pr_number: + description: 'Optional trusted PR to compare. The workflow executes its code.' + required: false + type: string + +permissions: + contents: read + +concurrency: + group: benchmarks-${{ github.event.inputs.seatunnel_ref || 'dev' }}-${{ github.event.inputs.pr_number || 'no-pr' }} + cancel-in-progress: false + +jobs: + benchmark: + name: Run benchmarks with Java ${{ matrix.java-version }} + runs-on: ubuntu-24.04 + timeout-minutes: 240 + + strategy: + fail-fast: false + matrix: + java-version: [ '8', '11' ] + + env: + BENCHMARKS: ${{ github.event.inputs.custom_benchmarks || github.event.inputs.benchmarks || '.*' }} + JAVA_VERSION: ${{ matrix.java-version }} + PR_NUMBER: ${{ github.event.inputs.pr_number || '' }} + SEATUNNEL_REF: ${{ github.event.inputs.seatunnel_ref || 'dev' }} + steps: + - name: Checkout baseline + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.seatunnel_ref || 'dev' }} + path: baseline + + - name: Checkout PR + if: env.PR_NUMBER != '' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: refs/pull/${{ github.event.inputs.pr_number }}/head + path: candidate + + - name: Set up JDK ${{ matrix.java-version }} with Maven cache + if: env.PR_NUMBER == '' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: ${{ matrix.java-version }} + cache: maven + + - name: Set up JDK ${{ matrix.java-version }} without Maven cache + if: env.PR_NUMBER != '' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: ${{ matrix.java-version }} + + - name: Build baseline benchmark module + working-directory: baseline + run: ./mvnw -Pbenchmark -pl seatunnel-benchmarks -am -DskipTests package + + - name: Build PR benchmark module + if: env.PR_NUMBER != '' + working-directory: candidate + run: ./mvnw -Pbenchmark -pl seatunnel-benchmarks -am -DskipTests package + + - name: Run benchmarks and build report + run: bash baseline/tools/benchmarks/run_benchmarks.sh + + - name: Upload benchmark result + if: always() + uses: actions/upload-artifact@v4 + with: + name: seatunnel-benchmarks-java${{ matrix.java-version }}-${{ github.run_id }} + retention-days: 90 + if-no-files-found: warn + path: benchmark-artifacts/** diff --git a/.github/workflows/build_main.yml b/.github/workflows/build_main.yml index f4816940a888..d5110c50a458 100644 --- a/.github/workflows/build_main.yml +++ b/.github/workflows/build_main.yml @@ -27,6 +27,7 @@ on: jobs: call-build-and-test: permissions: + contents: read packages: write name: Run uses: ./.github/workflows/backend.yml diff --git a/.github/workflows/labeler/label-scope-conf.yml b/.github/workflows/labeler/label-scope-conf.yml index e52c98ab56bf..77c2bc875efd 100644 --- a/.github/workflows/labeler/label-scope-conf.yml +++ b/.github/workflows/labeler/label-scope-conf.yml @@ -19,6 +19,12 @@ CI&CD: - changed-files: - any-glob-to-any-file: - .github/** +benchmarks: + - changed-files: + - any-glob-to-any-file: + - seatunnel-benchmarks/** + - tools/benchmarks/** + - .github/workflows/benchmarks.yml Zeta: - changed-files: - any-glob-to-any-file: seatunnel-engine/** @@ -143,6 +149,11 @@ file: - changed-files: - any-glob-to-any-file: seatunnel-connectors-v2/connector-file/** - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(file)/**' +fluss: + - all: + - changed-files: + - any-glob-to-any-file: seatunnel-connectors-v2/connector-fluss/** + - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(fluss)/**' google-firestore: - all: - changed-files: @@ -349,3 +360,13 @@ bigquery: - changed-files: - any-glob-to-any-file: seatunnel-connectors-v2/connector-bigquery/** - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(bigquery)/**' + +nats-jetstream: + - all: + - changed-files: + - any-glob-to-any-file: + - seatunnel-connectors-v2/connector-nats-jetstream/** + - seatunnel-e2e/seatunnel-connector-v2-e2e/connector-nats-jetstream-e2e/** + - all-globs-to-all-files: + - '!seatunnel-connectors-v2/connector-!(nats-jetstream)/**' + - '!seatunnel-e2e/seatunnel-connector-v2-e2e/connector-!(nats-jetstream)-e2e/**' diff --git a/.github/workflows/notify_test_workflow.yml b/.github/workflows/notify_test_workflow.yml index a517a03c168a..2a0d1729f35d 100644 --- a/.github/workflows/notify_test_workflow.yml +++ b/.github/workflows/notify_test_workflow.yml @@ -36,7 +36,7 @@ jobs: checks: write steps: - name: "Notify test workflow" - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -109,31 +109,33 @@ jobs: } }) } else { - const run_id = runs.data.workflow_runs[0].id + const build_run = runs.data.workflow_runs[0] + const run_id = build_run.id - if (runs.data.workflow_runs[0].head_sha != context.payload.pull_request.head.sha) { + if (build_run.head_sha != context.payload.pull_request.head.sha) { throw new Error('There was a new unsynced commit pushed. Please retrigger the workflow.'); } // Here we get check run ID to provide Check run view instead of Actions view, see also SPARK-37879. const check_runs = await github.request(check_run_endpoint, check_run_params) console.log('check_runs: ' + JSON.stringify(check_runs)) - const check_run_head = check_runs.data.check_runs.filter(r => r.name === "Run / License header")[0] - - console.log('check_run_head: ' + JSON.stringify(check_run_head)) - if (check_run_head.head_sha != context.payload.pull_request.head.sha) { - throw new Error('There was a new unsynced commit pushed. Please retrigger the workflow.'); - } - - const check_run_url = 'https://github.com/' - + context.payload.pull_request.head.repo.full_name - + '/runs/' - + check_run_head.id - const actions_url = 'https://github.com/' + context.payload.pull_request.head.repo.full_name + '/actions/runs/' + run_id + const run_url_part = '/actions/runs/' + run_id + const check_run_head = check_runs.data.check_runs.find(r => + r.head_sha === build_run.head_sha + && ( + (r.details_url && r.details_url.includes(run_url_part)) + || (r.html_url && r.html_url.includes(run_url_part)) + ) + ) + + console.log('check_run_head: ' + JSON.stringify(check_run_head)) + const check_run_url = check_run_head + ? (check_run_head.html_url || check_run_head.details_url || actions_url) + : actions_url await github.rest.checks.create({ owner: context.repo.owner, diff --git a/.github/workflows/publish-e2e-spark-images.yaml b/.github/workflows/publish-e2e-spark-images.yaml new file mode 100644 index 000000000000..1dc452b4d147 --- /dev/null +++ b/.github/workflows/publish-e2e-spark-images.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: publish-e2e-spark-images + +on: + workflow_dispatch: + +concurrency: + group: publish-e2e-spark-images + cancel-in-progress: false + +jobs: + build: + if: github.repository == 'apache/seatunnel' + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push Spark E2E images + working-directory: seatunnel-e2e/seatunnel-e2e-common/src/test/docker/spark + run: docker buildx bake --push diff --git a/.github/workflows/schedule_backend.yml b/.github/workflows/schedule_backend.yml index 9a6aa1cb90a2..b513ba62874d 100644 --- a/.github/workflows/schedule_backend.yml +++ b/.github/workflows/schedule_backend.yml @@ -27,8 +27,9 @@ concurrency: jobs: call-build-and-test: permissions: + contents: read packages: write name: Run uses: ./.github/workflows/backend.yml with: - TEST_IN_PR: false + TEST_IN_PR: ${{ 'false' }} diff --git a/.github/workflows/upgrade_compatibility.yml b/.github/workflows/upgrade_compatibility.yml new file mode 100644 index 000000000000..e76a33e81d51 --- /dev/null +++ b/.github/workflows/upgrade_compatibility.yml @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Upgrade Compatibility + +on: + workflow_dispatch: + inputs: + old_version: + description: "SeaTunnel release used to create the savepoint" + required: false + default: "2.3.13" + scenario: + description: "Restore scenario under tools/upgrade_compatibility/scenarios" + required: false + default: "all" + schedule: + - cron: "0 18 * * *" + +concurrency: + group: upgrade-compatibility-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + cross-version-restore: + name: Cross-version restore + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + OLD_SEATUNNEL_VERSION: ${{ github.event.inputs.old_version || '2.3.13' }} + SCENARIO: ${{ github.event.inputs.scenario || 'all' }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "8" + + - name: Free disk space + run: bash tools/github/free_disk_space.sh + + - name: Build current dev distribution + run: > + ./mvnw -B -T 1 package + -DskipTests + -DskipIT=true + -Dlicense.skipAddThirdParty=true + -Dskip.ui=true + -pl seatunnel-dist -am + + - name: Run upgrade compatibility scenarios + run: | + if [ "${SCENARIO}" = "all" ]; then + for scenario in generic-fake-localfile mysql-cdc-multitable-localfile; do + SCENARIO="${scenario}" bash tools/upgrade_compatibility/run_upgrade_compatibility.sh + done + else + bash tools/upgrade_compatibility/run_upgrade_compatibility.sh + fi + + - name: Upload compatibility logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: upgrade-compatibility-logs + path: target/upgrade-compatibility/runs/**/logs/*.log + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index bf83032a9c98..dfb9eefda891 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,12 @@ logs.zip *.iml .idea/* +# Eclipse / M2E files +.classpath +.factorypath +.project +.settings/ + .DS_Store metastore_db/ @@ -60,4 +66,11 @@ node/ dist/ -seatunnel-engine/seatunnel-engine-server/**/ui/* \ No newline at end of file +seatunnel-engine/seatunnel-engine-server/**/ui/* + +seatunnel-edge-agent/**/edge-agent.id +seatunnel-edge-agent/**/tmp/ + +# seatunnel-cli benchmark runtime artifacts +seatunnel-cli/benchmark/results/ +seatunnel-cli/benchmark/models.json diff --git a/.skills/seatunnel-test-suite/SKILL.md b/.skills/seatunnel-test-suite/SKILL.md new file mode 100644 index 000000000000..fc4c91bbd208 --- /dev/null +++ b/.skills/seatunnel-test-suite/SKILL.md @@ -0,0 +1,240 @@ +--- +name: seatunnel-test-suite +description: Write or review Apache SeaTunnel E2E (Testcontainers) and unit tests so they are stable, leak-free, and deterministic. Use when creating a new *IT/*Test class, reviewing a test diff, or refactoring a flaky or leaking test. +--- + +# SeaTunnel Test Suite + +Self-contained operating manual for writing and reviewing SeaTunnel tests: the rule tables, citable IDs, and +the patterns needed to act. Everything required to apply or cite a rule is in this file. + +## When to use + +- Writing a new connector E2E test (`*IT.java`) or unit test (`*Test.java`). +- Reviewing a test diff for flakiness, leaks, or layout violations before a PR. +- Stabilizing a flaky or leaking test, or bringing an older test up to these conventions. + +## Framework contract (E2E) + +Every E2E test **must** follow this structural contract — violating it means the test silently does nothing. + +| Requirement | Detail | +|-------------|--------| +| Extend `TestSuiteBase` | Provides the engine container lifecycle, `NETWORK`, and test dispatch | +| `@TestInstance(Lifecycle.PER_CLASS)` | Inherited from `TestSuiteBase` — one instance shared across all methods; fields need not be `static` | +| `@TestTemplate` on each test method | NOT `@Test`. The `TestCaseInvocationContextProvider` only dispatches `@TestTemplate` methods | +| `TestContainer` parameter | Each `@TestTemplate` method must accept a single `TestContainer container` parameter | +| `@DisabledOnContainer` (optional) | Add **only** when the scenario genuinely can't run on an engine — see below. Default to no annotation so the test runs on all engines. | + +Minimal skeleton — runs on every engine (Zeta, Flink, Spark), which is the default and correct case for +most batch source/sink tests: + +```java +public class MyConnectorIT extends TestSuiteBase { + + @TestTemplate + public void testSourceToSink(TestContainer container) throws Exception { + Container.ExecResult result = container.executeJob("/my_connector_to_assert.conf"); + Assertions.assertEquals(0, result.getExitCode()); + } +} +``` + +### When to add `@DisabledOnContainer` + +Do not exclude an engine by default. Add the annotation only when a scenario provably cannot run on that +engine, and word `disabledReason` for the *specific* scenario — not a blanket "engine X is unsupported". +The exclusion can go on the class (whole suite) or a single `@TestTemplate` method (just that case). Real +reasons from the codebase: + +| Excluded engines | Why (the real constraint) | +|------------------|---------------------------| +| Spark | CDC / streaming jobs — Spark doesn't support the continuous/changelog mode | +| Spark + Flink | checkpoint-restore tests; scenarios needing Zeta-only features | +| Spark + Flink (all but Zeta) | continuous-discovery long-running jobs; SeaTunnel-only behavior | +| Spark | drops the RowKind of a record, so changelog assertions fail | + +```java +// Only this CDC case can't run on Spark — exclude at the method, not the class: +@TestTemplate +@DisabledOnContainer( + value = {}, + type = {EngineType.SPARK}, + disabledReason = "Spark does not support the CDC streaming job") +public void testCdcStreaming(TestContainer container) throws Exception { + // ... +} +``` + +## The rules + +Two namespaces: **E1–E7 govern E2E (`*IT`) tests** (cite as `[E3]`); **U1–U6 govern unit (`*Test`) +tests** (cite as `[U5]`). E6 is Zeta-engine-only. Pick the namespace that matches the file under review — +never apply an E-rule to a unit test or vice versa. + +| # | Rule | Check (the smell) | Fix | +|---|------|-------------------|-----| +| E1 | Dynamic ports | literal `127.0.0.1:` / `localhost:` in Java code | `container.getHost()` + `container.getMappedPort(p)`; `.conf` files reference the network alias + internal port (see E7) | +| E2 | Condition-based waiting | any `Thread.sleep(...)` | `Awaitility.await().atMost(...).pollInterval(...).untilAsserted(...)`; timeout from the scenario table below | +| E3 | Release resources | an opened client/connection/container with no close | implement `TestResource`, close in `tearDown()` reverse-order + null-safe; try-with-resources for method-scoped | +| E4 | Async job submission | inline `executeJob` for a streaming/CDC job (never returns) | `CompletableFuture.supplyAsync(...)`, gate on `RUNNING`, act, verify, `cancel(true)` | +| E5 | Share one container | a fresh container started per test method | share one container across all methods in a class (`@TestInstance(PER_CLASS)`) | +| E5b | Pin image versions | `:latest` image tag | pin a specific version, e.g. `mysql:8.0.32`, never `mysql:latest` (see subsection) | +| E5c | Cover all data types | only `String`/`int` exercised | cover the connector's full data-type set, not just the easy ones | +| E6 | Thread whitelist (Zeta) | `There are still threads running in the container` | close the client first (E3); if the lib thread is unrecyclable, whitelist its prefix in `isIssueWeAlreadyKnow(...)` | +| E7 | Docker network | container not reachable from engine container | `.withNetwork(NETWORK).withNetworkAliases("my-alias")`; start with `Startables.deepStart(...)`; add `Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE))` for debug logs (full setup below) | + +### E7 — network setup + +The SeaTunnel engine runs **inside** its own Docker container. External service containers must share the +same Docker network so the engine can reach them (this is also why E1's `.conf` files use the alias, not the +mapped host port). Setup pattern: + +```java +// In your IT class @BeforeAll (NETWORK is inherited from TestSuiteBase): +container = new GenericContainer<>(DockerImageName.parse("my-service:1.2.3")) + .withNetwork(NETWORK) // inherited field from TestSuiteBase + .withNetworkAliases("my-service-host") // reachable by this name from engine + .withExposedPorts(3306) + .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger("my-service:1.2.3"))); +Startables.deepStart(Stream.of(container)).join(); // parallel startup +``` + +In your `.conf` file (runs inside the engine container): +```hocon +host = "my-service-host" +port = 3306 +``` + +In your Java test setup (runs on the host): +```java +String jdbcUrl = String.format("jdbc:mysql://%s:%d/test", + container.getHost(), container.getMappedPort(3306)); +``` + +### E2 — timeout reference + +| Scenario | atMost | pollInterval | +|----------|--------|--------------| +| Container / client readiness | 2 min | 1 s | +| Job reaches `RUNNING` | 1 min | 2 s | +| Batch job result verified | 60 s | 2 s | +| MQ / Kafka consumption | 30–60 s | 1 s | +| CDC / schema-change propagation | 60–120 s | 2–5 s | + +Chain `.ignoreExceptions()` onto the `Awaitility.await()` (the E2 fix) whenever the client throws until the +service is up — otherwise the first poll's exception fails the wait instead of retrying. Never pick a bare +round number without a scenario to justify it. + +### E5b — image version pinning + +Never use `:latest` — upstream images can change behavior without warning, breaking CI with no code diff to +bisect. Pin to a specific minor or patch version: + +```java +// Bad — breaks unpredictably +private static final String IMAGE = "postgres:latest"; + +// Good — reproducible +private static final String IMAGE = "postgres:14.5"; +``` + +### E6 — thread whitelist, in one paragraph + +After the last job finishes, `SeaTunnelContainer` snapshots the server JVM and fails if a non-system thread +survives 120s. Daemon threads from third-party clients (JDBC drivers, HTTP pools) are sometimes unrecyclable. +For those only, add a **specific name prefix** to `isIssueWeAlreadyKnow(String)` in `SeaTunnelContainer`, with +a comment naming the library. A thread you *can* close belongs in `tearDown()`, not the whitelist. System +threads (`hz.main`, `pool-N-thread-N`, …) are already handled by `isSystemThread(...)` — don't duplicate them. + +### UT rules + +Use this table for `*Test` classes (Surefire, no engine container): + +| # | Rule | Smell | Fix | +|---|------|-------|-----| +| U1 | Test behavior, not internals | asserts on private fields or internal call order | assert observable output/state; verify `void` calls only when side-effect IS the contract | +| U2 | No timing or IO dependence | `Thread.sleep`, `System.currentTimeMillis()`, filesystem, network | remove timing; mock/fake IO boundaries; inject clocks if needed | +| U3 | Mock at boundaries only | mocking domain objects or `new`-able value types | only mock external interfaces/clients/maps/services the class depends on via constructor or setter | +| U4 | One behavior per method | multi-branch `if`/`for` inside a single test | split into one method per branch; name each method as the behavior it proves | +| U5 | Negative paths assert message | `assertThrows(FooException.class, ...)` with no message check | also call `assertThat(ex.getMessage()).contains("expected fragment")` | +| U6 | Naming | `testFoo`, `testBar`, `test123` | class `*Test`; methods like `shouldReturn404WhenUserMissing`, `throwsOnNullHost` | + +**Mock setup shape** — each test method is Arrange-Act-Assert: +1. *Arrange* — in `@BeforeEach`, `mock(...)` every external boundary, wire the chain with `when(...).thenReturn(...)`, then construct the SUT. Per test, stub only the values that select the branch under test. +2. *Act* — call the one method under test. +3. *Assert* — check observable output (U1), not call chains. + +Pitfalls that make a mocked test pass for the wrong reason: +- When the SUT reaches a collaborator through a chain (`a.getB().getC(key)`), stub the whole chain, and stub each lookup with the *exact* key constant the production code uses — a mismatched key returns `null` and silently changes the branch taken. +- To exercise a downstream branch, stub every upstream guard non-null first; a null upstream value short-circuits early, so stubbing only the downstream value tests nothing. +- For negative paths, assert the exception message fragment, not just the type (U5) — a wrong-cause throw of the same type would otherwise pass. + +**When NOT to mock:** +- Simple value objects, POJOs, DTOs — use real instances. +- The class under test itself — never mock the SUT. +- When an in-memory fake (e.g. `HashMap` instead of an interface) is simpler and equally isolated. + +## Layout (where things go) + +One Maven module per connector under `seatunnel-e2e/seatunnel-connector-v2-e2e/connector--e2e/`. Files +load from the test **classpath root** (`src/test/resources/`): + +- Test class → `src/test/java/org/apache/seatunnel/e2e/connector//IT.java` + (CDC connectors use the `...connectors.seatunnel.cdc.` package). `*IT` = integration (Failsafe), + `*Test` = unit (Surefire). +- Job configs → `src/test/resources/_source_to_sink.conf`, referenced with a **leading slash** + (`executeJob("/_source_to_sink.conf")`). License header required (`#` comments). +- DDL → `src/test/resources/ddl/.sql` (CDC `UniqueDatabase` resolves `ddl/