diff --git a/.asf.yaml b/.asf.yaml index 8bd6ce570b5..fc525b20913 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -66,6 +66,16 @@ github: # strict means "Require branches to be up to date before merging". strict: true # contexts are the names of checks that must pass + contexts: + - frontend (ubuntu-latest, 18) + - frontend (windows-latest, 18) + - frontend (macos-latest, 18) + - scala (ubuntu-22.04, 11) + - python (ubuntu-latest, 3.10) + - python (ubuntu-latest, 3.11) + - python (ubuntu-latest, 3.12) + - Check License Headers + - Validate PR title required_pull_request_reviews: dismiss_stale_reviews: false require_code_owner_reviews: false @@ -74,10 +84,7 @@ github: notifications: commits: commits@texera.apache.org - issues: dev@texera.apache.org - pullrequests: dev@texera.apache.org + issues: notifications@texera.apache.org + pullrequests: notifications@texera.apache.org discussions: dev@texera.apache.org jobs: commits@texera.apache.org - -publish: - whoami: asf-site diff --git a/.github/ISSUE_TEMPLATE/bug-template.yaml b/.github/ISSUE_TEMPLATE/bug-template.yaml new file mode 100644 index 00000000000..7a3ebc92022 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-template.yaml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Bug Report +description: File a bug report. +labels: ["triage"] +type: "Bug" +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: textarea + id: what-happened + attributes: + label: What happened? + description: Also tell us, what did you expect to happen? + placeholder: Tell us what you see! + value: "A bug happened!" + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: How to reproduce? + description: Please include steps for a repro. + validations: + required: true + - type: dropdown + id: version + attributes: + label: Version + description: What version of Texera are you running? + options: + - 1.1.0-incubating (Pre-release/Master) + - 1.0.0 + default: 0 + validations: + required: true + - type: input + id: commit-hash + attributes: + label: Commit Hash (Optional) + description: If you know the specific commit that has the issue, please provide the commit hash here. + placeholder: e.g., a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6q7r8s9t0 + validations: + required: false + - type: dropdown + id: browsers + attributes: + label: What browsers are you seeing the problem on? + multiple: true + options: + - Chrome + - Safari + - Firefox + - Microsoft Edge + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: shell + + - type: markdown + attributes: + value: | + By submitting this issue, you agree to follow the [Apache Code of Conduct](https://www.apache.org/foundation/policies/conduct). diff --git a/.github/ISSUE_TEMPLATE/feature-template.yaml b/.github/ISSUE_TEMPLATE/feature-template.yaml new file mode 100644 index 00000000000..f88d85b0f67 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-template.yaml @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Feature Request +description: Suggest a new feature or improvement. +labels: ["triage"] +type: "Feature" +body: + - type: markdown + attributes: + value: | + Thanks for suggesting a feature! Please provide as much detail as possible to help us evaluate your idea. + + - type: textarea + id: summary + attributes: + label: Feature Summary + description: Clearly describe what the feature is and the problem it solves. + placeholder: Describe your feature idea and what problem it addresses. + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposed Solution or Design + description: Explain how you imagine this feature working. Include examples, diagrams, or pseudo-code if relevant. + placeholder: Describe your proposed solution or design approach. + validations: + required: true + + - type: dropdown + id: impact + attributes: + label: Impact / Priority + description: How important is this feature? + options: + - (P0)Critical – blocks existing use cases + - (P1)High – significantly improves user experience + - (P2)Medium – useful enhancement + - (P3)Low – nice to have + default: 2 + validations: + required: true + + - type: dropdown + id: affected-area + attributes: + label: Affected Area + description: Which part of the system does this feature relate to? + multiple: true + options: + - Workflow Engine (Amber) + - Workflow UI + - Hub + - Storage / Metadata + - Deployment / Infrastructure + - Other + + - type: markdown + attributes: + value: | + By submitting this issue, you agree to follow the [Apache Code of Conduct](https://www.apache.org/foundation/policies/conduct). \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/task-template.yaml b/.github/ISSUE_TEMPLATE/task-template.yaml new file mode 100644 index 00000000000..1e299fef7c6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/task-template.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Task +description: Create a new development or maintenance task. +labels: ["triage"] +type: "Task" +body: + - type: markdown + attributes: + value: | + Thanks for creating a task! Please describe what needs to be done and why. + + - type: textarea + id: task-summary + attributes: + label: Task Summary + description: Briefly describe what needs to be done, try to do a single step in a task. + placeholder: Example — Refactor workflow scheduler module for better modularity. + validations: + required: true + + - type: dropdown + id: priority + attributes: + label: Priority + description: How urgent or important is this task? + options: + - P0 – Critical + - P1 – High + - P2 – Medium + - P3 – Low + default: 2 + + - type: checkboxes + id: checklist + attributes: + label: Task Type + description: Select the type of work involved. + options: + - label: Code Implementation + - label: Documentation + - label: Refactor / Cleanup + - label: Testing / QA + - label: DevOps / Deployment + + - type: markdown + attributes: + value: | + By submitting this issue, you agree to follow the [Apache Code of Conduct](https://www.apache.org/foundation/policies/conduct). diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE new file mode 100644 index 00000000000..41287564ff3 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE @@ -0,0 +1,53 @@ + + +### What changes were proposed in this PR? + + + +### Any related issues, documentation, discussions? + + + +### How was this PR tested? + + + +### Was this PR authored or co-authored using generative AI tooling? + diff --git a/.github/labeler.yml b/.github/labeler.yml index 9d3d7a68fa2..72ff0ffd4a0 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -14,30 +14,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -gui: +frontend: - changed-files: - any-glob-to-any-file: - - 'core/gui/**' + - 'frontend/**' -backend: - - all: - - changed-files: - - any-glob-to-any-file: - - 'core/**' - - all-globs-to-all-files: - - '!core/gui/**' - - '!core/log/**' - - '!core/scripts/**' +common: + - changed-files: + - any-glob-to-any-file: + - 'common/**' + +service: + - changed-files: + - any-glob-to-any-file: + - '*-service/**' engine: - changed-files: - any-glob-to-any-file: - - 'core/amber/**' + - 'amber/**' python: - changed-files: - any-glob-to-any-file: - - 'core/amber/src/main/python/**' + - 'amber/src/main/python/**' - '**/*.py' docs: @@ -56,7 +56,7 @@ ci: build: - changed-files: - any-glob-to-any-file: - - 'core/scripts/**' + - 'bin/**' - 'deployment/**' dependencies: @@ -74,8 +74,8 @@ ddl-change: feature: - head-branch: - - '^feat' - - 'feature' + - '^feat' + - 'feature' fix: - head-branch: '^fix' diff --git a/.github/release/vote-email-template.md b/.github/release/vote-email-template.md new file mode 100644 index 00000000000..f3dd942a1d7 --- /dev/null +++ b/.github/release/vote-email-template.md @@ -0,0 +1,61 @@ +Subject: [VOTE] Release Apache Texera (incubating) ${VERSION} RC${RC_NUM} + +Hi Texera Community, + +This is a call for vote to release Apache Texera (incubating) ${VERSION}. + +== Release Candidate Artifacts == + +https://dist.apache.org/repos/dist/dev/incubator/texera/${VERSION}-RC${RC_NUM}/ + +The directory contains: +- Source tarball (.tar.gz) with GPG signature (.asc) and SHA512 checksum (.sha512) +- Docker Compose deployment bundle with GPG signature and SHA512 checksum +- Helm chart package with GPG signature and SHA512 checksum + +== Container Images == + +Container images are available at: + ${IMAGE_REGISTRY}/texera-dashboard-service:${VERSION} + ${IMAGE_REGISTRY}/texera-workflow-execution-coordinator:${VERSION} + ${IMAGE_REGISTRY}/texera-workflow-compiling-service:${VERSION} + ${IMAGE_REGISTRY}/texera-file-service:${VERSION} + ${IMAGE_REGISTRY}/texera-config-service:${VERSION} + ${IMAGE_REGISTRY}/texera-access-control-service:${VERSION} + ${IMAGE_REGISTRY}/texera-workflow-computing-unit-managing-service:${VERSION} + +These images are built from the source tarball included in this release. +The Dockerfiles are included in the source for audit and verification. + +== Git Tag == + +https://github.com/apache/texera/releases/tag/${TAG_NAME} +Commit: ${COMMIT_HASH} + +== Keys == + +The release was signed with GPG key [${GPG_KEY_ID}] (${GPG_EMAIL}) +KEYS file: https://downloads.apache.org/incubator/texera/KEYS + +== Vote == + +The vote will be open for at least 72 hours. + +[ ] +1 Approve the release +[ ] 0 No opinion +[ ] -1 Disapprove the release (please provide the reason) + +== Checklist == + +[ ] Checksums and PGP signatures are valid +[ ] LICENSE and NOTICE files are correct +[ ] All files have ASF license headers where appropriate +[ ] No unexpected binary files +[ ] Source tarball matches the Git tag +[ ] Can compile from source successfully +[ ] Docker Compose bundle deploys successfully with the published images +[ ] Helm chart deploys successfully (if applicable) + +Thanks, +[Your Name] +Apache Texera (incubating) PPMC diff --git a/.github/workflows/automatic-email-notif-on-ddl-change.yml b/.github/workflows/automatic-email-notif-on-ddl-change.yml new file mode 100644 index 00000000000..6d0fa62cd63 --- /dev/null +++ b/.github/workflows/automatic-email-notif-on-ddl-change.yml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Automatic email notification on DDL change + +on: + pull_request: + types: + - closed + +jobs: + notify: + if: >- + github.event.pull_request.merged == true && + contains(github.event.pull_request.labels.*.name, 'ddl-change') + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + sparse-checkout: sql/updates/ + + - name: Get added file in sql/updates/ + id: get_sql_file + run: | + FILE=$(git diff --name-only --diff-filter=A \ + ${{ github.event.pull_request.base.sha }} \ + ${{ github.event.pull_request.merge_commit_sha }} \ + -- 'sql/updates/') + echo "sql_file=$FILE" >> $GITHUB_OUTPUT + - name: Send email + uses: dawidd6/action-send-mail@v7 + with: + server_address: smtp.gmail.com + server_port: 465 + secure: true + username: ${{secrets.NOREPLY_EMAIL_USERNAME}} + password: ${{secrets.NOREPLY_EMAIL_PASSWORD}} + subject: Actions Required After Pulling Latest Update + to: dev@texera.apache.org + from: ${{ github.event.pull_request.user.login }} + body: | + Hi all, + We have merged PR #${{ github.event.pull_request.number }} (${{ github.event.pull_request.html_url }}): ${{github.event.pull_request.title}}. To incorporate the change, please apply ${{ steps.get_sql_file.outputs.sql_file }} to your local Postgres instance and run sbt jooqGenerate to generate jooq classes. + + Best, + ${{ github.event.pull_request.user.login }} + reply_to: dev@texera.apache.org \ No newline at end of file diff --git a/.github/workflows/build-and-push-images.yml b/.github/workflows/build-and-push-images.yml new file mode 100644 index 00000000000..e70c81a2d9f --- /dev/null +++ b/.github/workflows/build-and-push-images.yml @@ -0,0 +1,558 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build and push images + +on: + workflow_dispatch: + inputs: + branch: + description: 'Branch to checkout and build from' + required: false + default: 'main' + type: string + image_tag: + description: 'Docker image tag (e.g., latest, v1.0.0). Leave empty to use the short commit hash of the branch.' + required: false + default: '' + type: string + docker_registry: + description: 'Full image registry prefix (e.g., ghcr.io/apache, docker.io/apache)' + required: false + default: 'ghcr.io/apache' + type: string + services: + description: 'Services to build (comma-separated, "*" for all)' + required: false + default: '*' + type: string + platforms: + description: 'Target platforms to build' + required: false + default: 'both' + type: choice + options: + - both + - amd64 + - arm64 + with_r_support: + description: 'Enable R support for workflow-execution-coordinator' + required: false + default: false + type: boolean + schedule: + # Run nightly at 2:00 AM UTC + - cron: '0 2 * * *' + +permissions: + contents: read + packages: write # Required for pushing to ghcr.io + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs.image_tag || 'nightly' }} + cancel-in-progress: false + +jobs: + # Step 0: Set runtime parameters (handles both manual and scheduled runs) + set-parameters: + runs-on: ubuntu-latest + outputs: + branch: ${{ steps.set-params.outputs.branch }} + image_tag: ${{ steps.set-params.outputs.image_tag }} + docker_registry: ${{ steps.set-params.outputs.docker_registry }} + services: ${{ steps.set-params.outputs.services }} + platforms: ${{ steps.set-params.outputs.platforms }} + with_r_support: ${{ steps.set-params.outputs.with_r_support }} + steps: + - name: Set build parameters + id: set-params + env: + GH_TOKEN: ${{ github.token }} + run: | + # Detect if this is a scheduled run + if [[ "${{ github.event_name }}" == "schedule" ]]; then + echo "Nightly build detected - using nightly defaults" + echo "branch=main" >> $GITHUB_OUTPUT + echo "image_tag=nightly" >> $GITHUB_OUTPUT + echo "docker_registry=ghcr.io/apache" >> $GITHUB_OUTPUT + echo "services=*" >> $GITHUB_OUTPUT + echo "platforms=both" >> $GITHUB_OUTPUT + echo "with_r_support=false" >> $GITHUB_OUTPUT + else + echo "Manual workflow_dispatch - using user inputs" + BRANCH="${{ github.event.inputs.branch || 'main' }}" + IMAGE_TAG="${{ github.event.inputs.image_tag }}" + + # If image_tag is empty, resolve to the short commit hash of the branch + if [[ -z "$IMAGE_TAG" ]]; then + COMMIT_SHORT=$(gh api "repos/${{ github.repository }}/commits/${BRANCH}" --jq '.sha[:9]') + IMAGE_TAG="$COMMIT_SHORT" + echo "No image tag specified - using short commit hash: $IMAGE_TAG" + fi + + echo "branch=$BRANCH" >> $GITHUB_OUTPUT + echo "image_tag=$IMAGE_TAG" >> $GITHUB_OUTPUT + echo "docker_registry=${{ github.event.inputs.docker_registry || 'ghcr.io/apache' }}" >> $GITHUB_OUTPUT + echo "services=${{ github.event.inputs.services || '*' }}" >> $GITHUB_OUTPUT + echo "platforms=${{ github.event.inputs.platforms || 'both' }}" >> $GITHUB_OUTPUT + echo "with_r_support=${{ github.event.inputs.with_r_support || 'false' }}" >> $GITHUB_OUTPUT + fi + + # Step 1: Generate JOOQ code once and share it + generate-jooq: + needs: [set-parameters] + runs-on: ubuntu-latest + env: + JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + + steps: + - name: Checkout Texera + uses: actions/checkout@v5 + with: + ref: ${{ needs.set-parameters.outputs.branch }} + + - name: Setup JDK + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: 11 + + - name: Setup sbt launcher + uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14 + + - uses: coursier/cache-action@4e2615869d13561d626ed48655e1a39e5b192b3c # v6.4.9 + with: + extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' + + - name: Install PostgreSQL + run: sudo apt-get update && sudo apt-get install -y postgresql + + - name: Start PostgreSQL Service + run: sudo systemctl start postgresql + + - name: Configure PostgreSQL authentication + run: | + sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" + sudo sed -i 's/local all postgres peer/local all postgres md5/' /etc/postgresql/*/main/pg_hba.conf + sudo sed -i 's/host all all 127.0.0.1\/32 scram-sha-256/host all all 127.0.0.1\/32 md5/' /etc/postgresql/*/main/pg_hba.conf + sudo systemctl restart postgresql + sleep 2 + + - name: Create Databases + run: | + PGPASSWORD=postgres psql -h localhost -U postgres -f sql/texera_ddl.sql + PGPASSWORD=postgres psql -h localhost -U postgres -f sql/iceberg_postgres_catalog.sql + PGPASSWORD=postgres psql -h localhost -U postgres -f sql/texera_lakefs.sql + + - name: Generate JOOQ code + run: sbt DAO/jooqGenerate + + - name: Upload JOOQ generated code + uses: actions/upload-artifact@v4 + with: + name: jooq-code + path: | + common/dao/src/main/scala/org/apache/texera/dao/jooq/generated/ + retention-days: 1 + + # Step 2: Parse services and prepare build matrix + prepare-matrix: + needs: [set-parameters] + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + build_amd64: ${{ steps.set-platforms.outputs.build_amd64 }} + build_arm64: ${{ steps.set-platforms.outputs.build_arm64 }} + need_manifest: ${{ steps.set-platforms.outputs.need_manifest }} + steps: + - name: Checkout Texera + uses: actions/checkout@v5 + with: + ref: ${{ needs.set-parameters.outputs.branch }} + + - name: Set target platforms + id: set-platforms + run: | + PLATFORM_INPUT="${{ needs.set-parameters.outputs.platforms }}" + + case "$PLATFORM_INPUT" in + both) + echo "build_amd64=true" >> $GITHUB_OUTPUT + echo "build_arm64=true" >> $GITHUB_OUTPUT + echo "need_manifest=true" >> $GITHUB_OUTPUT + echo "Building for both platforms (parallel jobs)" + ;; + amd64) + echo "build_amd64=true" >> $GITHUB_OUTPUT + echo "build_arm64=false" >> $GITHUB_OUTPUT + echo "need_manifest=false" >> $GITHUB_OUTPUT + echo "Building for AMD64 only" + ;; + arm64) + echo "build_amd64=false" >> $GITHUB_OUTPUT + echo "build_arm64=true" >> $GITHUB_OUTPUT + echo "need_manifest=false" >> $GITHUB_OUTPUT + echo "Building for ARM64 only" + ;; + esac + + - name: Discover and parse services + id: set-matrix + run: | + SERVICES="${{ needs.set-parameters.outputs.services }}" + + # Discover all Dockerfiles in bin/ directory + echo "Discovering services from Dockerfiles..." + cd bin + + # Standard services from *.dockerfile pattern (excluding postgres17-pgroonga) + STANDARD_SERVICES=() + for dockerfile in *.dockerfile; do + if [[ -f "$dockerfile" ]]; then + service_name=$(basename "$dockerfile" .dockerfile) + # Skip postgres17-pgroonga + if [[ "$service_name" != "postgres17-pgroonga" ]]; then + STANDARD_SERVICES+=("$service_name") + fi + fi + done + + # All services are standard services only + ALL_SERVICES=("${STANDARD_SERVICES[@]}") + + echo "Found ${#ALL_SERVICES[@]} services: ${ALL_SERVICES[*]}" + + # Filter based on user input + if [[ "$SERVICES" == "*" ]]; then + SERVICES_LIST=("${ALL_SERVICES[@]}") + else + IFS=',' read -ra SERVICES_LIST <<< "$SERVICES" + # Trim whitespace + for i in "${!SERVICES_LIST[@]}"; do + SERVICES_LIST[$i]=$(echo "${SERVICES_LIST[$i]}" | xargs) + done + fi + + # Create JSON matrix with dockerfile info + JSON="[" + FIRST=true + for service in "${SERVICES_LIST[@]}"; do + # Determine dockerfile path and context + if [[ " ${STANDARD_SERVICES[@]} " =~ " ${service} " ]]; then + dockerfile="bin/${service}.dockerfile" + context="." + + # Map dockerfile service names to Docker image names + case "$service" in + "texera-web-application") + image_name="texera-dashboard-service" + ;; + "computing-unit-master") + image_name="texera-workflow-execution-coordinator" + ;; + "computing-unit-worker") + image_name="texera-workflow-execution-runner" + ;; + "access-control-service") + image_name="texera-access-control-service" + ;; + "config-service") + image_name="texera-config-service" + ;; + "file-service") + image_name="texera-file-service" + ;; + "workflow-compiling-service") + image_name="texera-workflow-compiling-service" + ;; + "workflow-computing-unit-managing-service") + image_name="texera-workflow-computing-unit-managing-service" + ;; + *) + # Default: use service name as-is + image_name="$service" + ;; + esac + else + echo "WARNING: Unknown service: $service, skipping" + continue + fi + + if [[ "$FIRST" == "true" ]]; then + FIRST=false + else + JSON+="," + fi + JSON+="{\"service\":\"$service\",\"image_name\":\"$image_name\",\"dockerfile\":\"$dockerfile\",\"context\":\"$context\"}" + done + JSON+="]" + + echo "Generated matrix: $JSON" + echo "matrix={\"include\":$JSON}" >> $GITHUB_OUTPUT + + # Step 3a: Build AMD64 images (runs in parallel with ARM64) + build-amd64: + runs-on: ubuntu-latest + needs: [set-parameters, generate-jooq, prepare-matrix] + if: needs.prepare-matrix.outputs.build_amd64 == 'true' + strategy: + matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} + fail-fast: false + max-parallel: 8 # Higher parallelism for native builds + env: + DOCKER_REGISTRY: ${{ needs.set-parameters.outputs.docker_registry }} + JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + + steps: + - name: Checkout Texera + uses: actions/checkout@v5 + with: + ref: ${{ needs.set-parameters.outputs.branch }} + + - name: Setup JDK + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: 11 + + - name: Setup sbt launcher + uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14 + + - uses: coursier/cache-action@4e2615869d13561d626ed48655e1a39e5b192b3c # v6.4.9 + with: + extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' + + - name: Download JOOQ generated code + uses: actions/download-artifact@v4 + with: + name: jooq-code + path: common/dao/src/main/scala/org/apache/texera/dao/jooq/generated/ + + - name: Free up disk space + run: | + sudo apt-get clean + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + df -h + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Log in to GitHub Container Registry + if: startsWith(needs.set-parameters.outputs.docker_registry, 'ghcr.io/') + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to Docker Hub + if: ${{ !startsWith(needs.set-parameters.outputs.docker_registry, 'ghcr.io/') }} + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push AMD64 image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + push: true + tags: ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-amd64 + cache-from: type=gha,scope=${{ matrix.image_name }}-amd64 + cache-to: type=gha,mode=max,scope=${{ matrix.image_name }}-amd64 + build-args: | + ${{ (matrix.service == 'computing-unit-master' || matrix.service == 'computing-unit-worker') && needs.set-parameters.outputs.with_r_support == 'true' && 'WITH_R_SUPPORT=true' || '' }} + labels: | + org.opencontainers.image.title=${{ matrix.image_name }} + org.opencontainers.image.description=Apache Texera ${{ matrix.image_name }} (AMD64) + org.opencontainers.image.vendor=Apache Texera + + # Step 3b: Build ARM64 images (runs in parallel with AMD64) + build-arm64: + runs-on: ubuntu-latest + needs: [set-parameters, generate-jooq, prepare-matrix] + if: needs.prepare-matrix.outputs.build_arm64 == 'true' + strategy: + matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} + fail-fast: false + max-parallel: 4 # Lower for QEMU builds + env: + DOCKER_REGISTRY: ${{ needs.set-parameters.outputs.docker_registry }} + JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + + steps: + - name: Checkout Texera + uses: actions/checkout@v5 + with: + ref: ${{ needs.set-parameters.outputs.branch }} + + - name: Setup JDK + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: 11 + + - name: Setup sbt launcher + uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14 + + - uses: coursier/cache-action@4e2615869d13561d626ed48655e1a39e5b192b3c # v6.4.9 + with: + extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' + + - name: Download JOOQ generated code + uses: actions/download-artifact@v4 + with: + name: jooq-code + path: common/dao/src/main/scala/org/apache/texera/dao/jooq/generated/ + + - name: Free up disk space + run: | + sudo apt-get clean + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + df -h + + # Set up QEMU for ARM64 emulation + - name: Set up QEMU + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + with: + platforms: linux/arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Log in to GitHub Container Registry + if: startsWith(needs.set-parameters.outputs.docker_registry, 'ghcr.io/') + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to Docker Hub + if: ${{ !startsWith(needs.set-parameters.outputs.docker_registry, 'ghcr.io/') }} + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push ARM64 image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: linux/arm64 + push: true + tags: ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-arm64 + cache-from: type=gha,scope=${{ matrix.image_name }}-arm64 + cache-to: type=gha,mode=max,scope=${{ matrix.image_name }}-arm64 + build-args: | + ${{ (matrix.service == 'computing-unit-master' || matrix.service == 'computing-unit-worker') && needs.set-parameters.outputs.with_r_support == 'true' && 'WITH_R_SUPPORT=true' || '' }} + labels: | + org.opencontainers.image.title=${{ matrix.image_name }} + org.opencontainers.image.description=Apache Texera ${{ matrix.image_name }} (ARM64) + org.opencontainers.image.vendor=Apache Texera + + # Step 4: Create multi-arch manifests (only if building both platforms) + create-manifests: + runs-on: ubuntu-latest + needs: [set-parameters, prepare-matrix, build-amd64, build-arm64] + if: always() && needs.prepare-matrix.outputs.need_manifest == 'true' + strategy: + matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} + fail-fast: false + env: + DOCKER_REGISTRY: ${{ needs.set-parameters.outputs.docker_registry }} + + steps: + - name: Log in to GitHub Container Registry + if: startsWith(needs.set-parameters.outputs.docker_registry, 'ghcr.io/') + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to Docker Hub + if: ${{ !startsWith(needs.set-parameters.outputs.docker_registry, 'ghcr.io/') }} + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Create and push multi-arch manifest + run: | + # Create manifest list combining both architectures + docker buildx imagetools create -t \ + ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }} \ + ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-amd64 \ + ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-arm64 + + - name: Inspect multi-arch manifest + run: | + docker buildx imagetools inspect ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }} + + # Step 5: Summary report + build-summary: + runs-on: ubuntu-latest + needs: [set-parameters, prepare-matrix, build-amd64, build-arm64, create-manifests] + if: always() + steps: + - name: Generate build summary + run: | + echo "# Texera Multi-Arch Build Complete (Parallel)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Build Configuration" >> $GITHUB_STEP_SUMMARY + echo "- **Trigger:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY + echo "- **Branch:** \`${{ needs.set-parameters.outputs.branch }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Registry:** \`${{ needs.set-parameters.outputs.docker_registry }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Tag:** \`${{ needs.set-parameters.outputs.image_tag }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Services:** ${{ needs.set-parameters.outputs.services }}" >> $GITHUB_STEP_SUMMARY + echo "- **Platforms:** ${{ needs.set-parameters.outputs.platforms }}" >> $GITHUB_STEP_SUMMARY + echo "- **R Support:** ${{ needs.set-parameters.outputs.with_r_support }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Build Method" >> $GITHUB_STEP_SUMMARY + echo "**Parallel platform builds** (faster)" >> $GITHUB_STEP_SUMMARY + echo "- AMD64: Native build on \`ubuntu-latest\`" >> $GITHUB_STEP_SUMMARY + echo "- ARM64: QEMU emulation on \`ubuntu-latest\` (runs in parallel)" >> $GITHUB_STEP_SUMMARY + echo "- Manifests: Combined into multi-arch images" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "> **Performance:** AMD64 and ARM64 now build simultaneously instead of sequentially!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Images Published" >> $GITHUB_STEP_SUMMARY + echo "All images are now available as multi-arch manifests at:" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "docker pull ${{ needs.set-parameters.outputs.docker_registry }}/:${{ needs.set-parameters.outputs.image_tag }}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Usage" >> $GITHUB_STEP_SUMMARY + echo "The images will automatically use the correct architecture:" >> $GITHUB_STEP_SUMMARY + echo "- On x86_64/AMD64: pulls linux/amd64 variant" >> $GITHUB_STEP_SUMMARY + echo "- On ARM64/M1/M2: pulls linux/arm64 variant" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Build Status" >> $GITHUB_STEP_SUMMARY + echo "- AMD64 builds: ${{ needs.build-amd64.result }}" >> $GITHUB_STEP_SUMMARY + echo "- ARM64 builds: ${{ needs.build-arm64.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Manifest creation: ${{ needs.create-manifests.result }}" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.github/workflows/check-header.yml b/.github/workflows/check-header.yml index 0f781236e9e..c2cbddc1333 100644 --- a/.github/workflows/check-header.yml +++ b/.github/workflows/check-header.yml @@ -30,4 +30,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: apache/skywalking-eyes@v0.7.0 + - uses: apache/skywalking-eyes@5c5b974209f0de5d905f37deb69369068ebfc15c # v0.7.0 diff --git a/.github/workflows/create-release-candidate.yml b/.github/workflows/create-release-candidate.yml new file mode 100644 index 00000000000..4b0ff4bb3a7 --- /dev/null +++ b/.github/workflows/create-release-candidate.yml @@ -0,0 +1,497 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Create and upload release candidate artifacts + +on: + workflow_dispatch: + inputs: + tag: + description: 'Existing Git tag (e.g., v1.1.0-incubating-rc1)' + required: true + type: string + rc_number: + description: 'Release candidate number for artifacts (e.g., 1 for RC1, 2 for RC2)' + required: true + type: string + default: '1' + image_registry: + description: 'Container image registry prefix (e.g., ghcr.io/apache, docker.io/apache)' + required: false + type: string + default: 'ghcr.io/apache' + +jobs: + create-rc: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.vars.outputs.version }} + rc_num: ${{ steps.vars.outputs.rc_num }} + tag_name: ${{ steps.vars.outputs.tag_name }} + rc_dir: ${{ steps.vars.outputs.rc_dir }} + commit_hash: ${{ steps.vars.outputs.commit_hash }} + src_tarball: ${{ steps.vars.outputs.src_tarball }} + compose_tarball: ${{ steps.vars.outputs.compose_tarball }} + helm_tarball: ${{ steps.vars.outputs.helm_tarball }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history for proper tagging + + - name: Validate tag exists + run: | + TAG_NAME="${{ github.event.inputs.tag }}" + + # Check if tag exists + if ! git rev-parse "$TAG_NAME" >/dev/null 2>&1; then + echo "Error: Tag '$TAG_NAME' does not exist" + echo "Available tags:" + git tag -l | tail -10 + exit 1 + fi + + echo "✓ Tag validation passed: $TAG_NAME" + + - name: Set up variables + id: vars + run: | + TAG_NAME="${{ github.event.inputs.tag }}" + RC_NUM="${{ github.event.inputs.rc_number }}" + IMAGE_REGISTRY="${{ github.event.inputs.image_registry }}" + + # Parse version from tag (format: v1.1.0-incubating or v1.1.0-incubating-rcN) + # Both formats are accepted, but we use the input rc_number for artifacts + if [[ "$TAG_NAME" =~ ^v([0-9]+\.[0-9]+\.[0-9]+-incubating)(-rc[0-9]+)?$ ]]; then + VERSION="${BASH_REMATCH[1]}" + else + echo "Error: Tag must be in format vX.Y.Z-incubating or vX.Y.Z-incubating-rcN (e.g., v1.1.0-incubating-rc1)" + exit 1 + fi + + COMMIT_HASH=$(git rev-parse "$TAG_NAME") + COMMIT_SHORT=$(git rev-parse "$TAG_NAME" | cut -c1-9) + RC_DIR="${VERSION}-RC${RC_NUM}" + SRC_TARBALL="apache-texera-${VERSION}-src.tar.gz" + COMPOSE_TARBALL="apache-texera-${VERSION}-docker-compose.tar.gz" + HELM_TARBALL="apache-texera-${VERSION}-helm.tgz" + + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "rc_num=$RC_NUM" >> $GITHUB_OUTPUT + echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT + echo "rc_dir=$RC_DIR" >> $GITHUB_OUTPUT + echo "commit_hash=$COMMIT_HASH" >> $GITHUB_OUTPUT + echo "commit_short=$COMMIT_SHORT" >> $GITHUB_OUTPUT + echo "image_registry=$IMAGE_REGISTRY" >> $GITHUB_OUTPUT + echo "src_tarball=$SRC_TARBALL" >> $GITHUB_OUTPUT + echo "compose_tarball=$COMPOSE_TARBALL" >> $GITHUB_OUTPUT + echo "helm_tarball=$HELM_TARBALL" >> $GITHUB_OUTPUT + + echo "Release Candidate: $TAG_NAME" + echo "Version: $VERSION" + echo "RC Number: $RC_NUM" + echo "Commit: $COMMIT_HASH ($COMMIT_SHORT)" + echo "Image Registry: $IMAGE_REGISTRY" + echo "Staging directory: dist/dev/incubator/texera/$RC_DIR" + + - name: Create source tarball + run: | + TAG_NAME="${{ steps.vars.outputs.tag_name }}" + SRC_TARBALL="${{ steps.vars.outputs.src_tarball }}" + VERSION="${{ steps.vars.outputs.version }}" + + TEMP_DIR=$(mktemp -d) + + # Export the git repository at the tag + git archive --format=tar --prefix="apache-texera-${VERSION}-src/" "$TAG_NAME" | tar -x -C "$TEMP_DIR" + + # Create tarball + cd "$TEMP_DIR" + tar -czf "$GITHUB_WORKSPACE/$SRC_TARBALL" "apache-texera-${VERSION}-src" + + cd "$GITHUB_WORKSPACE" + + # Verify tarball was created + if [[ ! -f "$SRC_TARBALL" ]]; then + echo "Error: Source tarball was not created" + exit 1 + fi + + # Show tarball info + ls -lh "$SRC_TARBALL" + echo "✓ Created source tarball: $SRC_TARBALL" + + - name: Create Docker Compose deployment bundle + run: | + VERSION="${{ steps.vars.outputs.version }}" + TAG_NAME="${{ steps.vars.outputs.tag_name }}" + IMAGE_REGISTRY="${{ steps.vars.outputs.image_registry }}" + COMMIT_SHORT="${{ steps.vars.outputs.commit_short }}" + COMPOSE_TARBALL="${{ steps.vars.outputs.compose_tarball }}" + + TEMP_DIR=$(mktemp -d) + BUNDLE_DIR="$TEMP_DIR/apache-texera-${VERSION}-docker-compose" + mkdir -p "$BUNDLE_DIR" + + # Export the single-node directory from the tagged source + mkdir -p "$TEMP_DIR/_raw" + git archive --format=tar "$TAG_NAME" -- bin/single-node/ sql/ | tar -x -C "$TEMP_DIR/_raw" + + # Copy deployment files + cp "$TEMP_DIR/_raw/bin/single-node/docker-compose.yml" "$BUNDLE_DIR/" + cp "$TEMP_DIR/_raw/bin/single-node/nginx.conf" "$BUNDLE_DIR/" + cp -r "$TEMP_DIR/_raw/sql" "$BUNDLE_DIR/" + + # Patch the SQL mount path for the self-contained bundle layout + # In the repo it's ../../sql (relative to bin/single-node/), in the bundle it's ./sql + sed -i 's|../../sql|./sql|g' "$BUNDLE_DIR/docker-compose.yml" + + # Generate a release-pinned .env file with the version tag + # Start from the source .env and ensure IMAGE_REGISTRY and IMAGE_TAG are set + cp "$TEMP_DIR/_raw/bin/single-node/.env" "$BUNDLE_DIR/.env" + # Replace if line exists, otherwise append + if grep -q '^IMAGE_REGISTRY=' "$BUNDLE_DIR/.env"; then + sed -i "s|^IMAGE_REGISTRY=.*|IMAGE_REGISTRY=${IMAGE_REGISTRY}|" "$BUNDLE_DIR/.env" + else + echo "IMAGE_REGISTRY=${IMAGE_REGISTRY}" >> "$BUNDLE_DIR/.env" + fi + if grep -q '^IMAGE_TAG=' "$BUNDLE_DIR/.env"; then + sed -i "s|^IMAGE_TAG=.*|IMAGE_TAG=${COMMIT_SHORT}|" "$BUNDLE_DIR/.env" + else + echo "IMAGE_TAG=${COMMIT_SHORT}" >> "$BUNDLE_DIR/.env" + fi + if grep -q '^TEXERA_SERVICE_LOG_LEVEL=' "$BUNDLE_DIR/.env"; then + sed -i "s|^TEXERA_SERVICE_LOG_LEVEL=.*|TEXERA_SERVICE_LOG_LEVEL=ERROR|" "$BUNDLE_DIR/.env" + else + echo "TEXERA_SERVICE_LOG_LEVEL=ERROR" >> "$BUNDLE_DIR/.env" + fi + + # Include the README from the repo + cp "$TEMP_DIR/_raw/bin/single-node/README.md" "$BUNDLE_DIR/" + + # Include example datasets, workflows, and the loader script + if [ -d "$TEMP_DIR/_raw/bin/single-node/examples" ]; then + cp -r "$TEMP_DIR/_raw/bin/single-node/examples" "$BUNDLE_DIR/" + echo "✓ Included examples directory (datasets, workflows, load-examples.sh)" + fi + + # Create tarball + cd "$TEMP_DIR" + tar -czf "$GITHUB_WORKSPACE/$COMPOSE_TARBALL" "apache-texera-${VERSION}-docker-compose" + + cd "$GITHUB_WORKSPACE" + ls -lh "$COMPOSE_TARBALL" + echo "✓ Created Docker Compose bundle: $COMPOSE_TARBALL" + + - name: Create Helm chart package + run: | + VERSION="${{ steps.vars.outputs.version }}" + TAG_NAME="${{ steps.vars.outputs.tag_name }}" + COMMIT_SHORT="${{ steps.vars.outputs.commit_short }}" + IMAGE_REGISTRY="${{ steps.vars.outputs.image_registry }}" + HELM_TARBALL="${{ steps.vars.outputs.helm_tarball }}" + + # Install Helm + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + + # Export the Helm chart from the tagged source + # Include sql/ because bin/k8s/files/ has symlinks to ../../../sql/ + TEMP_DIR=$(mktemp -d) + git archive --format=tar "$TAG_NAME" -- bin/k8s/ sql/ | tar -x -C "$TEMP_DIR" + + CHART_DIR="$TEMP_DIR/bin/k8s" + + # Resolve symlinks in the chart so it packages as a self-contained artifact + find "$CHART_DIR" -type l | while read -r link; do + target=$(readlink -f "$link") + rm "$link" + cp "$target" "$link" + done + + # Update Chart.yaml with release version + sed -i "s/^version:.*/version: ${VERSION}/" "$CHART_DIR/Chart.yaml" + sed -i "s/^appVersion:.*/appVersion: \"${VERSION}\"/" "$CHART_DIR/Chart.yaml" + + # Update values.yaml with release image registry and tag + sed -i "s|imageRegistry:.*|imageRegistry: ${IMAGE_REGISTRY}|" "$CHART_DIR/values.yaml" + sed -i "s|imageTag:.*|imageTag: ${COMMIT_SHORT}|" "$CHART_DIR/values.yaml" + + # Download chart dependencies declared in Chart.yaml + # These tarballs are .gitignored so they're not in git archive + helm dependency build "$CHART_DIR" + + # Package the Helm chart + helm package "$CHART_DIR" \ + --version "$VERSION" \ + --app-version "$VERSION" \ + --destination "$GITHUB_WORKSPACE" + + # Rename to our expected artifact name + HELM_PKG=$(ls "$GITHUB_WORKSPACE"/texera-helm-*.tgz 2>/dev/null | head -1) + if [[ -n "$HELM_PKG" && "$HELM_PKG" != "$GITHUB_WORKSPACE/$HELM_TARBALL" ]]; then + mv "$HELM_PKG" "$GITHUB_WORKSPACE/$HELM_TARBALL" + fi + + ls -lh "$GITHUB_WORKSPACE/$HELM_TARBALL" + echo "✓ Created Helm chart package: $HELM_TARBALL" + + - name: Import GPG key + run: | + echo "${{ secrets.GPG_PRIVATE_KEY }}" | gpg --batch --import + + # List imported keys + gpg --list-secret-keys + + echo "✓ GPG key imported successfully" + + - name: Sign and checksum all artifacts + run: | + for artifact in \ + "${{ steps.vars.outputs.src_tarball }}" \ + "${{ steps.vars.outputs.compose_tarball }}" \ + "${{ steps.vars.outputs.helm_tarball }}"; do + + # GPG signature + echo "${{ secrets.GPG_PASSPHRASE }}" | gpg --batch --yes --pinentry-mode loopback --passphrase-fd 0 \ + --armor --detach-sign --output "${artifact}.asc" "$artifact" + gpg --verify "${artifact}.asc" "$artifact" + echo "✓ Signed: ${artifact}" + + # SHA512 checksum + sha512sum "$artifact" > "${artifact}.sha512" + echo "✓ Checksum: ${artifact}.sha512" + done + + - name: Generate vote email template + id: vote_email + run: | + VERSION="${{ steps.vars.outputs.version }}" + RC_NUM="${{ steps.vars.outputs.rc_num }}" + TAG_NAME="${{ steps.vars.outputs.tag_name }}" + RC_DIR="${{ steps.vars.outputs.rc_dir }}" + COMMIT_HASH="${{ steps.vars.outputs.commit_hash }}" + IMAGE_REGISTRY="${{ steps.vars.outputs.image_registry }}" + + # Get GPG key ID from the imported key + GPG_KEY_ID=$(gpg --list-secret-keys --keyid-format LONG | grep 'sec' | head -n1 | awk '{print $2}' | cut -d'/' -f2) + GPG_EMAIL=$(gpg --list-secret-keys | grep 'uid' | head -n1 | grep -oP '[\w\.-]+@[\w\.-]+') + + # Copy template from repository + cp .github/release/vote-email-template.md vote-email.txt + + # Substitute variables in the template + sed -i "s|\${VERSION}|${VERSION}|g" vote-email.txt + sed -i "s|\${RC_NUM}|${RC_NUM}|g" vote-email.txt + sed -i "s|\${RC_DIR}|${RC_DIR}|g" vote-email.txt + sed -i "s|\${TAG_NAME}|${TAG_NAME}|g" vote-email.txt + sed -i "s|\${COMMIT_HASH}|${COMMIT_HASH}|g" vote-email.txt + sed -i "s|\${GPG_KEY_ID}|${GPG_KEY_ID}|g" vote-email.txt + sed -i "s|\${GPG_EMAIL}|${GPG_EMAIL}|g" vote-email.txt + sed -i "s|\${IMAGE_REGISTRY}|${IMAGE_REGISTRY}|g" vote-email.txt + + echo "✓ Vote email template generated!" + + - name: Upload RC artifacts + uses: actions/upload-artifact@v4 + with: + name: rc-artifacts + path: | + ${{ steps.vars.outputs.src_tarball }} + ${{ steps.vars.outputs.src_tarball }}.asc + ${{ steps.vars.outputs.src_tarball }}.sha512 + ${{ steps.vars.outputs.compose_tarball }} + ${{ steps.vars.outputs.compose_tarball }}.asc + ${{ steps.vars.outputs.compose_tarball }}.sha512 + ${{ steps.vars.outputs.helm_tarball }} + ${{ steps.vars.outputs.helm_tarball }}.asc + ${{ steps.vars.outputs.helm_tarball }}.sha512 + vote-email.txt + retention-days: 7 + + upload-rc: + runs-on: ubuntu-latest + needs: create-rc + + steps: + - name: Download RC artifacts + uses: actions/download-artifact@v4 + with: + name: rc-artifacts + + - name: Verify downloaded artifacts + run: | + SRC_TARBALL="${{ needs.create-rc.outputs.src_tarball }}" + COMPOSE_TARBALL="${{ needs.create-rc.outputs.compose_tarball }}" + HELM_TARBALL="${{ needs.create-rc.outputs.helm_tarball }}" + + echo "Verifying downloaded artifacts..." + ls -lh + + for artifact in "$SRC_TARBALL" "$COMPOSE_TARBALL" "$HELM_TARBALL"; do + if [[ ! -f "$artifact" ]] || [[ ! -f "${artifact}.asc" ]] || [[ ! -f "${artifact}.sha512" ]]; then + echo "Error: Missing artifact or signature/checksum for: $artifact" + exit 1 + fi + done + + echo "✓ All artifacts downloaded successfully" + + - name: Install SVN + run: | + sudo apt-get update + sudo apt-get install -y subversion + svn --version + + - name: Checkout SVN dev directory + run: | + RC_DIR="${{ needs.create-rc.outputs.rc_dir }}" + + # Checkout the dev directory with depth=empty (lightweight) + svn co --depth=empty https://dist.apache.org/repos/dist/dev/incubator/texera svn-texera \ + --username "${{ secrets.SVN_USERNAME }}" \ + --password "${{ secrets.SVN_PASSWORD }}" \ + --no-auth-cache + + cd svn-texera + + # Check if RC directory already exists on the remote + SVN_BASE="https://dist.apache.org/repos/dist/dev/incubator/texera" + if svn info "$SVN_BASE/$RC_DIR" >/dev/null 2>&1; then + # Directory exists remotely — update (checkout) it into the working copy + svn update --depth=infinity "$RC_DIR" \ + --username "${{ secrets.SVN_USERNAME }}" \ + --password "${{ secrets.SVN_PASSWORD }}" \ + --no-auth-cache || true + # If update didn't bring it down (empty parent checkout), do a sparse checkout + if [[ ! -d "$RC_DIR" ]]; then + svn update --set-depth=infinity "$RC_DIR" \ + --username "${{ secrets.SVN_USERNAME }}" \ + --password "${{ secrets.SVN_PASSWORD }}" \ + --no-auth-cache + fi + echo "✓ RC directory already exists remotely, checked out: $RC_DIR" + else + # Directory doesn't exist remotely — create and add it + mkdir -p "$RC_DIR" + svn add "$RC_DIR" + echo "✓ Created new RC directory: $RC_DIR" + fi + + - name: Stage artifacts to SVN + run: | + SRC_TARBALL="${{ needs.create-rc.outputs.src_tarball }}" + COMPOSE_TARBALL="${{ needs.create-rc.outputs.compose_tarball }}" + HELM_TARBALL="${{ needs.create-rc.outputs.helm_tarball }}" + RC_DIR="${{ needs.create-rc.outputs.rc_dir }}" + + cd svn-texera/"$RC_DIR" + + # Copy all artifacts + for artifact in "$SRC_TARBALL" "$COMPOSE_TARBALL" "$HELM_TARBALL"; do + cp "$GITHUB_WORKSPACE/$artifact" . + cp "$GITHUB_WORKSPACE/${artifact}.asc" . + cp "$GITHUB_WORKSPACE/${artifact}.sha512" . + done + + # Add files to SVN + svn add * --force + + # Check status + svn status + + echo "✓ Staged all artifacts to SVN" + + - name: Commit artifacts to dist/dev + run: | + VERSION="${{ needs.create-rc.outputs.version }}" + RC_NUM="${{ needs.create-rc.outputs.rc_num }}" + RC_DIR="${{ needs.create-rc.outputs.rc_dir }}" + + cd svn-texera + + # Commit with descriptive message + svn commit -m "Add Apache Texera ${VERSION} RC${RC_NUM} artifacts (source + docker-compose + helm)" \ + --username "${{ secrets.SVN_USERNAME }}" \ + --password "${{ secrets.SVN_PASSWORD }}" \ + --no-auth-cache + + echo "✓ Committed artifacts to dist/dev/incubator/texera/$RC_DIR" + + - name: Generate release summary + run: | + VERSION="${{ needs.create-rc.outputs.version }}" + RC_NUM="${{ needs.create-rc.outputs.rc_num }}" + TAG_NAME="${{ needs.create-rc.outputs.tag_name }}" + RC_DIR="${{ needs.create-rc.outputs.rc_dir }}" + COMMIT_HASH="${{ needs.create-rc.outputs.commit_hash }}" + SRC_TARBALL="${{ needs.create-rc.outputs.src_tarball }}" + COMPOSE_TARBALL="${{ needs.create-rc.outputs.compose_tarball }}" + HELM_TARBALL="${{ needs.create-rc.outputs.helm_tarball }}" + + echo "## Release Candidate Created Successfully!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Release Information" >> $GITHUB_STEP_SUMMARY + echo "- **Version:** ${VERSION}" >> $GITHUB_STEP_SUMMARY + echo "- **RC Number:** RC${RC_NUM}" >> $GITHUB_STEP_SUMMARY + echo "- **Git Tag:** \`${TAG_NAME}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Commit:** \`${COMMIT_HASH}\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Artifacts Location" >> $GITHUB_STEP_SUMMARY + echo "**Staging Directory:** https://dist.apache.org/repos/dist/dev/incubator/texera/${RC_DIR}/" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Artifacts Created" >> $GITHUB_STEP_SUMMARY + echo "| Artifact | Description |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------------|" >> $GITHUB_STEP_SUMMARY + echo "| \`${SRC_TARBALL}\` | Source code |" >> $GITHUB_STEP_SUMMARY + echo "| \`${COMPOSE_TARBALL}\` | Docker Compose deployment bundle |" >> $GITHUB_STEP_SUMMARY + echo "| \`${HELM_TARBALL}\` | Helm chart for Kubernetes deployment |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Each artifact has a corresponding \`.asc\` (GPG signature) and \`.sha512\` (checksum) file." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Next Steps" >> $GITHUB_STEP_SUMMARY + echo "1. Build and push container images using the \`Build and push images\` workflow with tag \`${VERSION}\`" >> $GITHUB_STEP_SUMMARY + echo "2. Verify the artifacts at the staging directory" >> $GITHUB_STEP_SUMMARY + echo "3. Send [VOTE] email to dev@texera.apache.org" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Verification" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY + echo "# Import KEYS and verify signatures" >> $GITHUB_STEP_SUMMARY + echo "gpg --import KEYS" >> $GITHUB_STEP_SUMMARY + echo "gpg --verify ${SRC_TARBALL}.asc ${SRC_TARBALL}" >> $GITHUB_STEP_SUMMARY + echo "gpg --verify ${COMPOSE_TARBALL}.asc ${COMPOSE_TARBALL}" >> $GITHUB_STEP_SUMMARY + echo "gpg --verify ${HELM_TARBALL}.asc ${HELM_TARBALL}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "# Verify SHA512 checksums" >> $GITHUB_STEP_SUMMARY + echo "sha512sum -c ${SRC_TARBALL}.sha512" >> $GITHUB_STEP_SUMMARY + echo "sha512sum -c ${COMPOSE_TARBALL}.sha512" >> $GITHUB_STEP_SUMMARY + echo "sha512sum -c ${HELM_TARBALL}.sha512" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**KEYS file:** https://downloads.apache.org/incubator/texera/KEYS" >> $GITHUB_STEP_SUMMARY + + echo "✓ Release candidate workflow completed successfully!" + + - name: Display vote email template + run: | + echo "## Vote Email Template" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Copy the content below to send to dev@texera.apache.org:" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + cat "$GITHUB_WORKSPACE/vote-email.txt" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/github-action-build.yml b/.github/workflows/github-action-build.yml index 14df23456a4..af2a60920d5 100644 --- a/.github/workflows/github-action-build.yml +++ b/.github/workflows/github-action-build.yml @@ -17,6 +17,9 @@ name: Build +env: + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + on: push: branches: @@ -30,7 +33,7 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - gui: + frontend: runs-on: ${{ matrix.os }} strategy: matrix: @@ -55,22 +58,29 @@ jobs: - uses: actions/cache@v4 with: path: | - core/gui/node_modules - key: ${{ runner.os }}-${{ matrix.arch }}-${{ matrix.node-version }}-yarn-cache-v1-${{ hashFiles('**/yarn.lock') }} + frontend/.yarn/cache + frontend/.yarn/unplugged + frontend/.yarn/install-state.gz + key: ${{ runner.os }}-${{ matrix.arch }}-${{ matrix.node-version }}-yarn-cache-v2-${{ hashFiles('**/yarn.lock') }} restore-keys: | - ${{ runner.os }}-${{ matrix.arch }}-${{ matrix.node-version }}-yarn-cache-v1- + ${{ runner.os }}-${{ matrix.arch }}-${{ matrix.node-version }}-yarn-cache-v2- - name: Prepare Yarn 4.5.1 - run: corepack enable && corepack prepare yarn@4.5.1 --activate && yarn --cwd core/gui set version 4.5.1 + run: corepack enable && corepack prepare yarn@4.5.1 --activate && yarn --cwd frontend set version 4.5.1 + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' - name: Install dependency - run: yarn --cwd core/gui install --immutable --network-timeout=100000 + timeout-minutes: 20 + run: yarn --cwd frontend install --immutable --inline-builds --network-timeout=100000 - name: Lint with Prettier & ESLint - run: yarn --cwd core/gui format:ci + run: yarn --cwd frontend format:ci - name: Run frontend unit tests - run: yarn --cwd core/gui run test:ci + run: yarn --cwd frontend run test:ci - name: Prod build - run: yarn --cwd core/gui run build:ci + run: yarn --cwd frontend run build:ci - core: + scala: strategy: matrix: os: [ ubuntu-22.04 ] @@ -101,27 +111,44 @@ jobs: with: distribution: 'temurin' java-version: 11 + - name: Setup Python for Scala tests + uses: actions/setup-python@v6 + with: + python-version: '3.11' + - name: Show Python + run: python --version || python3 --version - name: Setup sbt launcher - uses: sbt/setup-sbt@v1 - - uses: coursier/cache-action@v6 + uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14 + - uses: coursier/cache-action@4e2615869d13561d626ed48655e1a39e5b192b3c # v6.4.9 with: - extraSbtFiles: '["core/*.sbt", "core/project/**.{scala,sbt}", "core/project/build.properties" ]' + extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' - name: Lint with scalafmt - run: cd core && sbt scalafmtCheckAll - - name: Create Database - run: psql -h localhost -U postgres -f deployment/k8s/texera-helmchart/files/texera_ddl.sql + run: sbt scalafmtCheckAll + - name: Create Databases + run: | + psql -h localhost -U postgres -f sql/texera_ddl.sql + psql -h localhost -U postgres -f sql/iceberg_postgres_catalog.sql + psql -h localhost -U postgres -f sql/texera_lakefs.sql + env: + PGPASSWORD: postgres + - name: Create texera_db_for_test_cases + run: psql -h localhost -U postgres -v DB_NAME=texera_db_for_test_cases -f sql/texera_ddl.sql env: PGPASSWORD: postgres - name: Compile with sbt - run: cd core && sbt clean package + run: sbt clean package + - name: Set docker-java API version + run: | + echo "api.version=1.52" >> ~/.docker-java.properties + cat ~/.docker-java.properties - name: Run backend tests - run: cd core && sbt test + run: sbt test - python_udf: + python: strategy: matrix: os: [ ubuntu-latest ] - python-version: [ '3.9', '3.10', '3.11', '3.12' ] + python-version: [ '3.10', '3.11', '3.12' ] runs-on: ${{ matrix.os }} steps: - name: Checkout Texera @@ -133,18 +160,18 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - if [ -f core/amber/requirements.txt ]; then pip install -r core/amber/requirements.txt; fi - if [ -f core/amber/operator-requirements.txt ]; then pip install -r core/amber/operator-requirements.txt; fi + if [ -f amber/requirements.txt ]; then pip install -r amber/requirements.txt; fi + if [ -f amber/operator-requirements.txt ]; then pip install -r amber/operator-requirements.txt; fi - name: Install PostgreSQL run: sudo apt-get update && sudo apt-get install -y postgresql - name: Start PostgreSQL Service run: sudo systemctl start postgresql - name: Create Database and User run: | - cd core/scripts/sql && sudo -u postgres psql -f iceberg_postgres_catalog.sql - - name: Lint with flake8 and black + cd sql && sudo -u postgres psql -f iceberg_postgres_catalog.sql + - name: Lint with Ruff run: | - cd core/amber/src/main/python && flake8 && black . --check + cd amber/src/main/python && ruff check . && ruff format --check . - name: Test with pytest run: | - cd core/amber/src/main/python && pytest -sv --ignore=core/models/test_RTableExecutor.py + cd amber/src/main/python && pytest -sv diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml new file mode 100644 index 00000000000..0846b98ebe8 --- /dev/null +++ b/.github/workflows/issue-triage.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Issue triage +on: + issues: + types: [assigned, unassigned] + +permissions: + issues: write + pull-requests: read + +jobs: + # -------------------------------------------------------- + # 1) Issue triage: add/remove "triage" on assignment changes + # -------------------------------------------------------- + issue-triage: + if: github.event_name == 'issues' + runs-on: ubuntu-latest + steps: + - name: Remove 'triage' label when issue is assigned + if: github.event.action == 'assigned' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { owner, repo } = context.repo; + const issue_number = context.payload.issue.number; + + try { + await github.rest.issues.removeLabel({ + owner, + repo, + issue_number, + name: 'triage', + }); + core.info(`Removed 'triage' from issue #${issue_number}`); + } catch (e) { + if (e.status === 404) { + core.info(`Issue #${issue_number} has no 'triage' label, nothing to remove.`); + } else { + core.warning(`Failed to remove 'triage' from issue #${issue_number}: ${e.message}`); + throw e; + } + } + + - name: Add 'triage' label when issue is unassigned and has no assignees + if: github.event.action == 'unassigned' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issue = context.payload.issue; + const { owner, repo } = context.repo; + const issue_number = issue.number; + + const assignees = issue.assignees || []; + if (assignees.length > 0) { + core.info( + `Issue #${issue_number} still has ${assignees.length} assignee(s), not adding 'triage'.` + ); + return; + } + + core.info(`Issue #${issue_number} has no assignees, adding 'triage' label.`); + try { + await github.rest.issues.addLabels({ + owner, + repo, + issue_number, + labels: ['triage'], + }); + } catch (e) { + core.warning(`Failed to add 'triage' to issue #${issue_number}: ${e.message}`); + throw e; + } diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index f4151a19047..fcc5d46f0ca 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -31,6 +31,6 @@ jobs: permissions: pull-requests: read steps: - - uses: amannn/action-semantic-pull-request@v5 + - uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017 # v5.5.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 627c56f0874..62f85f0e2f0 100644 --- a/.gitignore +++ b/.gitignore @@ -87,7 +87,7 @@ rebel.xml *.log.gz # Ignoring the entire log folder -log/ +logs/ # Ignoring package-lock.json package-lock.json diff --git a/.licenserc.yaml b/.licenserc.yaml index 309f9d844fc..65e6c762904 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -16,6 +16,7 @@ header: - 'NOTICE' - '.dockerignore' - '.gitattributes' + - '.github/PULL_REQUEST_TEMPLATE' - 'yarn.lock' - '.nvmrc' - '.htaccess' @@ -23,4 +24,19 @@ header: - 'site.webmanifest' - '.gitignore' - '.licenserc.yaml' - - 'core/gui/.yarn/**' + - 'frontend/.yarn/**' + - 'amber/src/main/python/proto/**' + # Third-party code with MIT license - see LICENSE file for attribution + - 'common/workflow-operator/src/main/scala/com/kjetland/**' + # TypeFox monaco-languageclient derived files (MIT License) + - 'pyright-language-service/src/main.ts' + - 'pyright-language-service/src/language-server-runner.ts' + - 'pyright-language-service/src/server-commons.ts' + - 'frontend/src/app/common/formly/array.type.ts' + - 'frontend/src/app/common/formly/object.type.ts' + - 'frontend/src/app/common/formly/multischema.type.ts' + - 'frontend/src/app/common/formly/null.type.ts' + # Third-party SVG assets - see LICENSE file for attribution + - 'frontend/src/assets/svg/operator-view-result.svg' + - 'frontend/src/assets/svg/operator-reuse-cache-invalid.svg' + - 'frontend/src/assets/svg/operator-reuse-cache-valid.svg' diff --git a/.run/AccessControlService.run.xml b/.run/AccessControlService.run.xml new file mode 100644 index 00000000000..56ffbc50fbb --- /dev/null +++ b/.run/AccessControlService.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/ComputingUnitManagingService.run.xml b/.run/ComputingUnitManagingService.run.xml new file mode 100644 index 00000000000..8bbb5c1966f --- /dev/null +++ b/.run/ComputingUnitManagingService.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/ComputingUnitMaster.run.xml b/.run/ComputingUnitMaster.run.xml new file mode 100644 index 00000000000..5ffffb79e88 --- /dev/null +++ b/.run/ComputingUnitMaster.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/ComputingUnitWorker.run.xml b/.run/ComputingUnitWorker.run.xml new file mode 100644 index 00000000000..854da672287 --- /dev/null +++ b/.run/ComputingUnitWorker.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/ConfigService.run.xml b/.run/ConfigService.run.xml new file mode 100644 index 00000000000..395688afca4 --- /dev/null +++ b/.run/ConfigService.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/FileService.run.xml b/.run/FileService.run.xml new file mode 100644 index 00000000000..0a54e70aac8 --- /dev/null +++ b/.run/FileService.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/TexeraWebApplication.run.xml b/.run/TexeraWebApplication.run.xml new file mode 100644 index 00000000000..d9e3829fc22 --- /dev/null +++ b/.run/TexeraWebApplication.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/WorkflowCompilingService.run.xml b/.run/WorkflowCompilingService.run.xml new file mode 100644 index 00000000000..d41f60d9c54 --- /dev/null +++ b/.run/WorkflowCompilingService.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/frontend.run.xml b/.run/frontend.run.xml new file mode 100644 index 00000000000..8132f576027 --- /dev/null +++ b/.run/frontend.run.xml @@ -0,0 +1,30 @@ + + + + + + +