Update develop from main

.github/workflows/build_deploy_dev.yml

@@ -8,6 +8,13 @@ on:
     types: [opened, reopened, synchronize, labeled]
     branches:
       - 'main'
+      - 'develop'
+
+# Cancel any existing runs of this workflow on the same branch/pr
+# We always want to build/deploy/test a new commit over an older one
+concurrency:
+  group: ${{ github.workflow_ref }}
+  cancel-in-progress: true
 
 jobs:
 
@@ -16,6 +23,8 @@ jobs:
     runs-on: mdb-dev
     outputs:
       not-docs: ${{ steps.filter.outputs.not-docs }}
+    environment:
+      name: ${{ github.event.pull_request.head.repo.fork && 'manual-approval' || '' }}
     steps:
       - uses: dorny/paths-filter@v3
         id: filter
@@ -25,47 +34,44 @@ jobs:
             not-docs:
               - '!docs/**'  
               - '!**/*.md'
+              - '!.github/workflows/build_deploy_dev.yml'
+              - '!.github/workflows/test_on_deploy.yml'
+              - '!mindsdb/__about__.py'
+
+  # Start running unit tests early - we want to run them always
+  # and they don't depend on build or deployment
+  run_unit_tests:
+    name: Run Unit Tests
+    needs: [changes]
+    if: ${{ needs.changes.outputs.not-docs == 'true' }}
+    uses: ./.github/workflows/tests_unit.yml
+    with:
+      git-sha: ${{ github.event.pull_request.head.sha }}
+    secrets: inherit
 
   # Looks for labels like "deploy-to-<env>" attached to a PR so we can deploy to those envs
   get-deploy-labels:
     name: Get Deploy Envs
     runs-on: mdb-dev
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-labels
-      cancel-in-progress: true
-    environment:
-      name: ${{ github.event.pull_request.head.repo.fork && 'manual-approval' || '' }}
+    needs: [changes]
     outputs:
       deploy-envs: ${{ steps.get-labels.outputs.deploy-envs }}
     steps:
-      - name: Pull MindsDB Github Actions
-        uses: actions/checkout@v4
-        with:
-          repository: mindsdb/github-actions
-          path: github-actions
       - id: get-labels
-        uses: ./github-actions/get-deploy-labels
+        uses: mindsdb/github-actions/get-deploy-labels@main
 
   # Build our docker images based on our bake file
   build:
     name: Build Docker Images
     runs-on: mdb-dev
     needs: [get-deploy-labels]
     if: ${{ needs.get-deploy-labels.outputs.deploy-envs != '[]' }}
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-build
-      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-      - name: Pull MindsDB Github Actions
-        uses: actions/checkout@v4
-        with:
-          repository: mindsdb/github-actions
-          path: github-actions
       # Build the bakefile and push
-      - uses: ./github-actions/docker-bake
+      - uses: mindsdb/github-actions/docker-bake@main
         with:
           git-sha: ${{ github.event.pull_request.head.sha }}
           target: cloud-cpu
@@ -78,28 +84,19 @@ jobs:
     name: Push Docker Cache
     runs-on: mdb-dev
     needs: [build]
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-cache
-      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-      - name: Pull MindsDB Github Actions
-        uses: actions/checkout@v4
-        with:
-          repository: mindsdb/github-actions
-          path: github-actions
       # Build the bakefile and push
-      - uses: ./github-actions/docker-bake
+      - uses: mindsdb/github-actions/docker-bake@main
         with:
           git-sha: ${{ github.event.pull_request.head.sha }}
           target: cloud-cpu
           platforms: linux/amd64
           push-cache: true
           cache-only: true
 
-  # Call our deployment workflow, so long as this is not a forked PR
   # This will run the deployment workflow in the base branch, not in the PR.
   # So if you change the deploy workflow in your PR, the changes won't be reflected in this run.
   deploy:
@@ -113,7 +110,7 @@ jobs:
     secrets: inherit
 
   # Run integration tests against the deployed environment
-  run_tests:
+  run_integration_tests:
     name: Run Integration Tests
     needs: [deploy, get-deploy-labels]
     strategy:
@@ -123,18 +120,20 @@ jobs:
     concurrency:
       group: deploy-${{ matrix.deploy-env }}
       cancel-in-progress: false
-    uses: ./.github/workflows/test_on_deploy.yml
+    uses: ./.github/workflows/tests_integration.yml
     with:
       git-sha: ${{ github.event.pull_request.head.sha }}
       deploy-env: ${{ matrix.deploy-env }}
     secrets: inherit
 
+  # This is a collection point for all of the matrix tests above so we can have a single required job
   tests_completed:
-    needs: [run_tests, changes]
+    name: All Tests Succeeded
+    needs: [run_unit_tests, run_integration_tests, changes]
     runs-on: mdb-dev
     if: always()
     steps:
       - name: fail if tests failed or didnt run
-        if: ${{ needs.run_tests.result != 'success' && needs.changes.outputs.not-docs == 'true' }}
+        if: ${{ (needs.run_unit_tests.result != 'success' || needs.run_integration_tests.result != 'success') && needs.changes.outputs.not-docs == 'true' }}
         run: exit 1
       - run: echo "Tests ran successfully"

.github/workflows/build_deploy_prod.yml

@@ -13,15 +13,16 @@ on:
 env:
   UV_LINK_MODE: "symlink"
 
+concurrency:
+  group: release
+  cancel-in-progress: false
+
 jobs:
   # Check that the version defined in the github release is valid
   check-version:
     name: Check Code Version
     runs-on: mdb-dev
     if: github.actor != 'mindsdbadmin'
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-check-version
-      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v4
       - uses: FranzDiebold/github-env-vars-action@v2
@@ -39,18 +40,14 @@ jobs:
     runs-on: mdb-dev
     needs: check-version
     if: github.actor != 'mindsdbadmin'
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-pypi
-      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v4
       - name: Setup uv
         uses: astral-sh/setup-uv@v5
         with:
-          # Place cache in the tool dir because we mount this in our runnners
-          cache-local-path: "/home/runner/_work/_tool/uv-local-cache"
-          prune-cache: false
-          python-version: ${{ vars.CI_PYTHON_VERSION || '3.10' }}
+          cache-local-path: "/home/runner/_work/_tool/uv-local-cache"  # Place cache in the tool dir because we mount this in our runnners
+          prune-cache: false                                           # We want to save all cache because it's in the mount^
+          python-version: ${{ vars.CI_PYTHON_VERSION || '3.11' }}      # Default to 3.11 where vars aren't available (PRs from forks)
       - name: Install dependencies
         run: |
           uv pip install -r requirements/requirements-dev.txt
@@ -71,23 +68,15 @@ jobs:
     runs-on: mdb-dev
     needs: [check-version]
     if: github.actor != 'mindsdbadmin'
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-build
-      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v4
       - name: Docker Login
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Pull MindsDB Github Actions
-        uses: actions/checkout@v4
-        with:
-          repository: mindsdb/github-actions
-          path: github-actions
       # Build the bakefile and push
-      - uses: ./github-actions/docker-bake
+      - uses: mindsdb/github-actions/docker-bake@main
         with:
           push-to-dockerhub: true
           push-cache: false
@@ -98,20 +87,10 @@ jobs:
     name: Push Docker Cache
     runs-on: mdb-dev
     needs: [build]
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-cache
-      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-      - name: Pull MindsDB Github Actions
-        uses: actions/checkout@v4
-        with:
-          repository: mindsdb/github-actions
-          path: github-actions
       # Build the bakefile and push
-      - uses: ./github-actions/docker-bake
+      - uses: mindsdb/github-actions/docker-bake@main
         with:
           push-cache: true
           cache-only: true
@@ -133,11 +112,6 @@ jobs:
     runs-on: mdb-dev
     needs: [build]
     if: github.actor != 'mindsdbadmin'
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-extension
-      cancel-in-progress: true
-    environment:
-      name: prod
     steps:
       - uses: FranzDiebold/github-env-vars-action@v2
       - uses: convictional/trigger-workflow-and-wait@v1.6.5

.github/workflows/build_deploy_staging.yml

@@ -11,26 +11,34 @@ on:
       - closed
     branches:
       - 'main'
+      - 'release/*'
+
+concurrency:
+  group: ${{ github.workflow_ref }}
+  cancel-in-progress: true
 
 jobs:
 
+  run_unit_tests:
+    name: Run Unit Tests
+    if: github.event.pull_request.merged == true
+    uses: ./.github/workflows/tests_unit.yml
+    with:
+      git-sha: ${{ github.event.pull_request.merge_commit_sha }}
+    secrets: inherit
+
   # Build our docker images based on our bake file
   build:
     if: github.event.pull_request.merged == true
     name: Build Docker Images
     runs-on: mdb-dev
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-build
-      cancel-in-progress: true
     steps:
+      # Check out the merge commit on the base branch
       - uses: actions/checkout@v4
-      - name: Pull MindsDB Github Actions
-        uses: actions/checkout@v4
         with:
-          repository: mindsdb/github-actions
-          path: github-actions
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
       # Build the bakefile and push
-      - uses: ./github-actions/docker-bake
+      - uses: mindsdb/github-actions/docker-bake@main
         with:
           push-cache: false
 
@@ -40,20 +48,13 @@ jobs:
     name: Push Docker Cache
     runs-on: mdb-dev
     needs: [build]
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-cache
-      cancel-in-progress: true
     steps:
+      # Check out the merge commit on the base branch
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
-      - name: Pull MindsDB Github Actions
-        uses: actions/checkout@v4
-        with:
-          repository: mindsdb/github-actions
-          path: github-actions
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
       # Build the bakefile and push
-      - uses: ./github-actions/docker-bake
+      - uses: mindsdb/github-actions/docker-bake@main
         with:
           push-cache: true
           cache-only: true
@@ -65,32 +66,43 @@ jobs:
     uses: ./.github/workflows/deploy.yml
     with:
       deploy-envs: '["staging", "dev", "alpha-dev"]'
-      image-tag: ${{ github.sha }}
+      image-tag: ${{ github.event.pull_request.merge_commit_sha }}
     secrets: inherit
 
   # Run integration tests
-  # TODO: Run these against the deployed environment
-  run_tests:
+  run_integration_tests:
     if: github.event.pull_request.merged == true
     name: Run Integration Tests
     needs: [deploy]
     concurrency:
       group: deploy-staging
       cancel-in-progress: false
-    uses: ./.github/workflows/test_on_deploy.yml
+    uses: ./.github/workflows/tests_integration.yml
     with:
+      git-sha: ${{ github.event.pull_request.merge_commit_sha }}
       deploy-env: staging
     secrets: inherit
 
+  tests_completed:
+    name: All Tests Succeeded
+    needs: [run_unit_tests, run_integration_tests]
+    runs-on: mdb-dev
+    if: always()
+    steps:
+      - name: fail if tests failed or didnt run
+        if: ${{ needs.run_unit_tests.result != 'success' || needs.run_integration_tests.result != 'success'}}
+        run: exit 1
+      - run: echo "Tests ran successfully"
+
   slack_message:
     if: failure() && !cancelled()
     name: Notify Slack
     # Every previous job needs to be in here, because failure() will only return true if the job that failed is in 'needs'
-    needs: [build, build-cache, deploy, run_tests]
+    needs: [run_unit_tests, build, build-cache, deploy, run_integration_tests, tests_completed]
     runs-on: mdb-dev
     steps:
       - name: Notify of failing tests
-        if: ${{ needs.run_tests.result != 'success' && needs.run_tests.result != 'cancelled' }}
+        if: ${{ needs.tests_completed.result != 'success' && needs.tests_completed.result != 'cancelled' }}
         uses: slackapi/slack-github-action@v1.26.0
         with:
           channel-id: ${{ secrets.SLACK_ENG_CHANNEL_ID }}
@@ -104,7 +116,7 @@ jobs:
                       "type": "header",
                       "text": {
                         "type": "plain_text",
-                        "text": "TEST RUN FAILED ON MAIN",
+                        "text": "TEST RUN FAILED ON ${{ github.head_ref }}",
                         "emoji": true
                       }
                     },