diff --git a/.github/workflows/app-cd.yaml b/.github/workflows/app-cd.yaml deleted file mode 100644 index 46f662b..0000000 --- a/.github/workflows/app-cd.yaml +++ /dev/null @@ -1,169 +0,0 @@ -name: app-cd -on: - push: - tags: - - 'v*.*.*-dev*' - - 'v*.*.*-test*' - - 'v*.*.*-prd*' -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} -env: - HELM_REPO_URL: 'https://harbor.cvrgo.com/chartrepo/dds' - HELM_REPO_NAME: 'dds-deploy' - CHART_PATH: 'chart' - CHART_VALUES_PATH: 'chart_values' - DOCKER_REGISTRY: 'harbor.cvrgo.com/dds' -jobs: - build: - if: github.repository_owner == 'deepdataspace' - runs-on: self-hosted - outputs: - GIT_TAG: ${{ steps.check_env_vars.outputs.GIT_TAG }} - K8S_NAMESPACE: ${{ steps.check_env_vars.outputs.K8S_NAMESPACE }} - IMAGE_NAME: ${{ steps.check_env_vars.outputs.IMAGE_NAME }} - IMAGE_TAG: ${{ steps.check_env_vars.outputs.IMAGE_TAG }} - CHART_VERSION: ${{ steps.check_env_vars.outputs.CHART_VERSION }} - APP_VERSION: ${{ steps.check_env_vars.outputs.APP_VERSION }} - APP_NAME: ${{ steps.check_env_vars.outputs.APP_NAME }} - steps: - - name: Checkout deepdataspace - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Get Git Tag - id: get_git_tag - run: echo "tag=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT - - name: Set Environment Variables - run: | - # parse git tag - gitTag="${{ steps.get_git_tag.outputs.tag }}" - echo "GIT_TAG=${gitTag}" >> $GITHUB_ENV - - # parse k8s namespace - if [[ "${gitTag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-dev[0-9]+ ]]; then - k8sNamespace=dds-dev - elif [[ "${gitTag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-test[0-9]+ ]]; then - k8sNamespace=dds-test - elif [[ "${gitTag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-prd[0-9]+ ]]; then - k8sNamespace=dds-prd - else - echo "Invalid git tag. Pipeline terminated." - exit 1 - fi - echo "K8S_NAMESPACE=${k8sNamespace}" >> $GITHUB_ENV - - # parse image name and tag - imageName="${{ env.HELM_REPO_NAME }}" - echo "IMAGE_NAME=${imageName}" >> $GITHUB_ENV - imageTag="${gitTag}" - echo "IMAGE_TAG=${imageTag}" >> $GITHUB_ENV - - # parse chart version - chartVersion="${gitTag}+$(date +%s)" - echo "CHART_VERSION=${chartVersion}" >> $GITHUB_ENV - - # parse app name and version - appVersion="${gitTag}" - echo "APP_VERSION=${appVersion}" >> $GITHUB_ENV - appName=${k8sNamespace} - echo "APP_NAME=${appName}" >> $GITHUB_ENV - - name: Check Environment Variables - id: check_env_vars - run: | - echo env.GIT_TAG=${{ env.GIT_TAG }} - echo env.K8S_NAMESPACE=${{ env.K8S_NAMESPACE }} - echo env.IMAGE_NAME=${{ env.IMAGE_NAME }} - echo env.IMAGE_TAG=${{ env.IMAGE_TAG }} - echo env.CHART_VERSION=${{ env.CHART_VERSION }} - echo env.APP_VERSION=${{ env.APP_VERSION }} - echo env.APP_NAME=${{ env.APP_NAME }} - - echo "GIT_TAG=${{ env.GIT_TAG }}" >> $GITHUB_OUTPUT - echo "K8S_NAMESPACE=${{ env.K8S_NAMESPACE }}" >> $GITHUB_OUTPUT - echo "IMAGE_NAME=${{ env.IMAGE_NAME }}" >> $GITHUB_OUTPUT - echo "IMAGE_TAG=${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT - echo "CHART_VERSION=${{ env.CHART_VERSION }}" >> $GITHUB_OUTPUT - echo "APP_VERSION=${{ env.APP_VERSION }}" >> $GITHUB_OUTPUT - echo "APP_NAME=${{ env.APP_NAME }}" >> $GITHUB_OUTPUT - - - name: Checkout Devops Repo - uses: actions/checkout@v2 - with: - repository: deepdataspace/deepdataspace-helm - ref: 'main' - path: deepdataspace-helm - token: ${{ secrets.DDS_ORG_GITHUB_TOKEN }} - - name: Copy Frontend Build Env File - run: | - if [[ "${{ env.K8S_NAMESPACE }}" == 'dds-dev' ]]; then - cp deepdataspace-helm/frontend_envs/.env.dev packages/app/.env.local - elif [[ "${{ env.K8S_NAMESPACE }}" == 'dds-test' ]]; then - cp deepdataspace-helm/frontend_envs/.env.test packages/app/.env.local - elif [[ "${{ env.K8S_NAMESPACE }}" == 'dds-prd' ]]; then - cp deepdataspace-helm/frontend_envs/.env.prod packages/app/.env.local - fi - echo "TAG_VERSION=${{ env.GIT_TAG }}" >> packages/app/.env.local - - name: Copy Dockerfiles - run: | - cp deepdataspace-helm/dockerfiles/deploy.Dockerfile . - rm -rf deepdataspace-helm - - name: Build Docker Image - run: | - docker login ${{ env.DOCKER_REGISTRY }} -u 'robot$dds+robot-dds-deploy' --password ${{ secrets.HARBOR_CREDENTIAL }} - docker build -t ${{ env.DOCKER_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} -f deploy.Dockerfile . - docker push ${{ env.DOCKER_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ ENV.IMAGE_TAG }} - deploy: - if: github.repository_owner == 'deepdataspace' - runs-on: self-hosted - needs: build - steps: - - name: Checkout Devops Repo - uses: actions/checkout@v2 - with: - repository: deepdataspace/deepdataspace-helm - ref: 'main' - path: deepdataspace-helm - token: ${{ secrets.DDS_ORG_GITHUB_TOKEN }} - - name: Copy Helm Values File - working-directory: deepdataspace-helm - run: | - k8sNamespace=${{needs.build.outputs.K8S_NAMESPACE}} - chartPath=${{ env.CHART_PATH }} - chartValuesPath=${{ env.CHART_VALUES_PATH }} - if [[ "${k8sNamespace}" == 'dds-dev' ]]; then - cp ${chartValuesPath}/dev-values.yaml ${chartPath}/values.yaml - elif [[ "${k8sNamespace}" == 'dds-test' ]]; then - cp ${chartValuesPath}/test-values.yaml ${chartPath}/values.yaml - elif [[ "${k8sNamespace}" == 'dds-prd' ]]; then - cp ${chartValuesPath}/prd-values.yaml ${chartPath}/values.yaml - fi - rm -rf chartValuesPath - rm -rf deepdataspace-helm/dockerfiles - - name: Update Helm Chart - working-directory: deepdataspace-helm - run: | - appName=${{needs.build.outputs.APP_NAME}} - appVersion=${{needs.build.outputs.APP_VERSION}} - chartPath=${{ env.CHART_PATH }} - chartVersion=${{needs.build.outputs.CHART_VERSION}} - sed -i "s/^name:.*/name: ${appName}/" ${chartPath}/Chart.yaml - sed -i "s/^version:.*/version: ${chartVersion}/" ${chartPath}/Chart.yaml - sed -i "s/^appVersion:.*/appVersion: ${appVersion}/" ${chartPath}/Chart.yaml - - name: Push Helm Chart to Harbor - working-directory: deepdataspace-helm - run: | - chartPath=${{ env.CHART_PATH }} - if ! helm repo list | awk '{print $1}' | grep -q "^${{ env.HELM_REPO_NAME }}$"; then - helm repo add --username 'robot$dds+robot-dds-deploy' --password ${{ secrets.HARBOR_CREDENTIAL }} ${{ env.HELM_REPO_NAME }} ${{ env.HELM_REPO_URL }} - fi - helm repo update - helm package ${chartPath} - helm cm-push ${chartPath} ${{ env.HELM_REPO_NAME }} - - name: Deploy to Kubernetes - working-directory: deepdataspace-helm - run: | - appName=${{ needs.build.outputs.APP_NAME }} - imageTag=${{ needs.build.outputs.IMAGE_TAG }} - k8sNamespace=${{ needs.build.outputs.K8S_NAMESPACE }} - helm repo update - helm upgrade --install --devel --wait ${appName} ${{ env.HELM_REPO_NAME }}/${appName} --namespace ${k8sNamespace} --set django.imageTag=${imageTag},celery.imageTag=${imageTag} diff --git a/.github/workflows/sync-dev.yml b/.github/workflows/sync-dev.yml deleted file mode 100644 index 0105dae..0000000 --- a/.github/workflows/sync-dev.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Sync Dev to Private Repo -on: - push: - branches: - - dev -jobs: - merge-and-rebase: - if: github.repository_owner == 'IDEA-Research' - runs-on: ubuntu-latest - steps: - - name: Checkout Private Repo - uses: actions/checkout@v2 - with: - repository: deepdataspace/deepdataspace - ref: dev - token: ${{ secrets.PRIVATE_REPO_TOKEN }} - - - name: Add Public Remote - run: | - git remote add public https://github.com/IDEA-Research/deepdataspace.git - - - name: Rebase Private Repo - run: | - git pull public dev --rebase - - - name: Push Private Repo - run: | - git push diff --git a/.github/workflows/tool-ci.yaml b/.github/workflows/tool-ci.yaml index 85daa5c..bd46d64 100644 --- a/.github/workflows/tool-ci.yaml +++ b/.github/workflows/tool-ci.yaml @@ -2,67 +2,17 @@ name: tool-ci on: push: branches: - - '**' - tags: - - '' + - main + pull_request: + branches: + - dev jobs: - scan: - runs-on: self-hosted - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Scan Code - uses: sonarsource/sonarqube-scan-action@master - env: - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} - with: - args: > - -Dsonar.projectKey=dds-tool - test_on_windows_2019: - if: github.repository_owner == 'IDEA-Research' - needs: scan - runs-on: windows-2019 - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Setup Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: '3.10' - cache: 'pip' - cache-dependency-path: 'requirements-dev.txt' - - name: Run Test - run: | - pip install -r requirements-dev.txt - pytest --cov=deepdataspace -s --cov-report=xml - test_on_windows_2022: - if: github.repository_owner == 'IDEA-Research' - needs: scan - runs-on: windows-2022 - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Setup Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: '3.10' - cache: 'pip' - cache-dependency-path: 'requirements-dev.txt' - - name: Run Test - run: | - pip install -r requirements-dev.txt - pytest --cov=deepdataspace -s --cov-report=xml - test_on_macos_12: - if: github.repository_owner == 'IDEA-Research' - needs: scan - runs-on: macos-12 + test_python: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-2022, macos-12, ubuntu-22.04] steps: - name: Checkout Code uses: actions/checkout@v3 @@ -71,44 +21,34 @@ jobs: - name: Setup Python 3.10 uses: actions/setup-python@v4 with: - python-version: '3.10' - cache: 'pip' - cache-dependency-path: 'requirements-dev.txt' - - name: Run Test - run: | - pip install -r requirements-dev.txt - pytest --cov=deepdataspace -s --cov-report=xml - test_on_ubuntu_2204: - if: github.repository_owner == 'IDEA-Research' - needs: [test_on_windows_2019, test_on_windows_2022, test_on_macos_12 ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Setup Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: '3.10' - cache: 'pip' - cache-dependency-path: 'requirements-dev.txt' + python-version: "3.10" + cache: "pip" + cache-dependency-path: "requirements-dev.txt" - name: Run Test run: | pip install -r requirements-dev.txt pytest --cov=deepdataspace -s --cov-report=xml - name: Upload Test Report uses: codecov/codecov-action@v3 + if: matrix.os == 'ubuntu-22.04' with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true verbose: true flags: backend test_frontend: - if: github.repository_owner == 'IDEA-Research' - runs-on: self-hosted + runs-on: ubuntu-22.04 + env: + TZ: Asia/Shanghai steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: {} - name: Install requirements run: pnpm install --frozen-lockfile - name: Run pnpm audit diff --git a/.gitignore b/.gitignore index 0e25d68..de02444 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ venv .DS_Store /config.py .python-version +.venv-dds +.dds-home +.pypirc # for runtime file /data diff --git a/deepdataspace/server/static/index.html b/deepdataspace/server/static/index.html index f2490a7..1ed9a2a 100644 --- a/deepdataspace/server/static/index.html +++ b/deepdataspace/server/static/index.html @@ -9,6 +9,6 @@
- +