aperturedb-python/.github/workflows/pr.yaml at develop · aperture-data/aperturedb-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
name: pr

on:
  pull_request:
    branches:
    - develop

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  run_test:
    runs-on:
    - gcp

    steps:

    - name: Cleanup previous run
      run: |
        docker ps -q -a --filter "name=${{ runner.name }}_pr" | xargs -r docker rm -f || true
        docker run --rm -v ${{ github.workspace }}:/workspace alpine sh -c "rm -rf /workspace/test/aperturedb" || true
        sudo rm -rf ${{ github.workspace }}/test/aperturedb
      continue-on-error: true

    - uses: actions/checkout@v3

    - name: Login to DockerHub
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.DOCKER_USER }}
        password: ${{ secrets.DOCKER_PASS }}

    - name: Authenticate to Google Cloud
      uses: google-github-actions/auth@v2
      with:
        credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
        project_id: ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}

    - name: Set up Cloud SDK
      uses: google-github-actions/setup-gcloud@v2

    - name: Build and Run Tests
      env:
        # Run both protocols inside a single job. run_test_container.sh starts
        # the http and non_http stacks in parallel and runs both test
        # containers concurrently, so wall-clock time is roughly max(http,
        # non_http) instead of the previous matrix approach which duplicated
        # all docker image builds (notebook/tests/coverage/deps) per protocol.
        TEST_PROTOCOL: both
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
        RUNNER_NAME: ${{ runner.name }}_pr
        RUN_TESTS: true
        NO_PUSH: true
        # Skip building the notebook image and the coverage publishing image;
        # neither is required to execute the test suite and on PRs they would
        # be discarded (NO_PUSH=true). This removes a large chunk of the
        # build time without dropping any test coverage.
        BUILD_AUX_IMAGES: false
        # Use BuildKit for faster builds and proper layer caching from the
        # registry via --cache-from + BUILDKIT_INLINE_CACHE.
        DOCKER_BUILDKIT: 1
        COMPOSE_DOCKER_CLI_BUILD: 1
        BRANCH_NAME: ${{ github.event.pull_request.head.ref }}
        TARGET_BRANCH_NAME: ${{ github.event.pull_request.base.ref }}
        ADB_REPO: aperturedata/aperturedb
        ADB_TAG: dev
        LENZ_REPO: aperturedata/lenz
        LENZ_TAG: dev
        SKIP_SLOW_TESTS: true
      run: ./ci.sh
      shell: bash

  run_test_conda_gpu:

    runs-on:
    - self-hosted
    - deployer

    steps:

    - name: Cleanup previous run
      run: |
        docker ps -q -a --filter "name=${{ runner.name }}_${{ github.run_id }}" | xargs -r docker rm -f || true
        docker run --rm -v ${{ github.workspace }}:/workspace alpine sh -c "rm -rf /workspace/test/aperturedb" || true
        sudo rm -rf ${{ github.workspace }}/test/aperturedb
      continue-on-error: true

    - uses: actions/checkout@v3

    - name: Login to DockerHub
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.DOCKER_USER }}
        password: ${{ secrets.DOCKER_PASS }}

    - name: Authenticate to Google Cloud
      uses: google-github-actions/auth@v2
      with:
        credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
        project_id: ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}

    - name: Set up Cloud SDK
      uses: google-github-actions/setup-gcloud@v2
    - name: Build tests on pytorch GPU image
      run: |
        docker run --rm -v ${{ github.workspace }}:/workspace alpine sh -c "rm -rf /workspace/test/aperturedb" || true
        rm -rf docker/pytorch-gpu/aperturedata
        mkdir -p docker/pytorch-gpu/aperturedata/test
        cp -r aperturedb pyproject.toml README.md docker/pytorch-gpu/aperturedata
        cp -r test/*.py test/*.sh test/input test/pytest.ini docker/pytorch-gpu/aperturedata/test
        bash docker/pytorch-gpu/build.sh
      shell: bash

    - name: Run Tests
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
        RUNNER_NAME: ${{ runner.name }}_${{ github.run_id }}
        ADB_REPO: aperturedata/aperturedb
        ADB_TAG: dev
        LENZ_REPO: aperturedata/lenz
        LENZ_TAG: dev
        SKIP_SLOW_TESTS: true
      run: |
        cd test
        ./run_test_container.sh aperturedata/aperturedb-pytorch-gpu