arrowflow/.github/workflows/ci.yml at main · VTvito/arrowflow · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
name: CI — ETL Microservices Platform

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]

jobs:
  lint:
    name: Lint & Format Check
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.9"
      - name: Install ruff
        run: pip install ruff
      - name: Run linter
        run: ruff check .
        continue-on-error: true

  test-unit:
    name: Unit Tests
    runs-on: ubuntu-latest
    needs: lint
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.9"
          cache: pip
      - name: Install test dependencies
        run: |
          pip install pytest pytest-cov
          pip install flask pyarrow pandas numpy prometheus_client xlsxwriter openpyxl requests pyyaml sqlalchemy
      - name: Run unit tests
        run: pytest tests/unit/ -v --tb=short --cov=services --cov-report=xml
      - name: Upload coverage
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report
          path: coverage.xml

  test-integration:
    name: Integration Tests
    runs-on: ubuntu-latest
    needs: test-unit
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.9"
          cache: pip
      - name: Install test dependencies
        run: |
          pip install pytest
          pip install flask pyarrow pandas numpy prometheus_client requests
      - name: Run integration tests
        run: pytest tests/integration/ -v --tb=short

  build:
    name: Docker Build
    runs-on: ubuntu-latest
    needs: [test-unit, test-integration]
    steps:
      - uses: actions/checkout@v4
      - name: Build all services
        run: |
          docker compose build \
            extract-csv-service \
            clean-nan-service \
            delete-columns-service \
            extract-sql-service \
            extract-api-service \
            extract-excel-service \
            join-datasets-service \
            load-data-service \
            data-quality-service \
            outlier-detection-service \
            text-completion-llm-service \
            streamlit-app