feat: enhance Streamlit UI with Dataset Explorer and update pipeline … #16
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI — ETL Microservices Platform | |
| on: | |
| push: | |
| branches: [main, develop] | |
| pull_request: | |
| branches: [main] | |
| jobs: | |
| lint: | |
| name: Lint & Format Check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.9" | |
| - name: Install ruff | |
| run: pip install ruff | |
| - name: Run linter | |
| run: ruff check . | |
| continue-on-error: true | |
| test-unit: | |
| name: Unit Tests | |
| runs-on: ubuntu-latest | |
| needs: lint | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.9" | |
| cache: pip | |
| - name: Install test dependencies | |
| run: | | |
| pip install pytest pytest-cov | |
| pip install flask pyarrow pandas numpy prometheus_client xlsxwriter openpyxl requests pyyaml sqlalchemy | |
| - name: Run unit tests | |
| run: pytest tests/unit/ -v --tb=short --cov=services --cov-report=xml | |
| - name: Upload coverage | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report | |
| path: coverage.xml | |
| test-integration: | |
| name: Integration Tests | |
| runs-on: ubuntu-latest | |
| needs: test-unit | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.9" | |
| cache: pip | |
| - name: Install test dependencies | |
| run: | | |
| pip install pytest | |
| pip install flask pyarrow pandas numpy prometheus_client requests | |
| - name: Run integration tests | |
| run: pytest tests/integration/ -v --tb=short | |
| build: | |
| name: Docker Build | |
| runs-on: ubuntu-latest | |
| needs: [test-unit, test-integration] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Build all services | |
| run: | | |
| docker compose build \ | |
| extract-csv-service \ | |
| clean-nan-service \ | |
| delete-columns-service \ | |
| extract-sql-service \ | |
| extract-api-service \ | |
| extract-excel-service \ | |
| join-datasets-service \ | |
| load-data-service \ | |
| data-quality-service \ | |
| outlier-detection-service \ | |
| text-completion-llm-service \ | |
| streamlit-app |