Skip to content

Commit cc28d89

Browse files
khalilh2002khalilh2002
committed
github action
1 parent 7db3eab commit cc28d89

3 files changed

Lines changed: 145 additions & 9 deletions

File tree

‎.github/workflows/ci-cd.yml‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
name: CI/CD for Web-Scrapping
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
branches:
9+
- main
10+
11+
env:
12+
# Define image names centrally, including your Docker Hub username
13+
# Ensure these match your repositories on Docker Hub
14+
SHOPIFY_IMAGE_REPO: spamfake2022/shop-scraper # Your Docker Hub username / repo name
15+
WOO_IMAGE_REPO: spamfake2022/woo-scraper # Your Docker Hub username / repo name
16+
17+
jobs:
18+
build-and-push:
19+
name: Build and Push Docker Images
20+
runs-on: [self-hosted, Linux, minikube-local] # Runner needs Docker installed
21+
22+
steps:
23+
- name: Checkout code
24+
uses: actions/checkout@v4
25+
26+
- name: Login to Docker Hub
27+
uses: docker/login-action@v3
28+
with:
29+
username: ${{ secrets.DOCKERHUB_USERNAME }}
30+
password: ${{ secrets.DOCKERHUB_TOKEN }}
31+
32+
- name: Build and push Shopify Scrapper Docker image
33+
uses: docker/build-push-action@v5
34+
with:
35+
context: ./shopify # Path to the Dockerfile directory for Shopify
36+
file: ./shopify/Dockerfile
37+
push: true
38+
tags: ${{ env.SHOPIFY_IMAGE_REPO }}:latest # e.g., spamfake2022/shop-scraper:latest
39+
40+
- name: Build and push Woo Scrapper Docker image
41+
uses: docker/build-push-action@v5
42+
with:
43+
context: ./woo # Path to the Dockerfile directory for Woo
44+
file: ./woo/Dockerfile
45+
push: true
46+
tags: ${{ env.WOO_IMAGE_REPO }}:latest # e.g., spamfake2022/woo-scraper:latest
47+
48+
- name: List local Docker images (host Docker)
49+
run: |
50+
echo "Images built on host Docker:"
51+
docker images | grep -E "${{ env.SHOPIFY_IMAGE_REPO }}|${{ env.WOO_IMAGE_REPO }}"
52+
53+
deploy-to-minikube:
54+
name: Deploy to Minikube
55+
runs-on: [self-hosted, Linux, minikube-local] # Runner needs kubectl
56+
needs: build-and-push # This job runs only if 'build-and-push' job succeeds
57+
if: github.ref == 'refs/heads/main' && github.event_name == 'push' # Deploy only on push to main
58+
59+
steps:
60+
- name: Checkout code
61+
uses: actions/checkout@v4
62+
63+
- name: Ensure Kubernetes context is Minikube
64+
run: |
65+
kubectl config use-context kfp-cluster
66+
kubectl config current-context
67+
68+
- name: Apply K8s ConfigMap
69+
run: kubectl apply -f k8s/app-configmap.yaml
70+
71+
- name: Apply K8s DB Credentials Secret
72+
run: kubectl apply -f k8s/db-credentials.yaml
73+
74+
- name: Apply K8s Shopify Scrapper Deployment and Service
75+
run: |
76+
# The image name in the YAML MUST match ${{ env.SHOPIFY_IMAGE_REPO }}
77+
# imagePullPolicy should be Always or IfNotPresent (Always is better to ensure fresh image from registry)
78+
echo "Applying Shopify Scrapper K8s manifests (pulling from ${{ env.SHOPIFY_IMAGE_REPO }}:latest)..."
79+
kubectl apply -f k8s/shopify-scrapper-deployement.yaml
80+
81+
- name: Apply K8s Woo Scrapper Deployment and Service
82+
run: |
83+
# The image name in the YAML MUST match ${{ env.WOO_IMAGE_REPO }}
84+
# imagePullPolicy should be Always or IfNotPresent
85+
echo "Applying Woo Scrapper K8s manifests (pulling from ${{ env.WOO_IMAGE_REPO }}:latest)..."
86+
kubectl apply -f k8s/woo-scraper-deployment.yaml
87+
88+
- name: Apply K8s Ingress
89+
run: |
90+
echo "Applying Ingress..."
91+
kubectl apply -f k8s/ingress.yaml
92+
93+
- name: Wait for deployments to be ready
94+
run: |
95+
echo "Waiting for Shopify deployment..."
96+
kubectl rollout status deployment/shopify-scraper-api --timeout=180s
97+
kubectl wait --for=condition=available deployment/shopify-scraper-api --timeout=180s
98+
echo "Waiting for Woo deployment..."
99+
kubectl rollout status deployment/woo-scraper-api --timeout=180s
100+
kubectl wait --for=condition=available deployment/woo-scraper-api --timeout=180s
101+
102+
- name: Verify Deployments and Services
103+
run: |
104+
echo "--- Pods ---"
105+
kubectl get pods -l app=shopify-scraper-api
106+
kubectl get pods -l app=woo-scraper-api
107+
echo "--- Services ---"
108+
kubectl get services -l app=shopify-scraper-api
109+
kubectl get services -l app=woo-scraper-api
110+
echo "--- Ingress ---"
111+
kubectl get ingress ingress-service
112+
echo "--- Endpoints for Shopify ---"
113+
kubectl get endpoints shopify-scraper-api-service
114+
echo "--- Endpoints for Woo ---"
115+
kubectl get endpoints woo-scraper-api-service
116+
echo "--- Describe Pods for Image Check (Shopify) ---"
117+
kubectl describe pods -l app=shopify-scraper-api
118+
echo "--- Describe Pods for Image Check (Woo) ---"
119+
kubectl describe pods -l app=woo-scraper-api
120+
121+
- name: Check Ingress
122+
run: |
123+
echo "To access via Ingress 'dashboard2025.com', ensure 'minikube tunnel' is running in another terminal,"
124+
echo "or your Minikube Ingress controller is properly configured and you have DNS/hosts entry."
125+
echo "You might also need to enable the ingress addon: minikube -p kfp-cluster addons enable ingress"

‎k8s/app-configmap.yaml‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ metadata:
55
name: app-configs
66
data:
77
# --- Common DB Settings ---
8-
DB_HOST: "192.168.3.48" # For MySQL on host when running in Minikube
8+
DB_HOST: "192.168.3.224" # For MySQL on host when running in Minikube
99

1010
# --- Shopify Scraper Specific ---
1111
SHOP_DB_NAME: "scrap_test" # Example: specific DB name
@@ -18,11 +18,15 @@ data:
1818
FLASK_PORT_WOO: "5002" # Port Flask app listens on INSIDE container
1919

2020
# --- Analyzer Specific ---
21-
ANLZ_DB_NAME_SHOPIFY: "scrap_test_shopify" # Must match what shopify scraper uses
22-
ANLZ_DB_NAME_WOOCOMMERCE: "scrap_test" # Must match what woo scraper uses
23-
ANLZ_DB_NAME_ANALYSIS: "scrap_analyse"
24-
ANLZ_TOP_K_OVERALL: "20"
25-
ANLZ_FLAGSHIP_PER_STORE: "3"
26-
ANLZ_WEIGHT_AVAILABILITY: "0.6"
27-
ANLZ_WEIGHT_PRICE: "0.4"
28-
ANLZ_DB_BATCH_SIZE: "500"
21+
DB_NAME_SHOPIFY: "scrap_test" # Source DB for Shopify data
22+
DB_NAME_WOOCOMMERCE: "scrap_test" # Source DB for Woo data
23+
DB_NAME_ANALYSIS: "scrap_analyse" # Target DB for analysis results
24+
25+
ANALYZER_PARAM_TOP_K_OVERALL: "20"
26+
ANALYZER_PARAM_FLAGSHIP_PER_STORE: "3"
27+
ANALYZER_PARAM_WEIGHT_AVAILABILITY: "0.6"
28+
ANALYZER_PARAM_WEIGHT_PRICE: "0.4"
29+
ANALYZER_PARAM_DB_BATCH_SIZE: "500"
30+
31+
FLASK_PORT_ANALYZER: "5003"
32+

‎k8s/ingress.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,10 @@ spec:
2121
name: woo-scraper-api-service
2222
port:
2323
number: 80
24+
- path: /run_analysis
25+
pathType: Prefix
26+
backend:
27+
service:
28+
name: product-analyzer-svc
29+
port:
30+
number: 80

0 commit comments

Comments
 (0)