# Merge sprint-3 to main

.github/workflows/deploy.yml

@@ -0,0 +1,45 @@
+name: Continuous Integration - Stratify ETL
+
+on:
+  push:
+    branches:
+      - main
+      - sprint-3
+jobs:
+  test:
+    name: Teste unitário
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+         pip install -r requirements.txt
+    - name: Test with pytest
+      run: python3 -m unittest discover -s tests
+
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+    - name: Configure SSH
+      uses: webfactory/ssh-agent@v0.5.3
+      with:
+        ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+
+    - name: Add Remote Server to Known Hosts
+      run: |
+        ssh-keyscan -H ${{ secrets.SERVER_HOST }} >> ~/.ssh/known_hosts
+
+    - name: Git Pull on Remote Server
+      run: |
+        ssh ${{ secrets.SERVER_USER }}@${{ secrets.SERVER_HOST }} << EOF
+          cd apps/API_5SEM_ETL/
+          git pull origin $(git rev-parse --abbrev-ref HEAD)
+        EOF

src/extract/users.py

@@ -1,25 +1,9 @@
+import requests
+
 from src.config import config
 from src.utils.logger import Logger
 
 
-def get_all_users():
-    """Returns a list of users from Taiga."""
-    try:
-        client = config.TaigaClient()
-        users = client.api.users.list()
-
-        all_users = []
-
-        for user in users:
-            user_complete = client.api.users.get(resource_id=user.id)
-            all_users.append(user_complete)
-
-        return [vars(user) for user in all_users]
-    except Exception as e:
-        Logger.error(f"An error occurred while fetching users: {e}")
-        return []
-
-
 def get_all_users_by_project(project_id):
     """Returns a list of users by project ID from Taiga."""
     try:
@@ -30,6 +14,15 @@ def get_all_users_by_project(project_id):
 
         for user in users:
             user_complete = client.api.users.get(resource_id=user.id)
+
+            email = get_useremail_by_project_by_memberships(project_id, user.id)
+            if email:
+                user_complete.email = email
+            else:
+                Logger.error(
+                    f"Email not found for user {user.id} in project {project_id}."
+                )
+
             all_users.append(user_complete)
 
         return [vars(user) for user in all_users]
@@ -38,12 +31,60 @@ def get_all_users_by_project(project_id):
         return []
 
 
-def get_user_by_id(user_id):
+def get_user_by_id(user_id, project_id):
     """Returns a user by ID."""
     try:
         client = config.TaigaClient()
         user = client.api.users.get(resource_id=user_id)
+        email = get_useremail_by_project_by_memberships(project_id, user_id)
+        if email:
+            user.email = email
+        else:
+            Logger.error(f"Email not found for user {user_id} in project {project_id}.")
         return vars(user)
     except Exception as e:
         Logger.error(f"An error occurred while fetching user {user_id}: {e}")
         return None
+
+
+# In-memory cache for memberships
+membership_cache = {}
+
+
+def get_useremail_by_project_by_memberships(project_id, user_id):
+    """Fetches the email of a user by user ID from project memberships in Taiga."""
+
+    # Check if the email is already in the cache
+    if user_id in membership_cache:
+        return membership_cache[user_id]
+
+    token = config.TaigaClient().api.token
+    if not token:
+        Logger.error(
+            "No API token provided. Please set the TAIGA_API_TOKEN environment variable."
+        )
+        return None
+
+    url = f"https://api.taiga.io/api/v1/memberships?project={project_id}"
+    headers = {"Authorization": f"Bearer {token}"}
+
+    try:
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()
+        memberships = response.json()
+
+        for membership in memberships:
+            user_id_from_membership = membership.get("user")
+            user_email = membership.get("user_email")
+            if user_id_from_membership and user_email:
+                # Populate the cache with the user email
+                membership_cache[user_id_from_membership] = user_email
+
+            if user_id_from_membership == user_id:
+                return user_email
+
+        Logger.error(f"No user with ID {user_id} found in project {project_id}.")
+        return None
+    except requests.exceptions.RequestException as e:
+        Logger.error(f"An error occurred while fetching project memberships: {e}")
+        return None

src/map/fact_eficiencia_user_story.py

@@ -59,23 +59,23 @@ def process_data_2_fact_eficiencia():
 
                 created_date = datetime.datetime.strptime(
                     story["created_date"], "%Y-%m-%dT%H:%M:%S.%fZ"
-                )
+                ).replace(tzinfo=datetime.timezone.utc)
                 finish_date = story["finish_date"]
                 if finish_date is None:
                     Logger.warning(
-                        f"Finish date is None for story {story['id']}... skipping"
+                        f"Finish date is None for story {story['id']}... using current datetime"
                     )
-                    continue
+                    finish_date = datetime.datetime.now(datetime.timezone.utc)
                 else:
                     finish_date = datetime.datetime.strptime(
                         finish_date, "%Y-%m-%dT%H:%M:%S.%fZ"
-                    )
+                    ).replace(tzinfo=datetime.timezone.utc)
 
                 duration = (
                     finish_date - created_date
-                ).total_seconds() / 60  # Convert duration to minutes
+                ).total_seconds() / 3600  # Convert duration to hours
                 Logger.info(
-                    f"Story {story['id']} - Created Date: {created_date}, Finish Date: {finish_date}, Duration (minutes): {duration}"
+                    f"Story {story['id']} - Created Date: {created_date}, Finish Date: {finish_date}, Duration (hours): {duration}"
                 )
 
                 assigned_to_extra_info = story.get("assigned_to_extra_info")
@@ -89,7 +89,7 @@ def process_data_2_fact_eficiencia():
                     continue
 
                 Logger.info(f"Fetching user details for Taiga ID: {taiga_user_id}")
-                user_complete = get_user_by_id(taiga_user_id)
+                user_complete = get_user_by_id(taiga_user_id, project["id"])
                 if user_complete is None:
                     Logger.error(
                         f"User {taiga_user_id} not found in Taiga... skipping story {story['id']}"

src/map/fact_user_story_temporais.py

@@ -75,7 +75,7 @@ def process_data_2_fact_temporais():
                     continue
 
                 Logger.info(f"Fetching user details for Taiga ID: {taiga_user_id}")
-                user_complete = get_user_by_id(taiga_user_id)
+                user_complete = get_user_by_id(taiga_user_id, projetc["id"])
                 if user_complete is None:
                     Logger.error(
                         f"User {taiga_user_id} not found in Taiga... skipping story {story['id']}"

src/map/relation_project_user.py

@@ -56,7 +56,7 @@ def upsert_relation_project_user():
                     )
                     continue
 
-                user_complete = get_user_by_id(taiga_user_id)
+                user_complete = get_user_by_id(taiga_user_id, project["id"])
                 if user_complete is None:
                     Logger.error(
                         f"User {taiga_user_id} not found in Taiga... skipping project {project['id']}"

src/map/relation_tag_user_story.py

@@ -34,8 +34,13 @@ def upsert_relation_tag_us():
     )
 
     select_id_tag = "SELECT id FROM public.dim_tag WHERE LOWER(nome) = LOWER(%s)"
-
+
+
+    trucate_table = "TRUNCATE TABLE public.relacionamento_tag_user_story"
     conn = db.get_connection()
+    conn.cursor().execute(trucate_table)
+    conn.commit()
+
     Logger.info("Database connection established.")
     try:
         for project in projects: