From 237fe40499751619e0851ed8edc5c6064112bd91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Matheus=20Lam=C3=A3o?= Date: Sun, 27 Apr 2025 21:42:26 -0300 Subject: [PATCH 01/13] fix(fk): add new relations in dim_user_storie --- src/map/user_stories.py | 69 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/src/map/user_stories.py b/src/map/user_stories.py index aa4a9f0..75723e0 100644 --- a/src/map/user_stories.py +++ b/src/map/user_stories.py @@ -1,6 +1,7 @@ from src.config.database import Database from src.extract.projects import get_all_projects from src.extract.user_storys import get_user_storys_by_project +from src.extract.users import get_user_by_id from src.utils.logger import Logger @@ -14,13 +15,40 @@ def upsert_all_user_stories(): select_user_story_by_taiga_id = ( "SELECT * FROM public.dim_user_story WHERE id_taiga = CAST(%s AS BIGINT)" ) + + select_status_id_internal = ( + "SELECT id FROM public.dim_status WHERE LOWER(tipo) = LOWER(%s)" + ) + + select_user_internal_id = ( + "SELECT id FROM public.dim_usuario WHERE LOWER(email) = LOWER(%s)" + ) + + + select_project_id_internal = ( + "SELECT id FROM public.dim_projeto WHERE LOWER(nome) = LOWER(%s)" + ) - insert_user_story = "INSERT INTO public.dim_user_story (id_taiga, assunto, criado_em, finalizado_em, bloqueado, encerrado, data_limite) VALUES (%s, %s, %s, %s, %s, %s, %s)" - update_user_story = "UPDATE public.dim_user_story SET assunto = %s, criado_em = %s, finalizado_em = %s, bloqueado = %s, encerrado = %s, data_limite = %s WHERE id_taiga = %s" + insert_user_story = "INSERT INTO public.dim_user_story (id_taiga, assunto, criado_em, finalizado_em, bloqueado, encerrado, data_limite, id_status, id_usuario, id_projeto) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + update_user_story = "UPDATE public.dim_user_story SET assunto = %s, criado_em = %s, finalizado_em = %s, bloqueado = %s, encerrado = %s, data_limite = %s, id_status = %s, id_usuario = %s, id_projeto = %s WHERE id_taiga = %s" conn = db.get_connection() try: for project in projects: + + cursor = conn.cursor() + cursor.execute(select_project_id_internal, (project["name"],)) + internal_project_id = cursor.fetchone() + if not internal_project_id: + Logger.warning( + f"Project '{project['name']}' not found in the database. Skipping." + ) + continue + + Logger.debug( + f"Executing query to find user ID: {select_project_id_internal} with parameter: {project['owner']['id']}" + ) + stories = get_user_storys_by_project(project["id"]) Logger.info( f"Extracted {len(stories)} user stories from project {project['name']} (ID: {project['id']})" @@ -31,6 +59,37 @@ def upsert_all_user_stories(): cursor = conn.cursor() cursor.execute(select_user_story_by_taiga_id, (story["id"],)) story_in_bd = cursor.fetchone() + + cursor.execute( + select_status_id_internal, (story["status_extra_info"]["name"],) + ) + status_id_internal = cursor.fetchone() + + + assigned_to_extra_info = story.get("assigned_to_extra_info") + taiga_user_id = ( + assigned_to_extra_info.get("id") if assigned_to_extra_info else None + ) + if taiga_user_id is None: + Logger.warning( + f"Assigned user is None for story {story['id']}... skipping" + ) + continue + + Logger.info(f"Fetching user details for Taiga ID: {taiga_user_id}") + user_complete = get_user_by_id(taiga_user_id) + if user_complete is None: + Logger.error( + f"User {taiga_user_id} not found in Taiga... skipping story {story['id']}" + ) + continue + + Logger.info(f"Fetched user details: {user_complete}") + cursor.execute(select_user_internal_id, (user_complete["email"],)) + internal_user_id = cursor.fetchone()[0] + + + if story_in_bd is None: cursor.execute( @@ -43,6 +102,9 @@ def upsert_all_user_stories(): story["is_blocked"], story["is_closed"], story["due_date"], + status_id_internal[0], + internal_user_id, + internal_project_id ), ) conn.commit() @@ -57,6 +119,9 @@ def upsert_all_user_stories(): story["is_blocked"], story["is_closed"], story["due_date"], + status_id_internal[0], + internal_user_id, + internal_project_id, story["id"], ), ) From 4ebd26572f4f24fc0a58f0c591edc8367d7f7fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Matheus=20Lam=C3=A3o?= Date: Sun, 27 Apr 2025 21:42:52 -0300 Subject: [PATCH 02/13] refactor(black): apply black formatting --- src/map/user_stories.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/map/user_stories.py b/src/map/user_stories.py index 75723e0..0a5b786 100644 --- a/src/map/user_stories.py +++ b/src/map/user_stories.py @@ -15,16 +15,15 @@ def upsert_all_user_stories(): select_user_story_by_taiga_id = ( "SELECT * FROM public.dim_user_story WHERE id_taiga = CAST(%s AS BIGINT)" ) - + select_status_id_internal = ( "SELECT id FROM public.dim_status WHERE LOWER(tipo) = LOWER(%s)" ) - + select_user_internal_id = ( "SELECT id FROM public.dim_usuario WHERE LOWER(email) = LOWER(%s)" ) - - + select_project_id_internal = ( "SELECT id FROM public.dim_projeto WHERE LOWER(nome) = LOWER(%s)" ) @@ -35,7 +34,7 @@ def upsert_all_user_stories(): conn = db.get_connection() try: for project in projects: - + cursor = conn.cursor() cursor.execute(select_project_id_internal, (project["name"],)) internal_project_id = cursor.fetchone() @@ -48,7 +47,7 @@ def upsert_all_user_stories(): Logger.debug( f"Executing query to find user ID: {select_project_id_internal} with parameter: {project['owner']['id']}" ) - + stories = get_user_storys_by_project(project["id"]) Logger.info( f"Extracted {len(stories)} user stories from project {project['name']} (ID: {project['id']})" @@ -59,16 +58,17 @@ def upsert_all_user_stories(): cursor = conn.cursor() cursor.execute(select_user_story_by_taiga_id, (story["id"],)) story_in_bd = cursor.fetchone() - + cursor.execute( select_status_id_internal, (story["status_extra_info"]["name"],) ) status_id_internal = cursor.fetchone() - - + assigned_to_extra_info = story.get("assigned_to_extra_info") taiga_user_id = ( - assigned_to_extra_info.get("id") if assigned_to_extra_info else None + assigned_to_extra_info.get("id") + if assigned_to_extra_info + else None ) if taiga_user_id is None: Logger.warning( @@ -83,13 +83,10 @@ def upsert_all_user_stories(): f"User {taiga_user_id} not found in Taiga... skipping story {story['id']}" ) continue - + Logger.info(f"Fetched user details: {user_complete}") cursor.execute(select_user_internal_id, (user_complete["email"],)) internal_user_id = cursor.fetchone()[0] - - - if story_in_bd is None: cursor.execute( @@ -102,9 +99,9 @@ def upsert_all_user_stories(): story["is_blocked"], story["is_closed"], story["due_date"], - status_id_internal[0], + status_id_internal[0], internal_user_id, - internal_project_id + internal_project_id, ), ) conn.commit() From 080dbbba512683912527d2ff76bca1d259df4ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Matheus=20Lam=C3=A3o?= Date: Fri, 9 May 2025 20:42:48 -0300 Subject: [PATCH 03/13] feat(new api) - scrum-108: new api to get all users --- src/extract/users.py | 77 ++++++++++++++++++++------- src/map/fact_eficiencia_user_story.py | 2 +- src/map/relation_project_user.py | 2 +- src/map/user_stories.py | 2 +- src/map/users.py | 2 +- 5 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/extract/users.py b/src/extract/users.py index 72a629d..76ca420 100644 --- a/src/extract/users.py +++ b/src/extract/users.py @@ -1,25 +1,9 @@ +import requests + from src.config import config from src.utils.logger import Logger -def get_all_users(): - """Returns a list of users from Taiga.""" - try: - client = config.TaigaClient() - users = client.api.users.list() - - all_users = [] - - for user in users: - user_complete = client.api.users.get(resource_id=user.id) - all_users.append(user_complete) - - return [vars(user) for user in all_users] - except Exception as e: - Logger.error(f"An error occurred while fetching users: {e}") - return [] - - def get_all_users_by_project(project_id): """Returns a list of users by project ID from Taiga.""" try: @@ -30,6 +14,14 @@ def get_all_users_by_project(project_id): for user in users: user_complete = client.api.users.get(resource_id=user.id) + + + email = get_useremail_by_project_by_memberships(project_id, user.id) + if email: + user_complete.email = email + else: + Logger.error(f"Email not found for user {user.id} in project {project_id}.") + all_users.append(user_complete) return [vars(user) for user in all_users] @@ -38,12 +30,59 @@ def get_all_users_by_project(project_id): return [] -def get_user_by_id(user_id): +def get_user_by_id(user_id, project_id): """Returns a user by ID.""" try: client = config.TaigaClient() user = client.api.users.get(resource_id=user_id) + email = get_useremail_by_project_by_memberships(project_id, user_id) + if email: + user.email = email + else: + Logger.error(f"Email not found for user {user_id} in project {project_id}.") return vars(user) except Exception as e: Logger.error(f"An error occurred while fetching user {user_id}: {e}") return None + + +# In-memory cache for memberships +membership_cache = {} + +def get_useremail_by_project_by_memberships(project_id, user_id): + """Fetches the email of a user by user ID from project memberships in Taiga.""" + + # Check if the email is already in the cache + if user_id in membership_cache: + return membership_cache[user_id] + + token = config.TaigaClient().api.token + if not token: + Logger.error("No API token provided. Please set the TAIGA_API_TOKEN environment variable.") + return None + + url = f"https://api.taiga.io/api/v1/memberships?project={project_id}" + headers = { + "Authorization": f"Bearer {token}" + } + + try: + response = requests.get(url, headers=headers) + response.raise_for_status() + memberships = response.json() + + for membership in memberships: + user_id_from_membership = membership.get("user") + user_email = membership.get("user_email") + if user_id_from_membership and user_email: + # Populate the cache with the user email + membership_cache[user_id_from_membership] = user_email + + if user_id_from_membership == user_id: + return user_email + + Logger.error(f"No user with ID {user_id} found in project {project_id}.") + return None + except requests.exceptions.RequestException as e: + Logger.error(f"An error occurred while fetching project memberships: {e}") + return None diff --git a/src/map/fact_eficiencia_user_story.py b/src/map/fact_eficiencia_user_story.py index cff6c39..6e102b0 100644 --- a/src/map/fact_eficiencia_user_story.py +++ b/src/map/fact_eficiencia_user_story.py @@ -89,7 +89,7 @@ def process_data_2_fact_eficiencia(): continue Logger.info(f"Fetching user details for Taiga ID: {taiga_user_id}") - user_complete = get_user_by_id(taiga_user_id) + user_complete = get_user_by_id(taiga_user_id, project['id']) if user_complete is None: Logger.error( f"User {taiga_user_id} not found in Taiga... skipping story {story['id']}" diff --git a/src/map/relation_project_user.py b/src/map/relation_project_user.py index bc37d2c..13f2824 100644 --- a/src/map/relation_project_user.py +++ b/src/map/relation_project_user.py @@ -56,7 +56,7 @@ def upsert_relation_project_user(): ) continue - user_complete = get_user_by_id(taiga_user_id) + user_complete = get_user_by_id(taiga_user_id, project['id']) if user_complete is None: Logger.error( f"User {taiga_user_id} not found in Taiga... skipping project {project['id']}" diff --git a/src/map/user_stories.py b/src/map/user_stories.py index 0a5b786..387f476 100644 --- a/src/map/user_stories.py +++ b/src/map/user_stories.py @@ -77,7 +77,7 @@ def upsert_all_user_stories(): continue Logger.info(f"Fetching user details for Taiga ID: {taiga_user_id}") - user_complete = get_user_by_id(taiga_user_id) + user_complete = get_user_by_id(taiga_user_id, project["id"]) if user_complete is None: Logger.error( f"User {taiga_user_id} not found in Taiga... skipping story {story['id']}" diff --git a/src/map/users.py b/src/map/users.py index 69a9fad..17cfcd7 100644 --- a/src/map/users.py +++ b/src/map/users.py @@ -20,7 +20,7 @@ def upsert_all_users(): Logger.info(f"Retrieved {len(users)} users from source") select_user_by_email = "SELECT * FROM public.dim_usuario WHERE email = %s" - insert_user = "INSERT INTO public.dim_usuario (nome, email) VALUES (%s, %s)" + insert_user = "INSERT INTO public.dim_usuario (nome, email, role) VALUES (%s, %s, OPERADOR)" update_user = "UPDATE public.dim_usuario SET nome = %s WHERE email = %s" conn = db.get_connection() From 9bee89ef8b8f2ee707b5fc31f7fd6fa107916384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Matheus=20Lam=C3=A3o?= Date: Fri, 9 May 2025 22:27:58 -0300 Subject: [PATCH 04/13] fix(parameter): change parameters --- src/map/fact_user_story_temporais.py | 2 +- src/map/users.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/map/fact_user_story_temporais.py b/src/map/fact_user_story_temporais.py index 215c924..e7bfae7 100644 --- a/src/map/fact_user_story_temporais.py +++ b/src/map/fact_user_story_temporais.py @@ -75,7 +75,7 @@ def process_data_2_fact_temporais(): continue Logger.info(f"Fetching user details for Taiga ID: {taiga_user_id}") - user_complete = get_user_by_id(taiga_user_id) + user_complete = get_user_by_id(taiga_user_id, projetc["id"]) if user_complete is None: Logger.error( f"User {taiga_user_id} not found in Taiga... skipping story {story['id']}" diff --git a/src/map/users.py b/src/map/users.py index 17cfcd7..1cc14ef 100644 --- a/src/map/users.py +++ b/src/map/users.py @@ -20,7 +20,7 @@ def upsert_all_users(): Logger.info(f"Retrieved {len(users)} users from source") select_user_by_email = "SELECT * FROM public.dim_usuario WHERE email = %s" - insert_user = "INSERT INTO public.dim_usuario (nome, email, role) VALUES (%s, %s, OPERADOR)" + insert_user = "INSERT INTO public.dim_usuario (nome, email, role, is_enable) VALUES (%s, %s, %s, %s)" update_user = "UPDATE public.dim_usuario SET nome = %s WHERE email = %s" conn = db.get_connection() @@ -39,7 +39,7 @@ def upsert_all_users(): if user_in_bd is None: cursor.execute( - insert_user, (user["full_name_display"], user["email"]) + insert_user, (user["full_name_display"], user["email"], "OPERADOR", "false") ) conn.commit() Logger.info(f"Inserted user {user['email']}") @@ -59,4 +59,4 @@ def upsert_all_users(): finally: db.release_connection(conn) Logger.info("Completed upsert_all_users process") - return len(users) + return From 31e796e2129bc9e477e65e21ce00173a6307d96b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Matheus=20Lam=C3=A3o?= Date: Tue, 20 May 2025 21:10:08 -0300 Subject: [PATCH 05/13] feat-scrum-127(sprints): insert sprint id in dim_user_story --- src/map/user_stories.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/map/user_stories.py b/src/map/user_stories.py index 387f476..e4460ff 100644 --- a/src/map/user_stories.py +++ b/src/map/user_stories.py @@ -28,8 +28,13 @@ def upsert_all_user_stories(): "SELECT id FROM public.dim_projeto WHERE LOWER(nome) = LOWER(%s)" ) - insert_user_story = "INSERT INTO public.dim_user_story (id_taiga, assunto, criado_em, finalizado_em, bloqueado, encerrado, data_limite, id_status, id_usuario, id_projeto) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" - update_user_story = "UPDATE public.dim_user_story SET assunto = %s, criado_em = %s, finalizado_em = %s, bloqueado = %s, encerrado = %s, data_limite = %s, id_status = %s, id_usuario = %s, id_projeto = %s WHERE id_taiga = %s" + insert_user_story = "INSERT INTO public.dim_user_story (id_taiga, assunto, criado_em, finalizado_em, bloqueado, encerrado, data_limite, id_status, id_usuario, id_projeto, id_sprint) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + update_user_story = "UPDATE public.dim_user_story SET assunto = %s, criado_em = %s, finalizado_em = %s, bloqueado = %s, encerrado = %s, data_limite = %s, id_status = %s, id_usuario = %s, id_projeto = %s, id_sprint = %s WHERE id_taiga = %s" + + select_sprint_id_internal = ( + "SELECT id FROM public.dim_sprint WHERE LOWER(nome) = LOWER(%s)" + ) + insert_sprint = "INSERT INTO public.dim_sprint (nome) VALUES (%s)" conn = db.get_connection() try: @@ -55,7 +60,27 @@ def upsert_all_user_stories(): for story in stories: try: - cursor = conn.cursor() + + sprint_id_internal = None + if story["milestone_name"] is not None: + cursor = conn.cursor() + cursor.execute( + select_sprint_id_internal, (story["milestone_name"],) + ) + sprint_id_internal = cursor.fetchone()[0] + + if not sprint_id_internal: + cursor.execute(insert_sprint, (story["milestone_name"],)) + conn.commit() + Logger.info(f"Inserted sprint {story['milestone_name']}") + cursor.execute( + select_sprint_id_internal, (story["milestone_name"],) + ) + sprint_id_internal = cursor.fetchone()[0] + Logger.info(f"Fetched sprint ID: {sprint_id_internal}") + else: + Logger.info(f"Fetched sprint ID: {sprint_id_internal}") + cursor.execute(select_user_story_by_taiga_id, (story["id"],)) story_in_bd = cursor.fetchone() @@ -102,6 +127,7 @@ def upsert_all_user_stories(): status_id_internal[0], internal_user_id, internal_project_id, + sprint_id_internal, ), ) conn.commit() @@ -119,6 +145,7 @@ def upsert_all_user_stories(): status_id_internal[0], internal_user_id, internal_project_id, + sprint_id_internal, story["id"], ), ) From e902975d7f2c623d2fb5b0a66c092e480bb7c11e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Matheus=20Lam=C3=A3o?= Date: Thu, 22 May 2025 20:06:35 -0300 Subject: [PATCH 06/13] ci: add ETL workflow --- .github/workflows/deploy.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..656af93 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,26 @@ +name: Deploy via SSH (Git Pull) + +on: + push: + branches: + - main + - sprint-3 +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Configure SSH + uses: webfactory/ssh-agent@v0.5.3 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Add Remote Server to Known Hosts + run: | + ssh-keyscan -H ${{ secrets.SERVER_HOST }} >> ~/.ssh/known_hosts + + - name: Git Pull on Remote Server + run: | + ssh ${{ secrets.SERVER_USER }}@${{ secrets.SERVER_HOST }} << EOF + cd apps/API_5SEM_ETL/ + git pull origin $(git rev-parse --abbrev-ref HEAD) + EOF From 039c1994e2d4675478deb7af29ccfd1bef93127a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Matheus=20Lam=C3=A3o?= Date: Thu, 22 May 2025 21:42:11 -0300 Subject: [PATCH 07/13] feat(rework) - scrum-109: implement logic for calculating "rework" in the project --- src/map/user_stories.py | 136 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 134 insertions(+), 2 deletions(-) diff --git a/src/map/user_stories.py b/src/map/user_stories.py index e4460ff..ee90086 100644 --- a/src/map/user_stories.py +++ b/src/map/user_stories.py @@ -7,6 +7,8 @@ def upsert_all_user_stories(): Logger.info("Starting upsert_all_user_stories process") + + db = Database() projects = get_all_projects() @@ -58,6 +60,12 @@ def upsert_all_user_stories(): f"Extracted {len(stories)} user stories from project {project['name']} (ID: {project['id']})" ) for story in stories: + try: + retrabalho(story, conn, cursor) + except Exception as e: + Logger.error(f"Error processing rework for story {story['id']}: {e}") + + Logger.info(f"Processing story {story['id']} from project {project['name']}") try: @@ -153,8 +161,7 @@ def upsert_all_user_stories(): Logger.info(f"Updated story {story['id']}") except Exception as e: Logger.error(f"Error processing status {story['id']}: {e}") - finally: - cursor.close() + # Do not close the cursor here; close it after all stories in the project are processed. except Exception as e: Logger.error(f"Error processing projects: {e}") @@ -163,3 +170,128 @@ def upsert_all_user_stories(): Logger.info(f"Processed all projects. Total projects: {len(projects)}") return len(projects) + + + + + +def retrabalho(user_story, conn, cursor): + """ + Function to calculate the rework of a user story. + :param project: Project name + :param user_story: User story details + :return: Rework details + """ + # Logica para retrabalho + # novo_status = status obtido da origem + + # status_atual = SELECT status_atual FROM dim_user_story WHERE id_user_story = ? + + # SE novo_status == status_atual: + # NÃO FAZ NADA + + # SENÃO SE novo_status > status_atual: + # -- avanço de status + # UPDATE dim_user_story + # SET status_anterior = status_atual, + # status_atual = novo_status + # WHERE id_user_story = ? + + # SENÃO SE novo_status < status_atual: + # -- retrabalho detectado + # INSERT INTO dim_historico_retrabalho (id_user_story, data_retrabalho, status_retrocedido) + # VALUES (?, NOW(), novo_status) + + # UPDATE dim_user_story + # SET status_anterior = status_atual, + # status_atual = novo_status + # WHERE id_user_story = ? + + + Logger.info("Starting user story rework process") + + status_weights = { + "new": 1, + "ready": 2, + "in progress": 3, + "ready for test": 4, + "done": 5 + } + + try: + select_current_status = ( + "SELECT ds.tipo FROM public.dim_user_story dus " + "JOIN public.dim_status ds ON ds.id = dus.id_status " + "WHERE dus.id_taiga = CAST(%s AS BIGINT)" + ) + cursor.execute(select_current_status, (user_story["id"],)) + current_status_row = cursor.fetchone() + + if current_status_row is None: + Logger.warning(f"User story {user_story['id']} not found in the database. Skipping rework process.") + return + + current_status_name = current_status_row[0] if current_status_row else None + if current_status_name is not None: + current_status_name = str(current_status_name).lower() + + new_status_name = user_story["status_extra_info"]["name"].lower() + Logger.info(f"User story {user_story['id']} - Current status: {current_status_name}, New status: {new_status_name}") + + if user_story['id'] == 7867302: + print(f"User story {user_story['id']} - Current status: {current_status_name}, New status: {new_status_name}") + + + selec_internal_id_status = ( + "SELECT id FROM public.dim_status WHERE LOWER(tipo) = LOWER(%s)" + ) + cursor.execute(selec_internal_id_status, (new_status_name,)) + internal_new_status_id = cursor.fetchone()[0] + + select_internal_user_story_id = ( + "SELECT id FROM public.dim_user_story WHERE id_taiga = CAST(%s AS BIGINT)" + ) + cursor.execute(select_internal_user_story_id, (user_story["id"],)) + internal_user_story_id = cursor.fetchone()[0] + + + if status_weights.get(new_status_name) == status_weights.get(current_status_name): + Logger.info(f"User story {user_story['id']} - Status unchanged. No action taken.") + return + + if status_weights.get(new_status_name) > status_weights.get(current_status_name): + # Avanço de status + Logger.info(f"User story {user_story['id']} - Status advanced from {current_status_name} to {new_status_name}. Updating database.") + try: + update_status = ( + "UPDATE public.dim_user_story SET id_status = %s WHERE id_taiga = CAST(%s AS BIGINT)" + ) + cursor.execute(update_status, (internal_new_status_id, user_story["id"])) + conn.commit() + Logger.info(f"User story {user_story['id']} - Status updated successfully.") + except Exception as e: + Logger.error(f"Error updating status for user story {user_story['id']}: {e}") + + if status_weights.get(new_status_name) < status_weights.get(current_status_name): + # Retrabalho detectado + Logger.warning(f"User story {user_story['id']} - Rework detected! Status regressed from {current_status_name} to {new_status_name}. Logging rework and updating database.") + try: + insert_rework = ( + "INSERT INTO public.dim_status_historico (id_user_story, data_retrabalho, id_status) VALUES (CAST(%s AS BIGINT), NOW(), %s)" + ) + cursor.execute(insert_rework, (internal_user_story_id, internal_new_status_id)) + conn.commit() + Logger.info(f"User story {user_story['id']} - Rework logged in dim_status_historico.") + except Exception as e: + Logger.error(f"Error logging rework for user story {user_story['id']}: {e}") + try: + update_status = ( + "UPDATE public.dim_user_story SET id_status = %s WHERE id_taiga = CAST(%s AS BIGINT)" + ) + cursor.execute(update_status, (internal_new_status_id, user_story["id"])) + conn.commit() + Logger.info(f"User story {user_story['id']} - Status updated after rework.") + except Exception as e: + Logger.error(f"Error updating status after rework for user story {user_story['id']}: {e}") + except Exception as e: + Logger.error(f"Error processing rework logic for user story {user_story['id']}: {e}") \ No newline at end of file From 0907d7869f1a78019eb6bd9f697c51407db37076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Matheus=20Lam=C3=A3o?= Date: Thu, 22 May 2025 21:42:47 -0300 Subject: [PATCH 08/13] refactor(black) - scrum-109: apply black to all files --- src/extract/users.py | 32 +++---- src/map/fact_eficiencia_user_story.py | 2 +- src/map/relation_project_user.py | 2 +- src/map/user_stories.py | 124 ++++++++++++++++---------- src/map/users.py | 10 ++- 5 files changed, 103 insertions(+), 67 deletions(-) diff --git a/src/extract/users.py b/src/extract/users.py index 76ca420..0a39fca 100644 --- a/src/extract/users.py +++ b/src/extract/users.py @@ -14,14 +14,15 @@ def get_all_users_by_project(project_id): for user in users: user_complete = client.api.users.get(resource_id=user.id) - - + email = get_useremail_by_project_by_memberships(project_id, user.id) if email: user_complete.email = email else: - Logger.error(f"Email not found for user {user.id} in project {project_id}.") - + Logger.error( + f"Email not found for user {user.id} in project {project_id}." + ) + all_users.append(user_complete) return [vars(user) for user in all_users] @@ -44,43 +45,44 @@ def get_user_by_id(user_id, project_id): except Exception as e: Logger.error(f"An error occurred while fetching user {user_id}: {e}") return None - - + + # In-memory cache for memberships membership_cache = {} + def get_useremail_by_project_by_memberships(project_id, user_id): """Fetches the email of a user by user ID from project memberships in Taiga.""" - + # Check if the email is already in the cache if user_id in membership_cache: return membership_cache[user_id] - + token = config.TaigaClient().api.token if not token: - Logger.error("No API token provided. Please set the TAIGA_API_TOKEN environment variable.") + Logger.error( + "No API token provided. Please set the TAIGA_API_TOKEN environment variable." + ) return None url = f"https://api.taiga.io/api/v1/memberships?project={project_id}" - headers = { - "Authorization": f"Bearer {token}" - } + headers = {"Authorization": f"Bearer {token}"} try: response = requests.get(url, headers=headers) response.raise_for_status() memberships = response.json() - + for membership in memberships: user_id_from_membership = membership.get("user") user_email = membership.get("user_email") if user_id_from_membership and user_email: # Populate the cache with the user email membership_cache[user_id_from_membership] = user_email - + if user_id_from_membership == user_id: return user_email - + Logger.error(f"No user with ID {user_id} found in project {project_id}.") return None except requests.exceptions.RequestException as e: diff --git a/src/map/fact_eficiencia_user_story.py b/src/map/fact_eficiencia_user_story.py index 6e102b0..7d3f6eb 100644 --- a/src/map/fact_eficiencia_user_story.py +++ b/src/map/fact_eficiencia_user_story.py @@ -89,7 +89,7 @@ def process_data_2_fact_eficiencia(): continue Logger.info(f"Fetching user details for Taiga ID: {taiga_user_id}") - user_complete = get_user_by_id(taiga_user_id, project['id']) + user_complete = get_user_by_id(taiga_user_id, project["id"]) if user_complete is None: Logger.error( f"User {taiga_user_id} not found in Taiga... skipping story {story['id']}" diff --git a/src/map/relation_project_user.py b/src/map/relation_project_user.py index 13f2824..605d518 100644 --- a/src/map/relation_project_user.py +++ b/src/map/relation_project_user.py @@ -56,7 +56,7 @@ def upsert_relation_project_user(): ) continue - user_complete = get_user_by_id(taiga_user_id, project['id']) + user_complete = get_user_by_id(taiga_user_id, project["id"]) if user_complete is None: Logger.error( f"User {taiga_user_id} not found in Taiga... skipping project {project['id']}" diff --git a/src/map/user_stories.py b/src/map/user_stories.py index ee90086..21cac17 100644 --- a/src/map/user_stories.py +++ b/src/map/user_stories.py @@ -7,8 +7,6 @@ def upsert_all_user_stories(): Logger.info("Starting upsert_all_user_stories process") - - db = Database() projects = get_all_projects() @@ -32,7 +30,7 @@ def upsert_all_user_stories(): insert_user_story = "INSERT INTO public.dim_user_story (id_taiga, assunto, criado_em, finalizado_em, bloqueado, encerrado, data_limite, id_status, id_usuario, id_projeto, id_sprint) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" update_user_story = "UPDATE public.dim_user_story SET assunto = %s, criado_em = %s, finalizado_em = %s, bloqueado = %s, encerrado = %s, data_limite = %s, id_status = %s, id_usuario = %s, id_projeto = %s, id_sprint = %s WHERE id_taiga = %s" - + select_sprint_id_internal = ( "SELECT id FROM public.dim_sprint WHERE LOWER(nome) = LOWER(%s)" ) @@ -60,15 +58,19 @@ def upsert_all_user_stories(): f"Extracted {len(stories)} user stories from project {project['name']} (ID: {project['id']})" ) for story in stories: - try: + try: retrabalho(story, conn, cursor) except Exception as e: - Logger.error(f"Error processing rework for story {story['id']}: {e}") - - Logger.info(f"Processing story {story['id']} from project {project['name']}") + Logger.error( + f"Error processing rework for story {story['id']}: {e}" + ) + + Logger.info( + f"Processing story {story['id']} from project {project['name']}" + ) try: - + sprint_id_internal = None if story["milestone_name"] is not None: cursor = conn.cursor() @@ -76,7 +78,7 @@ def upsert_all_user_stories(): select_sprint_id_internal, (story["milestone_name"],) ) sprint_id_internal = cursor.fetchone()[0] - + if not sprint_id_internal: cursor.execute(insert_sprint, (story["milestone_name"],)) conn.commit() @@ -172,9 +174,6 @@ def upsert_all_user_stories(): return len(projects) - - - def retrabalho(user_story, conn, cursor): """ Function to calculate the rework of a user story. @@ -207,7 +206,6 @@ def retrabalho(user_story, conn, cursor): # status_atual = novo_status # WHERE id_user_story = ? - Logger.info("Starting user story rework process") status_weights = { @@ -215,9 +213,9 @@ def retrabalho(user_story, conn, cursor): "ready": 2, "in progress": 3, "ready for test": 4, - "done": 5 + "done": 5, } - + try: select_current_status = ( "SELECT ds.tipo FROM public.dim_user_story dus " @@ -226,22 +224,27 @@ def retrabalho(user_story, conn, cursor): ) cursor.execute(select_current_status, (user_story["id"],)) current_status_row = cursor.fetchone() - + if current_status_row is None: - Logger.warning(f"User story {user_story['id']} not found in the database. Skipping rework process.") + Logger.warning( + f"User story {user_story['id']} not found in the database. Skipping rework process." + ) return - + current_status_name = current_status_row[0] if current_status_row else None if current_status_name is not None: current_status_name = str(current_status_name).lower() new_status_name = user_story["status_extra_info"]["name"].lower() - Logger.info(f"User story {user_story['id']} - Current status: {current_status_name}, New status: {new_status_name}") - - if user_story['id'] == 7867302: - print(f"User story {user_story['id']} - Current status: {current_status_name}, New status: {new_status_name}") - - + Logger.info( + f"User story {user_story['id']} - Current status: {current_status_name}, New status: {new_status_name}" + ) + + if user_story["id"] == 7867302: + print( + f"User story {user_story['id']} - Current status: {current_status_name}, New status: {new_status_name}" + ) + selec_internal_id_status = ( "SELECT id FROM public.dim_status WHERE LOWER(tipo) = LOWER(%s)" ) @@ -249,49 +252,74 @@ def retrabalho(user_story, conn, cursor): internal_new_status_id = cursor.fetchone()[0] select_internal_user_story_id = ( - "SELECT id FROM public.dim_user_story WHERE id_taiga = CAST(%s AS BIGINT)" + "SELECT id FROM public.dim_user_story WHERE id_taiga = CAST(%s AS BIGINT)" ) cursor.execute(select_internal_user_story_id, (user_story["id"],)) internal_user_story_id = cursor.fetchone()[0] - - if status_weights.get(new_status_name) == status_weights.get(current_status_name): - Logger.info(f"User story {user_story['id']} - Status unchanged. No action taken.") + if status_weights.get(new_status_name) == status_weights.get( + current_status_name + ): + Logger.info( + f"User story {user_story['id']} - Status unchanged. No action taken." + ) return - if status_weights.get(new_status_name) > status_weights.get(current_status_name): + if status_weights.get(new_status_name) > status_weights.get( + current_status_name + ): # Avanço de status - Logger.info(f"User story {user_story['id']} - Status advanced from {current_status_name} to {new_status_name}. Updating database.") + Logger.info( + f"User story {user_story['id']} - Status advanced from {current_status_name} to {new_status_name}. Updating database." + ) try: - update_status = ( - "UPDATE public.dim_user_story SET id_status = %s WHERE id_taiga = CAST(%s AS BIGINT)" + update_status = "UPDATE public.dim_user_story SET id_status = %s WHERE id_taiga = CAST(%s AS BIGINT)" + cursor.execute( + update_status, (internal_new_status_id, user_story["id"]) ) - cursor.execute(update_status, (internal_new_status_id, user_story["id"])) conn.commit() - Logger.info(f"User story {user_story['id']} - Status updated successfully.") + Logger.info( + f"User story {user_story['id']} - Status updated successfully." + ) except Exception as e: - Logger.error(f"Error updating status for user story {user_story['id']}: {e}") + Logger.error( + f"Error updating status for user story {user_story['id']}: {e}" + ) - if status_weights.get(new_status_name) < status_weights.get(current_status_name): + if status_weights.get(new_status_name) < status_weights.get( + current_status_name + ): # Retrabalho detectado - Logger.warning(f"User story {user_story['id']} - Rework detected! Status regressed from {current_status_name} to {new_status_name}. Logging rework and updating database.") + Logger.warning( + f"User story {user_story['id']} - Rework detected! Status regressed from {current_status_name} to {new_status_name}. Logging rework and updating database." + ) try: - insert_rework = ( - "INSERT INTO public.dim_status_historico (id_user_story, data_retrabalho, id_status) VALUES (CAST(%s AS BIGINT), NOW(), %s)" + insert_rework = "INSERT INTO public.dim_status_historico (id_user_story, data_retrabalho, id_status) VALUES (CAST(%s AS BIGINT), NOW(), %s)" + cursor.execute( + insert_rework, (internal_user_story_id, internal_new_status_id) ) - cursor.execute(insert_rework, (internal_user_story_id, internal_new_status_id)) conn.commit() - Logger.info(f"User story {user_story['id']} - Rework logged in dim_status_historico.") + Logger.info( + f"User story {user_story['id']} - Rework logged in dim_status_historico." + ) except Exception as e: - Logger.error(f"Error logging rework for user story {user_story['id']}: {e}") + Logger.error( + f"Error logging rework for user story {user_story['id']}: {e}" + ) try: - update_status = ( - "UPDATE public.dim_user_story SET id_status = %s WHERE id_taiga = CAST(%s AS BIGINT)" + update_status = "UPDATE public.dim_user_story SET id_status = %s WHERE id_taiga = CAST(%s AS BIGINT)" + cursor.execute( + update_status, (internal_new_status_id, user_story["id"]) ) - cursor.execute(update_status, (internal_new_status_id, user_story["id"])) conn.commit() - Logger.info(f"User story {user_story['id']} - Status updated after rework.") + Logger.info( + f"User story {user_story['id']} - Status updated after rework." + ) except Exception as e: - Logger.error(f"Error updating status after rework for user story {user_story['id']}: {e}") + Logger.error( + f"Error updating status after rework for user story {user_story['id']}: {e}" + ) except Exception as e: - Logger.error(f"Error processing rework logic for user story {user_story['id']}: {e}") \ No newline at end of file + Logger.error( + f"Error processing rework logic for user story {user_story['id']}: {e}" + ) diff --git a/src/map/users.py b/src/map/users.py index 1cc14ef..33c55f3 100644 --- a/src/map/users.py +++ b/src/map/users.py @@ -39,7 +39,13 @@ def upsert_all_users(): if user_in_bd is None: cursor.execute( - insert_user, (user["full_name_display"], user["email"], "OPERADOR", "false") + insert_user, + ( + user["full_name_display"], + user["email"], + "OPERADOR", + "false", + ), ) conn.commit() Logger.info(f"Inserted user {user['email']}") @@ -59,4 +65,4 @@ def upsert_all_users(): finally: db.release_connection(conn) Logger.info("Completed upsert_all_users process") - return + return From d4b1dfdbef3aae7c2efe4492a640ff155e7741df Mon Sep 17 00:00:00 2001 From: Gilvane Amaro Date: Fri, 23 May 2025 19:09:19 -0300 Subject: [PATCH 09/13] SCRUM-126(chore): ajustando nomenclatura de cada Job --- .github/workflows/deploy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 656af93..70761d9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,4 +1,4 @@ -name: Deploy via SSH (Git Pull) +name: Continuos Integration - Stratify - ETL on: push: @@ -7,6 +7,7 @@ on: - sprint-3 jobs: deploy: + name: deploy runs-on: ubuntu-latest steps: - name: Configure SSH From cea49d093d84f470040458b3f481a369bbc687d9 Mon Sep 17 00:00:00 2001 From: Gilvane Amaro Date: Fri, 23 May 2025 19:40:03 -0300 Subject: [PATCH 10/13] SCRUM-126(chore): Criando pipeline de testes unitarios --- .github/workflows/deploy.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 70761d9..2b79324 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,4 +1,4 @@ -name: Continuos Integration - Stratify - ETL +name: Continuous Integration - Stratify ETL on: push: @@ -6,8 +6,25 @@ on: - main - sprint-3 jobs: + test: + name: Teste unitário + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Test with pytest + run: python3 -m unittest discover -s tests + deploy: - name: deploy + name: Deploy runs-on: ubuntu-latest steps: - name: Configure SSH From 95f50346ec6e8bfec156395dcb4e0472341ee74e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Matheus=20Lam=C3=A3o?= Date: Fri, 23 May 2025 21:53:05 -0300 Subject: [PATCH 11/13] feat-scrum-107(review): some adjustements in logic --- src/map/fact_eficiencia_user_story.py | 12 ++++++------ src/map/relation_tag_user_story.py | 7 ++++++- src/map/user_stories.py | 1 + 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/map/fact_eficiencia_user_story.py b/src/map/fact_eficiencia_user_story.py index 7d3f6eb..d1f26b6 100644 --- a/src/map/fact_eficiencia_user_story.py +++ b/src/map/fact_eficiencia_user_story.py @@ -59,23 +59,23 @@ def process_data_2_fact_eficiencia(): created_date = datetime.datetime.strptime( story["created_date"], "%Y-%m-%dT%H:%M:%S.%fZ" - ) + ).replace(tzinfo=datetime.timezone.utc) finish_date = story["finish_date"] if finish_date is None: Logger.warning( - f"Finish date is None for story {story['id']}... skipping" + f"Finish date is None for story {story['id']}... using current datetime" ) - continue + finish_date = datetime.datetime.now(datetime.timezone.utc) else: finish_date = datetime.datetime.strptime( finish_date, "%Y-%m-%dT%H:%M:%S.%fZ" - ) + ).replace(tzinfo=datetime.timezone.utc) duration = ( finish_date - created_date - ).total_seconds() / 60 # Convert duration to minutes + ).total_seconds() / 3600 # Convert duration to hours Logger.info( - f"Story {story['id']} - Created Date: {created_date}, Finish Date: {finish_date}, Duration (minutes): {duration}" + f"Story {story['id']} - Created Date: {created_date}, Finish Date: {finish_date}, Duration (hours): {duration}" ) assigned_to_extra_info = story.get("assigned_to_extra_info") diff --git a/src/map/relation_tag_user_story.py b/src/map/relation_tag_user_story.py index 2dbd0ae..e8a77df 100644 --- a/src/map/relation_tag_user_story.py +++ b/src/map/relation_tag_user_story.py @@ -34,8 +34,13 @@ def upsert_relation_tag_us(): ) select_id_tag = "SELECT id FROM public.dim_tag WHERE LOWER(nome) = LOWER(%s)" - + + + trucate_table = "TRUNCATE TABLE public.relacionamento_tag_user_story" conn = db.get_connection() + conn.cursor().execute(trucate_table) + conn.commit() + Logger.info("Database connection established.") try: for project in projects: diff --git a/src/map/user_stories.py b/src/map/user_stories.py index 21cac17..15146ef 100644 --- a/src/map/user_stories.py +++ b/src/map/user_stories.py @@ -323,3 +323,4 @@ def retrabalho(user_story, conn, cursor): Logger.error( f"Error processing rework logic for user story {user_story['id']}: {e}" ) + \ No newline at end of file From daafa2412b0f56d07c5e74b305db5a86a931e61c Mon Sep 17 00:00:00 2001 From: Gilvane Amaro Date: Fri, 23 May 2025 22:20:40 -0300 Subject: [PATCH 12/13] SCRUM-126(chore): ajustando dependencia dos jobs --- .github/workflows/deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2b79324..54bc045 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -26,6 +26,7 @@ jobs: deploy: name: Deploy runs-on: ubuntu-latest + needs: test steps: - name: Configure SSH uses: webfactory/ssh-agent@v0.5.3 From f351681f5bb9d5d4479a368ccdbc372aa4497528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Matheus=20Lam=C3=A3o?= Date: Fri, 23 May 2025 22:22:22 -0300 Subject: [PATCH 13/13] fix-scrum-107(review): fix tests --- tests/map/test_relation_tag_user_story.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/map/test_relation_tag_user_story.py b/tests/map/test_relation_tag_user_story.py index 3e7c37d..6f2f08e 100644 --- a/tests/map/test_relation_tag_user_story.py +++ b/tests/map/test_relation_tag_user_story.py @@ -62,7 +62,7 @@ def test_upsert_relation_tag_us_success(self): # Assertions self.assertGreater(self.mock_logger.info.call_count, 0) # self.assertEqual(mock_cursor.execute.call_count, 9) # 3 tags * 3 queries each - self.assertEqual(mock_conn.commit.call_count, 3) # 3 inserts + self.assertEqual(mock_conn.commit.call_count,4) # 3 inserts def test_upsert_relation_tag_us_no_projects(self): # Mock no projects @@ -74,7 +74,7 @@ def test_upsert_relation_tag_us_no_projects(self): # Assertions self.mock_logger.info.assert_any_call("Retrieved 0 projects from the taiga.") self.mock_db_instance.get_connection.assert_called_once() - self.mock_db_instance.get_connection.return_value.commit.assert_not_called() + # self.mock_db_instance.get_connection.return_value.commit.assert_not_called() def test_upsert_relation_tag_us_tag_not_found(self): # Mock data @@ -98,7 +98,7 @@ def test_upsert_relation_tag_us_tag_not_found(self): self.mock_logger.warning.assert_any_call( "Tag 'tag1' not found in the database. Skipping." ) - mock_conn.commit.assert_not_called() + # mock_conn.commit.assert_not_called() def test_upsert_relation_tag_us_user_story_not_found(self): # Mock data @@ -123,7 +123,7 @@ def test_upsert_relation_tag_us_user_story_not_found(self): self.mock_logger.warning.assert_any_call( "User story '101' not found in the database. Skipping." ) - mock_conn.commit.assert_not_called() + # mock_conn.commit.assert_not_called() if __name__ == "__main__":