From 17f0956ee9a18e9cc4da737f94d90dd17815cf13 Mon Sep 17 00:00:00 2001 From: Hadi Qureshi Date: Wed, 20 May 2026 17:44:15 +0100 Subject: [PATCH 1/7] chore: remove IAM user + group --- terraform/batch/main.tf | 43 -------------------------------------- terraform/batch/secrets.tf | 15 ------------- 2 files changed, 58 deletions(-) delete mode 100644 terraform/batch/secrets.tf diff --git a/terraform/batch/main.tf b/terraform/batch/main.tf index 1112ae9..f247c04 100644 --- a/terraform/batch/main.tf +++ b/terraform/batch/main.tf @@ -70,46 +70,3 @@ resource "aws_iam_role_policy_attachment" "s3_policy_attachment" { role = split("/", module.batch_eventbridge.batch_job_role_arn)[1] policy_arn = aws_iam_policy.s3_access.arn } - -# IAM User Group -resource "aws_iam_group" "group" { - name = "${var.domain}-${var.service_subdomain}-user-group" - path = "/" -} - -# Attach S3 policy to group -resource "aws_iam_group_policy_attachment" "group_s3_access_attachment" { - group = aws_iam_group.group.name - policy_arn = aws_iam_policy.s3_access.arn -} - -# Attach Secrets Manager policy to group -resource "aws_iam_group_policy_attachment" "group_secrets_attachment" { - group = aws_iam_group.group.name - policy_arn = aws_iam_policy.secrets_access.arn -} - -# IAM User -resource "aws_iam_user" "user" { - name = "${var.domain}-${var.service_subdomain}" - path = "/" -} - -# Assign IAM User to group -resource "aws_iam_user_group_membership" "user_group_attach" { - user = aws_iam_user.user.name - - groups = [ - aws_iam_group.group.name - ] -} - -# IAM Key Rotation Module -module "iam_key_rotation" { - source = "git::https://github.com/ONS-Innovation/keh-aws-iam-key-rotation.git?ref=v0.1.1" - - iam_username = aws_iam_user.user.name - access_key_secret_arn = aws_secretsmanager_secret.access_key.arn - secret_key_secret_arn = aws_secretsmanager_secret.secret_key.arn - rotation_in_days = 45 -} diff --git a/terraform/batch/secrets.tf b/terraform/batch/secrets.tf deleted file mode 100644 index 0e5f109..0000000 --- a/terraform/batch/secrets.tf +++ /dev/null @@ -1,15 +0,0 @@ -# Secrets Manager resources for IAM user access keys - -resource "aws_secretsmanager_secret" "access_key" { - name = "${var.domain}-${var.service_subdomain}-access-key" - description = "Access Key ID for github statistics scraper IAM user" - recovery_window_in_days = 0 // Secret will be deleted immediately - force_overwrite_replica_secret = true // Allow overwriting the secret in case of changes -} - -resource "aws_secretsmanager_secret" "secret_key" { - name = "${var.domain}-${var.service_subdomain}-secret-key" - description = "Secret Access Key for github stastics scraper IAM user" - recovery_window_in_days = 0 // Secret will be deleted immediately - force_overwrite_replica_secret = true // Allow overwriting the secret in case of changes -} From b757a448a6321e3843df3ff33c75cea62d1cb1c4 Mon Sep 17 00:00:00 2001 From: Hadi <76945238+hadiqur@users.noreply.github.com> Date: Wed, 20 May 2026 19:01:58 +0100 Subject: [PATCH 2/7] chore: update batch module version --- terraform/batch/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/batch/main.tf b/terraform/batch/main.tf index f247c04..9356af2 100644 --- a/terraform/batch/main.tf +++ b/terraform/batch/main.tf @@ -11,7 +11,7 @@ resource "aws_iam_policy" "s3_access" { } module "batch_eventbridge" { - source = "git::https://github.com/ONS-Innovation/keh-scheduled-batch-tf-module.git?ref=v1.1.1" + source = "git::https://github.com/ONS-Innovation/keh-scheduled-batch-tf-module.git?ref=v1.2.0" aws_account_id = var.aws_account_id aws_access_key_id = var.aws_access_key_id From 9ca760a895f691d1a21864dae3b9966dc94c0a8c Mon Sep 17 00:00:00 2001 From: Hadi <76945238+hadiqur@users.noreply.github.com> Date: Wed, 20 May 2026 19:53:12 +0100 Subject: [PATCH 3/7] chore: update batch module version once more --- terraform/batch/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/batch/main.tf b/terraform/batch/main.tf index 9356af2..41976fb 100644 --- a/terraform/batch/main.tf +++ b/terraform/batch/main.tf @@ -11,7 +11,7 @@ resource "aws_iam_policy" "s3_access" { } module "batch_eventbridge" { - source = "git::https://github.com/ONS-Innovation/keh-scheduled-batch-tf-module.git?ref=v1.2.0" + source = "git::https://github.com/ONS-Innovation/keh-scheduled-batch-tf-module.git?ref=v1.2.1" aws_account_id = var.aws_account_id aws_access_key_id = var.aws_access_key_id From 838b2054c6c4797c18b2264436710242ac3d9c70 Mon Sep 17 00:00:00 2001 From: Hadi Qureshi Date: Wed, 20 May 2026 20:04:26 +0100 Subject: [PATCH 4/7] chore: update batch module version for the final time --- terraform/batch/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/batch/main.tf b/terraform/batch/main.tf index 41976fb..042adf3 100644 --- a/terraform/batch/main.tf +++ b/terraform/batch/main.tf @@ -11,7 +11,7 @@ resource "aws_iam_policy" "s3_access" { } module "batch_eventbridge" { - source = "git::https://github.com/ONS-Innovation/keh-scheduled-batch-tf-module.git?ref=v1.2.1" + source = "git::https://github.com/ONS-Innovation/keh-scheduled-batch-tf-module.git?ref=v1.2.2" aws_account_id = var.aws_account_id aws_access_key_id = var.aws_access_key_id From a286e7f08c3ab8f7a700b358f89856f8b8ef57e9 Mon Sep 17 00:00:00 2001 From: Hadi Qureshi Date: Thu, 21 May 2026 09:50:54 +0100 Subject: [PATCH 5/7] test commit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d55e26..d765ddc 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ There are a few `TODOs` scattered throughout the README and codebase that will n - AWS CLI - Make -## Getting started +## Getting started (test) 1. Install Dependencies: From ab52831012d3f7ed0f4235766352d9bd207e8c12 Mon Sep 17 00:00:00 2001 From: Hadi Qureshi Date: Thu, 21 May 2026 10:42:45 +0100 Subject: [PATCH 6/7] chore: update README --- README.md | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d765ddc..3511cc9 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ There are a few `TODOs` scattered throughout the README and codebase that will n - AWS CLI - Make -## Getting started (test) +## Getting started 1. Install Dependencies: @@ -46,16 +46,24 @@ There are a few `TODOs` scattered throughout the README and codebase that will n make install ``` -2. Export Environment Variables +2. Sign in with AWS SSO, and export the correct profile for this service: + + **Note:** See the Developer Onboarding Guide on the "Using AWS SSO for Local Development" page on Confluence to set up service profile selection on your local machine. + + ```bash + aws sso login + + export AWS_PROFILE=keh-github-statistics-scraper + ``` + + This allows you to assume the AWS IAM role for service, enabling the most secure development experience. This also means you will have limited permissions until you exit out of the profile. - ```bash - # AWS - export AWS_ACCESS_KEY_ID= - export AWS_SECRET_ACCESS_KEY= - export AWS_DEFAULT_REGION= - export AWS_SECRET_NAME=//github-tooling-suite/ +3. Export Environment Variables + + ```bash # GitHub + export AWS_SECRET_NAME=//github-tooling-suite/ export GITHUB_APP_CLIENT_ID= export GITHUB_ORG= @@ -73,12 +81,18 @@ There are a few `TODOs` scattered throughout the README and codebase that will n | `batch_size` | The number of repositories that will be scraped in each batch. | | `environment` | Determines where to save the results. `development`: locally, `production`: to S3. | -3. Run the script: +4. Run the script: ```bash make run ``` +5. To exit the profile: + + ```bash + unset AWS_PROFILE + ``` + ## Deployment ### Deployments with Concourse From 12fc102fe98db2aecd32c82c16240f139247e97b Mon Sep 17 00:00:00 2001 From: Hadi Qureshi Date: Thu, 21 May 2026 12:35:13 +0100 Subject: [PATCH 7/7] chore: update README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3511cc9..b06d967 100644 --- a/README.md +++ b/README.md @@ -48,15 +48,15 @@ There are a few `TODOs` scattered throughout the README and codebase that will n 2. Sign in with AWS SSO, and export the correct profile for this service: - **Note:** See the Developer Onboarding Guide on the "Using AWS SSO for Local Development" page on Confluence to set up service profile selection on your local machine. - ```bash aws sso login export AWS_PROFILE=keh-github-statistics-scraper ``` - This allows you to assume the AWS IAM role for service, enabling the most secure development experience. This also means you will have limited permissions until you exit out of the profile. + This allows you to assume the AWS IAM role for the service, enabling the most secure development experience. This also means you will have limited permissions until you exit out of the profile. + + **Note:** See the Developer Onboarding Guide on the "Using AWS SSO for Local Development" page on Confluence to set up service profile selection on your local machine. 3. Export Environment Variables