diff --git a/codebundles/aws-c7n-service-usage/.runwhen/generation-rules/aws-c7n-service-usage.yaml b/codebundles/aws-c7n-service-usage/.runwhen/generation-rules/aws-c7n-service-usage.yaml new file mode 100644 index 0000000..35e860b --- /dev/null +++ b/codebundles/aws-c7n-service-usage/.runwhen/generation-rules/aws-c7n-service-usage.yaml @@ -0,0 +1,22 @@ +apiVersion: runwhen.com/v1 +kind: GenerationRules +spec: + platform: aws + generationRules: + - resourceTypes: + - aws_ec2_security_groups + matchRules: + - type: pattern + pattern: ".+" + properties: [name] + mode: substring + slxs: + - baseName: aws-c7n-service-usage + qualifiers: ["account_id"] + baseTemplateName: aws-c7n-service-usage + levelOfDetail: basic + outputItems: + - type: slx + - type: sli + - type: runbook + templateName: aws-c7n-service-usage-taskset.yaml \ No newline at end of file diff --git a/codebundles/aws-c7n-service-usage/.runwhen/templates/aws-c7n-service-usage-sli.yaml b/codebundles/aws-c7n-service-usage/.runwhen/templates/aws-c7n-service-usage-sli.yaml new file mode 100644 index 0000000..fffd295 --- /dev/null +++ b/codebundles/aws-c7n-service-usage/.runwhen/templates/aws-c7n-service-usage-sli.yaml @@ -0,0 +1,51 @@ +apiVersion: runwhen.com/v1 +kind: ServiceLevelIndicator +metadata: + name: {{slx_name}} + labels: + {% include "common-labels.yaml" %} + annotations: + {% include "common-annotations.yaml" %} +spec: + displayUnitsLong: OK + displayUnitsShort: ok + locations: + - {{default_location}} + description: Count AWS Service Usage Exceeding defined threshold in AWS account {{match_resource.resource.account_id}} + codeBundle: + {% if repo_url %} + repoUrl: {{repo_url}} + {% else %} + repoUrl: https://github.com/runwhen-contrib/rw-c7n-codecollection.git + {% endif %} + {% if ref %} + ref: {{ref}} + {% else %} + ref: main + {% endif %} + pathToRobot: codebundles/aws-c7n-service-usage/sli.robot + intervalStrategy: intermezzo + intervalSeconds: 600 + configProvided: + - name: AWS_REGION + value: "{{match_resource.resource.region}}" + - name: AWS_ACCOUNT_ID + value: "{{match_resource.resource.account_id}}" + secretsProvided: + - name: AWS_ACCESS_KEY_ID + workspaceKey: {{custom.aws_access_key_id}} + - name: AWS_SECRET_ACCESS_KEY + workspaceKey: {{custom.aws_secret_access_key}} + alerts: + warning: + operator: '>' + threshold: '1' + for: '20m' + ticket: + operator: '>' + threshold: '1' + for: '40m' + page: + operator: '==' + threshold: '0' + for: '' diff --git a/codebundles/aws-c7n-service-usage/.runwhen/templates/aws-c7n-service-usage-slx.yaml b/codebundles/aws-c7n-service-usage/.runwhen/templates/aws-c7n-service-usage-slx.yaml new file mode 100644 index 0000000..0d5e260 --- /dev/null +++ b/codebundles/aws-c7n-service-usage/.runwhen/templates/aws-c7n-service-usage-slx.yaml @@ -0,0 +1,21 @@ +apiVersion: runwhen.com/v1 +kind: ServiceLevelX +metadata: + name: {{slx_name}} + labels: + {% include "common-labels.yaml" %} + annotations: + {% include "common-annotations.yaml" %} +spec: + imageURL: https://storage.googleapis.com/runwhen-nonprod-shared-images/icons/aws/Resource-Icons_06072024/Res_Networking-Content-Delivery/Res_Amazon-VPC_Virtual-private-cloud-VPC_48.svg + alias: AWS Service Usage Exceeding defined threshold in AWS Account {{match_resource.resource.account_id}} + asMeasuredBy: The number of AWS Service Usage Exceeding defined threshold in AWS account {{match_resource.resource.account_id}} + configProvided: + - name: SLX_PLACEHOLDER + value: SLX_PLACEHOLDER + owners: + - {{workspace.owner_email}} + statement: List AWS Service Usage Exceeding defined threshold in the AWS account {{match_resource.resource.account_id}} + additionalContext: + region: "{{match_resource.resource.region}}" + account_id: "{{match_resource.resource.account_id}}" \ No newline at end of file diff --git a/codebundles/aws-c7n-service-usage/.runwhen/templates/aws-c7n-service-usage-taskset.yaml b/codebundles/aws-c7n-service-usage/.runwhen/templates/aws-c7n-service-usage-taskset.yaml new file mode 100644 index 0000000..15935d5 --- /dev/null +++ b/codebundles/aws-c7n-service-usage/.runwhen/templates/aws-c7n-service-usage-taskset.yaml @@ -0,0 +1,33 @@ +apiVersion: runwhen.com/v1 +kind: Runbook +metadata: + name: {{slx_name}} + labels: + {% include "common-labels.yaml" %} + annotations: + {% include "common-annotations.yaml" %} +spec: + location: {{default_location}} + description: List AWS Service Usage Exceeding defined threshold in the AWS account {{match_resource.resource.account_id}} + codeBundle: + {% if repo_url %} + repoUrl: {{repo_url}} + {% else %} + repoUrl: https://github.com/runwhen-contrib/rw-c7n-codecollection.git + {% endif %} + {% if ref %} + ref: {{ref}} + {% else %} + ref: main + {% endif %} + pathToRobot: codebundles/aws-c7n-service-usage/runbook.robot + configProvided: + - name: AWS_REGION + value: "{{match_resource.resource.region}}" + - name: AWS_ACCOUNT_ID + value: "{{match_resource.resource.account_id}}" + secretsProvided: + - name: AWS_ACCESS_KEY_ID + workspaceKey: {{custom.aws_access_key_id}} + - name: AWS_SECRET_ACCESS_KEY + workspaceKey: {{custom.aws_secret_access_key}} diff --git a/codebundles/aws-c7n-service-usage/.test/README.md b/codebundles/aws-c7n-service-usage/.test/README.md new file mode 100644 index 0000000..a568c3f --- /dev/null +++ b/codebundles/aws-c7n-service-usage/.test/README.md @@ -0,0 +1,91 @@ +### How to test this codebundle? + +#### IAM User Configuration + +We create two distinct AWS IAM users with carefully scoped access: + +**CloudCustodian IAM User** + +Purpose: Service Level Indicator (SLI) monitoring and runbook automation and configured with least privilege access principles + +with the following policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "tag:GetResources", + "ec2:DescribeRegions", + "ec2:DescribeSecurityGroups", + "servicequotas:ListServices", + "servicequotas:ListAWSDefaultServiceQuotas", + "servicequotas:ListServiceQuotas", + "cloudwatch:GetMetricStatistics" + ], + "Resource": "*" + } + ] +} +``` + +**Infrastructure Deployment User** + +Purpose: Cloud infrastructure provisioning and management using Terraform + +#### Credential Setup + +Navigate to the `.test/terraform` directory and configure two secret files for authentication: + +`cb.secret` - CloudCustodian and RunWhen Credentials + +Create this file with the following environment variables: + + ```sh + export RW_PAT="" + export RW_WORKSPACE="" + export RW_API_URL="papi.beta.runwhen.com" + + export AWS_DEFAULT_REGION="us-west-2" + export AWS_ACCESS_KEY_ID="" + export AWS_SECRET_ACCESS_KEY="" + export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) + ``` + + +`tf.secret` - Terraform Deployment Credentials + +Create this file with the following environment variables: + + ```sh + export AWS_DEFAULT_REGION="" + export AWS_ACCESS_KEY_ID="" + export AWS_SECRET_ACCESS_KEY="" + export AWS_SESSION_TOKEN="" # Optional: Include if using temporary credentials + ``` + +#### Testing Workflow + +1. Build test infra: + **Note** WIP + +2. Generate RunWhen Configurations + ```sh + tasks + ``` + +3. Upload generated SLx to RunWhen Platform + + ```sh + task upload-slxs + ``` + +4. At last, after testing, clean up the test infrastructure. + +```sh + task clean +``` + diff --git a/codebundles/aws-c7n-service-usage/.test/Taskfile.yaml b/codebundles/aws-c7n-service-usage/.test/Taskfile.yaml new file mode 100644 index 0000000..4319299 --- /dev/null +++ b/codebundles/aws-c7n-service-usage/.test/Taskfile.yaml @@ -0,0 +1,352 @@ +version: "3" + +tasks: + default: + desc: "Generate workspaceInfo and rebuild/test" + cmds: + - task: check-unpushed-commits + - task: generate-rwl-config + - task: run-rwl-discovery + + clean: + desc: "Run cleanup tasks" + cmds: + - task: check-and-cleanup-terraform + # - task: clean-rwl-discovery + + build-infra: + desc: "Build test infrastructure" + cmds: + - task: build-terraform-infra + + check-unpushed-commits: + desc: Check if outstanding commits or file updates need to be pushed before testing. + vars: + # Specify the base directory relative to your Taskfile location + BASE_DIR: "../" + cmds: + - | + echo "Checking for uncommitted changes in $BASE_DIR and $BASE_DIR.runwhen, excluding '.test'..." + UNCOMMITTED_FILES=$(git diff --name-only HEAD | grep -E "^${BASE_DIR}(\.runwhen|[^/]+)" | grep -v "/\.test/" || true) + if [ -n "$UNCOMMITTED_FILES" ]; then + echo "✗" + echo "Uncommitted changes found:" + echo "$UNCOMMITTED_FILES" + echo "Remember to commit & push changes before executing the `run-rwl-discovery` task." + echo "------------" + exit 1 + else + echo "√" + echo "No uncommitted changes in specified directories." + echo "------------" + fi + - | + echo "Checking for unpushed commits in $BASE_DIR and $BASE_DIR.runwhen, excluding '.test'..." + git fetch origin + UNPUSHED_FILES=$(git diff --name-only origin/$(git rev-parse --abbrev-ref HEAD) HEAD | grep -E "^${BASE_DIR}(\.runwhen|[^/]+)" | grep -v "/\.test/" || true) + if [ -n "$UNPUSHED_FILES" ]; then + echo "✗" + echo "Unpushed commits found:" + echo "$UNPUSHED_FILES" + echo "Remember to push changes before executing the `run-rwl-discovery` task." + echo "------------" + exit 1 + else + echo "√" + echo "No unpushed commits in specified directories." + echo "------------" + fi + silent: true + + generate-rwl-config: + desc: "Generate RunWhen Local configuration (workspaceInfo.yaml)" + env: + AWS_ACCESS_KEY_ID: "{{.AWS_ACCESS_KEY_ID}}" + AWS_SECRET_ACCESS_KEY: "{{.AWS_SECRET_ACCESS_KEY}}" + AWS_DEFAULT_REGION: "{{.AWS_DEFAULT_REGION}}" + RW_WORKSPACE: '{{.RW_WORKSPACE | default "my-workspace"}}' + cmds: + - | + source terraform/cb.secret + repo_url=$(git config --get remote.origin.url) + branch_name=$(git rev-parse --abbrev-ref HEAD) + codebundle=$(basename "$(dirname "$PWD")") + + # Fetch individual cluster details from Terraform state + # pushd terraform > /dev/null + # cluster1_name=$(terraform show -json terraform.tfstate | jq -r ' + # .values.outputs.cluster_1_name.value') + + # popd > /dev/null + + # # Check if any of the required cluster variables are empty + # if [ -z "$cluster1_name" ] || [ -z "$cluster1_server" ] || [ -z "$cluster1_resource_group" ]; then + # echo "Error: Missing cluster details. Ensure Terraform plan has been applied." + # exit 1 + # fi + + # Generate workspaceInfo.yaml with fetched cluster details + cat < workspaceInfo.yaml + workspaceName: "$RW_WORKSPACE" + workspaceOwnerEmail: authors@runwhen.com + defaultLocation: location-01-us-west1 + defaultLOD: detailed + cloudConfig: + aws: + awsAccessKeyId: "$AWS_ACCESS_KEY_ID" + awsSecretAccessKey: "$AWS_SECRET_ACCESS_KEY" + codeCollections: + - repoURL: "$repo_url" + branch: "$branch_name" + codeBundles: ["$codebundle"] + custom: + aws_access_key_id: AWS_ACCESS_KEY_ID + aws_secret_access_key: AWS_SECRET_ACCESS_KEY + EOF + silent: true + + run-rwl-discovery: + desc: "Run RunWhen Local Discovery on test infrastructure" + cmds: + - | + CONTAINER_NAME="RunWhenLocal" + if docker ps -q --filter "name=$CONTAINER_NAME" | grep -q .; then + echo "Stopping and removing existing container $CONTAINER_NAME..." + docker stop $CONTAINER_NAME && docker rm $CONTAINER_NAME + elif docker ps -a -q --filter "name=$CONTAINER_NAME" | grep -q .; then + echo "Removing existing stopped container $CONTAINER_NAME..." + docker rm $CONTAINER_NAME + else + echo "No existing container named $CONTAINER_NAME found." + fi + + echo "Cleaning up output directory..." + sudo rm -rf output || { echo "Failed to remove output directory"; exit 1; } + mkdir output && chmod 777 output || { echo "Failed to set permissions"; exit 1; } + + echo "Starting new container $CONTAINER_NAME..." + + docker run --name $CONTAINER_NAME -e DEBUG_LOGGING=true -p 8081:8081 -v "$(pwd)":/shared -d ghcr.io/runwhen-contrib/runwhen-local:latest || { + echo "Failed to start container"; exit 1; + } + + echo "Running workspace builder script in container..." + docker exec -w /workspace-builder $CONTAINER_NAME ./run.sh $1 --verbose || { + echo "Error executing script in container"; exit 1; + } + + echo "Review generated config files under output/workspaces/" + silent: true + + check-terraform-infra: + desc: "Check if Terraform has any deployed infrastructure in the terraform subdirectory" + cmds: + - | + # Source Envs for Auth + source terraform/tf.secret + + # Navigate to the Terraform directory + if [ ! -d "terraform" ]; then + echo "Terraform directory not found." + exit 1 + fi + cd terraform + + # Check if Terraform state file exists + if [ ! -f "terraform.tfstate" ]; then + echo "No Terraform state file found in the terraform directory. No infrastructure is deployed." + exit 0 + fi + + # List resources in Terraform state + resources=$(terraform state list) + + # Check if any resources are listed in the state file + if [ -n "$resources" ]; then + echo "Deployed infrastructure detected." + echo "$resources" + exit 0 + else + echo "No deployed infrastructure found in Terraform state." + exit 0 + fi + silent: true + + build-terraform-infra: + desc: "Run terraform apply" + cmds: + - | + # Source Envs for Auth + source terraform/tf.secret + + + # Navigate to the Terraform directory + if [ -d "terraform" ]; then + cd terraform + else + echo "Terraform directory not found. Terraform apply aborted." + exit 1 + fi + task format-and-init-terraform + echo "Starting Terraform Build of Terraform infrastructure..." + terraform apply -auto-approve || { + echo "Failed to clean up Terraform infrastructure." + exit 1 + } + echo "Terraform infrastructure build completed." + silent: true + + check-rwp-config: + desc: Check if env vars are set for RunWhen Platform + cmds: + - | + source terraform/cb.secret + missing_vars=() + + if [ -z "$RW_WORKSPACE" ]; then + missing_vars+=("RW_WORKSPACE") + fi + + if [ -z "$RW_API_URL" ]; then + missing_vars+=("RW_API_URL") + fi + + if [ -z "$RW_PAT" ]; then + missing_vars+=("RW_PAT") + fi + + if [ ${#missing_vars[@]} -ne 0 ]; then + echo "The following required environment variables are missing: ${missing_vars[*]}" + exit 1 + fi + silent: true + + upload-slxs: + desc: "Upload SLX files to the appropriate URL" + env: + RW_WORKSPACE: "{{.RW_WORKSPACE}}" + RW_API_URL: "{{.RW_API}}" + RW_PAT: "{{.RW_PAT}}" + cmds: + - task: check-rwp-config + - | + source terraform/cb.secret + BASE_DIR="output/workspaces/${RW_WORKSPACE}/slxs" + if [ ! -d "$BASE_DIR" ]; then + echo "Directory $BASE_DIR does not exist. Upload aborted." + exit 1 + fi + + for dir in "$BASE_DIR"/*; do + if [ -d "$dir" ]; then + SLX_NAME=$(basename "$dir") + PAYLOAD=$(jq -n --arg commitMsg "Creating new SLX $SLX_NAME" '{ commitMsg: $commitMsg, files: {} }') + for file in slx.yaml runbook.yaml sli.yaml; do + if [ -f "$dir/$file" ]; then + CONTENT=$(cat "$dir/$file") + PAYLOAD=$(echo "$PAYLOAD" | jq --arg fileContent "$CONTENT" --arg fileName "$file" '.files[$fileName] = $fileContent') + fi + done + + URL="https://${RW_API_URL}/api/v3/workspaces/${RW_WORKSPACE}/branches/main/slxs/${SLX_NAME}" + echo "Uploading SLX: $SLX_NAME to $URL" + response_code=$(curl -X POST "$URL" \ + -H "Authorization: Bearer $RW_PAT" \ + -H "Content-Type: application/json" \ + -d "$PAYLOAD" \ + -w "%{http_code}" -o /dev/null -s) + + if [[ "$response_code" == "200" || "$response_code" == "201" ]]; then + echo "Successfully uploaded SLX: $SLX_NAME to $URL" + elif [[ "$response_code" == "405" ]]; then + echo "Failed to upload SLX: $SLX_NAME to $URL. Method not allowed (405)." + else + echo "Failed to upload SLX: $SLX_NAME to $URL. Unexpected response code: $response_code" + fi + fi + done + silent: true + + delete-slxs: + desc: "Delete SLX objects from the appropriate URL" + env: + RW_WORKSPACE: '{{.RW_WORKSPACE | default "my-workspace"}}' + RW_API_URL: "{{.RW_API}}" + RW_PAT: "{{.RW_PAT}}" + cmds: + - task: check-rwp-config + - | + source terraform/cb.secret + BASE_DIR="output/workspaces/${RW_WORKSPACE}/slxs" + if [ ! -d "$BASE_DIR" ]; then + echo "Directory $BASE_DIR does not exist. Deletion aborted." + exit 1 + fi + + for dir in "$BASE_DIR"/*; do + if [ -d "$dir" ]; then + SLX_NAME=$(basename "$dir") + URL="https://${RW_API_URL}/api/v3/workspaces/${RW_WORKSPACE}/branches/main/slxs/${SLX_NAME}" + echo "Deleting SLX: $SLX_NAME from $URL" + response_code=$(curl -X DELETE "$URL" \ + -H "Authorization: Bearer $RW_PAT" \ + -H "Content-Type: application/json" \ + -w "%{http_code}" -o /dev/null -s) + + if [[ "$response_code" == "200" || "$response_code" == "204" ]]; then + echo "Successfully deleted SLX: $SLX_NAME from $URL" + elif [[ "$response_code" == "405" ]]; then + echo "Failed to delete SLX: $SLX_NAME from $URL. Method not allowed (405)." + else + echo "Failed to delete SLX: $SLX_NAME from $URL. Unexpected response code: $response_code" + fi + fi + done + silent: true + + cleanup-terraform-infra: + desc: "Cleanup deployed Terraform infrastructure" + cmds: + - | + # Source Envs for Auth + source terraform/tf.secret + + # Navigate to the Terraform directory + if [ -d "terraform" ]; then + cd terraform + else + echo "Terraform directory not found. Cleanup aborted." + exit 1 + fi + + echo "Starting cleanup of Terraform infrastructure..." + terraform destroy -auto-approve || { + echo "Failed to clean up Terraform infrastructure." + exit 1 + } + echo "Terraform infrastructure cleanup completed." + silent: true + + check-and-cleanup-terraform: + desc: "Check and clean up deployed Terraform infrastructure if it exists" + cmds: + - | + # Capture the output of check-terraform-infra + infra_output=$(task check-terraform-infra | tee /dev/tty) + + # Check if output contains indication of deployed infrastructure + if echo "$infra_output" | grep -q "Deployed infrastructure detected"; then + echo "Infrastructure detected; proceeding with cleanup." + task cleanup-terraform-infra + else + echo "No deployed infrastructure found; no cleanup required." + fi + silent: true + + clean-rwl-discovery: + desc: "Check and clean up RunWhen Local discovery output" + cmds: + - | + sudo rm -rf output + rm workspaceInfo.yaml + silent: true diff --git a/codebundles/aws-c7n-service-usage/.test/terraform/Taskfile.yaml b/codebundles/aws-c7n-service-usage/.test/terraform/Taskfile.yaml new file mode 100644 index 0000000..08e0e83 --- /dev/null +++ b/codebundles/aws-c7n-service-usage/.test/terraform/Taskfile.yaml @@ -0,0 +1,69 @@ +version: '3' + +env: + TERM: screen-256color + +tasks: + default: + cmds: + - task: test + + test: + desc: Run tests. + cmds: + - task: test-terraform + + clean: + desc: Clean the environment. + cmds: + - task: clean-go + - task: clean-terraform + + clean-terraform: + desc: Clean the terraform environment (remove terraform directories and files) + cmds: + - find . -type d -name .terraform -exec rm -rf {} + + - find . -type f -name .terraform.lock.hcl -delete + + format-and-init-terraform: + desc: Run Terraform fmt and init + cmds: + - | + terraform fmt + terraform init + test-terraform: + desc: Run tests for all terraform directories. + silent: true + env: + DIRECTORIES: + sh: find . -path '*/.terraform/*' -prune -o -name '*.tf' -type f -exec dirname {} \; | sort -u + cmds: + - | + BOLD=$(tput bold) + NORM=$(tput sgr0) + + CWD=$PWD + + for d in $DIRECTORIES; do + cd $d + echo "${BOLD}$PWD:${NORM}" + if ! terraform fmt -check=true -list=false -recursive=false; then + echo " ✗ terraform fmt" && exit 1 + else + echo " √ terraform fmt" + fi + + if ! terraform init -backend=false -input=false -get=true -no-color > /dev/null; then + echo " ✗ terraform init" && exit 1 + else + echo " √ terraform init" + fi + + if ! terraform validate > /dev/null; then + echo " ✗ terraform validate" && exit 1 + else + echo " √ terraform validate" + fi + + cd $CWD + done \ No newline at end of file diff --git a/codebundles/aws-c7n-service-usage/.test/terraform/main.tf b/codebundles/aws-c7n-service-usage/.test/terraform/main.tf new file mode 100644 index 0000000..f457233 --- /dev/null +++ b/codebundles/aws-c7n-service-usage/.test/terraform/main.tf @@ -0,0 +1 @@ +# placeholder \ No newline at end of file diff --git a/codebundles/aws-c7n-service-usage/.test/terraform/provider.tf b/codebundles/aws-c7n-service-usage/.test/terraform/provider.tf new file mode 100644 index 0000000..aa39e39 --- /dev/null +++ b/codebundles/aws-c7n-service-usage/.test/terraform/provider.tf @@ -0,0 +1,3 @@ +provider "aws" { + region = "us-west-2" # Replace with your desired region +} \ No newline at end of file diff --git a/codebundles/aws-c7n-service-usage/README.md b/codebundles/aws-c7n-service-usage/README.md new file mode 100644 index 0000000..a89cbb8 --- /dev/null +++ b/codebundles/aws-c7n-service-usage/README.md @@ -0,0 +1,27 @@ +# AWS Cloud Custodian Service Usage + +This CodeBundle evaluates the AWS Service Usage in AWS Account + +## SLI +The SLI metrics are generated by executing the following task: +- List AWS Service Usage Exceeding defined threshold + +The score of each check is added up and then divided by the total amount of checks. + + +## TaskSet +Similar to the SLI, but produces a report on the specific resources and raises issues for each volume that requires attention. + + +## Required Configuration + +``` +export AWS_ACCESS_KEY_ID=[] +export AWS_SECRET_ACCESS_KEY=[] +export AWS_DEFAULT_REGION=[] +export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) +``` + + +## Testing +See the .test directory for infrastructure test code. \ No newline at end of file diff --git a/codebundles/aws-c7n-service-usage/Util.py b/codebundles/aws-c7n-service-usage/Util.py new file mode 100644 index 0000000..ff71bdc --- /dev/null +++ b/codebundles/aws-c7n-service-usage/Util.py @@ -0,0 +1,68 @@ +import re +import json +from tabulate import tabulate + +def usage_table(resource_file_path): + # Load JSON data from file + print(resource_file_path) + try: + # Attempt to open and read the log file + with open(resource_file_path, 'r') as file: + data = json.load(file) + except FileNotFoundError: + return f"Error: The file '{resource_file_path}' was not found." + except PermissionError: + return f"Error: Permission denied when trying to read '{resource_file}'." + except Exception as e: + return f"An unexpected error occurred while reading the file: {e}" + + # Define table headers + headers = ["ServiceName", "QuotaName", "Usage", "Quota", "UsagePercentage", "MetricName", "Period"] + + # Prepare table rows + table = [] + for item in data: + service_name = item.get("ServiceName", "N/A") + quota_name = item.get("QuotaName", "N/A") + usage = item.get("c7n:UsageMetric", {}).get("metric", 0) + quota = item.get("c7n:UsageMetric", {}).get("quota", 1) # Avoid division by zero + usage_percentage = round((usage / quota) * 100, 2) if quota else 0 + metric_name = item.get("UsageMetric", {}).get("MetricName", "N/A") + period_value = item.get("Period", {}).get("PeriodValue", "N/A") + period_unit = item.get("Period", {}).get("PeriodUnit", "N/A") + period = f"{period_value} {period_unit}" + + table.append([service_name, quota_name, usage, quota, f"{usage_percentage}%", metric_name, period]) + + # Print the table + return tabulate(table, headers=headers, tablefmt="grid") + +# e.g aws usage logs + # "2025-02-06 06:27:33,611: custodian.filters:INFO Amazon Elastic Compute Cloud (Amazon EC2) Running On-Demand Standard (A, C, D, H, I, M, R, T, Z) instances usage: 2.0/512.0", + # "2025-02-06 06:28:05,175: custodian.filters:INFO Elastic Load Balancing (ELB) Targets per Network Load Balancer usage: 0.0/3000.0", + # "2025-02-06 06:32:00,460: custodian.filters:INFO Amazon CloudWatch Rate of GetMetricStatistics requests usage: 0.37/400.0", + # "2025-02-06 06:28:01,668: custodian.filters:INFO Elastic Load Balancing (ELB) Targets per Availability Zone per Network Load Balancer usage: 0.0/500.0", + # "2025-02-06 06:28:04,540: custodian.filters:INFO Elastic Load Balancing (ELB) Classic Load Balancers per Region usage: 0.0/20.0", + # "2025-02-06 06:28:05,175: custodian.filters:INFO Elastic Load Balancing (ELB) Targets per Network Load Balancer usage: 0.0/3000.0", + # "2025-02-06 06:28:06,787: custodian.filters:INFO Elastic Load Balancing (ELB) Listeners per Network Load Balancer usage: 0.0/50.0", + # "2025-02-06 06:28:05,969: custodian.filters:INFO Elastic Load Balancing (ELB) Target Groups per Region usage: 0.0/3000.0", + # "2025-02-06 06:28:07,590: custodian.filters:INFO Elastic Load Balancing (ELB) Targets per Target Group per Region usage: 0.0/1000.0", + # "2025-02-06 06:28:08,426: custodian.filters:INFO Elastic Load Balancing (ELB) Targets per Application Load Balancer usage: 0.0/1000.0", + # "2025-02-06 06:28:09,248: custodian.filters:INFO Elastic Load Balancing (ELB) Application Load Balancers per Region usage: 0.0/50.0", + # "2025-02-06 06:28:10,064: custodian.filters:INFO Elastic Load Balancing (ELB) Network Load Balancers per Region usage: 0.0/50.0", + # "2025-02-06 06:28:10,864: custodian.filters:INFO Elastic Load Balancing (ELB) Registered Instances per Classic Load Balancer usage: 0.0/1000.0", + # "2025-02-06 06:28:43,856: custodian.filters:INFO Amazon EMR The maximum number of ListSecurityConfigurations API requests that you can make per second. usage: 0.0/5.0", + # "2025-02-06 06:28:46,315: custodian.filters:INFO Amazon EventBridge (CloudWatch Events) Invocations throttle limit in transactions per second usage: 0.05/18750.0", + # "2025-02-06 06:28:52,341: custodian.filters:INFO Amazon Kinesis Data Firehose Rate of ListDeliveryStream requests usage: 1.0/5.0", + # "2025-02-06 06:30:26,459: custodian.filters:INFO AWS Key Management Service (AWS KMS) ListKeys request rate usage: 0.0/500.0", + # "2025-02-06 06:30:40,805: custodian.filters:INFO AWS Key Management Service (AWS KMS) GetKeyPolicy request rate usage: 0.01/1000.0", + # "2025-02-06 06:30:41,863: custodian.filters:INFO AWS Key Management Service (AWS KMS) ListAliases request rate usage: 0.0/500.0", + # "2025-02-06 06:30:54,569: custodian.filters:INFO AWS Key Management Service (AWS KMS) Customer Master Keys (CMKs) usage: 2.0/100000.0", + # "2025-02-06 06:30:57,280: custodian.filters:INFO AWS Key Management Service (AWS KMS) Cryptographic operations (symmetric) request rate usage: 0.01/100000.0", + # "2025-02-06 06:31:03,602: custodian.filters:INFO AWS Key Management Service (AWS KMS) DescribeKey request rate usage: 0.01/2000.0", + # "2025-02-06 06:31:07,009: custodian.filters:INFO AWS Lambda Concurrent executions usage: 1.0/1000.0", + # "2025-02-06 06:31:17,046: custodian.filters:INFO Amazon CloudWatch Logs DescribeLogGroups throttle limit in transactions per second usage: 0.0/10.0", + # "2025-02-06 06:31:20,730: custodian.filters:INFO Amazon CloudWatch Logs CreateLogStream throttle limit in transactions per second usage: 0.01/50.0", + # "2025-02-06 06:31:29,270: custodian.filters:INFO Amazon CloudWatch Logs DescribeDestinations throttle limit in transactions per second usage: 0.0/5.0", + # "2025-02-06 06:31:29,771: custodian.filters:INFO Amazon CloudWatch Logs PutLogEvents throttle limit in transactions per second usage: 0.03/5000.0", + # "2025-02-06 06:31:30,934: custodian.filters:INFO Amazon CloudWatch Logs DescribeMetricFilters throttle limit in transactions per second usage: 0.01/5.0" diff --git a/codebundles/aws-c7n-service-usage/runbook.robot b/codebundles/aws-c7n-service-usage/runbook.robot new file mode 100644 index 0000000..4d3456b --- /dev/null +++ b/codebundles/aws-c7n-service-usage/runbook.robot @@ -0,0 +1,100 @@ +*** Settings *** +Metadata Author saurabh3460 +Metadata Supports AWS Tag CloudCustodian +Metadata Display Name AWS Service Usage +Documentation List AWS Service Usage Exceeding defined threshold +Force Tags Tag AWS usage + +Library RW.Core +Library RW.CLI +Library CloudCustodian.Core +Library Util.py +Suite Setup Suite Initialization + + +*** Tasks *** +List AWS Service Usage Exceeding defined threshold in AWS Account ${AWS_ACCOUNT_ID} + [Documentation] List AWS services where usage exceeds a specified usage percentage + [Tags] aws service usage + ${result}= CloudCustodian.Core.Generate Policy + ... ${CURDIR}/service-usage.j2 + ... usage_percent=${USAGE_PERCENTAGE} + ... resource_providers=${AWS_RESOURCE_PROVIDERS} + ${c7n_output}= RW.CLI.Run Cli + ... cmd=custodian run -r ${AWS_REGION} --output-dir ${OUTPUT_DIR}/aws-c7n-service-usage ${CURDIR}/service-usage.yaml --cache-period 0 + ... secret__aws_access_key_id=${AWS_ACCESS_KEY_ID} + ... secret__aws_secret_access_key=${AWS_SECRET_ACCESS_KEY} + ... timeout_seconds=200 + ${report_data}= RW.CLI.Run Cli + ... cmd=cat ${OUTPUT_DIR}/aws-c7n-service-usage/service-usage/resources.json + + TRY + ${service_list}= Evaluate json.loads(r'''${report_data.stdout}''') json + EXCEPT + Log Failed to load JSON payload, defaulting to empty list. WARN + ${service_list}= Create List + END + + IF len(@{service_list}) > 0 + # Generate and format report + ${formatted_results}= USAGE TABLE ${OUTPUT_DIR}/aws-c7n-service-usage/service-usage/resources.json + RW.Core.Add Pre To Report ${formatted_results} + + FOR ${service} IN @{service_list} + ${usage_percentage}= Evaluate round(${service['c7n:UsageMetric']['metric']}/${service['c7n:UsageMetric']['quota']}*100, 2) + RW.Core.Add Issue + ... severity=3 + ... expected=Service `${service['ServiceName']}` usage should be below ${USAGE_PERCENTAGE}% of quota in AWS Account `${AWS_ACCOUNT_ID}` + ... actual=Service `${service['ServiceName']}` usage is at `${usage_percentage}%` of quota in AWS Account `${AWS_ACCOUNT_ID}` + ... title=Service `${service['ServiceName']}` usage exceeds threshold ${USAGE_PERCENTAGE}% in AWS Account `${AWS_ACCOUNT_ID}` + ... reproduce_hint=${c7n_output.cmd} + ... details=${service} + ... next_steps=Increase the limit of AWS service `${service['ServiceName']}` in AWS Account `${AWS_ACCOUNT_ID}` + END + ELSE + RW.Core.Add Pre To Report No services found with usage exceeding `${USAGE_PERCENTAGE}%` in AWS Account `${AWS_ACCOUNT_ID}` + END + +** Keywords *** +Suite Initialization + ${AWS_REGION}= RW.Core.Import User Variable AWS_REGION + ... type=string + ... description=AWS Region + ... pattern=\w* + ${AWS_ACCOUNT_ID}= RW.Core.Import User Variable AWS_ACCOUNT_ID + ... type=string + ... description=AWS Account ID + ... pattern=\w* + ${AWS_ACCESS_KEY_ID}= RW.Core.Import Secret AWS_ACCESS_KEY_ID + ... type=string + ... description=AWS Access Key ID + ... pattern=\w* + ${AWS_SECRET_ACCESS_KEY}= RW.Core.Import Secret AWS_SECRET_ACCESS_KEY + ... type=string + ... description=AWS Access Key Secret + ... pattern=\w* + ${AWS_RESOURCE_PROVIDERS}= RW.Core.Import User Variable AWS_RESOURCE_PROVIDERS + ... type=string + ... description=Comma separated list of AWS Resource Providers + ... pattern=^[a-zA-Z0-9,]+$ + ... example=ec2,firehose,lambda,logs,monitoring,rds,servicequotas,ssm,fargate,kms + ... default=ec2,firehose,lambda,logs,monitoring,rds,servicequotas,ssm,fargate,kms + ${USAGE_PERCENTAGE}= RW.Core.Import User Variable USAGE_PERCENTAGE + ... type=number + ... description=Threshold percentage for service usage monitoring. If usage exceeds this value, an issue will be raised. + ... pattern=^\d+$ + ... example=80 + ... default=80 + ${clean_workding_dir}= RW.CLI.Run Cli cmd=rm -rf ${OUTPUT_DIR}/aws-c7n-service-usage # Note: Clean out the cloud custoding report dir to ensure accurate data + ${AWS_ENABLED_REGIONS}= RW.CLI.Run Cli + ... cmd=aws ec2 describe-regions --region ${AWS_REGION} --query 'Regions[*].RegionName' --output json + ... secret__aws_access_key_id=${AWS_ACCESS_KEY_ID} + ... secret__aws_secret_access_key=${AWS_SECRET_ACCESS_KEY} + ${AWS_ENABLED_REGIONS}= Evaluate json.loads(r'''${AWS_ENABLED_REGIONS.stdout}''') json + Set Suite Variable ${AWS_ENABLED_REGIONS} ${AWS_ENABLED_REGIONS} + Set Suite Variable ${AWS_REGION} ${AWS_REGION} + Set Suite Variable ${AWS_ACCOUNT_ID} ${AWS_ACCOUNT_ID} + Set Suite Variable ${AWS_ACCESS_KEY_ID} ${AWS_ACCESS_KEY_ID} + Set Suite Variable ${AWS_SECRET_ACCESS_KEY} ${AWS_SECRET_ACCESS_KEY} + Set Suite Variable ${AWS_RESOURCE_PROVIDERS} ${AWS_RESOURCE_PROVIDERS} + Set Suite Variable ${USAGE_PERCENTAGE} ${USAGE_PERCENTAGE} \ No newline at end of file diff --git a/codebundles/aws-c7n-service-usage/service-usage.j2 b/codebundles/aws-c7n-service-usage/service-usage.j2 new file mode 100644 index 0000000..911ff46 --- /dev/null +++ b/codebundles/aws-c7n-service-usage/service-usage.j2 @@ -0,0 +1,15 @@ +policies: + - name: service-usage + description: AWS services usage stats + resource: aws.service-quota + {%- if resource_providers %} + query: + - include_service_codes: + {%- for provider in resource_providers.split(',') %} + - {{ provider.strip('"').strip("'").strip() }} + {%- endfor %} + {%- endif %} + filters: + - UsageMetric: present + - type: usage-metric + limit: {{ usage_percent }} \ No newline at end of file diff --git a/codebundles/aws-c7n-service-usage/sli.robot b/codebundles/aws-c7n-service-usage/sli.robot new file mode 100644 index 0000000..f860e8f --- /dev/null +++ b/codebundles/aws-c7n-service-usage/sli.robot @@ -0,0 +1,74 @@ +*** Settings *** +Metadata Author saurabh3460 +Metadata Supports AWS Tag CloudCustodian +Metadata Display Name AWS Service Usage +Documentation Count AWS Service Usage Exceeding defined threshold +Force Tags Tag AWS usage + +Library RW.Core +Library RW.CLI +Library CloudCustodian.Core +Library Util.py +Suite Setup Suite Initialization + + +*** Tasks *** +Check for AWS Service Usage Exceeding defined threshold in AWS Account `${AWS_ACCOUNT_ID}` + [Documentation] Check for AWS services where usage exceeds a specified usage percentage + [Tags] aws service usage + ${result}= CloudCustodian.Core.Generate Policy + ... ${CURDIR}/service-usage.j2 + ... usage_percent=${USAGE_PERCENTAGE} + ... resource_providers=${AWS_RESOURCE_PROVIDERS} + ${c7n_output}= RW.CLI.Run Cli + ... cmd=custodian run -r ${AWS_REGION} --output-dir ${OUTPUT_DIR}/aws-c7n-service-usage ${CURDIR}/service-usage.yaml --cache-period 0 + ... secret__aws_access_key_id=${AWS_ACCESS_KEY_ID} + ... secret__aws_secret_access_key=${AWS_SECRET_ACCESS_KEY} + ... timeout_seconds=200 + ${count}= RW.CLI.Run Cli + ... cmd=cat ${OUTPUT_DIR}/aws-c7n-service-usage/service-usage/metadata.json | jq '.metrics[] | select(.MetricName == "ResourceCount") | .Value' + RW.Core.Push Metric ${count.stdout} + +** Keywords *** +Suite Initialization + ${AWS_REGION}= RW.Core.Import User Variable AWS_REGION + ... type=string + ... description=AWS Region + ... pattern=\w* + ${AWS_ACCOUNT_ID}= RW.Core.Import User Variable AWS_ACCOUNT_ID + ... type=string + ... description=AWS Account ID + ... pattern=\w* + ${AWS_ACCESS_KEY_ID}= RW.Core.Import Secret AWS_ACCESS_KEY_ID + ... type=string + ... description=AWS Access Key ID + ... pattern=\w* + ${AWS_SECRET_ACCESS_KEY}= RW.Core.Import Secret AWS_SECRET_ACCESS_KEY + ... type=string + ... description=AWS Access Key Secret + ... pattern=\w* + ${AWS_RESOURCE_PROVIDERS}= RW.Core.Import User Variable AWS_RESOURCE_PROVIDERS + ... type=string + ... description=Comma separated list of AWS Resource Providers + ... pattern=^[a-zA-Z0-9,]+$ + ... example=ec2,firehose,lambda,logs,monitoring,rds,servicequotas,ssm,fargate,kms + ... default=ec2,firehose,lambda,logs,monitoring,rds,servicequotas,ssm,fargate,kms + ${USAGE_PERCENTAGE}= RW.Core.Import User Variable USAGE_PERCENTAGE + ... type=number + ... description=Threshold percentage for service usage monitoring. If usage exceeds this value, an issue will be raised. + ... pattern=^\d+$ + ... example=80 + ... default=80 + ${clean_workding_dir}= RW.CLI.Run Cli cmd=rm -rf ${OUTPUT_DIR}/aws-c7n-service-usage # Note: Clean out the cloud custoding report dir to ensure accurate data + ${AWS_ENABLED_REGIONS}= RW.CLI.Run Cli + ... cmd=aws ec2 describe-regions --region ${AWS_REGION} --query 'Regions[*].RegionName' --output json + ... secret__aws_access_key_id=${AWS_ACCESS_KEY_ID} + ... secret__aws_secret_access_key=${AWS_SECRET_ACCESS_KEY} + ${AWS_ENABLED_REGIONS}= Evaluate json.loads(r'''${AWS_ENABLED_REGIONS.stdout}''') json + Set Suite Variable ${AWS_ENABLED_REGIONS} ${AWS_ENABLED_REGIONS} + Set Suite Variable ${AWS_REGION} ${AWS_REGION} + Set Suite Variable ${AWS_ACCOUNT_ID} ${AWS_ACCOUNT_ID} + Set Suite Variable ${AWS_ACCESS_KEY_ID} ${AWS_ACCESS_KEY_ID} + Set Suite Variable ${AWS_SECRET_ACCESS_KEY} ${AWS_SECRET_ACCESS_KEY} + Set Suite Variable ${AWS_RESOURCE_PROVIDERS} ${AWS_RESOURCE_PROVIDERS} + Set Suite Variable ${USAGE_PERCENTAGE} ${USAGE_PERCENTAGE} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c737c58..2329fcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,5 @@ robotframework>=4.1.2 ruamel.base>=1.0.0 ruamel.yaml>=0.17.20 requests>=2 -c7n>=0.9.41 +c7n>=0.9.43 rw-cli-keywords>=0.0.19 \ No newline at end of file