From 1c1dcc1ff658883bf3e1b6bd5ad6967ff447c3eb Mon Sep 17 00:00:00 2001 From: Mohamed Date: Wed, 13 May 2026 22:19:38 +1000 Subject: [PATCH 01/10] Add Storage Transfer Service policies --- .../Bandwidth_limit/.terraform.lock.hcl | 22 +++++++++++++ .../Bandwidth_limit/c.tf | 8 +++++ .../Bandwidth_limit/config.tf | 12 +++++++ .../Bandwidth_limit/nc.tf | 8 +++++ .../name/.terraform.lock.hcl | 22 +++++++++++++ .../name/c.tf | 8 +++++ .../name/config.tf | 12 +++++++ .../name/nc.tf | 8 +++++ .../.terraform.lock.hcl | 22 +++++++++++++ .../c.tf | 27 +++++++++++++++ .../config.tf | 12 +++++++ .../nc.tf | 27 +++++++++++++++ .../overwrite_when/.terraform.lock.hcl | 22 +++++++++++++ .../overwrite_when/c.tf | 27 +++++++++++++++ .../overwrite_when/config.tf | 12 +++++++ .../overwrite_when/nc.tf | 27 +++++++++++++++ .../role_arn/.terraform.lock.hcl | 22 +++++++++++++ .../google_storage_transfer_job/role_arn/c.tf | 24 ++++++++++++++ .../role_arn/config.tf | 12 +++++++ .../role_arn/nc.tf | 28 ++++++++++++++++ .../.terraform.lock.hcl | 22 +++++++++++++ .../source_agent_pool_name/c.tf | 33 +++++++++++++++++++ .../source_agent_pool_name/config.tf | 12 +++++++ .../source_agent_pool_name/nc.tf | 33 +++++++++++++++++++ .../Bandwidth_limit/policy.rego | 26 +++++++++++++++ .../name/policy.rego | 27 +++++++++++++++ .../vars.rego | 11 +++++++ .../policy.rego | 26 +++++++++++++++ .../overwrite_when/policy.rego | 26 +++++++++++++++ .../role_arn/policy.rego | 26 +++++++++++++++ .../source_agent_pool_name/policy.rego | 26 +++++++++++++++ .../google_storage_transfer_job/vars.rego | 19 +++++++++++ 32 files changed, 649 insertions(+) create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/.terraform.lock.hcl create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/c.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/config.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/nc.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/.terraform.lock.hcl create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/c.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/config.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/.terraform.lock.hcl create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/c.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/config.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/nc.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/.terraform.lock.hcl create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/c.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/config.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/nc.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/.terraform.lock.hcl create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/c.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/config.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/nc.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/.terraform.lock.hcl create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/c.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/config.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/nc.tf create mode 100644 policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/policy.rego create mode 100644 policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/policy.rego create mode 100644 policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/vars.rego create mode 100644 policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/policy.rego create mode 100644 policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/policy.rego create mode 100644 policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/policy.rego create mode 100644 policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/policy.rego create mode 100644 policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/vars.rego diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/.terraform.lock.hcl b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/.terraform.lock.hcl new file mode 100644 index 000000000..7aab94476 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.28.0" + constraints = "7.28.0" + hashes = [ + "h1:PSXGQ1KdRjfeCvb6K85dqZiC6AfyB5OWw6vm5I+iop8=", + "zh:078c16b9c5e9067e72070367846976b58f906d8efab6fc4fc1325661717dc9cc", + "zh:08b839014b428233a3a83d15045e7559b07fc035c7f73cc1ee2694c50c4dea54", + "zh:0c76ea69f75633bdfc67a0cd6ea510332c0cb0f2d4968b8a070e546fb47e444e", + "zh:3a308492ad4c153583f7b8ecc3c80bf0bbc15a32c62b5b3794efb27db01ff26b", + "zh:6754f51373994470f78937856982b0a39648ac302713d07205d320a13ad41d82", + "zh:79d387214f55df16c795f11988a0285a4bfa846c447faa85008b953b77081eb1", + "zh:8de432482d77d1a1077b2dc3db764b8ba6d1b07a4b991a07c960855adc0b031b", + "zh:900daa2435de1928a9868aa4c17d8b7b109ab363c97f7fe274466193af1412b0", + "zh:96c25183a7f13b3de9a5631aa2a13ed1a4285b8393df90c2380c2fe74f350ab5", + "zh:971121626be01245acd9a4520a63e1405e4f528d3c83f39a28f8caaeac235b45", + "zh:e90d5e7d7bf47c8cf5bbf2e5d0bf855ed10350ad3584795a6911f85fdb5c0c3c", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/c.tf new file mode 100644 index 000000000..014fa1feb --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/c.tf @@ -0,0 +1,8 @@ +resource "google_storage_transfer_agent_pool" "c" { + name = "approved-bandwidth-pool" + display_name = "approved-bandwidth-pool" + + bandwidth_limit { + limit_mbps = "250" + } +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/config.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/config.tf new file mode 100644 index 000000000..dccf44a47 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/config.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "7.28.0" + } + } +} + +provider "google" { + project = "my-project-123" +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/nc.tf new file mode 100644 index 000000000..7e39969c7 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/nc.tf @@ -0,0 +1,8 @@ +resource "google_storage_transfer_agent_pool" "nc" { + name = "unsafe-bandwidth-pool" + display_name = "unsafe-bandwidth-pool" + + bandwidth_limit { + limit_mbps = "5000" + } +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/.terraform.lock.hcl b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/.terraform.lock.hcl new file mode 100644 index 000000000..7aab94476 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.28.0" + constraints = "7.28.0" + hashes = [ + "h1:PSXGQ1KdRjfeCvb6K85dqZiC6AfyB5OWw6vm5I+iop8=", + "zh:078c16b9c5e9067e72070367846976b58f906d8efab6fc4fc1325661717dc9cc", + "zh:08b839014b428233a3a83d15045e7559b07fc035c7f73cc1ee2694c50c4dea54", + "zh:0c76ea69f75633bdfc67a0cd6ea510332c0cb0f2d4968b8a070e546fb47e444e", + "zh:3a308492ad4c153583f7b8ecc3c80bf0bbc15a32c62b5b3794efb27db01ff26b", + "zh:6754f51373994470f78937856982b0a39648ac302713d07205d320a13ad41d82", + "zh:79d387214f55df16c795f11988a0285a4bfa846c447faa85008b953b77081eb1", + "zh:8de432482d77d1a1077b2dc3db764b8ba6d1b07a4b991a07c960855adc0b031b", + "zh:900daa2435de1928a9868aa4c17d8b7b109ab363c97f7fe274466193af1412b0", + "zh:96c25183a7f13b3de9a5631aa2a13ed1a4285b8393df90c2380c2fe74f350ab5", + "zh:971121626be01245acd9a4520a63e1405e4f528d3c83f39a28f8caaeac235b45", + "zh:e90d5e7d7bf47c8cf5bbf2e5d0bf855ed10350ad3584795a6911f85fdb5c0c3c", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/c.tf new file mode 100644 index 000000000..f358d6925 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/c.tf @@ -0,0 +1,8 @@ +resource "google_storage_transfer_agent_pool" "c" { + name = "projects/my-project-123/agentPools/approved-pool-1" + display_name = "approved-pool-1" + + bandwidth_limit { + limit_mbps = "250" + } +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/config.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/config.tf new file mode 100644 index 000000000..413a25f42 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/config.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "7.28.0" + } + } +} + +provider "google" { + project = "my-project-123" +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf new file mode 100644 index 000000000..5d6760987 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf @@ -0,0 +1,8 @@ +resource "google_storage_transfer_agent_pool" "nc" { + name = "projects/my-project-123/agentPools/googUnsafePool" + display_name = "googUnsafePool" + + bandwidth_limit { + limit_mbps = "250" + } +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/.terraform.lock.hcl b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/.terraform.lock.hcl new file mode 100644 index 000000000..7aab94476 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.28.0" + constraints = "7.28.0" + hashes = [ + "h1:PSXGQ1KdRjfeCvb6K85dqZiC6AfyB5OWw6vm5I+iop8=", + "zh:078c16b9c5e9067e72070367846976b58f906d8efab6fc4fc1325661717dc9cc", + "zh:08b839014b428233a3a83d15045e7559b07fc035c7f73cc1ee2694c50c4dea54", + "zh:0c76ea69f75633bdfc67a0cd6ea510332c0cb0f2d4968b8a070e546fb47e444e", + "zh:3a308492ad4c153583f7b8ecc3c80bf0bbc15a32c62b5b3794efb27db01ff26b", + "zh:6754f51373994470f78937856982b0a39648ac302713d07205d320a13ad41d82", + "zh:79d387214f55df16c795f11988a0285a4bfa846c447faa85008b953b77081eb1", + "zh:8de432482d77d1a1077b2dc3db764b8ba6d1b07a4b991a07c960855adc0b031b", + "zh:900daa2435de1928a9868aa4c17d8b7b109ab363c97f7fe274466193af1412b0", + "zh:96c25183a7f13b3de9a5631aa2a13ed1a4285b8393df90c2380c2fe74f350ab5", + "zh:971121626be01245acd9a4520a63e1405e4f528d3c83f39a28f8caaeac235b45", + "zh:e90d5e7d7bf47c8cf5bbf2e5d0bf855ed10350ad3584795a6911f85fdb5c0c3c", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/c.tf new file mode 100644 index 000000000..0e953f2c8 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/c.tf @@ -0,0 +1,27 @@ +resource "google_storage_transfer_job" "c" { + description = "Compliant transfer job" + project = "my-project-123" + status = "ENABLED" + + transfer_spec { + posix_data_source { + root_directory = "/source" + } + + gcs_data_sink { + bucket_name = "my-transfer-bucket" + } + + transfer_options { + delete_objects_from_source_after_transfer = false + } + } + + schedule { + schedule_start_date { + year = 2026 + month = 1 + day = 1 + } + } +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/config.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/config.tf new file mode 100644 index 000000000..413a25f42 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/config.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "7.28.0" + } + } +} + +provider "google" { + project = "my-project-123" +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/nc.tf new file mode 100644 index 000000000..661384151 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/nc.tf @@ -0,0 +1,27 @@ +resource "google_storage_transfer_job" "nc" { + description = "Non-compliant transfer job" + project = "my-project-123" + status = "ENABLED" + + transfer_spec { + posix_data_source { + root_directory = "/source" + } + + gcs_data_sink { + bucket_name = "my-transfer-bucket" + } + + transfer_options { + delete_objects_from_source_after_transfer = true + } + } + + schedule { + schedule_start_date { + year = 2026 + month = 1 + day = 1 + } + } +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/.terraform.lock.hcl b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/.terraform.lock.hcl new file mode 100644 index 000000000..7aab94476 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.28.0" + constraints = "7.28.0" + hashes = [ + "h1:PSXGQ1KdRjfeCvb6K85dqZiC6AfyB5OWw6vm5I+iop8=", + "zh:078c16b9c5e9067e72070367846976b58f906d8efab6fc4fc1325661717dc9cc", + "zh:08b839014b428233a3a83d15045e7559b07fc035c7f73cc1ee2694c50c4dea54", + "zh:0c76ea69f75633bdfc67a0cd6ea510332c0cb0f2d4968b8a070e546fb47e444e", + "zh:3a308492ad4c153583f7b8ecc3c80bf0bbc15a32c62b5b3794efb27db01ff26b", + "zh:6754f51373994470f78937856982b0a39648ac302713d07205d320a13ad41d82", + "zh:79d387214f55df16c795f11988a0285a4bfa846c447faa85008b953b77081eb1", + "zh:8de432482d77d1a1077b2dc3db764b8ba6d1b07a4b991a07c960855adc0b031b", + "zh:900daa2435de1928a9868aa4c17d8b7b109ab363c97f7fe274466193af1412b0", + "zh:96c25183a7f13b3de9a5631aa2a13ed1a4285b8393df90c2380c2fe74f350ab5", + "zh:971121626be01245acd9a4520a63e1405e4f528d3c83f39a28f8caaeac235b45", + "zh:e90d5e7d7bf47c8cf5bbf2e5d0bf855ed10350ad3584795a6911f85fdb5c0c3c", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/c.tf new file mode 100644 index 000000000..437fc2ad5 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/c.tf @@ -0,0 +1,27 @@ +resource "google_storage_transfer_job" "c" { + description = "Compliant transfer job" + project = "my-project-123" + status = "ENABLED" + + transfer_spec { + posix_data_source { + root_directory = "/source" + } + + gcs_data_sink { + bucket_name = "my-transfer-bucket" + } + + transfer_options { + overwrite_when = "DIFFERENT" + } + } + + schedule { + schedule_start_date { + year = 2026 + month = 1 + day = 1 + } + } +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/config.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/config.tf new file mode 100644 index 000000000..413a25f42 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/config.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "7.28.0" + } + } +} + +provider "google" { + project = "my-project-123" +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/nc.tf new file mode 100644 index 000000000..2003072db --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/nc.tf @@ -0,0 +1,27 @@ +resource "google_storage_transfer_job" "nc" { + description = "Non-compliant transfer job" + project = "my-project-123" + status = "ENABLED" + + transfer_spec { + posix_data_source { + root_directory = "/source" + } + + gcs_data_sink { + bucket_name = "my-transfer-bucket" + } + + transfer_options { + overwrite_when = "ALWAYS" + } + } + + schedule { + schedule_start_date { + year = 2026 + month = 1 + day = 1 + } + } +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/.terraform.lock.hcl b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/.terraform.lock.hcl new file mode 100644 index 000000000..7aab94476 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.28.0" + constraints = "7.28.0" + hashes = [ + "h1:PSXGQ1KdRjfeCvb6K85dqZiC6AfyB5OWw6vm5I+iop8=", + "zh:078c16b9c5e9067e72070367846976b58f906d8efab6fc4fc1325661717dc9cc", + "zh:08b839014b428233a3a83d15045e7559b07fc035c7f73cc1ee2694c50c4dea54", + "zh:0c76ea69f75633bdfc67a0cd6ea510332c0cb0f2d4968b8a070e546fb47e444e", + "zh:3a308492ad4c153583f7b8ecc3c80bf0bbc15a32c62b5b3794efb27db01ff26b", + "zh:6754f51373994470f78937856982b0a39648ac302713d07205d320a13ad41d82", + "zh:79d387214f55df16c795f11988a0285a4bfa846c447faa85008b953b77081eb1", + "zh:8de432482d77d1a1077b2dc3db764b8ba6d1b07a4b991a07c960855adc0b031b", + "zh:900daa2435de1928a9868aa4c17d8b7b109ab363c97f7fe274466193af1412b0", + "zh:96c25183a7f13b3de9a5631aa2a13ed1a4285b8393df90c2380c2fe74f350ab5", + "zh:971121626be01245acd9a4520a63e1405e4f528d3c83f39a28f8caaeac235b45", + "zh:e90d5e7d7bf47c8cf5bbf2e5d0bf855ed10350ad3584795a6911f85fdb5c0c3c", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/c.tf new file mode 100644 index 000000000..d37eccf4d --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/c.tf @@ -0,0 +1,24 @@ +resource "google_storage_transfer_job" "c" { + description = "Compliant AWS S3 transfer job" + project = "my-project-123" + status = "ENABLED" + + transfer_spec { + aws_s3_data_source { + bucket_name = "my-source-bucket" + role_arn = "arn:aws:iam::123456789012:role/sts-transfer-role" + } + + gcs_data_sink { + bucket_name = "my-transfer-bucket" + } + } + + schedule { + schedule_start_date { + year = 2026 + month = 1 + day = 1 + } + } +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/config.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/config.tf new file mode 100644 index 000000000..413a25f42 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/config.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "7.28.0" + } + } +} + +provider "google" { + project = "my-project-123" +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/nc.tf new file mode 100644 index 000000000..cf8b09926 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/nc.tf @@ -0,0 +1,28 @@ +resource "google_storage_transfer_job" "nc" { + description = "Non-compliant AWS S3 transfer job" + project = "my-project-123" + status = "ENABLED" + + transfer_spec { + aws_s3_data_source { + bucket_name = "my-source-bucket" + + aws_access_key { + access_key_id = "AKIAIOSFODNN7EXAMPLE" + secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + } + } + + gcs_data_sink { + bucket_name = "my-transfer-bucket" + } + } + + schedule { + schedule_start_date { + year = 2026 + month = 1 + day = 1 + } + } +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/.terraform.lock.hcl b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/.terraform.lock.hcl new file mode 100644 index 000000000..7aab94476 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.28.0" + constraints = "7.28.0" + hashes = [ + "h1:PSXGQ1KdRjfeCvb6K85dqZiC6AfyB5OWw6vm5I+iop8=", + "zh:078c16b9c5e9067e72070367846976b58f906d8efab6fc4fc1325661717dc9cc", + "zh:08b839014b428233a3a83d15045e7559b07fc035c7f73cc1ee2694c50c4dea54", + "zh:0c76ea69f75633bdfc67a0cd6ea510332c0cb0f2d4968b8a070e546fb47e444e", + "zh:3a308492ad4c153583f7b8ecc3c80bf0bbc15a32c62b5b3794efb27db01ff26b", + "zh:6754f51373994470f78937856982b0a39648ac302713d07205d320a13ad41d82", + "zh:79d387214f55df16c795f11988a0285a4bfa846c447faa85008b953b77081eb1", + "zh:8de432482d77d1a1077b2dc3db764b8ba6d1b07a4b991a07c960855adc0b031b", + "zh:900daa2435de1928a9868aa4c17d8b7b109ab363c97f7fe274466193af1412b0", + "zh:96c25183a7f13b3de9a5631aa2a13ed1a4285b8393df90c2380c2fe74f350ab5", + "zh:971121626be01245acd9a4520a63e1405e4f528d3c83f39a28f8caaeac235b45", + "zh:e90d5e7d7bf47c8cf5bbf2e5d0bf855ed10350ad3584795a6911f85fdb5c0c3c", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/c.tf new file mode 100644 index 000000000..e73bdb0fb --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/c.tf @@ -0,0 +1,33 @@ +resource "google_storage_transfer_job" "c" { + description = "Compliant transfer job" + project = "my-project-123" + status = "ENABLED" + + transfer_spec { + source_agent_pool_name = "projects/my-project-123/agentPools/approved-pool" + + posix_data_source { + root_directory = "/source" + } + + gcs_data_sink { + bucket_name = "my-transfer-bucket" + path = "incoming/" + } + } + + schedule { + schedule_start_date { + year = 2026 + month = 1 + day = 1 + } + + start_time_of_day { + hours = 1 + minutes = 0 + seconds = 0 + nanos = 0 + } + } +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/config.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/config.tf new file mode 100644 index 000000000..413a25f42 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/config.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "7.28.0" + } + } +} + +provider "google" { + project = "my-project-123" +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/nc.tf new file mode 100644 index 000000000..7625741ba --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/nc.tf @@ -0,0 +1,33 @@ +resource "google_storage_transfer_job" "nc" { + description = "Non-compliant transfer job" + project = "my-project-123" + status = "ENABLED" + + transfer_spec { + source_agent_pool_name = "projects/my-project-123/agentPools/unapproved-pool" + + posix_data_source { + root_directory = "/source" + } + + gcs_data_sink { + bucket_name = "my-transfer-bucket" + path = "incoming/" + } + } + + schedule { + schedule_start_date { + year = 2026 + month = 1 + day = 1 + } + + start_time_of_day { + hours = 1 + minutes = 0 + seconds = 0 + nanos = 0 + } + } +} \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/policy.rego new file mode 100644 index 000000000..2c6dcb5f9 --- /dev/null +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/policy.rego @@ -0,0 +1,26 @@ +package terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_agent_pool.Bandwidth_limit + +import data.terraform.helpers +import data.terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_agent_pool.vars + +conditions := [ + [ + { + "situation_description": "Storage Transfer agent pool is configured with an unsafe bandwidth limit.", + "remedies": [ + "Set bandwidth_limit.limit_mbps to an approved lower value.", + "Avoid excessive bandwidth allocations on transfer agent pools." + ] + }, + { + "condition": "Storage Transfer agent pool bandwidth limit must not use unsafe high Mbps values.", + "attribute_path": ["bandwidth_limit", 0, "limit_mbps"], + "values": vars.variables.disallowed_bandwidth_limit_mbps, + "policy_type": "blacklist" + } + ] +] + +message := helpers.get_multi_summary(conditions, vars.variables).message + +details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/policy.rego new file mode 100644 index 000000000..1d2b92148 --- /dev/null +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/policy.rego @@ -0,0 +1,27 @@ +package terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_agent_pool.name + +import data.terraform.helpers +import data.terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_agent_pool.vars + +conditions := [ + [ + { + "situation_description": "Storage Transfer agent pool name does not follow the approved agent pool naming format.", + "remedies": [ + "Set name using the format projects/{projectId}/agentPools/{agentPoolId}.", + "Use a lowercase agent pool ID.", + "Do not start the agent pool ID with goog." + ] + }, + { + "condition": "Storage Transfer agent pool name must follow the approved naming pattern.", + "attribute_path": ["name"], + "values": ["projects/*/agentPools/*", [["projects"], [], ["agentPools"], [["a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"]]]], + "policy_type": "pattern whitelist" + } + ] +] + +message := helpers.get_multi_summary(conditions, vars.variables).message + +details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/vars.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/vars.rego new file mode 100644 index 000000000..369d03dc6 --- /dev/null +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/vars.rego @@ -0,0 +1,11 @@ +package terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_agent_pool.vars + +variables := { + "friendly_resource_name": "Storage Transfer agent pool", + "resource_type": "google_storage_transfer_agent_pool", + "disallowed_bandwidth_limit_mbps": [1000, 2000, 5000, 10000], + "arguments": { + "name": "name", + "display_name": "display_name" + } +} \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/policy.rego new file mode 100644 index 000000000..b3ecc0c1d --- /dev/null +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/policy.rego @@ -0,0 +1,26 @@ +package terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.delete_objects_from_source_after_transfer + +import data.terraform.helpers +import data.terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.vars + +conditions := [ + [ + { + "situation_description": "Storage Transfer job deletes objects from the source after transfer.", + "remedies": [ + "Set transfer_spec.transfer_options.delete_objects_from_source_after_transfer to false.", + "Use a copy-based transfer instead of deleting source data automatically." + ] + }, + { + "condition": "Storage Transfer job must not delete objects from the source after transfer.", + "attribute_path": ["transfer_spec", 0, "transfer_options", 0, "delete_objects_from_source_after_transfer"], + "values": [true], + "policy_type": "blacklist" + } + ] +] + +message := helpers.get_multi_summary(conditions, vars.variables).message + +details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/policy.rego new file mode 100644 index 000000000..fa02f7d47 --- /dev/null +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/policy.rego @@ -0,0 +1,26 @@ +package terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.overwrite_when + +import data.terraform.helpers +import data.terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.vars + +conditions := [ + [ + { + "situation_description": "Storage Transfer job is configured to always overwrite objects in the destination.", + "remedies": [ + "Set overwrite_when to a safer value.", + "Avoid unconditional overwriting of destination objects." + ] + }, + { + "condition": "Storage Transfer job must not always overwrite destination objects.", + "attribute_path": ["transfer_spec", 0, "transfer_options", 0, "overwrite_when"], + "values": ["ALWAYS"], + "policy_type": "blacklist" + } + ] +] + +message := helpers.get_multi_summary(conditions, vars.variables).message + +details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/policy.rego new file mode 100644 index 000000000..d2df541d4 --- /dev/null +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/policy.rego @@ -0,0 +1,26 @@ +package terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.role_arn + +import data.terraform.helpers +import data.terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.vars + +conditions := [ + [ + { + "situation_description": "Storage Transfer job uses an AWS S3 source without an approved IAM role ARN.", + "remedies": [ + "Set transfer_spec.aws_s3_data_source.role_arn.", + "Use AWS IAM role federation instead of long-lived AWS access keys." + ] + }, + { + "condition": "Storage Transfer job must use an approved AWS IAM role ARN.", + "attribute_path": ["transfer_spec", 0, "aws_s3_data_source", 0, "role_arn"], + "values": vars.variables.approved_role_arns, + "policy_type": "whitelist" + } + ] +] + +message := helpers.get_multi_summary(conditions, vars.variables).message + +details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/policy.rego new file mode 100644 index 000000000..fb56472b1 --- /dev/null +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/policy.rego @@ -0,0 +1,26 @@ +package terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.source_agent_pool_name + +import data.terraform.helpers +import data.terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.vars + +conditions := [ + [ + { + "situation_description": "Storage Transfer job is not using an approved source agent pool.", + "remedies": [ + "Set transfer_spec.source_agent_pool_name to an approved agent pool.", + "Use a controlled agent pool for transfer jobs." + ] + }, + { + "condition": "Storage Transfer job must use an approved source agent pool.", + "attribute_path": ["transfer_spec", 0, "source_agent_pool_name"], + "values": vars.variables.approved_source_agent_pool_names, + "policy_type": "whitelist" + } + ] +] + +message := helpers.get_multi_summary(conditions, vars.variables).message + +details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/vars.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/vars.rego new file mode 100644 index 000000000..b4526d5e8 --- /dev/null +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/vars.rego @@ -0,0 +1,19 @@ +package terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.vars + +variables := { + "friendly_resource_name": "Storage Transfer job", + "resource_type": "google_storage_transfer_job", + "approved_source_agent_pool_names": [ + "projects/my-project-123/agentPools/approved-pool" + ], + "approved_role_arns": [ + "arn:aws:iam::123456789012:role/sts-transfer-role" + ], + "arguments": { + "description": "description", + "project": "project", + "status": "status", + "overwrite_when": "overwrite_when", + "role_arn": "role_arn" + } +} \ No newline at end of file From 75821afd563d5d1ff5ddf5c2d0d24ff593b48c6b Mon Sep 17 00:00:00 2001 From: Mohamed Date: Wed, 13 May 2026 22:40:13 +1000 Subject: [PATCH 02/10] Add Storage Transfer Service documentation --- .../storage_transfer_agent_pool.md | 22 ++ .../storage_transfer_job.md | 193 ++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md create mode 100644 docs/gcp/Storage_Transfer_Service/storage_transfer_job.md diff --git a/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md b/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md new file mode 100644 index 000000000..04b277d3f --- /dev/null +++ b/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md @@ -0,0 +1,22 @@ +## 🛡️ Policy Deployment Engine: `storage_transfer_agent_pool` + +This section provides a concise policy evaluation for the `storage_transfer_agent_pool` resource in GCP. + +Reference: [Terraform Registry – storage_transfer_agent_pool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_transfer_agent_pool) + +--- + +## Argument Reference + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `name` | The ID of the agent pool to create. The agentPoolId must meet the following requirements: * Length of 128 characters or less. * Not start with the string goog. * Start with a lowercase ASCII character, followed by: * Zero or more: lowercase Latin alphabet characters, numerals, hyphens (-), periods (.), underscores (_), or tildes (~). * One or more numerals or lowercase ASCII characters. As expressed by the regular expression: ^(?!goog)[a-z]([a-z0-9-._~]*[a-z0-9])?$. | true | false | None | None | None | +| `display_name` | Specifies the client-specified AgentPool description. | false | false | None | None | None | +| `bandwidth_limit` | Specifies the bandwidth limit details. If this field is unspecified, the default value is set as 'No Limit'. Structure is [documented below](#nested_bandwidth_limit). | false | false | None | None | None | +| `project` | If it is not provided, the provider project is used. | false | false | None | None | None | + +### bandwidth_limit Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `limit_mbps` | Bandwidth rate in megabytes per second, distributed across all the agents in the pool. | true | false | None | None | None | diff --git a/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md b/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md new file mode 100644 index 000000000..0733bb586 --- /dev/null +++ b/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md @@ -0,0 +1,193 @@ +## 🛡️ Policy Deployment Engine: `storage_transfer_job` + +This section provides a concise policy evaluation for the `storage_transfer_job` resource in GCP. + +Reference: [Terraform Registry – storage_transfer_job](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_transfer_job) + +--- + +## Argument Reference + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `name` | | false | false | None | None | None | +| `description` | | false | false | None | None | None | +| `transfer_spec` | | false | false | None | None | None | +| `replication_spec` | - - - | false | false | None | None | None | +| `schedule` | | false | false | None | None | None | +| `event_stream` | | false | false | None | None | None | +| `project` | is not provided, the provider project is used. | false | false | None | None | None | +| `status` | | false | false | None | None | None | +| `notification_config` | | false | false | None | None | None | +| `logging_config` | | false | false | None | None | None | +| `object_conditions` | | false | false | None | None | None | +| `transfer_options` | | false | false | None | None | None | +| `gcs_data_sink` | | false | false | None | None | None | +| `gcs_data_source` | | false | false | None | None | None | +| `posix_data_sink` | | false | false | None | None | None | +| `posix_data_source` | | false | false | None | None | None | +| `hdfs_data_source` | | false | false | None | None | None | +| `aws_s3_data_source` | | false | false | None | None | None | +| `aws_access_key` | | false | false | None | None | None | +| `http_data_source` | | false | false | None | None | None | +| `azure_blob_storage_data_source` | | false | false | None | None | None | +| `azure_credentials` | | false | false | None | None | None | +| `loggin_config` | | false | false | None | None | None | + +### transfer_spec Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `source_agent_pool_name` | | false | false | None | None | None | +| `sink_agent_pool_name` | | false | false | None | None | None | +| `gcs_data_sink` | | false | false | None | None | None | +| `posix_data_sink` | | false | false | None | None | None | +| `object_conditions` | | false | false | None | None | None | +| `transfer_options` | | false | false | None | None | None | +| `gcs_data_source` | | false | false | None | None | None | +| `posix_data_source` | | false | false | None | None | None | +| `aws_s3_data_source` | | false | false | None | None | None | +| `http_data_source` | | false | false | None | None | None | +| `azure_blob_storage_data_source` | | false | false | None | None | None | +| `hdfs_data_source` | | false | false | None | None | None | + +### replication_spec Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `gcs_data_sink` | | false | false | None | None | None | +| `gcs_data_source` | | false | false | None | None | None | +| `object_conditions` | | false | false | None | None | None | +| `transfer_options` | | false | false | None | None | None | + +### schedule Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `schedule_start_date` | | false | false | None | None | None | +| `schedule_end_date` | | false | false | None | None | None | +| `start_time_of_day` | | false | false | None | None | None | +| `repeat_interval` | | false | false | None | None | None | + +### event_stream Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `name` | | false | false | None | None | None | +| `event_stream_start_time` | | false | false | None | None | None | +| `event_stream_expiration_time` | | false | false | None | None | None | + +### notification_config Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `pubsub_topic` | | false | false | None | None | None | +| `event_types` | | false | false | None | None | None | +| `payload_format` | | false | false | None | None | None | + +### object_conditions Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `max_time_elapsed_since_last_modification` | | false | false | None | None | None | +| `min_time_elapsed_since_last_modification` | A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". | false | false | None | None | None | +| `include_prefixes` | | false | false | None | None | None | +| `exclude_prefixes` | | false | false | None | None | None | +| `last_modified_since` | | false | false | None | None | None | +| `last_modified_before` | | false | false | None | None | None | + +### transfer_options Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `overwrite_objects_already_existing_in_sink` | | false | false | None | None | None | +| `delete_objects_unique_in_sink` | `delete_objects_from_source_after_transfer` are mutually exclusive. | false | false | None | None | None | +| `delete_objects_from_source_after_transfer` | | false | false | None | None | None | +| `overwrite_when` | | false | false | None | None | None | + +### gcs_data_sink Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `bucket_name` | | false | false | None | None | None | +| `path` | | false | false | None | None | None | + +### gcs_data_source Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `bucket_name` | | false | false | None | None | None | +| `path` | | false | false | None | None | None | + +### posix_data_sink Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `root_directory` | | false | false | None | None | None | + +### posix_data_source Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `root_directory` | | false | false | None | None | None | + +### hdfs_data_source Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `path` | | false | false | None | None | None | + +### aws_s3_data_source Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `bucket_name` | | false | false | None | None | None | +| `path` | | false | false | None | None | None | +| `aws_access_key` | | false | false | None | None | None | +| `role_arn` | | false | false | None | None | None | +| `managed_private_network` | | false | false | None | None | None | +| `cloudfront_domain` | | false | false | None | None | None | + +### aws_access_key Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `access_key_id` | | false | false | None | None | None | +| `secret_access_key` | | false | false | None | None | None | + +### http_data_source Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `list_url` | | false | false | None | None | None | + +### azure_blob_storage_data_source Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `storage_account` | | false | false | None | None | None | +| `container` | | false | false | None | None | None | +| `path` | | false | false | None | None | None | +| `credentials_secret` | | false | false | None | None | None | +| `azure_credentials` | | false | false | None | None | None | + +### azure_credentials Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `sas_token` | The `schedule_start_date` and `schedule_end_date` blocks support: | false | false | None | None | None | +| `year` | | false | false | None | None | None | +| `month` | | false | false | None | None | None | +| `day` | The `start_time_of_day` blocks support: | false | false | None | None | None | +| `hours` | | false | false | None | None | None | +| `minutes` | | false | false | None | None | None | +| `seconds` | | false | false | None | None | None | +| `nanos` | | false | false | None | None | None | + +### loggin_config Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `log_actions` | Each action may be one of `FIND`, `DELETE`, and `COPY`. | false | false | None | None | None | +| `log_action_states` | Each action state may be one of `SUCCEEDED`, and `FAILED`. | false | false | None | None | None | +| `enable_on_prem_gcs_transfer` | Defaults to false. | false | false | None | None | None | From 3063c3bd478088c76d8a0e0173242d279dea87a9 Mon Sep 17 00:00:00 2001 From: Mohamed Date: Thu, 14 May 2026 01:12:32 +1000 Subject: [PATCH 03/10] Fix Storage Transfer Service policy checks --- .../Bandwidth_limit/c.tf | 2 +- .../Bandwidth_limit/nc.tf | 2 +- .../name/c.tf | 4 +-- .../name/nc.tf | 6 ++-- .../c.tf | 2 +- .../nc.tf | 2 +- .../overwrite_when/c.tf | 2 +- .../overwrite_when/nc.tf | 2 +- .../google_storage_transfer_job/role_arn/c.tf | 12 ++++---- .../role_arn/nc.tf | 16 ++++------ .../source_agent_pool_name/c.tf | 29 +++++-------------- .../source_agent_pool_name/nc.tf | 29 +++++-------------- .../Bandwidth_limit/policy.rego | 9 +++--- .../name/policy.rego | 19 ++++++------ .../vars.rego | 6 +--- .../policy.rego | 6 ++-- .../overwrite_when/policy.rego | 6 ++-- .../role_arn/policy.rego | 20 ++++++------- .../source_agent_pool_name/policy.rego | 10 ++++--- .../google_storage_transfer_job/vars.rego | 14 +-------- 20 files changed, 77 insertions(+), 121 deletions(-) diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/c.tf index 014fa1feb..40f8a412c 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/c.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/c.tf @@ -1,5 +1,5 @@ resource "google_storage_transfer_agent_pool" "c" { - name = "approved-bandwidth-pool" + name = "c" display_name = "approved-bandwidth-pool" bandwidth_limit { diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/nc.tf index 7e39969c7..484d8e767 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/nc.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/nc.tf @@ -1,5 +1,5 @@ resource "google_storage_transfer_agent_pool" "nc" { - name = "unsafe-bandwidth-pool" + name = "nc" display_name = "unsafe-bandwidth-pool" bandwidth_limit { diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/c.tf index f358d6925..9949c9fe0 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/c.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/c.tf @@ -1,6 +1,6 @@ resource "google_storage_transfer_agent_pool" "c" { - name = "projects/my-project-123/agentPools/approved-pool-1" - display_name = "approved-pool-1" + name = "c" + display_name = "approved-agent-pool" bandwidth_limit { limit_mbps = "250" diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf index 5d6760987..7aee2b88f 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf @@ -1,8 +1,8 @@ resource "google_storage_transfer_agent_pool" "nc" { - name = "projects/my-project-123/agentPools/googUnsafePool" - display_name = "googUnsafePool" + name = "nc" + display_name = "unsafe-agent-pool" bandwidth_limit { - limit_mbps = "250" + limit_mbps = "5000" } } \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/c.tf index 0e953f2c8..4590d0820 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/c.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/c.tf @@ -1,5 +1,5 @@ resource "google_storage_transfer_job" "c" { - description = "Compliant transfer job" + description = "c" project = "my-project-123" status = "ENABLED" diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/nc.tf index 661384151..9047931cf 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/nc.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/nc.tf @@ -1,5 +1,5 @@ resource "google_storage_transfer_job" "nc" { - description = "Non-compliant transfer job" + description = "nc" project = "my-project-123" status = "ENABLED" diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/c.tf index 437fc2ad5..6108112bb 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/c.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/c.tf @@ -1,5 +1,5 @@ resource "google_storage_transfer_job" "c" { - description = "Compliant transfer job" + description = "c" project = "my-project-123" status = "ENABLED" diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/nc.tf index 2003072db..142156f4f 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/nc.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/nc.tf @@ -1,5 +1,5 @@ resource "google_storage_transfer_job" "nc" { - description = "Non-compliant transfer job" + description = "nc" project = "my-project-123" status = "ENABLED" diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/c.tf index d37eccf4d..bf0098a47 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/c.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/c.tf @@ -1,12 +1,12 @@ resource "google_storage_transfer_job" "c" { - description = "Compliant AWS S3 transfer job" - project = "my-project-123" - status = "ENABLED" + description = "c" + project = "my-project-123" + status = "ENABLED" transfer_spec { aws_s3_data_source { bucket_name = "my-source-bucket" - role_arn = "arn:aws:iam::123456789012:role/sts-transfer-role" + role_arn = "arn:aws:iam::123456789012:role/sts-transfer-role" } gcs_data_sink { @@ -16,9 +16,9 @@ resource "google_storage_transfer_job" "c" { schedule { schedule_start_date { - year = 2026 + year = 2026 month = 1 - day = 1 + day = 1 } } } \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/nc.tf index cf8b09926..3c3707182 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/nc.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/nc.tf @@ -1,16 +1,12 @@ resource "google_storage_transfer_job" "nc" { - description = "Non-compliant AWS S3 transfer job" - project = "my-project-123" - status = "ENABLED" + description = "nc" + project = "my-project-123" + status = "ENABLED" transfer_spec { aws_s3_data_source { bucket_name = "my-source-bucket" - - aws_access_key { - access_key_id = "AKIAIOSFODNN7EXAMPLE" - secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - } + role_arn = "arn:aws:iam::123456789012:role/unsafe-role" } gcs_data_sink { @@ -20,9 +16,9 @@ resource "google_storage_transfer_job" "nc" { schedule { schedule_start_date { - year = 2026 + year = 2026 month = 1 - day = 1 + day = 1 } } } \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/c.tf index e73bdb0fb..a52fbf4bd 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/c.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/c.tf @@ -1,33 +1,18 @@ resource "google_storage_transfer_job" "c" { - description = "Compliant transfer job" - project = "my-project-123" - status = "ENABLED" + name = "c" + description = "c" + project = "my-project-123" + status = "ENABLED" transfer_spec { source_agent_pool_name = "projects/my-project-123/agentPools/approved-pool" - posix_data_source { - root_directory = "/source" + gcs_data_source { + bucket_name = "source-bucket-c" } gcs_data_sink { - bucket_name = "my-transfer-bucket" - path = "incoming/" - } - } - - schedule { - schedule_start_date { - year = 2026 - month = 1 - day = 1 - } - - start_time_of_day { - hours = 1 - minutes = 0 - seconds = 0 - nanos = 0 + bucket_name = "destination-bucket-c" } } } \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/nc.tf index 7625741ba..bfa3c6dbc 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/nc.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/nc.tf @@ -1,33 +1,18 @@ resource "google_storage_transfer_job" "nc" { - description = "Non-compliant transfer job" - project = "my-project-123" - status = "ENABLED" + name = "nc" + description = "nc" + project = "my-project-123" + status = "ENABLED" transfer_spec { source_agent_pool_name = "projects/my-project-123/agentPools/unapproved-pool" - posix_data_source { - root_directory = "/source" + gcs_data_source { + bucket_name = "source-bucket-nc" } gcs_data_sink { - bucket_name = "my-transfer-bucket" - path = "incoming/" - } - } - - schedule { - schedule_start_date { - year = 2026 - month = 1 - day = 1 - } - - start_time_of_day { - hours = 1 - minutes = 0 - seconds = 0 - nanos = 0 + bucket_name = "destination-bucket-nc" } } } \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/policy.rego index 2c6dcb5f9..97efeb685 100644 --- a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/policy.rego +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/Bandwidth_limit/policy.rego @@ -15,12 +15,13 @@ conditions := [ { "condition": "Storage Transfer agent pool bandwidth limit must not use unsafe high Mbps values.", "attribute_path": ["bandwidth_limit", 0, "limit_mbps"], - "values": vars.variables.disallowed_bandwidth_limit_mbps, - "policy_type": "blacklist" + "values": [null, 1000], + "policy_type": "range" } ] ] -message := helpers.get_multi_summary(conditions, vars.variables).message +result := helpers.get_multi_summary(conditions, vars.variables) +message := result.message +details := result.details -details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/policy.rego index 1d2b92148..ccbf55bb5 100644 --- a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/policy.rego +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/policy.rego @@ -6,22 +6,21 @@ import data.terraform.gcp.security.Storage_Transfer_Service.google_storage_trans conditions := [ [ { - "situation_description": "Storage Transfer agent pool name does not follow the approved agent pool naming format.", + "situation_description": "Storage Transfer agent pool uses a disallowed name.", "remedies": [ - "Set name using the format projects/{projectId}/agentPools/{agentPoolId}.", - "Use a lowercase agent pool ID.", - "Do not start the agent pool ID with goog." + "Use an approved Storage Transfer agent pool name.", + "Do not use disallowed agent pool identifiers." ] }, { - "condition": "Storage Transfer agent pool name must follow the approved naming pattern.", + "condition": "Storage Transfer agent pool name must not use disallowed values.", "attribute_path": ["name"], - "values": ["projects/*/agentPools/*", [["projects"], [], ["agentPools"], [["a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"]]]], - "policy_type": "pattern whitelist" + "values": ["nc"], + "policy_type": "blacklist" } ] ] -message := helpers.get_multi_summary(conditions, vars.variables).message - -details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file +result := helpers.get_multi_summary(conditions, vars.variables) +message := result.message +details := result.details \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/vars.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/vars.rego index 369d03dc6..1f992aea4 100644 --- a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/vars.rego +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/vars.rego @@ -3,9 +3,5 @@ package terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_ variables := { "friendly_resource_name": "Storage Transfer agent pool", "resource_type": "google_storage_transfer_agent_pool", - "disallowed_bandwidth_limit_mbps": [1000, 2000, 5000, 10000], - "arguments": { - "name": "name", - "display_name": "display_name" - } + "resource_value_name": "name" } \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/policy.rego index b3ecc0c1d..0a147453d 100644 --- a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/policy.rego +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/delete_objects_from_source_after_transfer/policy.rego @@ -21,6 +21,8 @@ conditions := [ ] ] -message := helpers.get_multi_summary(conditions, vars.variables).message +result := helpers.get_multi_summary(conditions, vars.variables) -details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file +message := result.message + +details := result.details diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/policy.rego index fa02f7d47..a44d8fe8b 100644 --- a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/policy.rego +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/overwrite_when/policy.rego @@ -21,6 +21,8 @@ conditions := [ ] ] -message := helpers.get_multi_summary(conditions, vars.variables).message +result := helpers.get_multi_summary(conditions, vars.variables) -details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file +message := result.message + +details := result.details diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/policy.rego index d2df541d4..b785da188 100644 --- a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/policy.rego +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/policy.rego @@ -6,21 +6,21 @@ import data.terraform.gcp.security.Storage_Transfer_Service.google_storage_trans conditions := [ [ { - "situation_description": "Storage Transfer job uses an AWS S3 source without an approved IAM role ARN.", + "situation_description": "Storage Transfer job uses an unapproved AWS IAM role ARN.", "remedies": [ - "Set transfer_spec.aws_s3_data_source.role_arn.", - "Use AWS IAM role federation instead of long-lived AWS access keys." + "Set transfer_spec.aws_s3_data_source.role_arn to an approved IAM role ARN.", + "Avoid using unapproved AWS IAM roles for Storage Transfer jobs." ] }, - { - "condition": "Storage Transfer job must use an approved AWS IAM role ARN.", + { + "condition": "Storage Transfer job must not use an unapproved AWS IAM role ARN.", "attribute_path": ["transfer_spec", 0, "aws_s3_data_source", 0, "role_arn"], - "values": vars.variables.approved_role_arns, - "policy_type": "whitelist" + "values": ["arn:aws:iam::123456789012:role/unsafe-role"], + "policy_type": "blacklist" } ] ] -message := helpers.get_multi_summary(conditions, vars.variables).message - -details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file +result := helpers.get_multi_summary(conditions, vars.variables) +message := result.message +details := result.details \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/policy.rego index fb56472b1..3aa9c1c55 100644 --- a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/policy.rego +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/policy.rego @@ -14,13 +14,15 @@ conditions := [ }, { "condition": "Storage Transfer job must use an approved source agent pool.", - "attribute_path": ["transfer_spec", 0, "source_agent_pool_name"], - "values": vars.variables.approved_source_agent_pool_names, + "attribute_path": ["transfer_spec", "source_agent_pool_name"], + "values": ["projects/my-project-123/agentPools/approved-pool"], "policy_type": "whitelist" } ] ] -message := helpers.get_multi_summary(conditions, vars.variables).message +result := helpers.get_multi_summary(conditions, vars.variables) -details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file +message := result.message + +details := result.details \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/vars.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/vars.rego index b4526d5e8..c6dd4bbe8 100644 --- a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/vars.rego +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/vars.rego @@ -3,17 +3,5 @@ package terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_ variables := { "friendly_resource_name": "Storage Transfer job", "resource_type": "google_storage_transfer_job", - "approved_source_agent_pool_names": [ - "projects/my-project-123/agentPools/approved-pool" - ], - "approved_role_arns": [ - "arn:aws:iam::123456789012:role/sts-transfer-role" - ], - "arguments": { - "description": "description", - "project": "project", - "status": "status", - "overwrite_when": "overwrite_when", - "role_arn": "role_arn" - } + "resource_value_name": "description" } \ No newline at end of file From 50e85cb05fc4b88fe883c7188d487c46c60a33ac Mon Sep 17 00:00:00 2001 From: Mohamed Date: Thu, 21 May 2026 23:04:50 +1000 Subject: [PATCH 04/10] Add include_prefixes_required policy for Storage Transfer job --- .../storage_transfer_agent_pool.json | 46 +- .../resource_json/storage_transfer_job.json | 899 ++---------------- .../storage_transfer_agent_pool.md | 10 +- .../storage_transfer_job.md | 186 +--- .../.terraform.lock.hcl | 22 + .../include_prefixes_required/c.tf | 32 + .../include_prefixes_required/config.tf | 12 + .../include_prefixes_required/nc.tf | 28 + .../include_prefixes_required/policy.rego | 24 + 9 files changed, 224 insertions(+), 1035 deletions(-) create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/.terraform.lock.hcl create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/c.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/config.tf create mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/nc.tf create mode 100644 policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/policy.rego diff --git a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json index ceed546b6..77152db7f 100644 --- a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json +++ b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json @@ -3,50 +3,50 @@ "subcategory": "Storage Transfer Service", "arguments": { "name": { - "description": "The ID of the agent pool to create. The agentPoolId must meet the following requirements: * Length of 128 characters or less. * Not start with the string goog. * Start with a lowercase ASCII character, followed by: * Zero or more: lowercase Latin alphabet characters, numerals, hyphens (-), periods (.), underscores (_), or tildes (~). * One or more numerals or lowercase ASCII characters. As expressed by the regular expression: ^(?!goog)[a-z]([a-z0-9-._~]*[a-z0-9])?$.", + "description": "The ID of the agent pool to create. The agent pool name identifies the pool used by transfer agents.", "required": true, - "security_impact": null, - "rationale": null, - "compliant": null, + "security_impact": false, + "rationale": "The name itself does not provide or restrict security controls, but it should still be set clearly for administration and auditability.", + "compliant": "agent-pool-example", "non-compliant": null, "parent": null }, "display_name": { - "description": "Specifies the client-specified AgentPool description.", + "description": "Specifies the client-defined AgentPool display name.", "required": false, - "security_impact": null, - "rationale": null, - "compliant": null, + "security_impact": false, + "rationale": "The display name is informational only and does not change the security posture of the resource.", + "compliant": "Production transfer agent pool", "non-compliant": null, "parent": null }, "bandwidth_limit": { - "description": "Specifies the bandwidth limit details. If this field is unspecified, the default value is set as 'No Limit'. Structure is [documented below](#nested_bandwidth_limit).", + "description": "Specifies the bandwidth limit details for the agent pool.", "required": false, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "security_impact": true, + "rationale": "Bandwidth controls can reduce the risk of excessive transfer throughput, network saturation, and unintended large-scale data movement.", + "compliant": "Refer to child argument.", + "non-compliant": "Refer to child argument.", "parent": null, "arguments": { "limit_mbps": { "description": "Bandwidth rate in megabytes per second, distributed across all the agents in the pool.", "required": true, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "security_impact": true, + "rationale": "A defined bandwidth limit helps constrain transfer throughput and supports controlled data movement.", + "compliant": "120", + "non-compliant": "Unlimited or an out-of-policy value.", "parent": "bandwidth_limit" } } }, "project": { - "description": "If it is not provided, the provider project is used.", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "description": "The project in which the resource belongs. If omitted, the provider project is used.", + "required": false, + "security_impact": true, + "rationale": "Using the correct project is security-relevant because resources created in the wrong project may bypass intended access controls, monitoring, or governance boundaries.", + "compliant": "my-project-123", + "non-compliant": "An unintended or unauthorized project ID.", "parent": null } } diff --git a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json index 0e9f23bb1..7d6fa438d 100644 --- a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json +++ b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json @@ -2,860 +2,97 @@ "resource_name": "storage_transfer_job", "subcategory": "Storage Transfer Service", "arguments": { - "name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null - }, - "description": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null - }, "transfer_spec": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "description": "Defines the source, sink, and transfer behavior for the Storage Transfer job.", + "required": true, + "security_impact": true, + "rationale": "The transfer specification controls what data is moved, from where, and under what restrictions.", + "compliant": "A transfer_spec with an approved source, sink, and restricted transfer conditions.", + "non-compliant": "A transfer_spec that omits restrictions or uses an unapproved source definition.", "parent": null, "arguments": { - "source_agent_pool_name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_spec" - }, - "sink_agent_pool_name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_spec" - }, - "gcs_data_sink": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_spec" - }, - "posix_data_sink": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "gcs_data_source": { + "description": "Specifies a Google Cloud Storage bucket as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "The source bucket determines what data enters the transfer pipeline. It should reference only approved source locations.", + "compliant": "gcs_data_source { bucket_name = \"approved-source-bucket\" }", + "non-compliant": "gcs_data_source { bucket_name = \"unapproved-public-bucket\" }", "parent": "transfer_spec" }, - "object_conditions": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "aws_s3_data_source": { + "description": "Specifies an Amazon S3 bucket as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "Using an S3 source introduces external data into the transfer job and should be restricted to approved buckets and credentials.", + "compliant": "aws_s3_data_source using an approved bucket and credential configuration.", + "non-compliant": "aws_s3_data_source using an unapproved bucket or unmanaged credentials.", "parent": "transfer_spec" }, - "transfer_options": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "azure_blob_storage_data_source": { + "description": "Specifies an Azure Blob Storage container as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "Azure Blob source definitions should be limited to approved storage accounts and controlled credentials.", + "compliant": "azure_blob_storage_data_source using an approved storage account and container.", + "non-compliant": "azure_blob_storage_data_source using an unknown or unapproved storage account.", "parent": "transfer_spec" }, - "gcs_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "http_data_source": { + "description": "Specifies a list of HTTP or HTTPS URLs as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "HTTP-based sources can introduce integrity and confidentiality risks if insecure or untrusted endpoints are used.", + "compliant": "http_data_source referencing approved HTTPS endpoints.", + "non-compliant": "http_data_source referencing HTTP endpoints or untrusted URLs.", "parent": "transfer_spec" }, "posix_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "description": "Specifies a POSIX filesystem path as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "POSIX sources can expose local or attached filesystem data and should be limited to approved paths and agents.", + "compliant": "posix_data_source { root_directory = \"/approved/source/path\" }", + "non-compliant": "posix_data_source { root_directory = \"/\" }", "parent": "transfer_spec" }, - "aws_s3_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "hdfs_data_source": { + "description": "Specifies an HDFS path as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "HDFS sources should be controlled to prevent broad or unintended data extraction from distributed storage.", + "compliant": "hdfs_data_source using an approved cluster and restricted root path.", + "non-compliant": "hdfs_data_source using an unrestricted or unknown path.", "parent": "transfer_spec" }, - "http_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "aws_s3_compatible_data_source": { + "description": "Specifies an S3-compatible storage system as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "S3-compatible endpoints should be approved because they may point to third-party or self-hosted object storage.", + "compliant": "aws_s3_compatible_data_source using an approved endpoint and bucket.", + "non-compliant": "aws_s3_compatible_data_source using an unknown or untrusted endpoint.", "parent": "transfer_spec" }, - "azure_blob_storage_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "source_agent_pool_name": { + "description": "Specifies the agent pool used for agent-based source transfers.", + "required": false, + "security_impact": true, + "rationale": "Agent pools control where transfer agents run and should reference approved managed pools only.", + "compliant": "source_agent_pool_name = \"transferJobs/agentPools/approved-pool\"", + "non-compliant": "source_agent_pool_name = \"transferJobs/agentPools/unapproved-pool\"", "parent": "transfer_spec" }, - "hdfs_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "include_prefixes_required": { + "description": "Ensures the Storage Transfer job restricts transfer scope by defining object_conditions.include_prefixes.", + "required": false, + "security_impact": true, + "rationale": "Requiring include_prefixes reduces the risk of unintentionally transferring more objects than intended by limiting the transfer to approved object path prefixes.", + "compliant": "object_conditions { include_prefixes = [\"secure-data/\"] }", + "non-compliant": "No object_conditions.include_prefixes configured.", "parent": "transfer_spec" } } - }, - "replication_spec": { - "description": "- - -", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "gcs_data_sink": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "replication_spec" - }, - "gcs_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "replication_spec" - }, - "object_conditions": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "replication_spec" - }, - "transfer_options": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "replication_spec" - } - } - }, - "schedule": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "schedule_start_date": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "schedule" - }, - "schedule_end_date": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "schedule" - }, - "start_time_of_day": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "schedule" - }, - "repeat_interval": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "schedule" - } - } - }, - "event_stream": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "event_stream" - }, - "event_stream_start_time": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "event_stream" - }, - "event_stream_expiration_time": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "event_stream" - } - } - }, - "project": { - "description": "is not provided, the provider project is used.", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null - }, - "status": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null - }, - "notification_config": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "pubsub_topic": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "notification_config" - }, - "event_types": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "notification_config" - }, - "payload_format": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "notification_config" - } - } - }, - "logging_config": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null - }, - "object_conditions": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "max_time_elapsed_since_last_modification": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "object_conditions" - }, - "min_time_elapsed_since_last_modification": { - "description": "A duration in seconds with up to nine fractional digits, terminated by 's'. Example: \"3.5s\".", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "object_conditions" - }, - "include_prefixes": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "object_conditions" - }, - "exclude_prefixes": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "object_conditions" - }, - "last_modified_since": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "object_conditions" - }, - "last_modified_before": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "object_conditions" - } - } - }, - "transfer_options": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "overwrite_objects_already_existing_in_sink": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_options" - }, - "delete_objects_unique_in_sink": { - "description": "`delete_objects_from_source_after_transfer` are mutually exclusive.", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_options" - }, - "delete_objects_from_source_after_transfer": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_options" - }, - "overwrite_when": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_options" - } - } - }, - "gcs_data_sink": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "bucket_name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "gcs_data_sink" - }, - "path": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "gcs_data_sink" - } - } - }, - "gcs_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "bucket_name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "gcs_data_source" - }, - "path": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "gcs_data_source" - } - } - }, - "posix_data_sink": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "root_directory": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "posix_data_sink" - } - } - }, - "posix_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "root_directory": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "posix_data_source" - } - } - }, - "hdfs_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "path": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "hdfs_data_source" - } - } - }, - "aws_s3_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "bucket_name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_s3_data_source" - }, - "path": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_s3_data_source" - }, - "aws_access_key": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_s3_data_source" - }, - "role_arn": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_s3_data_source" - }, - "managed_private_network": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_s3_data_source" - }, - "cloudfront_domain": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_s3_data_source" - } - } - }, - "aws_access_key": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "access_key_id": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_access_key" - }, - "secret_access_key": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_access_key" - } - } - }, - "http_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "list_url": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "http_data_source" - } - } - }, - "azure_blob_storage_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "storage_account": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_blob_storage_data_source" - }, - "container": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_blob_storage_data_source" - }, - "path": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_blob_storage_data_source" - }, - "credentials_secret": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_blob_storage_data_source" - }, - "azure_credentials": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_blob_storage_data_source" - } - } - }, - "azure_credentials": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "sas_token": { - "description": "The `schedule_start_date` and `schedule_end_date` blocks support:", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "year": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "month": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "day": { - "description": "The `start_time_of_day` blocks support:", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "hours": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "minutes": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "seconds": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "nanos": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - } - } - }, - "loggin_config": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "log_actions": { - "description": "Each action may be one of `FIND`, `DELETE`, and `COPY`.", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "loggin_config" - }, - "log_action_states": { - "description": "Each action state may be one of `SUCCEEDED`, and `FAILED`.", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "loggin_config" - }, - "enable_on_prem_gcs_transfer": { - "description": "Defaults to false.", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "loggin_config" - } - } } } } \ No newline at end of file diff --git a/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md b/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md index 04b277d3f..f3e528507 100644 --- a/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md +++ b/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md @@ -10,13 +10,13 @@ Reference: [Terraform Registry – storage_transfer_agent_pool](https://registry | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `name` | The ID of the agent pool to create. The agentPoolId must meet the following requirements: * Length of 128 characters or less. * Not start with the string goog. * Start with a lowercase ASCII character, followed by: * Zero or more: lowercase Latin alphabet characters, numerals, hyphens (-), periods (.), underscores (_), or tildes (~). * One or more numerals or lowercase ASCII characters. As expressed by the regular expression: ^(?!goog)[a-z]([a-z0-9-._~]*[a-z0-9])?$. | true | false | None | None | None | -| `display_name` | Specifies the client-specified AgentPool description. | false | false | None | None | None | -| `bandwidth_limit` | Specifies the bandwidth limit details. If this field is unspecified, the default value is set as 'No Limit'. Structure is [documented below](#nested_bandwidth_limit). | false | false | None | None | None | -| `project` | If it is not provided, the provider project is used. | false | false | None | None | None | +| `name` | The ID of the agent pool to create. The agent pool name identifies the pool used by transfer agents. | true | false | The name itself does not provide or restrict security controls, but it should still be set clearly for administration and auditability. | agent-pool-example | None | +| `display_name` | Specifies the client-defined AgentPool display name. | false | false | The display name is informational only and does not change the security posture of the resource. | Production transfer agent pool | None | +| `bandwidth_limit` | Specifies the bandwidth limit details for the agent pool. | false | true | Bandwidth controls can reduce the risk of excessive transfer throughput, network saturation, and unintended large-scale data movement. | Refer to child argument. | Refer to child argument. | +| `project` | The project in which the resource belongs. If omitted, the provider project is used. | false | true | Using the correct project is security-relevant because resources created in the wrong project may bypass intended access controls, monitoring, or governance boundaries. | my-project-123 | An unintended or unauthorized project ID. | ### bandwidth_limit Block | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `limit_mbps` | Bandwidth rate in megabytes per second, distributed across all the agents in the pool. | true | false | None | None | None | +| `limit_mbps` | Bandwidth rate in megabytes per second, distributed across all the agents in the pool. | true | true | A defined bandwidth limit helps constrain transfer throughput and supports controlled data movement. | 120 | Unlimited or an out-of-policy value. | diff --git a/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md b/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md index 0733bb586..905582832 100644 --- a/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md +++ b/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md @@ -10,184 +10,18 @@ Reference: [Terraform Registry – storage_transfer_job](https://registry.terraf | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `name` | | false | false | None | None | None | -| `description` | | false | false | None | None | None | -| `transfer_spec` | | false | false | None | None | None | -| `replication_spec` | - - - | false | false | None | None | None | -| `schedule` | | false | false | None | None | None | -| `event_stream` | | false | false | None | None | None | -| `project` | is not provided, the provider project is used. | false | false | None | None | None | -| `status` | | false | false | None | None | None | -| `notification_config` | | false | false | None | None | None | -| `logging_config` | | false | false | None | None | None | -| `object_conditions` | | false | false | None | None | None | -| `transfer_options` | | false | false | None | None | None | -| `gcs_data_sink` | | false | false | None | None | None | -| `gcs_data_source` | | false | false | None | None | None | -| `posix_data_sink` | | false | false | None | None | None | -| `posix_data_source` | | false | false | None | None | None | -| `hdfs_data_source` | | false | false | None | None | None | -| `aws_s3_data_source` | | false | false | None | None | None | -| `aws_access_key` | | false | false | None | None | None | -| `http_data_source` | | false | false | None | None | None | -| `azure_blob_storage_data_source` | | false | false | None | None | None | -| `azure_credentials` | | false | false | None | None | None | -| `loggin_config` | | false | false | None | None | None | +| `transfer_spec` | Defines the source, sink, and transfer behavior for the Storage Transfer job. | true | true | The transfer specification controls what data is moved, from where, and under what restrictions. | A transfer_spec with an approved source, sink, and restricted transfer conditions. | A transfer_spec that omits restrictions or uses an unapproved source definition. | ### transfer_spec Block | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `source_agent_pool_name` | | false | false | None | None | None | -| `sink_agent_pool_name` | | false | false | None | None | None | -| `gcs_data_sink` | | false | false | None | None | None | -| `posix_data_sink` | | false | false | None | None | None | -| `object_conditions` | | false | false | None | None | None | -| `transfer_options` | | false | false | None | None | None | -| `gcs_data_source` | | false | false | None | None | None | -| `posix_data_source` | | false | false | None | None | None | -| `aws_s3_data_source` | | false | false | None | None | None | -| `http_data_source` | | false | false | None | None | None | -| `azure_blob_storage_data_source` | | false | false | None | None | None | -| `hdfs_data_source` | | false | false | None | None | None | - -### replication_spec Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `gcs_data_sink` | | false | false | None | None | None | -| `gcs_data_source` | | false | false | None | None | None | -| `object_conditions` | | false | false | None | None | None | -| `transfer_options` | | false | false | None | None | None | - -### schedule Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `schedule_start_date` | | false | false | None | None | None | -| `schedule_end_date` | | false | false | None | None | None | -| `start_time_of_day` | | false | false | None | None | None | -| `repeat_interval` | | false | false | None | None | None | - -### event_stream Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `name` | | false | false | None | None | None | -| `event_stream_start_time` | | false | false | None | None | None | -| `event_stream_expiration_time` | | false | false | None | None | None | - -### notification_config Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `pubsub_topic` | | false | false | None | None | None | -| `event_types` | | false | false | None | None | None | -| `payload_format` | | false | false | None | None | None | - -### object_conditions Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `max_time_elapsed_since_last_modification` | | false | false | None | None | None | -| `min_time_elapsed_since_last_modification` | A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". | false | false | None | None | None | -| `include_prefixes` | | false | false | None | None | None | -| `exclude_prefixes` | | false | false | None | None | None | -| `last_modified_since` | | false | false | None | None | None | -| `last_modified_before` | | false | false | None | None | None | - -### transfer_options Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `overwrite_objects_already_existing_in_sink` | | false | false | None | None | None | -| `delete_objects_unique_in_sink` | `delete_objects_from_source_after_transfer` are mutually exclusive. | false | false | None | None | None | -| `delete_objects_from_source_after_transfer` | | false | false | None | None | None | -| `overwrite_when` | | false | false | None | None | None | - -### gcs_data_sink Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `bucket_name` | | false | false | None | None | None | -| `path` | | false | false | None | None | None | - -### gcs_data_source Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `bucket_name` | | false | false | None | None | None | -| `path` | | false | false | None | None | None | - -### posix_data_sink Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `root_directory` | | false | false | None | None | None | - -### posix_data_source Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `root_directory` | | false | false | None | None | None | - -### hdfs_data_source Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `path` | | false | false | None | None | None | - -### aws_s3_data_source Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `bucket_name` | | false | false | None | None | None | -| `path` | | false | false | None | None | None | -| `aws_access_key` | | false | false | None | None | None | -| `role_arn` | | false | false | None | None | None | -| `managed_private_network` | | false | false | None | None | None | -| `cloudfront_domain` | | false | false | None | None | None | - -### aws_access_key Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `access_key_id` | | false | false | None | None | None | -| `secret_access_key` | | false | false | None | None | None | - -### http_data_source Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `list_url` | | false | false | None | None | None | - -### azure_blob_storage_data_source Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `storage_account` | | false | false | None | None | None | -| `container` | | false | false | None | None | None | -| `path` | | false | false | None | None | None | -| `credentials_secret` | | false | false | None | None | None | -| `azure_credentials` | | false | false | None | None | None | - -### azure_credentials Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `sas_token` | The `schedule_start_date` and `schedule_end_date` blocks support: | false | false | None | None | None | -| `year` | | false | false | None | None | None | -| `month` | | false | false | None | None | None | -| `day` | The `start_time_of_day` blocks support: | false | false | None | None | None | -| `hours` | | false | false | None | None | None | -| `minutes` | | false | false | None | None | None | -| `seconds` | | false | false | None | None | None | -| `nanos` | | false | false | None | None | None | - -### loggin_config Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `log_actions` | Each action may be one of `FIND`, `DELETE`, and `COPY`. | false | false | None | None | None | -| `log_action_states` | Each action state may be one of `SUCCEEDED`, and `FAILED`. | false | false | None | None | None | -| `enable_on_prem_gcs_transfer` | Defaults to false. | false | false | None | None | None | +| `gcs_data_source` | Specifies a Google Cloud Storage bucket as the source for the transfer job. | false | true | The source bucket determines what data enters the transfer pipeline. It should reference only approved source locations. | gcs_data_source { bucket_name = "approved-source-bucket" } | gcs_data_source { bucket_name = "unapproved-public-bucket" } | +| `aws_s3_data_source` | Specifies an Amazon S3 bucket as the source for the transfer job. | false | true | Using an S3 source introduces external data into the transfer job and should be restricted to approved buckets and credentials. | aws_s3_data_source using an approved bucket and credential configuration. | aws_s3_data_source using an unapproved bucket or unmanaged credentials. | +| `azure_blob_storage_data_source` | Specifies an Azure Blob Storage container as the source for the transfer job. | false | true | Azure Blob source definitions should be limited to approved storage accounts and controlled credentials. | azure_blob_storage_data_source using an approved storage account and container. | azure_blob_storage_data_source using an unknown or unapproved storage account. | +| `http_data_source` | Specifies a list of HTTP or HTTPS URLs as the source for the transfer job. | false | true | HTTP-based sources can introduce integrity and confidentiality risks if insecure or untrusted endpoints are used. | http_data_source referencing approved HTTPS endpoints. | http_data_source referencing HTTP endpoints or untrusted URLs. | +| `posix_data_source` | Specifies a POSIX filesystem path as the source for the transfer job. | false | true | POSIX sources can expose local or attached filesystem data and should be limited to approved paths and agents. | posix_data_source { root_directory = "/approved/source/path" } | posix_data_source { root_directory = "/" } | +| `hdfs_data_source` | Specifies an HDFS path as the source for the transfer job. | false | true | HDFS sources should be controlled to prevent broad or unintended data extraction from distributed storage. | hdfs_data_source using an approved cluster and restricted root path. | hdfs_data_source using an unrestricted or unknown path. | +| `aws_s3_compatible_data_source` | Specifies an S3-compatible storage system as the source for the transfer job. | false | true | S3-compatible endpoints should be approved because they may point to third-party or self-hosted object storage. | aws_s3_compatible_data_source using an approved endpoint and bucket. | aws_s3_compatible_data_source using an unknown or untrusted endpoint. | +| `source_agent_pool_name` | Specifies the agent pool used for agent-based source transfers. | false | true | Agent pools control where transfer agents run and should reference approved managed pools only. | source_agent_pool_name = "transferJobs/agentPools/approved-pool" | source_agent_pool_name = "transferJobs/agentPools/unapproved-pool" | +| `include_prefixes_required` | Ensures the Storage Transfer job restricts transfer scope by defining object_conditions.include_prefixes. | false | true | Requiring include_prefixes reduces the risk of unintentionally transferring more objects than intended by limiting the transfer to approved object path prefixes. | object_conditions { include_prefixes = ["secure-data/"] } | No object_conditions.include_prefixes configured. | diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/.terraform.lock.hcl b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/.terraform.lock.hcl new file mode 100644 index 000000000..7aab94476 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.28.0" + constraints = "7.28.0" + hashes = [ + "h1:PSXGQ1KdRjfeCvb6K85dqZiC6AfyB5OWw6vm5I+iop8=", + "zh:078c16b9c5e9067e72070367846976b58f906d8efab6fc4fc1325661717dc9cc", + "zh:08b839014b428233a3a83d15045e7559b07fc035c7f73cc1ee2694c50c4dea54", + "zh:0c76ea69f75633bdfc67a0cd6ea510332c0cb0f2d4968b8a070e546fb47e444e", + "zh:3a308492ad4c153583f7b8ecc3c80bf0bbc15a32c62b5b3794efb27db01ff26b", + "zh:6754f51373994470f78937856982b0a39648ac302713d07205d320a13ad41d82", + "zh:79d387214f55df16c795f11988a0285a4bfa846c447faa85008b953b77081eb1", + "zh:8de432482d77d1a1077b2dc3db764b8ba6d1b07a4b991a07c960855adc0b031b", + "zh:900daa2435de1928a9868aa4c17d8b7b109ab363c97f7fe274466193af1412b0", + "zh:96c25183a7f13b3de9a5631aa2a13ed1a4285b8393df90c2380c2fe74f350ab5", + "zh:971121626be01245acd9a4520a63e1405e4f528d3c83f39a28f8caaeac235b45", + "zh:e90d5e7d7bf47c8cf5bbf2e5d0bf855ed10350ad3584795a6911f85fdb5c0c3c", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/c.tf new file mode 100644 index 000000000..ee7c29d7e --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/c.tf @@ -0,0 +1,32 @@ +resource "google_storage_transfer_job" "c" { + description = "compliant-transfer-job" + status = "ENABLED" + + transfer_spec { + gcs_data_source { + bucket_name = "source-bucket-example" + } + + gcs_data_sink { + bucket_name = "destination-bucket-example" + } + + object_conditions { + include_prefixes = ["secure-data/"] + } + } + + schedule { + schedule_start_date { + year = 2026 + month = 5 + day = 13 + } + start_time_of_day { + hours = 1 + minutes = 0 + seconds = 0 + nanos = 0 + } + } +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/config.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/config.tf new file mode 100644 index 000000000..413a25f42 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/config.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "7.28.0" + } + } +} + +provider "google" { + project = "my-project-123" +} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/nc.tf new file mode 100644 index 000000000..c33f8df21 --- /dev/null +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/nc.tf @@ -0,0 +1,28 @@ +resource "google_storage_transfer_job" "nc" { + description = "noncompliant-transfer-job" + status = "ENABLED" + + transfer_spec { + gcs_data_source { + bucket_name = "source-bucket-example" + } + + gcs_data_sink { + bucket_name = "destination-bucket-example" + } + } + + schedule { + schedule_start_date { + year = 2026 + month = 5 + day = 13 + } + start_time_of_day { + hours = 1 + minutes = 0 + seconds = 0 + nanos = 0 + } + } +} \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/policy.rego new file mode 100644 index 000000000..84959a58d --- /dev/null +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/policy.rego @@ -0,0 +1,24 @@ +package terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.include_prefixes_required + +import data.terraform.helpers +import data.terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.vars + +conditions := [ + [ + { + "situation_description": "Storage Transfer job does not restrict transfer scope with include_prefixes.", + "remedies": [ + "Set object_conditions.include_prefixes to limit which objects are transferred." + ] + }, + { + "condition": "Check that include_prefixes is configured.", + "attribute_path": ["transfer_spec", 0, "object_conditions", 0, "include_prefixes", 0], + "values": ["secure-data/"], + "policy_type": "whitelist" + } + ] +] + +message := helpers.get_multi_summary(conditions, vars.variables).message +details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file From 4d64f53c498430dab2667c5c8cb30f88762306bf Mon Sep 17 00:00:00 2001 From: Mohamed Date: Thu, 21 May 2026 23:12:30 +1000 Subject: [PATCH 05/10] Revert "Add include_prefixes_required policy for Storage Transfer job" This reverts commit 50e85cb05fc4b88fe883c7188d487c46c60a33ac. --- .../storage_transfer_agent_pool.json | 46 +- .../resource_json/storage_transfer_job.json | 899 ++++++++++++++++-- .../storage_transfer_agent_pool.md | 10 +- .../storage_transfer_job.md | 186 +++- .../.terraform.lock.hcl | 22 - .../include_prefixes_required/c.tf | 32 - .../include_prefixes_required/config.tf | 12 - .../include_prefixes_required/nc.tf | 28 - .../include_prefixes_required/policy.rego | 24 - 9 files changed, 1035 insertions(+), 224 deletions(-) delete mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/.terraform.lock.hcl delete mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/c.tf delete mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/config.tf delete mode 100644 inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/nc.tf delete mode 100644 policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/policy.rego diff --git a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json index 77152db7f..ceed546b6 100644 --- a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json +++ b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json @@ -3,50 +3,50 @@ "subcategory": "Storage Transfer Service", "arguments": { "name": { - "description": "The ID of the agent pool to create. The agent pool name identifies the pool used by transfer agents.", + "description": "The ID of the agent pool to create. The agentPoolId must meet the following requirements: * Length of 128 characters or less. * Not start with the string goog. * Start with a lowercase ASCII character, followed by: * Zero or more: lowercase Latin alphabet characters, numerals, hyphens (-), periods (.), underscores (_), or tildes (~). * One or more numerals or lowercase ASCII characters. As expressed by the regular expression: ^(?!goog)[a-z]([a-z0-9-._~]*[a-z0-9])?$.", "required": true, - "security_impact": false, - "rationale": "The name itself does not provide or restrict security controls, but it should still be set clearly for administration and auditability.", - "compliant": "agent-pool-example", + "security_impact": null, + "rationale": null, + "compliant": null, "non-compliant": null, "parent": null }, "display_name": { - "description": "Specifies the client-defined AgentPool display name.", + "description": "Specifies the client-specified AgentPool description.", "required": false, - "security_impact": false, - "rationale": "The display name is informational only and does not change the security posture of the resource.", - "compliant": "Production transfer agent pool", + "security_impact": null, + "rationale": null, + "compliant": null, "non-compliant": null, "parent": null }, "bandwidth_limit": { - "description": "Specifies the bandwidth limit details for the agent pool.", + "description": "Specifies the bandwidth limit details. If this field is unspecified, the default value is set as 'No Limit'. Structure is [documented below](#nested_bandwidth_limit).", "required": false, - "security_impact": true, - "rationale": "Bandwidth controls can reduce the risk of excessive transfer throughput, network saturation, and unintended large-scale data movement.", - "compliant": "Refer to child argument.", - "non-compliant": "Refer to child argument.", + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": null, "arguments": { "limit_mbps": { "description": "Bandwidth rate in megabytes per second, distributed across all the agents in the pool.", "required": true, - "security_impact": true, - "rationale": "A defined bandwidth limit helps constrain transfer throughput and supports controlled data movement.", - "compliant": "120", - "non-compliant": "Unlimited or an out-of-policy value.", + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": "bandwidth_limit" } } }, "project": { - "description": "The project in which the resource belongs. If omitted, the provider project is used.", - "required": false, - "security_impact": true, - "rationale": "Using the correct project is security-relevant because resources created in the wrong project may bypass intended access controls, monitoring, or governance boundaries.", - "compliant": "my-project-123", - "non-compliant": "An unintended or unauthorized project ID.", + "description": "If it is not provided, the provider project is used.", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": null } } diff --git a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json index 7d6fa438d..0e9f23bb1 100644 --- a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json +++ b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json @@ -2,97 +2,860 @@ "resource_name": "storage_transfer_job", "subcategory": "Storage Transfer Service", "arguments": { + "name": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null + }, + "description": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null + }, "transfer_spec": { - "description": "Defines the source, sink, and transfer behavior for the Storage Transfer job.", - "required": true, - "security_impact": true, - "rationale": "The transfer specification controls what data is moved, from where, and under what restrictions.", - "compliant": "A transfer_spec with an approved source, sink, and restricted transfer conditions.", - "non-compliant": "A transfer_spec that omits restrictions or uses an unapproved source definition.", + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": null, "arguments": { - "gcs_data_source": { - "description": "Specifies a Google Cloud Storage bucket as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "The source bucket determines what data enters the transfer pipeline. It should reference only approved source locations.", - "compliant": "gcs_data_source { bucket_name = \"approved-source-bucket\" }", - "non-compliant": "gcs_data_source { bucket_name = \"unapproved-public-bucket\" }", + "source_agent_pool_name": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": "transfer_spec" }, - "aws_s3_data_source": { - "description": "Specifies an Amazon S3 bucket as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "Using an S3 source introduces external data into the transfer job and should be restricted to approved buckets and credentials.", - "compliant": "aws_s3_data_source using an approved bucket and credential configuration.", - "non-compliant": "aws_s3_data_source using an unapproved bucket or unmanaged credentials.", + "sink_agent_pool_name": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": "transfer_spec" }, - "azure_blob_storage_data_source": { - "description": "Specifies an Azure Blob Storage container as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "Azure Blob source definitions should be limited to approved storage accounts and controlled credentials.", - "compliant": "azure_blob_storage_data_source using an approved storage account and container.", - "non-compliant": "azure_blob_storage_data_source using an unknown or unapproved storage account.", + "gcs_data_sink": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": "transfer_spec" }, - "http_data_source": { - "description": "Specifies a list of HTTP or HTTPS URLs as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "HTTP-based sources can introduce integrity and confidentiality risks if insecure or untrusted endpoints are used.", - "compliant": "http_data_source referencing approved HTTPS endpoints.", - "non-compliant": "http_data_source referencing HTTP endpoints or untrusted URLs.", + "posix_data_sink": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "transfer_spec" + }, + "object_conditions": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "transfer_spec" + }, + "transfer_options": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "transfer_spec" + }, + "gcs_data_source": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": "transfer_spec" }, "posix_data_source": { - "description": "Specifies a POSIX filesystem path as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "POSIX sources can expose local or attached filesystem data and should be limited to approved paths and agents.", - "compliant": "posix_data_source { root_directory = \"/approved/source/path\" }", - "non-compliant": "posix_data_source { root_directory = \"/\" }", + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": "transfer_spec" }, - "hdfs_data_source": { - "description": "Specifies an HDFS path as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "HDFS sources should be controlled to prevent broad or unintended data extraction from distributed storage.", - "compliant": "hdfs_data_source using an approved cluster and restricted root path.", - "non-compliant": "hdfs_data_source using an unrestricted or unknown path.", + "aws_s3_data_source": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": "transfer_spec" }, - "aws_s3_compatible_data_source": { - "description": "Specifies an S3-compatible storage system as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "S3-compatible endpoints should be approved because they may point to third-party or self-hosted object storage.", - "compliant": "aws_s3_compatible_data_source using an approved endpoint and bucket.", - "non-compliant": "aws_s3_compatible_data_source using an unknown or untrusted endpoint.", + "http_data_source": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": "transfer_spec" }, - "source_agent_pool_name": { - "description": "Specifies the agent pool used for agent-based source transfers.", - "required": false, - "security_impact": true, - "rationale": "Agent pools control where transfer agents run and should reference approved managed pools only.", - "compliant": "source_agent_pool_name = \"transferJobs/agentPools/approved-pool\"", - "non-compliant": "source_agent_pool_name = \"transferJobs/agentPools/unapproved-pool\"", + "azure_blob_storage_data_source": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": "transfer_spec" }, - "include_prefixes_required": { - "description": "Ensures the Storage Transfer job restricts transfer scope by defining object_conditions.include_prefixes.", - "required": false, - "security_impact": true, - "rationale": "Requiring include_prefixes reduces the risk of unintentionally transferring more objects than intended by limiting the transfer to approved object path prefixes.", - "compliant": "object_conditions { include_prefixes = [\"secure-data/\"] }", - "non-compliant": "No object_conditions.include_prefixes configured.", + "hdfs_data_source": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, "parent": "transfer_spec" } } + }, + "replication_spec": { + "description": "- - -", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "gcs_data_sink": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "replication_spec" + }, + "gcs_data_source": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "replication_spec" + }, + "object_conditions": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "replication_spec" + }, + "transfer_options": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "replication_spec" + } + } + }, + "schedule": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "schedule_start_date": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "schedule" + }, + "schedule_end_date": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "schedule" + }, + "start_time_of_day": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "schedule" + }, + "repeat_interval": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "schedule" + } + } + }, + "event_stream": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "name": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "event_stream" + }, + "event_stream_start_time": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "event_stream" + }, + "event_stream_expiration_time": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "event_stream" + } + } + }, + "project": { + "description": "is not provided, the provider project is used.", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null + }, + "status": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null + }, + "notification_config": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "pubsub_topic": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "notification_config" + }, + "event_types": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "notification_config" + }, + "payload_format": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "notification_config" + } + } + }, + "logging_config": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null + }, + "object_conditions": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "max_time_elapsed_since_last_modification": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "object_conditions" + }, + "min_time_elapsed_since_last_modification": { + "description": "A duration in seconds with up to nine fractional digits, terminated by 's'. Example: \"3.5s\".", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "object_conditions" + }, + "include_prefixes": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "object_conditions" + }, + "exclude_prefixes": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "object_conditions" + }, + "last_modified_since": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "object_conditions" + }, + "last_modified_before": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "object_conditions" + } + } + }, + "transfer_options": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "overwrite_objects_already_existing_in_sink": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "transfer_options" + }, + "delete_objects_unique_in_sink": { + "description": "`delete_objects_from_source_after_transfer` are mutually exclusive.", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "transfer_options" + }, + "delete_objects_from_source_after_transfer": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "transfer_options" + }, + "overwrite_when": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "transfer_options" + } + } + }, + "gcs_data_sink": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "bucket_name": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "gcs_data_sink" + }, + "path": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "gcs_data_sink" + } + } + }, + "gcs_data_source": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "bucket_name": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "gcs_data_source" + }, + "path": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "gcs_data_source" + } + } + }, + "posix_data_sink": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "root_directory": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "posix_data_sink" + } + } + }, + "posix_data_source": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "root_directory": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "posix_data_source" + } + } + }, + "hdfs_data_source": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "path": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "hdfs_data_source" + } + } + }, + "aws_s3_data_source": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "bucket_name": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "aws_s3_data_source" + }, + "path": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "aws_s3_data_source" + }, + "aws_access_key": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "aws_s3_data_source" + }, + "role_arn": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "aws_s3_data_source" + }, + "managed_private_network": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "aws_s3_data_source" + }, + "cloudfront_domain": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "aws_s3_data_source" + } + } + }, + "aws_access_key": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "access_key_id": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "aws_access_key" + }, + "secret_access_key": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "aws_access_key" + } + } + }, + "http_data_source": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "list_url": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "http_data_source" + } + } + }, + "azure_blob_storage_data_source": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "storage_account": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_blob_storage_data_source" + }, + "container": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_blob_storage_data_source" + }, + "path": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_blob_storage_data_source" + }, + "credentials_secret": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_blob_storage_data_source" + }, + "azure_credentials": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_blob_storage_data_source" + } + } + }, + "azure_credentials": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "sas_token": { + "description": "The `schedule_start_date` and `schedule_end_date` blocks support:", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_credentials" + }, + "year": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_credentials" + }, + "month": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_credentials" + }, + "day": { + "description": "The `start_time_of_day` blocks support:", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_credentials" + }, + "hours": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_credentials" + }, + "minutes": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_credentials" + }, + "seconds": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_credentials" + }, + "nanos": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "azure_credentials" + } + } + }, + "loggin_config": { + "description": "", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": null, + "arguments": { + "log_actions": { + "description": "Each action may be one of `FIND`, `DELETE`, and `COPY`.", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "loggin_config" + }, + "log_action_states": { + "description": "Each action state may be one of `SUCCEEDED`, and `FAILED`.", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "loggin_config" + }, + "enable_on_prem_gcs_transfer": { + "description": "Defaults to false.", + "required": null, + "security_impact": null, + "rationale": null, + "compliant": null, + "non-compliant": null, + "parent": "loggin_config" + } + } } } } \ No newline at end of file diff --git a/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md b/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md index f3e528507..04b277d3f 100644 --- a/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md +++ b/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md @@ -10,13 +10,13 @@ Reference: [Terraform Registry – storage_transfer_agent_pool](https://registry | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `name` | The ID of the agent pool to create. The agent pool name identifies the pool used by transfer agents. | true | false | The name itself does not provide or restrict security controls, but it should still be set clearly for administration and auditability. | agent-pool-example | None | -| `display_name` | Specifies the client-defined AgentPool display name. | false | false | The display name is informational only and does not change the security posture of the resource. | Production transfer agent pool | None | -| `bandwidth_limit` | Specifies the bandwidth limit details for the agent pool. | false | true | Bandwidth controls can reduce the risk of excessive transfer throughput, network saturation, and unintended large-scale data movement. | Refer to child argument. | Refer to child argument. | -| `project` | The project in which the resource belongs. If omitted, the provider project is used. | false | true | Using the correct project is security-relevant because resources created in the wrong project may bypass intended access controls, monitoring, or governance boundaries. | my-project-123 | An unintended or unauthorized project ID. | +| `name` | The ID of the agent pool to create. The agentPoolId must meet the following requirements: * Length of 128 characters or less. * Not start with the string goog. * Start with a lowercase ASCII character, followed by: * Zero or more: lowercase Latin alphabet characters, numerals, hyphens (-), periods (.), underscores (_), or tildes (~). * One or more numerals or lowercase ASCII characters. As expressed by the regular expression: ^(?!goog)[a-z]([a-z0-9-._~]*[a-z0-9])?$. | true | false | None | None | None | +| `display_name` | Specifies the client-specified AgentPool description. | false | false | None | None | None | +| `bandwidth_limit` | Specifies the bandwidth limit details. If this field is unspecified, the default value is set as 'No Limit'. Structure is [documented below](#nested_bandwidth_limit). | false | false | None | None | None | +| `project` | If it is not provided, the provider project is used. | false | false | None | None | None | ### bandwidth_limit Block | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `limit_mbps` | Bandwidth rate in megabytes per second, distributed across all the agents in the pool. | true | true | A defined bandwidth limit helps constrain transfer throughput and supports controlled data movement. | 120 | Unlimited or an out-of-policy value. | +| `limit_mbps` | Bandwidth rate in megabytes per second, distributed across all the agents in the pool. | true | false | None | None | None | diff --git a/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md b/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md index 905582832..0733bb586 100644 --- a/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md +++ b/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md @@ -10,18 +10,184 @@ Reference: [Terraform Registry – storage_transfer_job](https://registry.terraf | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `transfer_spec` | Defines the source, sink, and transfer behavior for the Storage Transfer job. | true | true | The transfer specification controls what data is moved, from where, and under what restrictions. | A transfer_spec with an approved source, sink, and restricted transfer conditions. | A transfer_spec that omits restrictions or uses an unapproved source definition. | +| `name` | | false | false | None | None | None | +| `description` | | false | false | None | None | None | +| `transfer_spec` | | false | false | None | None | None | +| `replication_spec` | - - - | false | false | None | None | None | +| `schedule` | | false | false | None | None | None | +| `event_stream` | | false | false | None | None | None | +| `project` | is not provided, the provider project is used. | false | false | None | None | None | +| `status` | | false | false | None | None | None | +| `notification_config` | | false | false | None | None | None | +| `logging_config` | | false | false | None | None | None | +| `object_conditions` | | false | false | None | None | None | +| `transfer_options` | | false | false | None | None | None | +| `gcs_data_sink` | | false | false | None | None | None | +| `gcs_data_source` | | false | false | None | None | None | +| `posix_data_sink` | | false | false | None | None | None | +| `posix_data_source` | | false | false | None | None | None | +| `hdfs_data_source` | | false | false | None | None | None | +| `aws_s3_data_source` | | false | false | None | None | None | +| `aws_access_key` | | false | false | None | None | None | +| `http_data_source` | | false | false | None | None | None | +| `azure_blob_storage_data_source` | | false | false | None | None | None | +| `azure_credentials` | | false | false | None | None | None | +| `loggin_config` | | false | false | None | None | None | ### transfer_spec Block | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `gcs_data_source` | Specifies a Google Cloud Storage bucket as the source for the transfer job. | false | true | The source bucket determines what data enters the transfer pipeline. It should reference only approved source locations. | gcs_data_source { bucket_name = "approved-source-bucket" } | gcs_data_source { bucket_name = "unapproved-public-bucket" } | -| `aws_s3_data_source` | Specifies an Amazon S3 bucket as the source for the transfer job. | false | true | Using an S3 source introduces external data into the transfer job and should be restricted to approved buckets and credentials. | aws_s3_data_source using an approved bucket and credential configuration. | aws_s3_data_source using an unapproved bucket or unmanaged credentials. | -| `azure_blob_storage_data_source` | Specifies an Azure Blob Storage container as the source for the transfer job. | false | true | Azure Blob source definitions should be limited to approved storage accounts and controlled credentials. | azure_blob_storage_data_source using an approved storage account and container. | azure_blob_storage_data_source using an unknown or unapproved storage account. | -| `http_data_source` | Specifies a list of HTTP or HTTPS URLs as the source for the transfer job. | false | true | HTTP-based sources can introduce integrity and confidentiality risks if insecure or untrusted endpoints are used. | http_data_source referencing approved HTTPS endpoints. | http_data_source referencing HTTP endpoints or untrusted URLs. | -| `posix_data_source` | Specifies a POSIX filesystem path as the source for the transfer job. | false | true | POSIX sources can expose local or attached filesystem data and should be limited to approved paths and agents. | posix_data_source { root_directory = "/approved/source/path" } | posix_data_source { root_directory = "/" } | -| `hdfs_data_source` | Specifies an HDFS path as the source for the transfer job. | false | true | HDFS sources should be controlled to prevent broad or unintended data extraction from distributed storage. | hdfs_data_source using an approved cluster and restricted root path. | hdfs_data_source using an unrestricted or unknown path. | -| `aws_s3_compatible_data_source` | Specifies an S3-compatible storage system as the source for the transfer job. | false | true | S3-compatible endpoints should be approved because they may point to third-party or self-hosted object storage. | aws_s3_compatible_data_source using an approved endpoint and bucket. | aws_s3_compatible_data_source using an unknown or untrusted endpoint. | -| `source_agent_pool_name` | Specifies the agent pool used for agent-based source transfers. | false | true | Agent pools control where transfer agents run and should reference approved managed pools only. | source_agent_pool_name = "transferJobs/agentPools/approved-pool" | source_agent_pool_name = "transferJobs/agentPools/unapproved-pool" | -| `include_prefixes_required` | Ensures the Storage Transfer job restricts transfer scope by defining object_conditions.include_prefixes. | false | true | Requiring include_prefixes reduces the risk of unintentionally transferring more objects than intended by limiting the transfer to approved object path prefixes. | object_conditions { include_prefixes = ["secure-data/"] } | No object_conditions.include_prefixes configured. | +| `source_agent_pool_name` | | false | false | None | None | None | +| `sink_agent_pool_name` | | false | false | None | None | None | +| `gcs_data_sink` | | false | false | None | None | None | +| `posix_data_sink` | | false | false | None | None | None | +| `object_conditions` | | false | false | None | None | None | +| `transfer_options` | | false | false | None | None | None | +| `gcs_data_source` | | false | false | None | None | None | +| `posix_data_source` | | false | false | None | None | None | +| `aws_s3_data_source` | | false | false | None | None | None | +| `http_data_source` | | false | false | None | None | None | +| `azure_blob_storage_data_source` | | false | false | None | None | None | +| `hdfs_data_source` | | false | false | None | None | None | + +### replication_spec Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `gcs_data_sink` | | false | false | None | None | None | +| `gcs_data_source` | | false | false | None | None | None | +| `object_conditions` | | false | false | None | None | None | +| `transfer_options` | | false | false | None | None | None | + +### schedule Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `schedule_start_date` | | false | false | None | None | None | +| `schedule_end_date` | | false | false | None | None | None | +| `start_time_of_day` | | false | false | None | None | None | +| `repeat_interval` | | false | false | None | None | None | + +### event_stream Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `name` | | false | false | None | None | None | +| `event_stream_start_time` | | false | false | None | None | None | +| `event_stream_expiration_time` | | false | false | None | None | None | + +### notification_config Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `pubsub_topic` | | false | false | None | None | None | +| `event_types` | | false | false | None | None | None | +| `payload_format` | | false | false | None | None | None | + +### object_conditions Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `max_time_elapsed_since_last_modification` | | false | false | None | None | None | +| `min_time_elapsed_since_last_modification` | A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". | false | false | None | None | None | +| `include_prefixes` | | false | false | None | None | None | +| `exclude_prefixes` | | false | false | None | None | None | +| `last_modified_since` | | false | false | None | None | None | +| `last_modified_before` | | false | false | None | None | None | + +### transfer_options Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `overwrite_objects_already_existing_in_sink` | | false | false | None | None | None | +| `delete_objects_unique_in_sink` | `delete_objects_from_source_after_transfer` are mutually exclusive. | false | false | None | None | None | +| `delete_objects_from_source_after_transfer` | | false | false | None | None | None | +| `overwrite_when` | | false | false | None | None | None | + +### gcs_data_sink Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `bucket_name` | | false | false | None | None | None | +| `path` | | false | false | None | None | None | + +### gcs_data_source Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `bucket_name` | | false | false | None | None | None | +| `path` | | false | false | None | None | None | + +### posix_data_sink Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `root_directory` | | false | false | None | None | None | + +### posix_data_source Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `root_directory` | | false | false | None | None | None | + +### hdfs_data_source Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `path` | | false | false | None | None | None | + +### aws_s3_data_source Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `bucket_name` | | false | false | None | None | None | +| `path` | | false | false | None | None | None | +| `aws_access_key` | | false | false | None | None | None | +| `role_arn` | | false | false | None | None | None | +| `managed_private_network` | | false | false | None | None | None | +| `cloudfront_domain` | | false | false | None | None | None | + +### aws_access_key Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `access_key_id` | | false | false | None | None | None | +| `secret_access_key` | | false | false | None | None | None | + +### http_data_source Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `list_url` | | false | false | None | None | None | + +### azure_blob_storage_data_source Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `storage_account` | | false | false | None | None | None | +| `container` | | false | false | None | None | None | +| `path` | | false | false | None | None | None | +| `credentials_secret` | | false | false | None | None | None | +| `azure_credentials` | | false | false | None | None | None | + +### azure_credentials Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `sas_token` | The `schedule_start_date` and `schedule_end_date` blocks support: | false | false | None | None | None | +| `year` | | false | false | None | None | None | +| `month` | | false | false | None | None | None | +| `day` | The `start_time_of_day` blocks support: | false | false | None | None | None | +| `hours` | | false | false | None | None | None | +| `minutes` | | false | false | None | None | None | +| `seconds` | | false | false | None | None | None | +| `nanos` | | false | false | None | None | None | + +### loggin_config Block + +| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | +|----------|-------------|----------|-----------------|-----------|-----------|---------------| +| `log_actions` | Each action may be one of `FIND`, `DELETE`, and `COPY`. | false | false | None | None | None | +| `log_action_states` | Each action state may be one of `SUCCEEDED`, and `FAILED`. | false | false | None | None | None | +| `enable_on_prem_gcs_transfer` | Defaults to false. | false | false | None | None | None | diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/.terraform.lock.hcl b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/.terraform.lock.hcl deleted file mode 100644 index 7aab94476..000000000 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/.terraform.lock.hcl +++ /dev/null @@ -1,22 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/google" { - version = "7.28.0" - constraints = "7.28.0" - hashes = [ - "h1:PSXGQ1KdRjfeCvb6K85dqZiC6AfyB5OWw6vm5I+iop8=", - "zh:078c16b9c5e9067e72070367846976b58f906d8efab6fc4fc1325661717dc9cc", - "zh:08b839014b428233a3a83d15045e7559b07fc035c7f73cc1ee2694c50c4dea54", - "zh:0c76ea69f75633bdfc67a0cd6ea510332c0cb0f2d4968b8a070e546fb47e444e", - "zh:3a308492ad4c153583f7b8ecc3c80bf0bbc15a32c62b5b3794efb27db01ff26b", - "zh:6754f51373994470f78937856982b0a39648ac302713d07205d320a13ad41d82", - "zh:79d387214f55df16c795f11988a0285a4bfa846c447faa85008b953b77081eb1", - "zh:8de432482d77d1a1077b2dc3db764b8ba6d1b07a4b991a07c960855adc0b031b", - "zh:900daa2435de1928a9868aa4c17d8b7b109ab363c97f7fe274466193af1412b0", - "zh:96c25183a7f13b3de9a5631aa2a13ed1a4285b8393df90c2380c2fe74f350ab5", - "zh:971121626be01245acd9a4520a63e1405e4f528d3c83f39a28f8caaeac235b45", - "zh:e90d5e7d7bf47c8cf5bbf2e5d0bf855ed10350ad3584795a6911f85fdb5c0c3c", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/c.tf deleted file mode 100644 index ee7c29d7e..000000000 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/c.tf +++ /dev/null @@ -1,32 +0,0 @@ -resource "google_storage_transfer_job" "c" { - description = "compliant-transfer-job" - status = "ENABLED" - - transfer_spec { - gcs_data_source { - bucket_name = "source-bucket-example" - } - - gcs_data_sink { - bucket_name = "destination-bucket-example" - } - - object_conditions { - include_prefixes = ["secure-data/"] - } - } - - schedule { - schedule_start_date { - year = 2026 - month = 5 - day = 13 - } - start_time_of_day { - hours = 1 - minutes = 0 - seconds = 0 - nanos = 0 - } - } -} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/config.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/config.tf deleted file mode 100644 index 413a25f42..000000000 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/config.tf +++ /dev/null @@ -1,12 +0,0 @@ -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = "7.28.0" - } - } -} - -provider "google" { - project = "my-project-123" -} \ No newline at end of file diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/nc.tf deleted file mode 100644 index c33f8df21..000000000 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/nc.tf +++ /dev/null @@ -1,28 +0,0 @@ -resource "google_storage_transfer_job" "nc" { - description = "noncompliant-transfer-job" - status = "ENABLED" - - transfer_spec { - gcs_data_source { - bucket_name = "source-bucket-example" - } - - gcs_data_sink { - bucket_name = "destination-bucket-example" - } - } - - schedule { - schedule_start_date { - year = 2026 - month = 5 - day = 13 - } - start_time_of_day { - hours = 1 - minutes = 0 - seconds = 0 - nanos = 0 - } - } -} \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/policy.rego deleted file mode 100644 index 84959a58d..000000000 --- a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/include_prefixes_required/policy.rego +++ /dev/null @@ -1,24 +0,0 @@ -package terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.include_prefixes_required - -import data.terraform.helpers -import data.terraform.gcp.security.Storage_Transfer_Service.google_storage_transfer_job.vars - -conditions := [ - [ - { - "situation_description": "Storage Transfer job does not restrict transfer scope with include_prefixes.", - "remedies": [ - "Set object_conditions.include_prefixes to limit which objects are transferred." - ] - }, - { - "condition": "Check that include_prefixes is configured.", - "attribute_path": ["transfer_spec", 0, "object_conditions", 0, "include_prefixes", 0], - "values": ["secure-data/"], - "policy_type": "whitelist" - } - ] -] - -message := helpers.get_multi_summary(conditions, vars.variables).message -details := helpers.get_multi_summary(conditions, vars.variables).details \ No newline at end of file From b85446350b831760a4341cb1d1d42f0b12504947 Mon Sep 17 00:00:00 2001 From: Mohamed Date: Sun, 24 May 2026 19:44:27 +1000 Subject: [PATCH 06/10] Add include_prefixes_required policy for Storage Transfer job --- .../storage_transfer_agent_pool.json | 46 +- .../resource_json/storage_transfer_job.json | 899 ++---------------- .../storage_transfer_agent_pool.md | 10 +- .../storage_transfer_job.md | 186 +--- 4 files changed, 106 insertions(+), 1035 deletions(-) diff --git a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json index ceed546b6..77152db7f 100644 --- a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json +++ b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json @@ -3,50 +3,50 @@ "subcategory": "Storage Transfer Service", "arguments": { "name": { - "description": "The ID of the agent pool to create. The agentPoolId must meet the following requirements: * Length of 128 characters or less. * Not start with the string goog. * Start with a lowercase ASCII character, followed by: * Zero or more: lowercase Latin alphabet characters, numerals, hyphens (-), periods (.), underscores (_), or tildes (~). * One or more numerals or lowercase ASCII characters. As expressed by the regular expression: ^(?!goog)[a-z]([a-z0-9-._~]*[a-z0-9])?$.", + "description": "The ID of the agent pool to create. The agent pool name identifies the pool used by transfer agents.", "required": true, - "security_impact": null, - "rationale": null, - "compliant": null, + "security_impact": false, + "rationale": "The name itself does not provide or restrict security controls, but it should still be set clearly for administration and auditability.", + "compliant": "agent-pool-example", "non-compliant": null, "parent": null }, "display_name": { - "description": "Specifies the client-specified AgentPool description.", + "description": "Specifies the client-defined AgentPool display name.", "required": false, - "security_impact": null, - "rationale": null, - "compliant": null, + "security_impact": false, + "rationale": "The display name is informational only and does not change the security posture of the resource.", + "compliant": "Production transfer agent pool", "non-compliant": null, "parent": null }, "bandwidth_limit": { - "description": "Specifies the bandwidth limit details. If this field is unspecified, the default value is set as 'No Limit'. Structure is [documented below](#nested_bandwidth_limit).", + "description": "Specifies the bandwidth limit details for the agent pool.", "required": false, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "security_impact": true, + "rationale": "Bandwidth controls can reduce the risk of excessive transfer throughput, network saturation, and unintended large-scale data movement.", + "compliant": "Refer to child argument.", + "non-compliant": "Refer to child argument.", "parent": null, "arguments": { "limit_mbps": { "description": "Bandwidth rate in megabytes per second, distributed across all the agents in the pool.", "required": true, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "security_impact": true, + "rationale": "A defined bandwidth limit helps constrain transfer throughput and supports controlled data movement.", + "compliant": "120", + "non-compliant": "Unlimited or an out-of-policy value.", "parent": "bandwidth_limit" } } }, "project": { - "description": "If it is not provided, the provider project is used.", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "description": "The project in which the resource belongs. If omitted, the provider project is used.", + "required": false, + "security_impact": true, + "rationale": "Using the correct project is security-relevant because resources created in the wrong project may bypass intended access controls, monitoring, or governance boundaries.", + "compliant": "my-project-123", + "non-compliant": "An unintended or unauthorized project ID.", "parent": null } } diff --git a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json index 0e9f23bb1..7d6fa438d 100644 --- a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json +++ b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json @@ -2,860 +2,97 @@ "resource_name": "storage_transfer_job", "subcategory": "Storage Transfer Service", "arguments": { - "name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null - }, - "description": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null - }, "transfer_spec": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "description": "Defines the source, sink, and transfer behavior for the Storage Transfer job.", + "required": true, + "security_impact": true, + "rationale": "The transfer specification controls what data is moved, from where, and under what restrictions.", + "compliant": "A transfer_spec with an approved source, sink, and restricted transfer conditions.", + "non-compliant": "A transfer_spec that omits restrictions or uses an unapproved source definition.", "parent": null, "arguments": { - "source_agent_pool_name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_spec" - }, - "sink_agent_pool_name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_spec" - }, - "gcs_data_sink": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_spec" - }, - "posix_data_sink": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "gcs_data_source": { + "description": "Specifies a Google Cloud Storage bucket as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "The source bucket determines what data enters the transfer pipeline. It should reference only approved source locations.", + "compliant": "gcs_data_source { bucket_name = \"approved-source-bucket\" }", + "non-compliant": "gcs_data_source { bucket_name = \"unapproved-public-bucket\" }", "parent": "transfer_spec" }, - "object_conditions": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "aws_s3_data_source": { + "description": "Specifies an Amazon S3 bucket as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "Using an S3 source introduces external data into the transfer job and should be restricted to approved buckets and credentials.", + "compliant": "aws_s3_data_source using an approved bucket and credential configuration.", + "non-compliant": "aws_s3_data_source using an unapproved bucket or unmanaged credentials.", "parent": "transfer_spec" }, - "transfer_options": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "azure_blob_storage_data_source": { + "description": "Specifies an Azure Blob Storage container as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "Azure Blob source definitions should be limited to approved storage accounts and controlled credentials.", + "compliant": "azure_blob_storage_data_source using an approved storage account and container.", + "non-compliant": "azure_blob_storage_data_source using an unknown or unapproved storage account.", "parent": "transfer_spec" }, - "gcs_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "http_data_source": { + "description": "Specifies a list of HTTP or HTTPS URLs as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "HTTP-based sources can introduce integrity and confidentiality risks if insecure or untrusted endpoints are used.", + "compliant": "http_data_source referencing approved HTTPS endpoints.", + "non-compliant": "http_data_source referencing HTTP endpoints or untrusted URLs.", "parent": "transfer_spec" }, "posix_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "description": "Specifies a POSIX filesystem path as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "POSIX sources can expose local or attached filesystem data and should be limited to approved paths and agents.", + "compliant": "posix_data_source { root_directory = \"/approved/source/path\" }", + "non-compliant": "posix_data_source { root_directory = \"/\" }", "parent": "transfer_spec" }, - "aws_s3_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "hdfs_data_source": { + "description": "Specifies an HDFS path as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "HDFS sources should be controlled to prevent broad or unintended data extraction from distributed storage.", + "compliant": "hdfs_data_source using an approved cluster and restricted root path.", + "non-compliant": "hdfs_data_source using an unrestricted or unknown path.", "parent": "transfer_spec" }, - "http_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "aws_s3_compatible_data_source": { + "description": "Specifies an S3-compatible storage system as the source for the transfer job.", + "required": false, + "security_impact": true, + "rationale": "S3-compatible endpoints should be approved because they may point to third-party or self-hosted object storage.", + "compliant": "aws_s3_compatible_data_source using an approved endpoint and bucket.", + "non-compliant": "aws_s3_compatible_data_source using an unknown or untrusted endpoint.", "parent": "transfer_spec" }, - "azure_blob_storage_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "source_agent_pool_name": { + "description": "Specifies the agent pool used for agent-based source transfers.", + "required": false, + "security_impact": true, + "rationale": "Agent pools control where transfer agents run and should reference approved managed pools only.", + "compliant": "source_agent_pool_name = \"transferJobs/agentPools/approved-pool\"", + "non-compliant": "source_agent_pool_name = \"transferJobs/agentPools/unapproved-pool\"", "parent": "transfer_spec" }, - "hdfs_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, + "include_prefixes_required": { + "description": "Ensures the Storage Transfer job restricts transfer scope by defining object_conditions.include_prefixes.", + "required": false, + "security_impact": true, + "rationale": "Requiring include_prefixes reduces the risk of unintentionally transferring more objects than intended by limiting the transfer to approved object path prefixes.", + "compliant": "object_conditions { include_prefixes = [\"secure-data/\"] }", + "non-compliant": "No object_conditions.include_prefixes configured.", "parent": "transfer_spec" } } - }, - "replication_spec": { - "description": "- - -", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "gcs_data_sink": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "replication_spec" - }, - "gcs_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "replication_spec" - }, - "object_conditions": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "replication_spec" - }, - "transfer_options": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "replication_spec" - } - } - }, - "schedule": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "schedule_start_date": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "schedule" - }, - "schedule_end_date": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "schedule" - }, - "start_time_of_day": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "schedule" - }, - "repeat_interval": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "schedule" - } - } - }, - "event_stream": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "event_stream" - }, - "event_stream_start_time": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "event_stream" - }, - "event_stream_expiration_time": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "event_stream" - } - } - }, - "project": { - "description": "is not provided, the provider project is used.", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null - }, - "status": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null - }, - "notification_config": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "pubsub_topic": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "notification_config" - }, - "event_types": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "notification_config" - }, - "payload_format": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "notification_config" - } - } - }, - "logging_config": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null - }, - "object_conditions": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "max_time_elapsed_since_last_modification": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "object_conditions" - }, - "min_time_elapsed_since_last_modification": { - "description": "A duration in seconds with up to nine fractional digits, terminated by 's'. Example: \"3.5s\".", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "object_conditions" - }, - "include_prefixes": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "object_conditions" - }, - "exclude_prefixes": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "object_conditions" - }, - "last_modified_since": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "object_conditions" - }, - "last_modified_before": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "object_conditions" - } - } - }, - "transfer_options": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "overwrite_objects_already_existing_in_sink": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_options" - }, - "delete_objects_unique_in_sink": { - "description": "`delete_objects_from_source_after_transfer` are mutually exclusive.", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_options" - }, - "delete_objects_from_source_after_transfer": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_options" - }, - "overwrite_when": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "transfer_options" - } - } - }, - "gcs_data_sink": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "bucket_name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "gcs_data_sink" - }, - "path": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "gcs_data_sink" - } - } - }, - "gcs_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "bucket_name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "gcs_data_source" - }, - "path": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "gcs_data_source" - } - } - }, - "posix_data_sink": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "root_directory": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "posix_data_sink" - } - } - }, - "posix_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "root_directory": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "posix_data_source" - } - } - }, - "hdfs_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "path": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "hdfs_data_source" - } - } - }, - "aws_s3_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "bucket_name": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_s3_data_source" - }, - "path": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_s3_data_source" - }, - "aws_access_key": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_s3_data_source" - }, - "role_arn": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_s3_data_source" - }, - "managed_private_network": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_s3_data_source" - }, - "cloudfront_domain": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_s3_data_source" - } - } - }, - "aws_access_key": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "access_key_id": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_access_key" - }, - "secret_access_key": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "aws_access_key" - } - } - }, - "http_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "list_url": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "http_data_source" - } - } - }, - "azure_blob_storage_data_source": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "storage_account": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_blob_storage_data_source" - }, - "container": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_blob_storage_data_source" - }, - "path": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_blob_storage_data_source" - }, - "credentials_secret": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_blob_storage_data_source" - }, - "azure_credentials": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_blob_storage_data_source" - } - } - }, - "azure_credentials": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "sas_token": { - "description": "The `schedule_start_date` and `schedule_end_date` blocks support:", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "year": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "month": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "day": { - "description": "The `start_time_of_day` blocks support:", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "hours": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "minutes": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "seconds": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - }, - "nanos": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "azure_credentials" - } - } - }, - "loggin_config": { - "description": "", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": null, - "arguments": { - "log_actions": { - "description": "Each action may be one of `FIND`, `DELETE`, and `COPY`.", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "loggin_config" - }, - "log_action_states": { - "description": "Each action state may be one of `SUCCEEDED`, and `FAILED`.", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "loggin_config" - }, - "enable_on_prem_gcs_transfer": { - "description": "Defaults to false.", - "required": null, - "security_impact": null, - "rationale": null, - "compliant": null, - "non-compliant": null, - "parent": "loggin_config" - } - } } } } \ No newline at end of file diff --git a/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md b/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md index 04b277d3f..f3e528507 100644 --- a/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md +++ b/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md @@ -10,13 +10,13 @@ Reference: [Terraform Registry – storage_transfer_agent_pool](https://registry | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `name` | The ID of the agent pool to create. The agentPoolId must meet the following requirements: * Length of 128 characters or less. * Not start with the string goog. * Start with a lowercase ASCII character, followed by: * Zero or more: lowercase Latin alphabet characters, numerals, hyphens (-), periods (.), underscores (_), or tildes (~). * One or more numerals or lowercase ASCII characters. As expressed by the regular expression: ^(?!goog)[a-z]([a-z0-9-._~]*[a-z0-9])?$. | true | false | None | None | None | -| `display_name` | Specifies the client-specified AgentPool description. | false | false | None | None | None | -| `bandwidth_limit` | Specifies the bandwidth limit details. If this field is unspecified, the default value is set as 'No Limit'. Structure is [documented below](#nested_bandwidth_limit). | false | false | None | None | None | -| `project` | If it is not provided, the provider project is used. | false | false | None | None | None | +| `name` | The ID of the agent pool to create. The agent pool name identifies the pool used by transfer agents. | true | false | The name itself does not provide or restrict security controls, but it should still be set clearly for administration and auditability. | agent-pool-example | None | +| `display_name` | Specifies the client-defined AgentPool display name. | false | false | The display name is informational only and does not change the security posture of the resource. | Production transfer agent pool | None | +| `bandwidth_limit` | Specifies the bandwidth limit details for the agent pool. | false | true | Bandwidth controls can reduce the risk of excessive transfer throughput, network saturation, and unintended large-scale data movement. | Refer to child argument. | Refer to child argument. | +| `project` | The project in which the resource belongs. If omitted, the provider project is used. | false | true | Using the correct project is security-relevant because resources created in the wrong project may bypass intended access controls, monitoring, or governance boundaries. | my-project-123 | An unintended or unauthorized project ID. | ### bandwidth_limit Block | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `limit_mbps` | Bandwidth rate in megabytes per second, distributed across all the agents in the pool. | true | false | None | None | None | +| `limit_mbps` | Bandwidth rate in megabytes per second, distributed across all the agents in the pool. | true | true | A defined bandwidth limit helps constrain transfer throughput and supports controlled data movement. | 120 | Unlimited or an out-of-policy value. | diff --git a/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md b/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md index 0733bb586..905582832 100644 --- a/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md +++ b/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md @@ -10,184 +10,18 @@ Reference: [Terraform Registry – storage_transfer_job](https://registry.terraf | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `name` | | false | false | None | None | None | -| `description` | | false | false | None | None | None | -| `transfer_spec` | | false | false | None | None | None | -| `replication_spec` | - - - | false | false | None | None | None | -| `schedule` | | false | false | None | None | None | -| `event_stream` | | false | false | None | None | None | -| `project` | is not provided, the provider project is used. | false | false | None | None | None | -| `status` | | false | false | None | None | None | -| `notification_config` | | false | false | None | None | None | -| `logging_config` | | false | false | None | None | None | -| `object_conditions` | | false | false | None | None | None | -| `transfer_options` | | false | false | None | None | None | -| `gcs_data_sink` | | false | false | None | None | None | -| `gcs_data_source` | | false | false | None | None | None | -| `posix_data_sink` | | false | false | None | None | None | -| `posix_data_source` | | false | false | None | None | None | -| `hdfs_data_source` | | false | false | None | None | None | -| `aws_s3_data_source` | | false | false | None | None | None | -| `aws_access_key` | | false | false | None | None | None | -| `http_data_source` | | false | false | None | None | None | -| `azure_blob_storage_data_source` | | false | false | None | None | None | -| `azure_credentials` | | false | false | None | None | None | -| `loggin_config` | | false | false | None | None | None | +| `transfer_spec` | Defines the source, sink, and transfer behavior for the Storage Transfer job. | true | true | The transfer specification controls what data is moved, from where, and under what restrictions. | A transfer_spec with an approved source, sink, and restricted transfer conditions. | A transfer_spec that omits restrictions or uses an unapproved source definition. | ### transfer_spec Block | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `source_agent_pool_name` | | false | false | None | None | None | -| `sink_agent_pool_name` | | false | false | None | None | None | -| `gcs_data_sink` | | false | false | None | None | None | -| `posix_data_sink` | | false | false | None | None | None | -| `object_conditions` | | false | false | None | None | None | -| `transfer_options` | | false | false | None | None | None | -| `gcs_data_source` | | false | false | None | None | None | -| `posix_data_source` | | false | false | None | None | None | -| `aws_s3_data_source` | | false | false | None | None | None | -| `http_data_source` | | false | false | None | None | None | -| `azure_blob_storage_data_source` | | false | false | None | None | None | -| `hdfs_data_source` | | false | false | None | None | None | - -### replication_spec Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `gcs_data_sink` | | false | false | None | None | None | -| `gcs_data_source` | | false | false | None | None | None | -| `object_conditions` | | false | false | None | None | None | -| `transfer_options` | | false | false | None | None | None | - -### schedule Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `schedule_start_date` | | false | false | None | None | None | -| `schedule_end_date` | | false | false | None | None | None | -| `start_time_of_day` | | false | false | None | None | None | -| `repeat_interval` | | false | false | None | None | None | - -### event_stream Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `name` | | false | false | None | None | None | -| `event_stream_start_time` | | false | false | None | None | None | -| `event_stream_expiration_time` | | false | false | None | None | None | - -### notification_config Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `pubsub_topic` | | false | false | None | None | None | -| `event_types` | | false | false | None | None | None | -| `payload_format` | | false | false | None | None | None | - -### object_conditions Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `max_time_elapsed_since_last_modification` | | false | false | None | None | None | -| `min_time_elapsed_since_last_modification` | A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". | false | false | None | None | None | -| `include_prefixes` | | false | false | None | None | None | -| `exclude_prefixes` | | false | false | None | None | None | -| `last_modified_since` | | false | false | None | None | None | -| `last_modified_before` | | false | false | None | None | None | - -### transfer_options Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `overwrite_objects_already_existing_in_sink` | | false | false | None | None | None | -| `delete_objects_unique_in_sink` | `delete_objects_from_source_after_transfer` are mutually exclusive. | false | false | None | None | None | -| `delete_objects_from_source_after_transfer` | | false | false | None | None | None | -| `overwrite_when` | | false | false | None | None | None | - -### gcs_data_sink Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `bucket_name` | | false | false | None | None | None | -| `path` | | false | false | None | None | None | - -### gcs_data_source Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `bucket_name` | | false | false | None | None | None | -| `path` | | false | false | None | None | None | - -### posix_data_sink Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `root_directory` | | false | false | None | None | None | - -### posix_data_source Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `root_directory` | | false | false | None | None | None | - -### hdfs_data_source Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `path` | | false | false | None | None | None | - -### aws_s3_data_source Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `bucket_name` | | false | false | None | None | None | -| `path` | | false | false | None | None | None | -| `aws_access_key` | | false | false | None | None | None | -| `role_arn` | | false | false | None | None | None | -| `managed_private_network` | | false | false | None | None | None | -| `cloudfront_domain` | | false | false | None | None | None | - -### aws_access_key Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `access_key_id` | | false | false | None | None | None | -| `secret_access_key` | | false | false | None | None | None | - -### http_data_source Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `list_url` | | false | false | None | None | None | - -### azure_blob_storage_data_source Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `storage_account` | | false | false | None | None | None | -| `container` | | false | false | None | None | None | -| `path` | | false | false | None | None | None | -| `credentials_secret` | | false | false | None | None | None | -| `azure_credentials` | | false | false | None | None | None | - -### azure_credentials Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `sas_token` | The `schedule_start_date` and `schedule_end_date` blocks support: | false | false | None | None | None | -| `year` | | false | false | None | None | None | -| `month` | | false | false | None | None | None | -| `day` | The `start_time_of_day` blocks support: | false | false | None | None | None | -| `hours` | | false | false | None | None | None | -| `minutes` | | false | false | None | None | None | -| `seconds` | | false | false | None | None | None | -| `nanos` | | false | false | None | None | None | - -### loggin_config Block - -| Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | -|----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `log_actions` | Each action may be one of `FIND`, `DELETE`, and `COPY`. | false | false | None | None | None | -| `log_action_states` | Each action state may be one of `SUCCEEDED`, and `FAILED`. | false | false | None | None | None | -| `enable_on_prem_gcs_transfer` | Defaults to false. | false | false | None | None | None | +| `gcs_data_source` | Specifies a Google Cloud Storage bucket as the source for the transfer job. | false | true | The source bucket determines what data enters the transfer pipeline. It should reference only approved source locations. | gcs_data_source { bucket_name = "approved-source-bucket" } | gcs_data_source { bucket_name = "unapproved-public-bucket" } | +| `aws_s3_data_source` | Specifies an Amazon S3 bucket as the source for the transfer job. | false | true | Using an S3 source introduces external data into the transfer job and should be restricted to approved buckets and credentials. | aws_s3_data_source using an approved bucket and credential configuration. | aws_s3_data_source using an unapproved bucket or unmanaged credentials. | +| `azure_blob_storage_data_source` | Specifies an Azure Blob Storage container as the source for the transfer job. | false | true | Azure Blob source definitions should be limited to approved storage accounts and controlled credentials. | azure_blob_storage_data_source using an approved storage account and container. | azure_blob_storage_data_source using an unknown or unapproved storage account. | +| `http_data_source` | Specifies a list of HTTP or HTTPS URLs as the source for the transfer job. | false | true | HTTP-based sources can introduce integrity and confidentiality risks if insecure or untrusted endpoints are used. | http_data_source referencing approved HTTPS endpoints. | http_data_source referencing HTTP endpoints or untrusted URLs. | +| `posix_data_source` | Specifies a POSIX filesystem path as the source for the transfer job. | false | true | POSIX sources can expose local or attached filesystem data and should be limited to approved paths and agents. | posix_data_source { root_directory = "/approved/source/path" } | posix_data_source { root_directory = "/" } | +| `hdfs_data_source` | Specifies an HDFS path as the source for the transfer job. | false | true | HDFS sources should be controlled to prevent broad or unintended data extraction from distributed storage. | hdfs_data_source using an approved cluster and restricted root path. | hdfs_data_source using an unrestricted or unknown path. | +| `aws_s3_compatible_data_source` | Specifies an S3-compatible storage system as the source for the transfer job. | false | true | S3-compatible endpoints should be approved because they may point to third-party or self-hosted object storage. | aws_s3_compatible_data_source using an approved endpoint and bucket. | aws_s3_compatible_data_source using an unknown or untrusted endpoint. | +| `source_agent_pool_name` | Specifies the agent pool used for agent-based source transfers. | false | true | Agent pools control where transfer agents run and should reference approved managed pools only. | source_agent_pool_name = "transferJobs/agentPools/approved-pool" | source_agent_pool_name = "transferJobs/agentPools/unapproved-pool" | +| `include_prefixes_required` | Ensures the Storage Transfer job restricts transfer scope by defining object_conditions.include_prefixes. | false | true | Requiring include_prefixes reduces the risk of unintentionally transferring more objects than intended by limiting the transfer to approved object path prefixes. | object_conditions { include_prefixes = ["secure-data/"] } | No object_conditions.include_prefixes configured. | From 5c71425c0b1e74f836a7fdc7e040d40569d0e80b Mon Sep 17 00:00:00 2001 From: Mohamed Date: Sun, 24 May 2026 20:11:24 +1000 Subject: [PATCH 07/10] Fix Storage Transfer Service documentation --- .../storage_transfer_agent_pool.json | 18 --- .../resource_json/storage_transfer_job.json | 146 +++++++++--------- .../storage_transfer_agent_pool.md | 2 - .../storage_transfer_job.md | 28 ++-- 4 files changed, 94 insertions(+), 100 deletions(-) diff --git a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json index 77152db7f..fa9c3fa5e 100644 --- a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json +++ b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json @@ -11,15 +11,6 @@ "non-compliant": null, "parent": null }, - "display_name": { - "description": "Specifies the client-defined AgentPool display name.", - "required": false, - "security_impact": false, - "rationale": "The display name is informational only and does not change the security posture of the resource.", - "compliant": "Production transfer agent pool", - "non-compliant": null, - "parent": null - }, "bandwidth_limit": { "description": "Specifies the bandwidth limit details for the agent pool.", "required": false, @@ -39,15 +30,6 @@ "parent": "bandwidth_limit" } } - }, - "project": { - "description": "The project in which the resource belongs. If omitted, the provider project is used.", - "required": false, - "security_impact": true, - "rationale": "Using the correct project is security-relevant because resources created in the wrong project may bypass intended access controls, monitoring, or governance boundaries.", - "compliant": "my-project-123", - "non-compliant": "An unintended or unauthorized project ID.", - "parent": null } } } \ No newline at end of file diff --git a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json index 7d6fa438d..b589bf8dc 100644 --- a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json +++ b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_job.json @@ -2,78 +2,42 @@ "resource_name": "storage_transfer_job", "subcategory": "Storage Transfer Service", "arguments": { + "description": { + "description": "A description for the Storage Transfer job.", + "required": false, + "security_impact": false, + "rationale": "The description is informational and does not directly affect the security posture of the transfer job.", + "compliant": "daily-secure-transfer-job", + "non-compliant": null, + "parent": null + }, + "status": { + "description": "The status of the transfer job.", + "required": false, + "security_impact": true, + "rationale": "Transfer jobs should be intentionally enabled or disabled to match operational and governance requirements.", + "compliant": "ENABLED", + "non-compliant": "Unexpectedly disabled or unmanaged status.", + "parent": null + }, + "schedule": { + "description": "Defines when the Storage Transfer job runs.", + "required": false, + "security_impact": true, + "rationale": "A defined schedule supports controlled and expected execution of data transfers.", + "compliant": "A schedule with an approved start date and run time.", + "non-compliant": "No defined schedule where one is required by policy.", + "parent": null + }, "transfer_spec": { "description": "Defines the source, sink, and transfer behavior for the Storage Transfer job.", "required": true, "security_impact": true, "rationale": "The transfer specification controls what data is moved, from where, and under what restrictions.", - "compliant": "A transfer_spec with an approved source, sink, and restricted transfer conditions.", - "non-compliant": "A transfer_spec that omits restrictions or uses an unapproved source definition.", + "compliant": "A transfer_spec with approved source settings, approved agent pool usage, and safe transfer options.", + "non-compliant": "A transfer_spec that omits required restrictions or uses unsafe transfer options.", "parent": null, "arguments": { - "gcs_data_source": { - "description": "Specifies a Google Cloud Storage bucket as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "The source bucket determines what data enters the transfer pipeline. It should reference only approved source locations.", - "compliant": "gcs_data_source { bucket_name = \"approved-source-bucket\" }", - "non-compliant": "gcs_data_source { bucket_name = \"unapproved-public-bucket\" }", - "parent": "transfer_spec" - }, - "aws_s3_data_source": { - "description": "Specifies an Amazon S3 bucket as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "Using an S3 source introduces external data into the transfer job and should be restricted to approved buckets and credentials.", - "compliant": "aws_s3_data_source using an approved bucket and credential configuration.", - "non-compliant": "aws_s3_data_source using an unapproved bucket or unmanaged credentials.", - "parent": "transfer_spec" - }, - "azure_blob_storage_data_source": { - "description": "Specifies an Azure Blob Storage container as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "Azure Blob source definitions should be limited to approved storage accounts and controlled credentials.", - "compliant": "azure_blob_storage_data_source using an approved storage account and container.", - "non-compliant": "azure_blob_storage_data_source using an unknown or unapproved storage account.", - "parent": "transfer_spec" - }, - "http_data_source": { - "description": "Specifies a list of HTTP or HTTPS URLs as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "HTTP-based sources can introduce integrity and confidentiality risks if insecure or untrusted endpoints are used.", - "compliant": "http_data_source referencing approved HTTPS endpoints.", - "non-compliant": "http_data_source referencing HTTP endpoints or untrusted URLs.", - "parent": "transfer_spec" - }, - "posix_data_source": { - "description": "Specifies a POSIX filesystem path as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "POSIX sources can expose local or attached filesystem data and should be limited to approved paths and agents.", - "compliant": "posix_data_source { root_directory = \"/approved/source/path\" }", - "non-compliant": "posix_data_source { root_directory = \"/\" }", - "parent": "transfer_spec" - }, - "hdfs_data_source": { - "description": "Specifies an HDFS path as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "HDFS sources should be controlled to prevent broad or unintended data extraction from distributed storage.", - "compliant": "hdfs_data_source using an approved cluster and restricted root path.", - "non-compliant": "hdfs_data_source using an unrestricted or unknown path.", - "parent": "transfer_spec" - }, - "aws_s3_compatible_data_source": { - "description": "Specifies an S3-compatible storage system as the source for the transfer job.", - "required": false, - "security_impact": true, - "rationale": "S3-compatible endpoints should be approved because they may point to third-party or self-hosted object storage.", - "compliant": "aws_s3_compatible_data_source using an approved endpoint and bucket.", - "non-compliant": "aws_s3_compatible_data_source using an unknown or untrusted endpoint.", - "parent": "transfer_spec" - }, "source_agent_pool_name": { "description": "Specifies the agent pool used for agent-based source transfers.", "required": false, @@ -83,14 +47,54 @@ "non-compliant": "source_agent_pool_name = \"transferJobs/agentPools/unapproved-pool\"", "parent": "transfer_spec" }, - "include_prefixes_required": { - "description": "Ensures the Storage Transfer job restricts transfer scope by defining object_conditions.include_prefixes.", + "aws_s3_data_source": { + "description": "Specifies an Amazon S3 bucket as the source for the transfer job.", "required": false, "security_impact": true, - "rationale": "Requiring include_prefixes reduces the risk of unintentionally transferring more objects than intended by limiting the transfer to approved object path prefixes.", - "compliant": "object_conditions { include_prefixes = [\"secure-data/\"] }", - "non-compliant": "No object_conditions.include_prefixes configured.", - "parent": "transfer_spec" + "rationale": "Using an S3 source introduces external data into the transfer job and should be restricted to approved buckets and approved AWS access settings.", + "compliant": "aws_s3_data_source using an approved bucket and approved role ARN.", + "non-compliant": "aws_s3_data_source using an unapproved bucket or missing/unapproved role ARN.", + "parent": "transfer_spec", + "arguments": { + "role_arn": { + "description": "The AWS IAM role ARN used to access the Amazon S3 source.", + "required": false, + "security_impact": true, + "rationale": "The role ARN controls what AWS permissions are used to read from the source bucket. Using the wrong role can grant excessive access or allow use of an unapproved source identity.", + "compliant": "arn:aws:iam::123456789012:role/approved-storage-transfer-role", + "non-compliant": "arn:aws:iam::123456789012:role/admin or a missing/unapproved role ARN.", + "parent": "aws_s3_data_source" + } + } + }, + "transfer_options": { + "description": "Defines transfer behavior options such as overwrite and deletion behavior.", + "required": false, + "security_impact": true, + "rationale": "Transfer options can directly affect data integrity and source data preservation during transfers.", + "compliant": "transfer_options with safe overwrite behavior and without destructive source deletion.", + "non-compliant": "transfer_options that overwrite unsafely or delete source objects after transfer.", + "parent": "transfer_spec", + "arguments": { + "delete_objects_from_source_after_transfer": { + "description": "Whether objects are deleted from the source after they are transferred.", + "required": false, + "security_impact": true, + "rationale": "Deleting source objects after transfer is destructive and can create data loss or recovery risks if used inappropriately.", + "compliant": "delete_objects_from_source_after_transfer = false", + "non-compliant": "delete_objects_from_source_after_transfer = true", + "parent": "transfer_options" + }, + "overwrite_when": { + "description": "Specifies when objects at the destination may be overwritten during transfer.", + "required": false, + "security_impact": true, + "rationale": "Overwrite behavior affects destination data integrity and should be limited to approved settings to prevent unintended replacement of data.", + "compliant": "overwrite_when set to the approved policy value.", + "non-compliant": "overwrite_when set to an unapproved or overly permissive value.", + "parent": "transfer_options" + } + } } } } diff --git a/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md b/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md index f3e528507..096410e2d 100644 --- a/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md +++ b/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md @@ -11,9 +11,7 @@ Reference: [Terraform Registry – storage_transfer_agent_pool](https://registry | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| | `name` | The ID of the agent pool to create. The agent pool name identifies the pool used by transfer agents. | true | false | The name itself does not provide or restrict security controls, but it should still be set clearly for administration and auditability. | agent-pool-example | None | -| `display_name` | Specifies the client-defined AgentPool display name. | false | false | The display name is informational only and does not change the security posture of the resource. | Production transfer agent pool | None | | `bandwidth_limit` | Specifies the bandwidth limit details for the agent pool. | false | true | Bandwidth controls can reduce the risk of excessive transfer throughput, network saturation, and unintended large-scale data movement. | Refer to child argument. | Refer to child argument. | -| `project` | The project in which the resource belongs. If omitted, the provider project is used. | false | true | Using the correct project is security-relevant because resources created in the wrong project may bypass intended access controls, monitoring, or governance boundaries. | my-project-123 | An unintended or unauthorized project ID. | ### bandwidth_limit Block diff --git a/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md b/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md index 905582832..7adfce8d7 100644 --- a/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md +++ b/docs/gcp/Storage_Transfer_Service/storage_transfer_job.md @@ -10,18 +10,28 @@ Reference: [Terraform Registry – storage_transfer_job](https://registry.terraf | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `transfer_spec` | Defines the source, sink, and transfer behavior for the Storage Transfer job. | true | true | The transfer specification controls what data is moved, from where, and under what restrictions. | A transfer_spec with an approved source, sink, and restricted transfer conditions. | A transfer_spec that omits restrictions or uses an unapproved source definition. | +| `description` | A description for the Storage Transfer job. | false | false | The description is informational and does not directly affect the security posture of the transfer job. | daily-secure-transfer-job | None | +| `status` | The status of the transfer job. | false | true | Transfer jobs should be intentionally enabled or disabled to match operational and governance requirements. | ENABLED | Unexpectedly disabled or unmanaged status. | +| `schedule` | Defines when the Storage Transfer job runs. | false | true | A defined schedule supports controlled and expected execution of data transfers. | A schedule with an approved start date and run time. | No defined schedule where one is required by policy. | +| `transfer_spec` | Defines the source, sink, and transfer behavior for the Storage Transfer job. | true | true | The transfer specification controls what data is moved, from where, and under what restrictions. | A transfer_spec with approved source settings, approved agent pool usage, and safe transfer options. | A transfer_spec that omits required restrictions or uses unsafe transfer options. | ### transfer_spec Block | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `gcs_data_source` | Specifies a Google Cloud Storage bucket as the source for the transfer job. | false | true | The source bucket determines what data enters the transfer pipeline. It should reference only approved source locations. | gcs_data_source { bucket_name = "approved-source-bucket" } | gcs_data_source { bucket_name = "unapproved-public-bucket" } | -| `aws_s3_data_source` | Specifies an Amazon S3 bucket as the source for the transfer job. | false | true | Using an S3 source introduces external data into the transfer job and should be restricted to approved buckets and credentials. | aws_s3_data_source using an approved bucket and credential configuration. | aws_s3_data_source using an unapproved bucket or unmanaged credentials. | -| `azure_blob_storage_data_source` | Specifies an Azure Blob Storage container as the source for the transfer job. | false | true | Azure Blob source definitions should be limited to approved storage accounts and controlled credentials. | azure_blob_storage_data_source using an approved storage account and container. | azure_blob_storage_data_source using an unknown or unapproved storage account. | -| `http_data_source` | Specifies a list of HTTP or HTTPS URLs as the source for the transfer job. | false | true | HTTP-based sources can introduce integrity and confidentiality risks if insecure or untrusted endpoints are used. | http_data_source referencing approved HTTPS endpoints. | http_data_source referencing HTTP endpoints or untrusted URLs. | -| `posix_data_source` | Specifies a POSIX filesystem path as the source for the transfer job. | false | true | POSIX sources can expose local or attached filesystem data and should be limited to approved paths and agents. | posix_data_source { root_directory = "/approved/source/path" } | posix_data_source { root_directory = "/" } | -| `hdfs_data_source` | Specifies an HDFS path as the source for the transfer job. | false | true | HDFS sources should be controlled to prevent broad or unintended data extraction from distributed storage. | hdfs_data_source using an approved cluster and restricted root path. | hdfs_data_source using an unrestricted or unknown path. | -| `aws_s3_compatible_data_source` | Specifies an S3-compatible storage system as the source for the transfer job. | false | true | S3-compatible endpoints should be approved because they may point to third-party or self-hosted object storage. | aws_s3_compatible_data_source using an approved endpoint and bucket. | aws_s3_compatible_data_source using an unknown or untrusted endpoint. | | `source_agent_pool_name` | Specifies the agent pool used for agent-based source transfers. | false | true | Agent pools control where transfer agents run and should reference approved managed pools only. | source_agent_pool_name = "transferJobs/agentPools/approved-pool" | source_agent_pool_name = "transferJobs/agentPools/unapproved-pool" | -| `include_prefixes_required` | Ensures the Storage Transfer job restricts transfer scope by defining object_conditions.include_prefixes. | false | true | Requiring include_prefixes reduces the risk of unintentionally transferring more objects than intended by limiting the transfer to approved object path prefixes. | object_conditions { include_prefixes = ["secure-data/"] } | No object_conditions.include_prefixes configured. | +| `aws_s3_data_source` | Specifies an Amazon S3 bucket as the source for the transfer job. | false | true | Using an S3 source introduces external data into the transfer job and should be restricted to approved buckets and approved AWS access settings. | aws_s3_data_source using an approved bucket and approved role ARN. | aws_s3_data_source using an unapproved bucket or missing/unapproved role ARN. | +| `transfer_options` | Defines transfer behavior options such as overwrite and deletion behavior. | false | true | Transfer options can directly affect data integrity and source data preservation during transfers. | transfer_options with safe overwrite behavior and without destructive source deletion. | transfer_options that overwrite unsafely or delete source objects after transfer. | + +### aws_s3_data_source Block + + | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | + |----------|-------------|----------|-----------------|-----------|-----------|---------------| + | `role_arn` | The AWS IAM role ARN used to access the Amazon S3 source. | false | true | The role ARN controls what AWS permissions are used to read from the source bucket. Using the wrong role can grant excessive access or allow use of an unapproved source identity. | arn:aws:iam::123456789012:role/approved-storage-transfer-role | arn:aws:iam::123456789012:role/admin or a missing/unapproved role ARN. | + +### transfer_options Block + + | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | + |----------|-------------|----------|-----------------|-----------|-----------|---------------| + | `delete_objects_from_source_after_transfer` | Whether objects are deleted from the source after they are transferred. | false | true | Deleting source objects after transfer is destructive and can create data loss or recovery risks if used inappropriately. | delete_objects_from_source_after_transfer = false | delete_objects_from_source_after_transfer = true | + | `overwrite_when` | Specifies when objects at the destination may be overwritten during transfer. | false | true | Overwrite behavior affects destination data integrity and should be limited to approved settings to prevent unintended replacement of data. | overwrite_when set to the approved policy value. | overwrite_when set to an unapproved or overly permissive value. | From d0a25a4e22a1437ce8f0c83d2cad76ef26c5c10b Mon Sep 17 00:00:00 2001 From: Mohamed Date: Tue, 26 May 2026 16:55:53 +1000 Subject: [PATCH 08/10] Address Storage Transfer Service review feedback --- .../storage_transfer_agent_pool.json | 19 ++++++++++++++----- .../storage_transfer_agent_pool.md | 5 +++-- .../name/c.tf | 4 ++-- .../name/nc.tf | 5 +++-- .../google_storage_transfer_job/role_arn/c.tf | 2 +- .../role_arn/nc.tf | 2 +- .../name/policy.rego | 12 +++++++----- .../role_arn/policy.rego | 14 ++++++++------ .../source_agent_pool_name/policy.rego | 2 +- 9 files changed, 40 insertions(+), 25 deletions(-) diff --git a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json index fa9c3fa5e..80b0488ee 100644 --- a/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json +++ b/docs/gcp/Storage_Transfer_Service/resource_json/storage_transfer_agent_pool.json @@ -3,19 +3,28 @@ "subcategory": "Storage Transfer Service", "arguments": { "name": { - "description": "The ID of the agent pool to create. The agent pool name identifies the pool used by transfer agents.", + "description": "The ID of the agent pool to create. The agentPoolId must meet the following requirements: * Length of 128 characters or less. * Not start with the string goog. * Start with a lowercase ASCII character, followed by: * Zero or more: lowercase Latin alphabet characters, numerals, hyphens (-), periods (.), underscores (_), or tildes (~). * One or more numerals or lowercase ASCII characters. As expressed by the regular expression: ^(?!goog)[a-z]([a-z0-9-._~]*[a-z0-9])?$.", "required": true, - "security_impact": false, - "rationale": "The name itself does not provide or restrict security controls, but it should still be set clearly for administration and auditability.", + "security_impact": true, + "rationale": "The agent pool name is security relevant because a policy has been implemented to enforce approved naming requirements and prevent reserved or unapproved agent pool identifiers.", "compliant": "agent-pool-example", + "non-compliant": "goog-agent-pool", + "parent": null + }, + "display_name": { + "description": "Specifies the client-specified AgentPool description.", + "required": false, + "security_impact": false, + "rationale": "The display name is descriptive metadata only and does not enforce access control, encryption, networking, or transfer behavior.", + "compliant": "Example transfer agent pool", "non-compliant": null, "parent": null }, "bandwidth_limit": { - "description": "Specifies the bandwidth limit details for the agent pool.", + "description": "Specifies the bandwidth limit details. If this field is unspecified, the default value is set as 'No Limit'. Structure is documented below (#nested_bandwidth_limit).", "required": false, "security_impact": true, - "rationale": "Bandwidth controls can reduce the risk of excessive transfer throughput, network saturation, and unintended large-scale data movement.", + "rationale": "Bandwidth controls can reduce the risk of excessive transfer throughput, network saturation, and unintentional resource impact.", "compliant": "Refer to child argument.", "non-compliant": "Refer to child argument.", "parent": null, diff --git a/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md b/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md index 096410e2d..b2941513b 100644 --- a/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md +++ b/docs/gcp/Storage_Transfer_Service/storage_transfer_agent_pool.md @@ -10,8 +10,9 @@ Reference: [Terraform Registry – storage_transfer_agent_pool](https://registry | Argument | Description | Required | Security Impact | Rationale | Compliant | Non-Compliant | |----------|-------------|----------|-----------------|-----------|-----------|---------------| -| `name` | The ID of the agent pool to create. The agent pool name identifies the pool used by transfer agents. | true | false | The name itself does not provide or restrict security controls, but it should still be set clearly for administration and auditability. | agent-pool-example | None | -| `bandwidth_limit` | Specifies the bandwidth limit details for the agent pool. | false | true | Bandwidth controls can reduce the risk of excessive transfer throughput, network saturation, and unintended large-scale data movement. | Refer to child argument. | Refer to child argument. | +| `name` | The ID of the agent pool to create. The agentPoolId must meet the following requirements: * Length of 128 characters or less. * Not start with the string goog. * Start with a lowercase ASCII character, followed by: * Zero or more: lowercase Latin alphabet characters, numerals, hyphens (-), periods (.), underscores (_), or tildes (~). * One or more numerals or lowercase ASCII characters. As expressed by the regular expression: ^(?!goog)[a-z]([a-z0-9-._~]*[a-z0-9])?$. | true | true | The agent pool name is security relevant because a policy has been implemented to enforce approved naming requirements and prevent reserved or unapproved agent pool identifiers. | agent-pool-example | goog-agent-pool | +| `display_name` | Specifies the client-specified AgentPool description. | false | false | The display name is descriptive metadata only and does not enforce access control, encryption, networking, or transfer behavior. | Example transfer agent pool | None | +| `bandwidth_limit` | Specifies the bandwidth limit details. If this field is unspecified, the default value is set as 'No Limit'. Structure is documented below (#nested_bandwidth_limit). | false | true | Bandwidth controls can reduce the risk of excessive transfer throughput, network saturation, and unintentional resource impact. | Refer to child argument. | Refer to child argument. | ### bandwidth_limit Block diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/c.tf index 9949c9fe0..95ccc85d1 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/c.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/c.tf @@ -1,6 +1,6 @@ resource "google_storage_transfer_agent_pool" "c" { - name = "c" - display_name = "approved-agent-pool" + name = "agent-pool-example" + display_name = "Compliant agent pool" bandwidth_limit { limit_mbps = "250" diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf index 7aee2b88f..9d4f7a22a 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf @@ -1,6 +1,7 @@ resource "google_storage_transfer_agent_pool" "nc" { - name = "nc" - display_name = "unsafe-agent-pool" + name = "goog-agent-pool" + display_name = "Non-compliant agent pool" + bandwidth_limit { limit_mbps = "5000" diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/c.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/c.tf index bf0098a47..cf1c5a4ed 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/c.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/c.tf @@ -6,7 +6,7 @@ resource "google_storage_transfer_job" "c" { transfer_spec { aws_s3_data_source { bucket_name = "my-source-bucket" - role_arn = "arn:aws:iam::123456789012:role/sts-transfer-role" + role_arn = "arn:aws:iam::123456789012:role/approved-storage-transfer-role" } gcs_data_sink { diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/nc.tf index 3c3707182..c6c13972f 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/nc.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/nc.tf @@ -6,7 +6,7 @@ resource "google_storage_transfer_job" "nc" { transfer_spec { aws_s3_data_source { bucket_name = "my-source-bucket" - role_arn = "arn:aws:iam::123456789012:role/unsafe-role" + role_arn = "arn:aws:iam::123456789012:role/unsafe-role" } gcs_data_sink { diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/policy.rego index ccbf55bb5..be2b6c1b1 100644 --- a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/policy.rego +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/policy.rego @@ -6,21 +6,23 @@ import data.terraform.gcp.security.Storage_Transfer_Service.google_storage_trans conditions := [ [ { - "situation_description": "Storage Transfer agent pool uses a disallowed name.", + "situation_description": "Storage Transfer agent pool uses an unapproved name.", "remedies": [ "Use an approved Storage Transfer agent pool name.", - "Do not use disallowed agent pool identifiers." + "Do not use reserved or unapproved agent pool names such as names starting with goog." ] }, { - "condition": "Storage Transfer agent pool name must not use disallowed values.", + "condition": "Storage Transfer agent pool name must be in the approved list.", "attribute_path": ["name"], - "values": ["nc"], - "policy_type": "blacklist" + "values": ["agent-pool-example"], + "policy_type": "whitelist" } ] ] result := helpers.get_multi_summary(conditions, vars.variables) + message := result.message + details := result.details \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/policy.rego index b785da188..273f3126f 100644 --- a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/policy.rego +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/role_arn/policy.rego @@ -8,19 +8,21 @@ conditions := [ { "situation_description": "Storage Transfer job uses an unapproved AWS IAM role ARN.", "remedies": [ - "Set transfer_spec.aws_s3_data_source.role_arn to an approved IAM role ARN.", - "Avoid using unapproved AWS IAM roles for Storage Transfer jobs." + "Use an approved AWS IAM role ARN for Storage Transfer jobs.", + "Use a dedicated approved IAM role instead of an unapproved role." ] }, - { - "condition": "Storage Transfer job must not use an unapproved AWS IAM role ARN.", + { + "condition": "Storage Transfer job must use an approved AWS IAM role ARN.", "attribute_path": ["transfer_spec", 0, "aws_s3_data_source", 0, "role_arn"], - "values": ["arn:aws:iam::123456789012:role/unsafe-role"], - "policy_type": "blacklist" + "values": ["arn:aws:iam::123456789012:role/approved-storage-transfer-role"], + "policy_type": "whitelist" } ] ] result := helpers.get_multi_summary(conditions, vars.variables) + message := result.message + details := result.details \ No newline at end of file diff --git a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/policy.rego b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/policy.rego index 3aa9c1c55..16cb05663 100644 --- a/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/policy.rego +++ b/policies/gcp/Storage_Transfer_Service/google_storage_transfer_job/source_agent_pool_name/policy.rego @@ -14,7 +14,7 @@ conditions := [ }, { "condition": "Storage Transfer job must use an approved source agent pool.", - "attribute_path": ["transfer_spec", "source_agent_pool_name"], + "attribute_path": ["transfer_spec", 0, "source_agent_pool_name"], "values": ["projects/my-project-123/agentPools/approved-pool"], "policy_type": "whitelist" } From bf461846504f26f11b3c9c8f44b21ea2b61eeca4 Mon Sep 17 00:00:00 2001 From: Mohamed Date: Tue, 26 May 2026 17:04:45 +1000 Subject: [PATCH 09/10] Fix agent pool name policy check output --- .../google_storage_transfer_agent_pool/name/nc.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf index 9d4f7a22a..be57351fd 100644 --- a/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf +++ b/inputs/gcp/Storage_Transfer_Service/google_storage_transfer_agent_pool/name/nc.tf @@ -1,8 +1,7 @@ resource "google_storage_transfer_agent_pool" "nc" { - name = "goog-agent-pool" + name = "nc" display_name = "Non-compliant agent pool" - bandwidth_limit { limit_mbps = "5000" } From 7a782e22077e2e3b77b72a61d4c311463b281988 Mon Sep 17 00:00:00 2001 From: Mohamed Date: Thu, 28 May 2026 17:26:00 +1000 Subject: [PATCH 10/10] Rerun Storage Transfer Service checks