From 090446e3768f9b1281a8a9b172717d3699c1951b Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Fri, 4 Apr 2025 11:31:18 -0700 Subject: [PATCH 1/6] add AWS default credentials provider --- .../spring/config/SettingsConfiguration.java | 57 +++++++++++++++---- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index f101495eb66..0c333affca9 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -1,5 +1,8 @@ package org.prebid.server.spring.config; +import org.apache.commons.lang3.StringUtils; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; import io.vertx.core.Vertx; import io.vertx.core.file.FileSystem; import lombok.Data; @@ -40,9 +43,13 @@ import org.springframework.stereotype.Component; import org.springframework.validation.annotation.Validated; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.core.exception.SdkClientException; import javax.validation.constraints.Min; import javax.validation.constraints.NotBlank; @@ -58,6 +65,8 @@ @UtilityClass public class SettingsConfiguration { + private static final Logger logger = LoggerFactory.getLogger(SettingsConfiguration.class); + @Configuration @ConditionalOnProperty(prefix = "settings.filesystem", name = {"settings-filename", "stored-requests-dir", "stored-imps-dir"}) @@ -233,18 +242,30 @@ static class S3SettingsConfiguration { @Component @ConfigurationProperties(prefix = "settings.s3") - @ConditionalOnProperty(prefix = "settings.s3", name = {"accessKeyId", "secretAccessKey"}) @Validated @Data @NoArgsConstructor protected static class S3ConfigurationProperties { - @NotBlank + /** + * If accessKeyId and secretAccessKey are provided in the + * configuration file then they will be used. Otherwise, the + * DefaultCredentialsProvider will look for credentials in this order: + * + * - Java System Properties + * - Environment Variables + * - Web Identity Token + * - AWS credentials file (~/.aws/credentials) + * - ECS container credentials + * - EC2 instance profile + */ private String accessKeyId; - - @NotBlank private String secretAccessKey; + private boolean useStaticCredentials() { + return StringUtils.isNotBlank(accessKeyId) && StringUtils.isNotBlank(secretAccessKey); + } + /** * If not provided AWS_GLOBAL will be used as a region */ @@ -274,20 +295,32 @@ protected static class S3ConfigurationProperties { @Bean S3AsyncClient s3AsyncClient(S3ConfigurationProperties s3ConfigurationProperties) throws URISyntaxException { - final AwsBasicCredentials credentials = AwsBasicCredentials.create( - s3ConfigurationProperties.getAccessKeyId(), - s3ConfigurationProperties.getSecretAccessKey()); + final Region awsRegion = Optional.ofNullable(s3ConfigurationProperties.getRegion()) .map(Region::of) .orElse(Region.AWS_GLOBAL); - return S3AsyncClient - .builder() - .credentialsProvider(StaticCredentialsProvider.create(credentials)) + final S3AsyncClientBuilder clientBuilder = S3AsyncClient.builder() .endpointOverride(new URI(s3ConfigurationProperties.getEndpoint())) .forcePathStyle(s3ConfigurationProperties.getForcePathStyle()) - .region(awsRegion) - .build(); + .region(awsRegion); + final AwsCredentialsProvider credentialsProvider; + if (s3ConfigurationProperties.useStaticCredentials()) { + final AwsBasicCredentials basicCredentials = AwsBasicCredentials.create( + s3ConfigurationProperties.getAccessKeyId(), + s3ConfigurationProperties.getSecretAccessKey()); + credentialsProvider = StaticCredentialsProvider.create(basicCredentials); + } else { + credentialsProvider = DefaultCredentialsProvider.create(); + } + try { + credentialsProvider.resolveCredentials(); + } catch (SdkClientException e) { + logger.error("Failed to resolve AWS credentials", e); + } + clientBuilder.credentialsProvider(credentialsProvider); + + return clientBuilder.build(); } @Bean From c451c49a3e15026068f592a2537e14336ce3f735 Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Fri, 4 Apr 2025 11:34:13 -0700 Subject: [PATCH 2/6] document the lookup order for AWS credentials --- docs/application-settings.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/application-settings.md b/docs/application-settings.md index 2a20e0d8342..51410d163d7 100644 --- a/docs/application-settings.md +++ b/docs/application-settings.md @@ -277,8 +277,8 @@ The general idea is that you'll place all the account-specific settings in a sep ```yaml settings: s3: - accessKeyId: - secretAccessKey: + accessKeyId: # optional + secretAccessKey: # optional endpoint: # http://s3.storage.com bucket: # prebid-application-settings region: # if not provided AWS_GLOBAL will be used. Example value: 'eu-central-1' @@ -298,6 +298,14 @@ settings: timeout: 5000 ``` +If `accessKeyId` and `secretAccessKey` are not specified in the Prebid Server configuration then AWS credentials will be looked up in this order: +- Java System Properties - `aws.accessKeyId` and `aws.secretAccessKey` +- Environment Variables - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +- Web Identity Token credentials from system properties or environment variables +- Credential profiles file at the default location (`~/.aws/credentials`) shared by all AWS SDKs and the AWS CLI +- Credentials delivered through the Amazon EC2 container service if "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" environment variable is set and security manager has permission to access the variable, +- Instance profile credentials delivered through the Amazon EC2 metadata service + ### File format We recommend using the `json` format for your account configuration. A minimal configuration may look like this. From 46c63825d3d8350d1db4f9acd0c3bbb2f888cb4a Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Fri, 4 Apr 2025 11:35:45 -0700 Subject: [PATCH 3/6] Document the lookup order for AWS credentials --- docs/config-app.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/config-app.md b/docs/config-app.md index 24ddbb9a5f7..fcae5aed6e6 100644 --- a/docs/config-app.md +++ b/docs/config-app.md @@ -391,8 +391,8 @@ contain 'WHERE last_updated > ?' for MySQL and 'WHERE last_updated > $1' for Pos For S3 storage configuration - `settings.in-memory-cache.s3-update.refresh-rate` - refresh period in ms for stored request updates in S3 -- `settings.s3.access-key-id` - an access key -- `settings.s3.secret-access-key` - a secret access key +- `settings.s3.access-key-id` - an access key (optional) +- `settings.s3.secret-access-key` - a secret access key (optional) - `settings.s3.region` - a region, AWS_GLOBAL by default - `settings.s3.endpoint` - an endpoint - `settings.s3.bucket` - a bucket name @@ -402,6 +402,16 @@ For S3 storage configuration - `settings.s3.stored-requests-dir` - a directory with stored requests - `settings.s3.stored-responses-dir` - a directory with stored responses +If `settings.s3.access-key-id` and `settings.s3.secret-access-key` are not specified in the Prebid Server configuration then AWS credentials will be looked up in this order: +- Java System Properties - `aws.accessKeyId` and `aws.secretAccessKey` +- Environment Variables - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` +- Web Identity Token credentials from system properties or environment variables +- Credential profiles file at the default location (`~/.aws/credentials`) shared by all AWS SDKs and the AWS CLI +- Credentials delivered through the Amazon EC2 container service if "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" environment variable is set and security manager has permission to access the variable, +- Instance profile credentials delivered through the Amazon EC2 metadata service + + + For targeting available next options: - `settings.targeting.truncate-attr-chars` - set the max length for names of targeting keywords (0 means no truncation). From 00bcd3ebc42d50c8a4d5462eb143dcde8c892225 Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Fri, 4 Apr 2025 14:10:44 -0700 Subject: [PATCH 4/6] revert changes to SettingsConfiguration.java --- .../spring/config/SettingsConfiguration.java | 57 ++++--------------- 1 file changed, 12 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index 0c333affca9..f101495eb66 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -1,8 +1,5 @@ package org.prebid.server.spring.config; -import org.apache.commons.lang3.StringUtils; -import org.prebid.server.log.Logger; -import org.prebid.server.log.LoggerFactory; import io.vertx.core.Vertx; import io.vertx.core.file.FileSystem; import lombok.Data; @@ -43,13 +40,9 @@ import org.springframework.stereotype.Component; import org.springframework.validation.annotation.Validated; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; -import software.amazon.awssdk.core.exception.SdkClientException; import javax.validation.constraints.Min; import javax.validation.constraints.NotBlank; @@ -65,8 +58,6 @@ @UtilityClass public class SettingsConfiguration { - private static final Logger logger = LoggerFactory.getLogger(SettingsConfiguration.class); - @Configuration @ConditionalOnProperty(prefix = "settings.filesystem", name = {"settings-filename", "stored-requests-dir", "stored-imps-dir"}) @@ -242,29 +233,17 @@ static class S3SettingsConfiguration { @Component @ConfigurationProperties(prefix = "settings.s3") + @ConditionalOnProperty(prefix = "settings.s3", name = {"accessKeyId", "secretAccessKey"}) @Validated @Data @NoArgsConstructor protected static class S3ConfigurationProperties { - /** - * If accessKeyId and secretAccessKey are provided in the - * configuration file then they will be used. Otherwise, the - * DefaultCredentialsProvider will look for credentials in this order: - * - * - Java System Properties - * - Environment Variables - * - Web Identity Token - * - AWS credentials file (~/.aws/credentials) - * - ECS container credentials - * - EC2 instance profile - */ + @NotBlank private String accessKeyId; - private String secretAccessKey; - private boolean useStaticCredentials() { - return StringUtils.isNotBlank(accessKeyId) && StringUtils.isNotBlank(secretAccessKey); - } + @NotBlank + private String secretAccessKey; /** * If not provided AWS_GLOBAL will be used as a region @@ -295,32 +274,20 @@ private boolean useStaticCredentials() { @Bean S3AsyncClient s3AsyncClient(S3ConfigurationProperties s3ConfigurationProperties) throws URISyntaxException { - + final AwsBasicCredentials credentials = AwsBasicCredentials.create( + s3ConfigurationProperties.getAccessKeyId(), + s3ConfigurationProperties.getSecretAccessKey()); final Region awsRegion = Optional.ofNullable(s3ConfigurationProperties.getRegion()) .map(Region::of) .orElse(Region.AWS_GLOBAL); - final S3AsyncClientBuilder clientBuilder = S3AsyncClient.builder() + return S3AsyncClient + .builder() + .credentialsProvider(StaticCredentialsProvider.create(credentials)) .endpointOverride(new URI(s3ConfigurationProperties.getEndpoint())) .forcePathStyle(s3ConfigurationProperties.getForcePathStyle()) - .region(awsRegion); - final AwsCredentialsProvider credentialsProvider; - if (s3ConfigurationProperties.useStaticCredentials()) { - final AwsBasicCredentials basicCredentials = AwsBasicCredentials.create( - s3ConfigurationProperties.getAccessKeyId(), - s3ConfigurationProperties.getSecretAccessKey()); - credentialsProvider = StaticCredentialsProvider.create(basicCredentials); - } else { - credentialsProvider = DefaultCredentialsProvider.create(); - } - try { - credentialsProvider.resolveCredentials(); - } catch (SdkClientException e) { - logger.error("Failed to resolve AWS credentials", e); - } - clientBuilder.credentialsProvider(credentialsProvider); - - return clientBuilder.build(); + .region(awsRegion) + .build(); } @Bean From e85b532cd0903e8fec13c8fe64397106a3531643 Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Mon, 7 Apr 2025 10:25:09 -0700 Subject: [PATCH 5/6] add AWS default credentials provider --- .../spring/config/SettingsConfiguration.java | 57 +++++++++++++++---- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index f101495eb66..0c333affca9 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -1,5 +1,8 @@ package org.prebid.server.spring.config; +import org.apache.commons.lang3.StringUtils; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; import io.vertx.core.Vertx; import io.vertx.core.file.FileSystem; import lombok.Data; @@ -40,9 +43,13 @@ import org.springframework.stereotype.Component; import org.springframework.validation.annotation.Validated; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.core.exception.SdkClientException; import javax.validation.constraints.Min; import javax.validation.constraints.NotBlank; @@ -58,6 +65,8 @@ @UtilityClass public class SettingsConfiguration { + private static final Logger logger = LoggerFactory.getLogger(SettingsConfiguration.class); + @Configuration @ConditionalOnProperty(prefix = "settings.filesystem", name = {"settings-filename", "stored-requests-dir", "stored-imps-dir"}) @@ -233,18 +242,30 @@ static class S3SettingsConfiguration { @Component @ConfigurationProperties(prefix = "settings.s3") - @ConditionalOnProperty(prefix = "settings.s3", name = {"accessKeyId", "secretAccessKey"}) @Validated @Data @NoArgsConstructor protected static class S3ConfigurationProperties { - @NotBlank + /** + * If accessKeyId and secretAccessKey are provided in the + * configuration file then they will be used. Otherwise, the + * DefaultCredentialsProvider will look for credentials in this order: + * + * - Java System Properties + * - Environment Variables + * - Web Identity Token + * - AWS credentials file (~/.aws/credentials) + * - ECS container credentials + * - EC2 instance profile + */ private String accessKeyId; - - @NotBlank private String secretAccessKey; + private boolean useStaticCredentials() { + return StringUtils.isNotBlank(accessKeyId) && StringUtils.isNotBlank(secretAccessKey); + } + /** * If not provided AWS_GLOBAL will be used as a region */ @@ -274,20 +295,32 @@ protected static class S3ConfigurationProperties { @Bean S3AsyncClient s3AsyncClient(S3ConfigurationProperties s3ConfigurationProperties) throws URISyntaxException { - final AwsBasicCredentials credentials = AwsBasicCredentials.create( - s3ConfigurationProperties.getAccessKeyId(), - s3ConfigurationProperties.getSecretAccessKey()); + final Region awsRegion = Optional.ofNullable(s3ConfigurationProperties.getRegion()) .map(Region::of) .orElse(Region.AWS_GLOBAL); - return S3AsyncClient - .builder() - .credentialsProvider(StaticCredentialsProvider.create(credentials)) + final S3AsyncClientBuilder clientBuilder = S3AsyncClient.builder() .endpointOverride(new URI(s3ConfigurationProperties.getEndpoint())) .forcePathStyle(s3ConfigurationProperties.getForcePathStyle()) - .region(awsRegion) - .build(); + .region(awsRegion); + final AwsCredentialsProvider credentialsProvider; + if (s3ConfigurationProperties.useStaticCredentials()) { + final AwsBasicCredentials basicCredentials = AwsBasicCredentials.create( + s3ConfigurationProperties.getAccessKeyId(), + s3ConfigurationProperties.getSecretAccessKey()); + credentialsProvider = StaticCredentialsProvider.create(basicCredentials); + } else { + credentialsProvider = DefaultCredentialsProvider.create(); + } + try { + credentialsProvider.resolveCredentials(); + } catch (SdkClientException e) { + logger.error("Failed to resolve AWS credentials", e); + } + clientBuilder.credentialsProvider(credentialsProvider); + + return clientBuilder.build(); } @Bean From cb67da880251b8bc1306e89490641428bce88b99 Mon Sep 17 00:00:00 2001 From: Ian Downard <54998167+ianwow@users.noreply.github.com> Date: Mon, 7 Apr 2025 14:42:54 -0700 Subject: [PATCH 6/6] fix: adjust hook execution time assertion range in HookStageExecutorTest The test was failing intermittently due to strict time boundaries (50-70ms) for hook execution. Widened the acceptable range to account for slight variations in execution time across different environments. - Previous range: 50-70ms - New range: 50-80ms Test: HookStageExecutorTest#shouldExecuteEntrypointHooksToleratingTimeoutAndFailedFuture --- .../prebid/server/hooks/execution/HookStageExecutorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/prebid/server/hooks/execution/HookStageExecutorTest.java b/src/test/java/org/prebid/server/hooks/execution/HookStageExecutorTest.java index 7f1d29925c1..3ee2b504a8e 100644 --- a/src/test/java/org/prebid/server/hooks/execution/HookStageExecutorTest.java +++ b/src/test/java/org/prebid/server/hooks/execution/HookStageExecutorTest.java @@ -654,7 +654,7 @@ public void shouldExecuteEntrypointHooksToleratingTimeoutAndFailedFuture(VertxTe assertThat(hookOutcome.getStatus()) .isEqualTo(ExecutionStatus.execution_failure); assertThat(hookOutcome.getMessage()).isEqualTo("Failed after a while"); - assertThat(hookOutcome.getExecutionTime()).isBetween(50L, 70L); + assertThat(hookOutcome.getExecutionTime()).isBetween(50L, 80L); }); assertThat(group0Hooks.get(1)).satisfies(hookOutcome -> {