Test/buildkite improvements

.buildkite/hooks/pre-command

@@ -1,25 +1,3 @@
 #!/bin/bash
-
-set -e
-
-# Export secrets for Docker containers.
-# Restrict exposing secrets only to the steps that need them
-export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917")
-export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917")
-export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917")
-export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917")
-export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917")
-export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917")
-export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917")
-export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917")
-export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917")
-export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917")
-export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917")
-export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917")
-export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917")
-export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917")
-export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
+curl -s -o /tmp/build-setup.sh "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/build_setup.sh"
+source /tmp/build-setup.sh

.buildkite/pipeline.yml

@@ -1,74 +1,107 @@
+# Each test downloads a centralized test runner script from the dbt_package_automations repo,
+# which handles environment setup, dependency installation, and dbt test execution.
+
+env:
+  RUN_MODELS_URL: "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_models.sh"
+
 steps:
   - label: ":postgres: Run Tests - Postgres"
-    key: "run-dbt-postgres"
+    key: "run_dbt_postgres"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "CI_POSTGRES_DBT_DBNAME"
             - "CI_POSTGRES_DBT_HOST"
             - "CI_POSTGRES_DBT_PASS"
             - "CI_POSTGRES_DBT_USER"
     commands: |
-      bash .buildkite/scripts/run_models.sh postgres 
+      curl -s "${RUN_MODELS_URL}" | bash -s postgres
 
   - label: ":snowflake-db: Run Tests - Snowflake"
     key: "run_dbt_snowflake"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "CI_SNOWFLAKE_DBT_ACCOUNT"
             - "CI_SNOWFLAKE_DBT_DATABASE"
             - "CI_SNOWFLAKE_DBT_PASS"
             - "CI_SNOWFLAKE_DBT_ROLE"
             - "CI_SNOWFLAKE_DBT_USER"
             - "CI_SNOWFLAKE_DBT_WAREHOUSE"
     commands: |
-      bash .buildkite/scripts/run_models.sh snowflake
+      curl -s "${RUN_MODELS_URL}" | bash -s snowflake
 
   - label: ":gcloud: Run Tests - BigQuery"
     key: "run_dbt_bigquery"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "GCLOUD_SERVICE_KEY"
     commands: |
-      bash .buildkite/scripts/run_models.sh bigquery
+      curl -s "${RUN_MODELS_URL}" | bash -s bigquery
 
   - label: ":amazon-redshift: Run Tests - Redshift"
     key: "run_dbt_redshift"
+    concurrency: 3
+    concurrency_group: "warehouse/redshift"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "CI_REDSHIFT_DBT_DBNAME"
             - "CI_REDSHIFT_DBT_HOST"
             - "CI_REDSHIFT_DBT_PASS"
             - "CI_REDSHIFT_DBT_USER"
     commands: |
-      bash .buildkite/scripts/run_models.sh redshift
+      curl -s "${RUN_MODELS_URL}" | bash -s redshift
 
   - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
             - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
-      bash .buildkite/scripts/run_models.sh databricks
+      curl -s "${RUN_MODELS_URL}" | bash -s databricks

integration_tests/ci/test_scenarios.yml

@@ -0,0 +1,18 @@
+# Name of the dbt variable used to set the schema where data lives
+# This is the variable name you use in dbt (e.g., amazon_ads_schema)
+schema_variable_name: "amazon_ads_schema"
+default_include_incremental: false  # Whether default scenario should test incremental runs
+
+# Additional test scenarios (default run happens automatically)
+test_scenarios:
+  - name: "portfolio history disabled"
+    vars:
+      amazon_ads__portfolio_history_enabled: false
+    include_incremental: false
+
+# Add more scenarios as needed:
+# - name: "some other scenario"
+#   vars:
+#     amazon_ads__some_other_flag: true
+#     amazon_ads__another_setting: "custom_value"
+#   include_incremental: false

integration_tests/dbt_project.yml

@@ -7,7 +7,7 @@ profile: 'integration_tests'
 
 vars:
   amazon_ads:
-    amazon_ads_schema: amazon_ads_integration_tests_03
+    amazon_ads_schema: "amazon_ads_integration_tests"
     amazon_ads_ad_group_history_identifier: "ad_group_history_data"
     amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data"
     amazon_ads_advertised_product_report_identifier: "advertised_product_report_data"
@@ -43,12 +43,43 @@ seeds:
   amazon_ads_integration_tests:
     +column_types:
       _fivetran_synced: "timestamp"
-      campaign_budget_amount: "float"
-      click_through_rate: "float"
-      keyword_bid: "float"
-      clicks: "float"
-      impressions: "float"
-      cost: "float"
+    # Seeds with reporting metrics (clicks, impressions, cost)
+    ad_group_level_report_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+        clicks: "float"
+        impressions: "float"
+        cost: "float"
+    campaign_level_report_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+        campaign_budget_amount: "float"
+        clicks: "float"
+        impressions: "float"
+        cost: "float"
+    targeting_keyword_report_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+        campaign_budget_amount: "float"
+        keyword_bid: "float"
+        clicks: "float"
+        impressions: "float"
+        cost: "float"
+    search_term_ad_keyword_report_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+        campaign_budget_amount: "float"
+        keyword_bid: "float"
+        clicks: "float"
+        impressions: "float"
+        cost: "float"
+    advertised_product_report_data:
+      +column_types:
+        _fivetran_synced: "timestamp"
+        campaign_budget_amount: "float"
+        clicks: "float"
+        impressions: "float"
+        cost: "float"
 flags:
   send_anonymous_usage_stats: False
   use_colors: True