Consolidate source + remove combo tests for amazon_ads

CHANGELOG.md

@@ -1,3 +1,24 @@
+# dbt_amazon_ads v1.0.0
+[PR #27](https://github.com/fivetran/dbt_amazon_ads/pull/27) includes the following updates:
+
+## Breaking Changes
+> A `--full-refresh` is required after upgrading to this release.
+
+### Source Package Consolidation
+- Consolidated `dbt_amazon_ads_source` into this package.
+- All functionality from the source package has been merged into this transformation package. Be sure to:
+  - Remove `fivetran/amazon_ads_source` from `packages.yml`
+  - Delete any `source:` overrides that reference the source package
+
+### dbt Fusion Compatibility Updates
+- For compatibility with dbt Fusion versions >= 1.10.6:
+    - Removed `dbt_utils.unique_combination_of_columns` tests
+    - Removed `accepted_values` tests
+    - In src_amazon_ads.yml, moved `loaded_at_field: _fivetran_synced` under the `config:` block.
+
+### Schema Configuration Updates
+- Some customizations have changed, refer to the [README](https://github.com/fivetran/dbt_amazon_ads/blob/main/README.md) for details.
+
 # dbt_amazon_ads v0.5.0
 
 [PR #23](https://github.com/fivetran/dbt_amazon_ads/pull/23) includes the following updates:

README.md

@@ -13,7 +13,7 @@
 </p>
 
 ## What does this dbt package do?
-- Produces modeled tables that leverage Amazon Ads data from [Fivetran's connector](https://fivetran.com/docs/applications/amazon-ads) in the format described by [this ERD](https://fivetran.com/docs/applications/amazon-ads#schemainformation) and builds off the output of our [Amazon Ads source package](https://github.com/fivetran/dbt_amazon_ads_source).
+- Produces modeled tables that leverage Amazon Ads data from [Fivetran's connector](https://fivetran.com/docs/applications/amazon-ads) in the format described by [this ERD](https://fivetran.com/docs/applications/amazon-ads#schemainformation).
 - Provides insight into your ad performance across the following grains:
   - Account, portfolio, campaign, ad group, ad, keyword, and search term
 - Materializes output models designed to work simultaneously with our [multi-platform Ad Reporting package](https://github.com/fivetran/dbt_ad_reporting).
@@ -58,10 +58,10 @@ Include the following amazon_ads package version in your `packages.yml` file _if
 ```yaml
 packages:
   - package: fivetran/amazon_ads
-    version: [">=0.5.0", "<0.6.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=1.0.0", "<1.1.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
-Do NOT include the `amazon_ads_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.
+> All required sources are now bundled into this transformation package. Do not include `fivetran/amazon_ads_source` in your `packages.yml`.
 
 
 ### Step 3: Define database and schema variables
@@ -126,10 +126,10 @@ By default, this package will build the Amazon Ads staging models (11 views, 11
 
 ```yml
 models:
-    amazon_ads_source:
-      +schema: my_new_schema_name # leave blank for just the target_schema
     amazon_ads:
-      +schema: my_new_schema_name # leave blank for just the target_schema
+      +schema: my_new_schema_name # Leave +schema: blank to use the default target_schema.
+      staging:
+        +schema: my_new_schema_name # Leave +schema: blank to use the default target_schema.
 ```
 
 #### Change the source table references
@@ -157,9 +157,6 @@ This dbt package is dependent on the following dbt packages. Be aware that these
 
 ```yml
 packages:
-    - package: fivetran/amazon_ads_source
-      version: [">=0.5.0", "<0.6.0"]
-
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'amazon_ads'
-version: '0.5.0'
+version: '1.0.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
@@ -8,16 +8,25 @@ models:
     +materialized: table
     intermediate:
       +materialized: view
+    staging:
+      +schema: amazon_ads_source
+      +materialized: view
 vars:
   amazon_ads:    
-    ad_group_history: "{{ ref('stg_amazon_ads__ad_group_history') }}"
-    ad_group_level_report: "{{ ref('stg_amazon_ads__ad_group_level_report') }}"
-    advertised_product_report: "{{ ref('stg_amazon_ads__advertised_product_report') }}"
-    campaign_history: "{{ ref('stg_amazon_ads__campaign_history') }}"
-    campaign_level_report: "{{ ref('stg_amazon_ads__campaign_level_report') }}"
-    portfolio_history: "{{ ref('stg_amazon_ads__portfolio_history') }}"
-    product_ad_history: "{{ ref('stg_amazon_ads__product_ad_history') }}"
-    profile: "{{ ref('stg_amazon_ads__profile') }}"
-    keyword_history: "{{ ref('stg_amazon_ads__keyword_history') }}"
-    targeting_keyword_report: "{{ ref('stg_amazon_ads__targeting_keyword_report') }}"
-    search_term_ad_keyword_report: "{{ ref('stg_amazon_ads__search_term_ad_keyword_report') }}"
+    ad_group_history: "{{ source('amazon_ads', 'ad_group_history') }}"
+    ad_group_level_report: "{{ source('amazon_ads', 'ad_group_level_report') }}"
+    advertised_product_report: "{{ source('amazon_ads', 'advertised_product_report') }}"
+    campaign_history: "{{ source('amazon_ads', 'campaign_history') }}"
+    campaign_level_report: "{{ source('amazon_ads', 'campaign_level_report') }}"
+    portfolio_history: "{{ source('amazon_ads', 'portfolio_history') }}"
+    product_ad_history: "{{ source('amazon_ads', 'product_ad_history') }}"
+    profile: "{{ source('amazon_ads', 'profile') }}"
+    keyword_history: "{{ source('amazon_ads', 'keyword_history') }}"
+    targeting_keyword_report: "{{ source('amazon_ads', 'targeting_keyword_report') }}"
+    search_term_ad_keyword_report: "{{ source('amazon_ads', 'search_term_ad_keyword_report') }}"
+
+    amazon_ads__ad_group_passthrough_metrics: []
+    amazon_ads__advertised_product_passthrough_metrics: []
+    amazon_ads__campaign_passthrough_metrics: []
+    amazon_ads__search_term_ad_keyword_passthrough_metrics: []
+    amazon_ads__targeting_keyword_passthrough_metrics: []

integration_tests/dbt_project.yml

@@ -1,12 +1,12 @@
 config-version: 2
 
 name: 'amazon_ads_integration_tests'
-version: '0.5.0'
+version: '1.0.0'
 
 profile: 'integration_tests' 
 
 vars:
-  amazon_ads_source:
+  amazon_ads:
     amazon_ads_schema: amazon_ads_integration_tests_03
     amazon_ads_ad_group_history_identifier: "ad_group_history_data"
     amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data"
@@ -43,4 +43,4 @@ seeds:
       keyword_bid: "float"
       clicks: "float"
       impressions: "float"
-      cost: "float"
+      cost: "float"

macros/staging/amazon_ads_add_pass_though_columns.sql

@@ -0,0 +1,28 @@
+{# Backwards compatible version of fivetran_utils.add_pass_through_columns #}
+
+{% macro amazon_ads_add_pass_through_columns(base_columns, pass_through_fields, except_fields=[]) %}
+
+{% if pass_through_fields %}
+    {% for column in pass_through_fields %}
+
+        {% if column is mapping %}
+        {% set col_name = column.alias|default(column.name)|lower %}
+
+            {% if col_name not in except_fields %}
+                {% if column.alias %}
+                    {% do base_columns.append({ "name": column.name, "alias": column.alias, "datatype": column.datatype if column.datatype else dbt.type_string()}) %}
+                {% else %}
+                    {% do base_columns.append({ "name": column.name, "datatype": column.datatype if column.datatype else dbt.type_string()}) %}
+                {% endif %}
+            {% endif %}
+
+        {% else %}
+            {% if column|lower not in except_fields %}
+                {% do base_columns.append({ "name": column, "datatype": dbt.type_string()}) %}
+            {% endif %}
+        {% endif %}
+
+    {% endfor %}
+{% endif %}
+
+{% endmacro %}

macros/staging/amazon_ads_fill_pass_through_columns.sql

@@ -0,0 +1,17 @@
+{# Backwards compatible version of fivetran_utils.fill_pass_through_columns #}
+
+{% macro amazon_ads_fill_pass_through_columns(pass_through_fields, except=[]) %}
+
+{% if pass_through_fields %}
+    {% for field in pass_through_fields %}
+        {% if (field.alias if field.alias else field.name) not in except %}
+            {% if field.transform_sql %}
+                , coalesce(cast({{ field.transform_sql }} as {{ dbt.type_float() }}), 0) as {{ field.alias if field.alias else field.name }}
+            {% else %}
+                , coalesce(cast({{ field.alias if field.alias else field.name }} as {{ dbt.type_float() }}), 0) as {{ field.alias if field.alias else field.name }}
+            {% endif %}
+        {% endif %}
+    {% endfor %}
+{% endif %}
+
+{% endmacro %}

macros/staging/get_ad_group_history_columns.sql

@@ -0,0 +1,16 @@
+{% macro get_ad_group_history_columns() %}
+
+{% set columns = [
+    {"name": "campaign_id", "datatype": dbt.type_int()},
+    {"name": "creation_date", "datatype": dbt.type_timestamp()},
+    {"name": "default_bid", "datatype": dbt.type_float()},
+    {"name": "id", "datatype": dbt.type_string()},
+    {"name": "last_updated_date", "datatype": dbt.type_timestamp()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "serving_status", "datatype": dbt.type_string()},
+    {"name": "state", "datatype": dbt.type_string()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/staging/get_ad_group_level_report_columns.sql

@@ -0,0 +1,19 @@
+{% macro get_ad_group_level_report_columns() %}
+
+{% set columns = [
+    {"name": "ad_group_id", "datatype": dbt.type_int()},
+    {"name": "campaign_bidding_strategy", "datatype": dbt.type_string()},
+    {"name": "clicks", "datatype": dbt.type_int()},
+    {"name": "cost", "datatype": dbt.type_float()},
+    {"name": "date", "datatype": "date"},
+    {"name": "impressions", "datatype": dbt.type_int()},
+    {"name": "purchases_30_d", "datatype": dbt.type_int()},
+    {"name": "sales_30_d", "datatype": dbt.type_float()}
+] %}
+
+{# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #}
+{{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__ad_group_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/staging/get_advertised_product_report_columns.sql

@@ -0,0 +1,25 @@
+{% macro get_advertised_product_report_columns() %}
+
+{% set columns = [
+    {"name": "ad_group_id", "datatype": dbt.type_int()},
+    {"name": "ad_id", "datatype": dbt.type_int()},
+    {"name": "advertised_asin", "datatype": dbt.type_string()},
+    {"name": "advertised_sku", "datatype": dbt.type_string()},
+    {"name": "campaign_budget_amount", "datatype": dbt.type_float()},
+    {"name": "campaign_budget_currency_code", "datatype": dbt.type_string()},
+    {"name": "campaign_budget_type", "datatype": dbt.type_string()},
+    {"name": "campaign_id", "datatype": dbt.type_int()},
+    {"name": "clicks", "datatype": dbt.type_int()},
+    {"name": "cost", "datatype": dbt.type_float()},
+    {"name": "date", "datatype": "date"},
+    {"name": "impressions", "datatype": dbt.type_int()},
+    {"name": "purchases_30_d", "datatype": dbt.type_int()},
+    {"name": "sales_30_d", "datatype": dbt.type_float()}
+] %}
+
+{# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #}
+{{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__advertised_product_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/staging/get_campaign_history_columns.sql

@@ -0,0 +1,23 @@
+{% macro get_campaign_history_columns() %}
+
+{% set columns = [
+    {"name": "bidding_strategy", "datatype": dbt.type_string()},
+    {"name": "creation_date", "datatype": dbt.type_timestamp()},
+    {"name": "end_date", "datatype": "date"},
+    {"name": "id", "datatype": dbt.type_string()},
+    {"name": "last_updated_date", "datatype": dbt.type_timestamp()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "portfolio_id", "datatype": dbt.type_int()},
+    {"name": "profile_id", "datatype": dbt.type_int()},
+    {"name": "serving_status", "datatype": dbt.type_string()},
+    {"name": "start_date", "datatype": "date"},
+    {"name": "state", "datatype": dbt.type_string()},
+    {"name": "targeting_type", "datatype": dbt.type_string()},
+    {"name": "budget", "datatype": dbt.type_float()},
+    {"name": "budget_type", "datatype": dbt.type_string()},
+    {"name": "effective_budget", "datatype": dbt.type_float()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/staging/get_campaign_level_report_columns.sql

@@ -0,0 +1,25 @@
+{% macro get_campaign_level_report_columns() %}
+
+{% set columns = [
+    {"name": "campaign_applicable_budget_rule_id", "datatype": dbt.type_string()},
+    {"name": "campaign_applicable_budget_rule_name", "datatype": dbt.type_string()},
+    {"name": "campaign_bidding_strategy", "datatype": dbt.type_string()},
+    {"name": "campaign_budget_amount", "datatype": dbt.type_float()},
+    {"name": "campaign_budget_currency_code", "datatype": dbt.type_string()},
+    {"name": "campaign_budget_type", "datatype": dbt.type_string()},
+    {"name": "campaign_id", "datatype": dbt.type_int()},
+    {"name": "campaign_rule_based_budget_amount", "datatype": dbt.type_float()},
+    {"name": "clicks", "datatype": dbt.type_int()},
+    {"name": "cost", "datatype": dbt.type_float()},
+    {"name": "date", "datatype": "date"},
+    {"name": "impressions", "datatype": dbt.type_int()},
+    {"name": "purchases_30_d", "datatype": dbt.type_int()},
+    {"name": "sales_30_d", "datatype": dbt.type_float()}
+] %}
+
+{# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #}
+{{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__campaign_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/staging/get_keyword_history_columns.sql

@@ -0,0 +1,20 @@
+{% macro get_keyword_history_columns() %}
+
+{% set columns = [
+    {"name": "ad_group_id", "datatype": dbt.type_int()},
+    {"name": "bid", "datatype": dbt.type_float()},
+    {"name": "campaign_id", "datatype": dbt.type_int()},
+    {"name": "creation_date", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_string()},
+    {"name": "keyword_text", "datatype": dbt.type_string()},
+    {"name": "last_updated_date", "datatype": dbt.type_timestamp()},
+    {"name": "match_type", "datatype": dbt.type_string()},
+    {"name": "native_language_keyword", "datatype": dbt.type_string()},
+    {"name": "serving_status", "datatype": dbt.type_string()},
+    {"name": "state", "datatype": dbt.type_string()},
+    {"name": "native_language_locale", "datatype": dbt.type_string()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/staging/get_portfolio_history_columns.sql

@@ -0,0 +1,21 @@
+{% macro get_portfolio_history_columns() %}
+
+{% set columns = [
+    {"name": "budget_amount", "datatype": dbt.type_float()},
+    {"name": "budget_currency_code", "datatype": dbt.type_string()},
+    {"name": "budget_end_date", "datatype": "date"},
+    {"name": "budget_policy", "datatype": dbt.type_string()},
+    {"name": "budget_start_date", "datatype": "date"},
+    {"name": "creation_date", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "in_budget", "datatype": "boolean"},
+    {"name": "last_updated_date", "datatype": dbt.type_timestamp()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "profile_id", "datatype": dbt.type_int()},
+    {"name": "serving_status", "datatype": dbt.type_string()},
+    {"name": "state", "datatype": dbt.type_string()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/staging/get_product_ad_history_columns.sql

@@ -0,0 +1,17 @@
+{% macro get_product_ad_history_columns() %}
+
+{% set columns = [
+    {"name": "ad_group_id", "datatype": dbt.type_int()},
+    {"name": "asin", "datatype": dbt.type_string()},
+    {"name": "campaign_id", "datatype": dbt.type_int()},
+    {"name": "creation_date", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_string()},
+    {"name": "last_updated_date", "datatype": dbt.type_timestamp()},
+    {"name": "serving_status", "datatype": dbt.type_string()},
+    {"name": "sku", "datatype": dbt.type_string()},
+    {"name": "state", "datatype": dbt.type_string()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

macros/staging/get_profile_columns.sql

@@ -0,0 +1,20 @@
+{% macro get_profile_columns() %}
+
+{% set columns = [
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "account_id", "datatype": dbt.type_string()},
+    {"name": "account_marketplace_string_id", "datatype": dbt.type_string()},
+    {"name": "account_name", "datatype": dbt.type_string()},
+    {"name": "account_sub_type", "datatype": dbt.type_string()},
+    {"name": "account_type", "datatype": dbt.type_string()},
+    {"name": "account_valid_payment_method", "datatype": dbt.type_boolean()},
+    {"name": "country_code", "datatype": dbt.type_string()},
+    {"name": "currency_code", "datatype": dbt.type_string()},
+    {"name": "daily_budget", "datatype": dbt.type_int()},
+    {"name": "timezone", "datatype": dbt.type_string()},
+    {"name": "_fivetran_deleted", "datatype": dbt.type_boolean()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}