diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 281f1d42..26036027 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,7 +3,7 @@ steps: key: "run-dbt-postgres" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -18,7 +18,7 @@ steps: key: "run_dbt_snowflake" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -35,7 +35,7 @@ steps: key: "run_dbt_bigquery" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -47,7 +47,7 @@ steps: key: "run_dbt_redshift" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -62,7 +62,7 @@ steps: key: "run_dbt_databricks" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index e509e997..7766949f 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -17,6 +17,7 @@ echo `pwd` cd integration_tests dbt deps dbt seed --target "$db" --full-refresh +dbt source freshness --target "$db" || echo "...Only verifying freshness runs..." dbt run --target "$db" --full-refresh dbt test --target "$db" dbt run --vars '{stripe__using_invoice_line_sub_filter: false, stripe__using_credit_notes: true, stripe__using_price: false, stripe__using_subscription_history: false, stripe__convert_values: true}' --full-refresh --target "$db" diff --git a/CHANGELOG.md b/CHANGELOG.md index 294754c2..a8231ac6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,30 @@ +# dbt_stripe_source v0.15.0 + +[PR #96](https://github.com/fivetran/dbt_stripe_source/pull/96) includes the following updates: + +## Breaking Change for dbt Core < 1.9.6 + +> *Note: This is not relevant to Fivetran Quickstart users.* + +Migrated `freshness` from a top-level source property to a source `config` in alignment with [recent updates](https://github.com/dbt-labs/dbt-core/issues/11506) from dbt Core. This will resolve the following deprecation warning that users running dbt >= 1.9.6 may have received: + +``` +[WARNING]: Deprecated functionality +Found `freshness` as a top-level property of `stripe` in file +`models/src_stripe.yml`. The `freshness` top-level property should be moved +into the `config` of `stripe`. +``` + +**IMPORTANT:** Users running dbt Core < 1.9.6 will not be able to utilize freshness tests in this release or any subsequent releases, as older versions of dbt will not recognize freshness as a source `config` and therefore not run the tests. + +If you are using dbt Core < 1.9.6 and want to continue running Stripe freshness tests, please elect **one** of the following options: + 1. (Recommended) Upgrade to dbt Core >= 1.9.6 + 2. Do not upgrade your installed version of the `stripe_source` package. Pin your dependency on v0.14.2 in your `packages.yml` file. + 3. Utilize a dbt [override](https://docs.getdbt.com/reference/resource-properties/overrides) to overwrite the package's `stripe` source and apply freshness via the previous release top-level property route. This will require you to copy and paste the entirety of the previous release `src_stripe.yml` file and add an `overrides: stripe_source` property. + +## Under the Hood +- Updates to ensure integration tests use latest version of dbt. + # dbt_stripe_source v0.14.2 ## Under the Hood diff --git a/README.md b/README.md index 86df204c..18cfcc10 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ - Materializes [Stripe staging tables](https://fivetran.github.io/dbt_stripe_source/#!/overview/stripe_source/models/?g_v=1&g_e=seeds) which leverage data in the format described by [this ERD](https://fivetran.com/docs/applications/stripe/#schemainformation). These staging tables clean, test, and prepare your Stripe data from [Fivetran's connector](https://fivetran.com/docs/applications/stripe) for analysis by doing the following: - Name columns for consistency across all packages and for easier analysis - - Adds freshness tests to source data + - Adds freshness tests to source data. dbt Core >= 1.9.6 is required to run freshness tests out of the box. - Adds column-level testing where applicable. For example, all primary keys are tested for uniqueness and non-null values. - Generates a comprehensive data dictionary of your Stripe data through the [dbt docs site](https://fivetran.github.io/dbt_stripe_source/). - These tables are designed to work simultaneously with our [Stripe transformation package](https://github.com/fivetran/dbt_stripe). @@ -46,7 +46,7 @@ If you are **not** using the [Stripe transformation package](https://github.com ```yaml packages: - package: fivetran/stripe_source - version: [">=0.14.0", "<0.15.0"] + version: [">=0.15.0", "<0.16.0"] ``` ### Step 3: Define database and schema variables By default, this package runs using your destination and the `stripe` schema. If this is not where your stripe data is (for example, if your stripe schema is named `stripe_fivetran`), add the following configuration to your root `dbt_project.yml` file: diff --git a/dbt_project.yml b/dbt_project.yml index cbbc0d15..80e5d26e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,6 @@ config-version: 2 name: 'stripe_source' -version: '0.14.2' +version: '0.15.0' require-dbt-version: [">=1.3.0", "<2.0.0"] models: diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 647536b7..5b561fd7 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -2,10 +2,6 @@ # HEY! This file is used in the dbt package integrations tests with BuildKite. # You should __NEVER__ check credentials into version control. Thanks for reading :) -config: - send_anonymous_usage_stats: False - use_colors: True - integration_tests: target: redshift outputs: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 0f465b82..f90de267 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'stripe_source_integration_tests' -version: '0.14.2' +version: '0.15.0' profile: 'integration_tests' @@ -72,4 +72,7 @@ seeds: dispatch: - macro_namespace: dbt_utils - search_order: ['spark_utils', 'dbt_utils'] \ No newline at end of file + search_order: ['spark_utils', 'dbt_utils'] + +flags: + send_anonymous_usage_stats: False diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 04b5ebfc..c9f6024f 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -5,4 +5,4 @@ dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 dbt-databricks>=1.3.0,<2.0.0 -certifi==2025.1.31 \ No newline at end of file +certifi==2025.1.31 diff --git a/models/src_stripe.yml b/models/src_stripe.yml index b91aff13..10acc616 100644 --- a/models/src_stripe.yml +++ b/models/src_stripe.yml @@ -7,9 +7,10 @@ sources: loader: fivetran loaded_at_field: _fivetran_synced - freshness: - warn_after: { count: 72, period: hour } - error_after: { count: 168, period: hour } + config: + freshness: + warn_after: { count: 72, period: hour } + error_after: { count: 168, period: hour } tables: - name: balance_transaction @@ -194,7 +195,8 @@ sources: - name: customer identifier: "{{ var('stripe_customer_identifier', 'customer')}}" description: Customer objects allow you to perform recurring charges, and to track multiple charges, that are associated with the same customer. - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -368,7 +370,8 @@ sources: - name: payment_intent identifier: "{{ var('stripe_payment_intent_identifier', 'payment_intent')}}" description: A Payment Intent guides you through the process of collecting a payment from your customer. - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -429,7 +432,8 @@ sources: - name: payment_method identifier: "{{ var('stripe_payment_method_identifier', 'payment_method')}}" description: PaymentMethod objects represent your customer's payment instruments. They can be used with PaymentIntents to collect payments or saved to Customer objects to store instrument details for future payments. - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -497,7 +501,8 @@ sources: - name: refund identifier: "{{ var('stripe_refund_identifier', 'refund')}}" description: Details of transactions that have been refunded - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -529,7 +534,8 @@ sources: - name: invoice_line_item identifier: "{{ var('stripe_invoice_line_item_identifier', 'invoice_line_item')}}" description: The different items that an invoice contains - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -567,7 +573,8 @@ sources: - name: invoice identifier: "{{ var('stripe_invoice_identifier', 'invoice')}}" description: Invoices are statements of amounts owed by a customer, and are either generated one-off, or generated periodically from a subscription. - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -643,7 +650,8 @@ sources: - name: subscription_history identifier: "{{ var('stripe_subscription_history_identifier', 'subscription_history')}}" description: Subscriptions allow you to charge a customer on a recurring basis. Please note this source table is used only on connectors setup **after** February 09, 2022. - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -695,7 +703,8 @@ sources: - name: subscription identifier: "{{ var('stripe_subscription_identifier', 'subscription')}}" description: Subscriptions allow you to charge a customer on a recurring basis. Please note this source table is only present in connectors setup **before** February 09, 2022. - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -737,7 +746,8 @@ sources: - name: plan identifier: "{{ var('stripe_plan_identifier', 'plan')}}" description: Plans define the base price, currency, and billing cycle for recurring purchases of products. - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -763,7 +773,8 @@ sources: - name: credit_note identifier: "{{ var('stripe_credit_note_identifier', 'credit_note')}}" description: Credit notes are documents that decrease the amount owed on an invoice. They’re the only way to adjust the amount of a finalized invoice other than voiding and recreating the invoice. - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -807,7 +818,8 @@ sources: - name: credit_note_line_item identifier: "{{ var('stripe_credit_note_line_item_identifier', 'credit_note_line_item')}}" description: The different items that a credit note contains. - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -833,7 +845,8 @@ sources: - name: price identifier: "{{ var('stripe_price_identifier', 'price')}}" description: Prices define the unit cost, currency, and (optional) billing cycle for both recurring and one-time purchases of products. - freshness: null + config: + freshness: null columns: - name: active description: Whether the price can be used for new purchases. @@ -883,7 +896,8 @@ sources: - name: account identifier: "{{ var('stripe_account_identifier', 'account')}}" description: Prices define the unit cost, currency, and (optional) billing cycle for both recurring and one-time purchases of products. - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -971,7 +985,8 @@ sources: - name: product identifier: "{{ var('stripe_product_identifier', 'product') }}" description: A product object represents an individual product to be sold, with various attributes detailing its properties and behaviors. - freshness: null + config: + freshness: null columns: - name: id description: Unique identifier for the object. @@ -1012,7 +1027,8 @@ sources: - name: discount identifier: "{{ var('stripe_discount_identifier', 'discount') }}" description: A discount represents the actual application of a coupon or promotion code. It contains information about when the discount began, when it will end, and what it is applied to. - freshness: null + config: + freshness: null columns: - name: id description: The ID of the discount object.