From 7d84d03ef0df9aa3234f3bd776f9ca12dbecc233 Mon Sep 17 00:00:00 2001 From: Ismael Boukhars Date: Thu, 21 Aug 2025 23:07:35 +0200 Subject: [PATCH 1/6] [handle-multi-crm] WIP handle multi CRM --- app/jobs/etlify/sync_job.rb | 16 +- lib/etlify.rb | 9 +- lib/etlify/config.rb | 10 +- lib/etlify/crm.rb | 43 ++++ lib/etlify/engine.rb | 24 +++ lib/etlify/errors.rb | 15 +- lib/etlify/model.rb | 203 ++++++++++++------ .../create_crm_synchronisations.rb.tt | 6 + 8 files changed, 235 insertions(+), 91 deletions(-) create mode 100644 lib/etlify/crm.rb diff --git a/app/jobs/etlify/sync_job.rb b/app/jobs/etlify/sync_job.rb index 1860fb6..621340c 100644 --- a/app/jobs/etlify/sync_job.rb +++ b/app/jobs/etlify/sync_job.rb @@ -16,19 +16,17 @@ class SyncJob < ActiveJob::Base end around_perform do |job, block| - begin - block.call - ensure - Etlify.config.cache_store.delete(enqueue_lock_key(job)) - end + block.call + ensure + Etlify.config.cache_store.delete(enqueue_lock_key(job)) end - def perform(record_class, id) - model = record_class.constantize - record = model.find_by(id: id) + def perform(model_class_name, record_id, crm_name) + model = model_class_name.constantize + record = model.find_by(id: record_id) return unless record - Etlify::Synchronizer.call(record) + Etlify::Synchronizer.call(record, crm: crm_name.to_sym) end private diff --git a/lib/etlify.rb b/lib/etlify.rb index 5380044..dbef534 100644 --- a/lib/etlify.rb +++ b/lib/etlify.rb @@ -9,6 +9,7 @@ require_relative "etlify/config" require_relative "etlify/errors" require_relative "etlify/digest" +require_relative "etlify/crm" require_relative "etlify/model" require_relative "etlify/synchronizer" require_relative "etlify/deleter" @@ -17,17 +18,15 @@ require_relative "etlify/adapters/null_adapter" require_relative "etlify/adapters/hubspot_v3_adapter" require_relative "etlify/serializers/base_serializer" -require_relative "./generators/etlify/install/install_generator" -require_relative "./generators/etlify/migration/migration_generator" -require_relative "./generators/etlify/serializer/serializer_generator" - +require_relative "generators/etlify/install/install_generator" +require_relative "generators/etlify/migration/migration_generator" +require_relative "generators/etlify/serializer/serializer_generator" require_relative "etlify/railtie" if defined?(Rails) require_relative "etlify/engine" if defined?(Rails) module Etlify class << self - def config @configuration ||= Etlify::Config.new end diff --git a/lib/etlify/config.rb b/lib/etlify/config.rb index f87fed1..700ecdd 100644 --- a/lib/etlify/config.rb +++ b/lib/etlify/config.rb @@ -1,25 +1,21 @@ module Etlify class Config attr_accessor( - :crm_adapter, :digest_strategy, :logger, :job_queue_name, - :sync_job_class, :cache_store ) def initialize - @crm_adapter = Etlify::Adapters::NullAdapter.new @digest_strategy = Etlify::Digest.method(:stable_sha256) - @job_queue_name = "low" + @job_queue_name = "low" - rails_logger = defined?(Rails) && Rails.respond_to?(:logger) ? Rails.logger : nil + rails_logger = (defined?(Rails) && Rails.respond_to?(:logger)) ? Rails.logger : nil @logger = rails_logger || Logger.new($stdout) - rails_cache = defined?(Rails) && Rails.respond_to?(:cache) ? Rails.cache : nil + rails_cache = (defined?(Rails) && Rails.respond_to?(:cache)) ? Rails.cache : nil @cache_store = rails_cache || ActiveSupport::Cache::MemoryStore.new - @sync_job_class = "Etlify::SyncJob" end end end diff --git a/lib/etlify/crm.rb b/lib/etlify/crm.rb new file mode 100644 index 0000000..475f793 --- /dev/null +++ b/lib/etlify/crm.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Etlify + module CRM + RegistryItem = Struct.new( + :name, + :adapter, + :options, + keyword_init: true + ) + + class << self + # Holds { Symbol => RegistryItem } + def registry + @registry ||= {} + end + + # Public API: register a new CRM + # Etlify::CRM.register(:my_crm, adapter: MyAdapter, options: { job_class: X }) + def register(name, adapter:, options: {}) + key = name.to_sym + registry[key] = RegistryItem.new( + name: key, + adapter: adapter, + options: options || {} + ) + + # Install DSL on all classes that already included Etlify::Model + Etlify::Model.install_dsl_for_crm(key) + end + + # Internal: fetch a RegistryItem + def fetch(name) + registry.fetch(name.to_sym) + end + + # Internal: list all registered CRM names + def names + registry.keys + end + end + end +end diff --git a/lib/etlify/engine.rb b/lib/etlify/engine.rb index e060811..023b9b4 100644 --- a/lib/etlify/engine.rb +++ b/lib/etlify/engine.rb @@ -4,5 +4,29 @@ module Etlify class Engine < ::Rails::Engine isolate_namespace Etlify + + initializer "etlify.check_crm_name_column" do + ActiveSupport.on_load(:active_record) do + if defined?(CrmSynchronisation) && + CrmSynchronisation.table_exists? && + !CrmSynchronisation.column_names.include?("crm_name") + raise( + Etlify::Errors::MissingColumnError, + <<~MSG.squish + Missing column "crm_name" on table "crm_synchronisations". + Please generate a migration with: + + rails g migration AddCrmNameToCrmSynchronisations \ + crm_name:string:index + + Then run: rails db:migrate + MSG + ) + end + rescue ActiveRecord::NoDatabaseError, ActiveRecord::StatementInvalid + # Happens during `rails db:create` or before schema is loaded. + # Silently ignore; check will run again once DB is ready. + end + end end end diff --git a/lib/etlify/errors.rb b/lib/etlify/errors.rb index 1a4490a..4dc6d25 100644 --- a/lib/etlify/errors.rb +++ b/lib/etlify/errors.rb @@ -33,8 +33,15 @@ class TransportError < Error; end # HTTP errors class ApiError < Error; end - class Unauthorized < ApiError; end # 401/403 - class NotFound < ApiError; end # 404 - class RateLimited < ApiError; end # 429 - class ValidationFailed < ApiError; end # 409/422 + + class Unauthorized < ApiError; end # 401/403 + + class NotFound < ApiError; end # 404 + + class RateLimited < ApiError; end # 429 + + class ValidationFailed < ApiError; end # 409/422 + + # Configuration error (update) + class MissingColumnError < StandardError; end end diff --git a/lib/etlify/model.rb b/lib/etlify/model.rb index c0d6400..4ddf855 100644 --- a/lib/etlify/model.rb +++ b/lib/etlify/model.rb @@ -3,98 +3,169 @@ module Model extend ActiveSupport::Concern included do + # Track classes that included this concern to backfill DSL on register. + Etlify::Model.__included_klasses__ << self + + # Hash keyed by CRM name, with config per CRM + class_attribute :etlify_crms, instance_writer: false, default: {} + + Etlify::CRM.names.each do |crm_name| + Etlify::Model.define_crm_dsl_on(self, crm_name) + Etlify::Model.define_crm_instance_helpers_on(self, crm_name) + end end - class_methods do - # DSL: etlified_with( - # serializer:, - # crm_object_type:, - # id_property:, - # sync_if: ->(r){ true } - # ) - def etlified_with( - serializer:, - crm_object_type:, - dependencies: [], - id_property:, - sync_if: ->(_r) { true }) - class_attribute( - :etlify_serializer, - instance_accessor: false, - default: serializer - ) - class_attribute( - :etlify_guard, - instance_accessor: false, - default: sync_if - ) - class_attribute( - :etlify_crm_object_type, - instance_accessor: true, - default: crm_object_type - ) - class_attribute( - :etlify_id_property, - instance_accessor: true, - default: id_property - ) - class_attribute( - :etlify_dependencies, - instance_accessor: false, - default: Array(dependencies).map(&:to_sym) - ) - has_one( - :crm_synchronisation, - as: :resource, - dependent: :destroy, - class_name: "CrmSynchronisation" - ) + class << self + # Internal: store all including classes + def __included_klasses__ + @__included_klasses__ ||= [] + end + # Called by Etlify::CRM.register to (re)install DSL on all classes + def install_dsl_for_crm(crm_name) + __included_klasses__.each do |klass| + define_crm_dsl_on(klass, crm_name) + define_crm_instance_helpers_on(klass, crm_name) + end + end + + # Define the class-level DSL method: "_etlified_with" + def define_crm_dsl_on(klass, crm_name) + dsl_name = "#{crm_name}_etlified_with" + + # Avoid redefining if already defined + return if klass.respond_to?(dsl_name) + + klass.define_singleton_method(dsl_name) do | + serializer:, + crm_object_type:, + id_property:, + dependencies: [], + sync_if: ->(_r) { true }, + job_class: nil + | + # Fetch registered CRM (adapter, options) + reg = Etlify::CRM.fetch(crm_name) + + # Merge model-level config for this CRM + conf = { + serializer: serializer, + guard: sync_if, + crm_object_type: crm_object_type, + id_property: id_property, + dependencies: Array(dependencies).map(&:to_sym), + adapter: reg.adapter, + # Job class priority: method arg > registry options > nil + job_class: job_class || reg.options[:job_class], + } + + # Store into class attribute hash + new_hash = (etlify_crms || {}).dup + new_hash[crm_name.to_sym] = conf + self.etlify_crms = new_hash + + # Ensure instance helpers exist + Etlify::Model.define_crm_instance_helpers_on(self, crm_name) + end + end + + # Define instance helpers: "_build_payload", "_sync!", "_delete!" + def define_crm_instance_helpers_on(klass, crm_name) + payload_m = "#{crm_name}_build_payload" + sync_m = "#{crm_name}_sync!" + delete_m = "#{crm_name}_delete!" + + unless klass.method_defined?(payload_m) + klass.define_method(payload_m) do + build_crm_payload(crm: crm_name) + end + end + + unless klass.method_defined?(sync_m) + klass.define_method(sync_m) do |async: true, job_class: nil| + crm_sync!(crm: crm_name, async: async, job_class: job_class) + end + end + + unless klass.method_defined?(delete_m) + klass.define_method(delete_m) do + crm_delete!(crm: crm_name) + end + end end end - # Public API injected - def crm_synced? + # ---------- Public generic API (now CRM-aware) ---------- + + def crm_synced?(crm: nil) + # If you have per-CRM synchronisation records, adapt accordingly. + # For now keep a single association; adjust when your schema changes. crm_synchronisation.present? end - def build_crm_payload - raise_unless_crm_is_configured + def build_crm_payload(crm:) + raise_unless_crm_is_configured(crm) - self.class.etlify_serializer.new(self).as_crm_payload + conf = self.class.etlify_crms.fetch(crm.to_sym) + conf[:serializer].new(self).as_crm_payload end - # @param async [Boolean, nil] prioritaire sur la config globale - def crm_sync!(async: true) - return false if self.class.respond_to?(:etlify_guard) && !self.class.etlify_guard.call(self) + # @param crm [Symbol] which CRM to use + # @param async [Boolean] whether to enqueue or run inline + # @param job_class [Class,String,nil] explicit override + def crm_sync!(crm:, async: true, job_class: nil) + return false unless allow_sync_for?(crm) if async - if job_class.respond_to?(:perform_later) - job_class.perform_later(self.class.name, id) - elsif job_class.respond_to?(:perform_async) - job_class.perform_async(self.class.name, id) + jc = resolve_job_class_for(crm, override: job_class) + if jc.respond_to?(:perform_later) + jc.perform_later(self.class.name, id, crm.to_s) + elsif jc.respond_to?(:perform_async) + jc.perform_async(self.class.name, id, crm.to_s) else raise ArgumentError, "No job class available for CRM sync" end else - Etlify::Synchronizer.call(self) + Etlify::Synchronizer.call(self, crm: crm) end end - def crm_delete! - Etlify::Deleter.call(self) + def crm_delete!(crm:) + Etlify::Deleter.call(self, crm: crm) end private - def job_class - given_class = Etlify.config.sync_job_class - given_class.is_a?(String) ? given_class.constantize : given_class - end - def raise_unless_crm_is_configured - return if self.class.respond_to?(:etlify_serializer) && self.class.etlify_serializer + # Guard evaluation per CRM + def allow_sync_for?(crm) + conf = self.class.etlify_crms[crm.to_sym] + return false unless conf + + guard = conf[:guard] + guard ? guard.call(self) : true + end + + def resolve_job_class_for(crm, override:) + return constantize_if_needed(override) if override + + conf = self.class.etlify_crms.fetch(crm.to_sym) + given = conf[:job_class] + return constantize_if_needed(given) if given + + # Fallback to default sync job name if you want one + constantize_if_needed("Etlify::SyncJob") + end + + def constantize_if_needed(klass_or_name) + return klass_or_name unless klass_or_name.is_a?(String) + + klass_or_name.constantize + end - raise ArgumentError, "crm_synced not configured" + def raise_unless_crm_is_configured(crm) + unless self.class.etlify_crms && self.class.etlify_crms[crm.to_sym] + raise ArgumentError, "crm not configured for #{crm}" end + end end end diff --git a/lib/generators/etlify/migration/templates/create_crm_synchronisations.rb.tt b/lib/generators/etlify/migration/templates/create_crm_synchronisations.rb.tt index 86cb63a..ea68e75 100644 --- a/lib/generators/etlify/migration/templates/create_crm_synchronisations.rb.tt +++ b/lib/generators/etlify/migration/templates/create_crm_synchronisations.rb.tt @@ -7,6 +7,7 @@ class <%= file_name.camelize %> < ActiveRecord::Migration[<%= ActiveRecord::VERS t.string :last_digest t.datetime :last_synced_at t.string :last_error + t.string :crm t.timestamps end @@ -16,5 +17,10 @@ class <%= file_name.camelize %> < ActiveRecord::Migration[<%= ActiveRecord::VERS add_index :crm_synchronisations, :last_synced_at add_index :crm_synchronisations, :resource_type add_index :crm_synchronisations, :resource_id + add_index ( + :crm_synchronisations, + [:crm_name, :resource_type, :resource_id], + name: "idx_crm_sync_on_crm_and_resource" + ) end end From 27a0a93bc88565810acff4db31af882eb37afec7 Mon Sep 17 00:00:00 2001 From: Ismael Boukhars Date: Mon, 25 Aug 2025 21:20:41 +0200 Subject: [PATCH 2/6] [handle-multi-crm] feat: handle many crm on one app --- Gemfile.lock | 134 ++-- app/jobs/etlify/sync_job.rb | 12 +- app/models/crm_synchronisation.rb | 3 +- etlify.gemspec | 2 +- lib/etlify.rb | 9 +- lib/etlify/adapters/null_adapter.rb | 4 +- lib/etlify/deleter.rb | 40 +- lib/etlify/engine.rb | 2 +- lib/etlify/{errors.rb => error.rb} | 7 +- lib/etlify/model.rb | 36 +- lib/etlify/stale_records/batch_sync.rb | 96 +-- lib/etlify/stale_records/finder.rb | 104 ++- lib/etlify/synchronizer.rb | 46 +- .../etlify/install/templates/initializer.rb | 14 - .../install/templates/initializer.rb.tt | 24 + .../etlify/migration/migration_generator.rb | 1 - .../create_crm_synchronisations.rb.tt | 3 +- spec/adapters/null_adapter_spec.rb | 5 +- spec/etlify/config_spec.rb | 31 + spec/etlify/crm_registry_spec.rb | 34 + spec/etlify/deleter_spec.rb | 111 +++ spec/etlify/digest_spec.rb | 154 ++++ spec/etlify/engine_initializer_spec.rb | 92 +++ spec/etlify/error_spec.rb | 36 + spec/etlify/model_spec.rb | 275 +++++++ spec/etlify/stale_records/batch_sync_spec.rb | 275 +++++++ spec/etlify/stale_records/finder_spec.rb | 702 ++++++++++++++++++ spec/etlify/synchronizer_spec.rb | 163 ++++ spec/factories.rb | 24 - .../etlify/install_generator_spec.rb | 85 +++ .../etlify/migration_generator_spec.rb | 151 ++++ spec/generators/install_generator_spec.rb | 11 - spec/generators/migration_generator_spec.rb | 140 ---- spec/generators/serializer_generator_spec.rb | 76 -- spec/jobs/config_job_class_spec.rb | 37 - spec/jobs/etlify/sync_job_spec.rb | 131 ++++ spec/jobs/sync_jobs_spec.rb | 69 -- spec/lib/stale_records/batch_sync_spec.rb | 206 ----- spec/lib/stale_records/finder_spec.rb | 212 ------ spec/models/crm_synchronisation_spec.rb | 155 +++- spec/models/etlify_model_spec.rb | 149 ---- spec/rails_helper.rb | 203 +++-- spec/serializers/base_serializer_spec.rb | 8 + spec/services/deleter_spec.rb | 23 - spec/services/synchronizer_spec.rb | 112 --- spec/support/aj_test_adapter_helpers.rb | 32 + spec/support/time_helpers.rb | 63 ++ 47 files changed, 2876 insertions(+), 1426 deletions(-) rename lib/etlify/{errors.rb => error.rb} (94%) delete mode 100644 lib/generators/etlify/install/templates/initializer.rb create mode 100644 lib/generators/etlify/install/templates/initializer.rb.tt create mode 100644 spec/etlify/config_spec.rb create mode 100644 spec/etlify/crm_registry_spec.rb create mode 100644 spec/etlify/deleter_spec.rb create mode 100644 spec/etlify/digest_spec.rb create mode 100644 spec/etlify/engine_initializer_spec.rb create mode 100644 spec/etlify/error_spec.rb create mode 100644 spec/etlify/model_spec.rb create mode 100644 spec/etlify/stale_records/batch_sync_spec.rb create mode 100644 spec/etlify/stale_records/finder_spec.rb create mode 100644 spec/etlify/synchronizer_spec.rb delete mode 100644 spec/factories.rb create mode 100644 spec/generators/etlify/install_generator_spec.rb create mode 100644 spec/generators/etlify/migration_generator_spec.rb delete mode 100644 spec/generators/install_generator_spec.rb delete mode 100644 spec/generators/migration_generator_spec.rb delete mode 100644 spec/generators/serializer_generator_spec.rb delete mode 100644 spec/jobs/config_job_class_spec.rb create mode 100644 spec/jobs/etlify/sync_job_spec.rb delete mode 100644 spec/jobs/sync_jobs_spec.rb delete mode 100644 spec/lib/stale_records/batch_sync_spec.rb delete mode 100644 spec/lib/stale_records/finder_spec.rb delete mode 100644 spec/models/etlify_model_spec.rb create mode 100644 spec/serializers/base_serializer_spec.rb delete mode 100644 spec/services/deleter_spec.rb delete mode 100644 spec/services/synchronizer_spec.rb create mode 100644 spec/support/aj_test_adapter_helpers.rb create mode 100644 spec/support/time_helpers.rb diff --git a/Gemfile.lock b/Gemfile.lock index 51ec0ee..520adc0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,89 +1,83 @@ PATH remote: . specs: - etlify (0.4.0) - rails (>= 7.0, < 8) + etlify (0.6.0) + rails (>= 7.2, < 8) GEM remote: https://rubygems.org/ specs: - actioncable (7.1.5.1) - actionpack (= 7.1.5.1) - activesupport (= 7.1.5.1) + actioncable (7.2.2.2) + actionpack (= 7.2.2.2) + activesupport (= 7.2.2.2) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.1.5.1) - actionpack (= 7.1.5.1) - activejob (= 7.1.5.1) - activerecord (= 7.1.5.1) - activestorage (= 7.1.5.1) - activesupport (= 7.1.5.1) - mail (>= 2.7.1) - net-imap - net-pop - net-smtp - actionmailer (7.1.5.1) - actionpack (= 7.1.5.1) - actionview (= 7.1.5.1) - activejob (= 7.1.5.1) - activesupport (= 7.1.5.1) - mail (~> 2.5, >= 2.5.4) - net-imap - net-pop - net-smtp + actionmailbox (7.2.2.2) + actionpack (= 7.2.2.2) + activejob (= 7.2.2.2) + activerecord (= 7.2.2.2) + activestorage (= 7.2.2.2) + activesupport (= 7.2.2.2) + mail (>= 2.8.0) + actionmailer (7.2.2.2) + actionpack (= 7.2.2.2) + actionview (= 7.2.2.2) + activejob (= 7.2.2.2) + activesupport (= 7.2.2.2) + mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (7.1.5.1) - actionview (= 7.1.5.1) - activesupport (= 7.1.5.1) + actionpack (7.2.2.2) + actionview (= 7.2.2.2) + activesupport (= 7.2.2.2) nokogiri (>= 1.8.5) racc - rack (>= 2.2.4) + rack (>= 2.2.4, < 3.2) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actiontext (7.1.5.1) - actionpack (= 7.1.5.1) - activerecord (= 7.1.5.1) - activestorage (= 7.1.5.1) - activesupport (= 7.1.5.1) + useragent (~> 0.16) + actiontext (7.2.2.2) + actionpack (= 7.2.2.2) + activerecord (= 7.2.2.2) + activestorage (= 7.2.2.2) + activesupport (= 7.2.2.2) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.1.5.1) - activesupport (= 7.1.5.1) + actionview (7.2.2.2) + activesupport (= 7.2.2.2) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.1.5.1) - activesupport (= 7.1.5.1) + activejob (7.2.2.2) + activesupport (= 7.2.2.2) globalid (>= 0.3.6) - activemodel (7.1.5.1) - activesupport (= 7.1.5.1) - activerecord (7.1.5.1) - activemodel (= 7.1.5.1) - activesupport (= 7.1.5.1) + activemodel (7.2.2.2) + activesupport (= 7.2.2.2) + activerecord (7.2.2.2) + activemodel (= 7.2.2.2) + activesupport (= 7.2.2.2) timeout (>= 0.4.0) - activestorage (7.1.5.1) - actionpack (= 7.1.5.1) - activejob (= 7.1.5.1) - activerecord (= 7.1.5.1) - activesupport (= 7.1.5.1) + activestorage (7.2.2.2) + actionpack (= 7.2.2.2) + activejob (= 7.2.2.2) + activerecord (= 7.2.2.2) + activesupport (= 7.2.2.2) marcel (~> 1.0) - activesupport (7.1.5.1) + activesupport (7.2.2.2) base64 benchmark (>= 0.3) bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) + concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) logger (>= 1.4.2) minitest (>= 5.1) - mutex_m securerandom (>= 0.3) - tzinfo (~> 2.0) + tzinfo (~> 2.0, >= 2.0.5) ast (2.4.3) base64 (0.3.0) benchmark (0.4.1) @@ -122,7 +116,6 @@ GEM marcel (1.0.4) mini_mime (1.1.5) minitest (5.25.5) - mutex_m (0.3.0) net-imap (0.5.9) date net-protocol @@ -161,7 +154,7 @@ GEM date stringio racc (1.8.1) - rack (3.2.0) + rack (3.1.16) rack-session (2.1.1) base64 (>= 0.1.0) rack (>= 3.0.0) @@ -169,20 +162,20 @@ GEM rack (>= 1.3) rackup (2.2.1) rack (>= 3) - rails (7.1.5.1) - actioncable (= 7.1.5.1) - actionmailbox (= 7.1.5.1) - actionmailer (= 7.1.5.1) - actionpack (= 7.1.5.1) - actiontext (= 7.1.5.1) - actionview (= 7.1.5.1) - activejob (= 7.1.5.1) - activemodel (= 7.1.5.1) - activerecord (= 7.1.5.1) - activestorage (= 7.1.5.1) - activesupport (= 7.1.5.1) + rails (7.2.2.2) + actioncable (= 7.2.2.2) + actionmailbox (= 7.2.2.2) + actionmailer (= 7.2.2.2) + actionpack (= 7.2.2.2) + actiontext (= 7.2.2.2) + actionview (= 7.2.2.2) + activejob (= 7.2.2.2) + activemodel (= 7.2.2.2) + activerecord (= 7.2.2.2) + activestorage (= 7.2.2.2) + activesupport (= 7.2.2.2) bundler (>= 1.15.0) - railties (= 7.1.5.1) + railties (= 7.2.2.2) rails-dom-testing (2.3.0) activesupport (>= 5.0.0) minitest @@ -190,10 +183,10 @@ GEM rails-html-sanitizer (1.6.2) loofah (~> 2.21) nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) - railties (7.1.5.1) - actionpack (= 7.1.5.1) - activesupport (= 7.1.5.1) - irb + railties (7.2.2.2) + actionpack (= 7.2.2.2) + activesupport (= 7.2.2.2) + irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) @@ -266,6 +259,7 @@ GEM unicode-display_width (3.1.4) unicode-emoji (~> 4.0, >= 4.0.4) unicode-emoji (4.0.4) + useragent (0.16.11) websocket-driver (0.8.0) base64 websocket-extensions (>= 0.1.0) diff --git a/app/jobs/etlify/sync_job.rb b/app/jobs/etlify/sync_job.rb index 621340c..37d0b06 100644 --- a/app/jobs/etlify/sync_job.rb +++ b/app/jobs/etlify/sync_job.rb @@ -1,6 +1,6 @@ module Etlify class SyncJob < ActiveJob::Base - queue_as Etlify.config.job_queue_name + queue_as { Etlify.config.job_queue_name } retry_on(StandardError, attempts: 3, wait: :polynomially_longer) ENQUEUE_LOCK_TTL = 15.minutes @@ -21,12 +21,12 @@ class SyncJob < ActiveJob::Base Etlify.config.cache_store.delete(enqueue_lock_key(job)) end - def perform(model_class_name, record_id, crm_name) - model = model_class_name.constantize - record = model.find_by(id: record_id) - return unless record + def perform(model_class_name, resource_id, crm_name) + model = model_class_name.constantize + resource = model.find_by(id: resource_id) + return unless resource - Etlify::Synchronizer.call(record, crm: crm_name.to_sym) + Etlify::Synchronizer.call(resource, crm_name: crm_name.to_sym) end private diff --git a/app/models/crm_synchronisation.rb b/app/models/crm_synchronisation.rb index 32ac93e..aba48b0 100644 --- a/app/models/crm_synchronisation.rb +++ b/app/models/crm_synchronisation.rb @@ -6,7 +6,8 @@ class CrmSynchronisation < ApplicationRecord validates :crm_id, uniqueness: true, allow_nil: true validates :resource_type, presence: true validates :resource_id, presence: true - validates :resource_id, uniqueness: { scope: :resource_type } + validates :resource_id, uniqueness: {scope: [:resource_type, :crm_name]} + validates :crm_name, presence: true, uniqueness: {scope: :resource} def stale?(digest) last_digest != digest diff --git a/etlify.gemspec b/etlify.gemspec index 114db4a..93eb74f 100644 --- a/etlify.gemspec +++ b/etlify.gemspec @@ -16,7 +16,7 @@ Gem::Specification.new do |spec| ).select { |f| File.file?(f) } + %w[README.md] spec.require_paths = ["lib"] - spec.add_dependency "rails", ">= 7.0", "< 8" + spec.add_dependency "rails", ">= 7.2", "< 8" spec.add_development_dependency "rspec", "~> 3.13" spec.add_development_dependency "rspec-rails", "~> 6.1" spec.add_development_dependency "sqlite3", "~> 2.7" diff --git a/lib/etlify.rb b/lib/etlify.rb index dbef534..f2abc6a 100644 --- a/lib/etlify.rb +++ b/lib/etlify.rb @@ -7,7 +7,7 @@ require_relative "etlify/version" require_relative "etlify/config" -require_relative "etlify/errors" +require_relative "etlify/error" require_relative "etlify/digest" require_relative "etlify/crm" require_relative "etlify/model" @@ -22,9 +22,6 @@ require_relative "generators/etlify/migration/migration_generator" require_relative "generators/etlify/serializer/serializer_generator" -require_relative "etlify/railtie" if defined?(Rails) -require_relative "etlify/engine" if defined?(Rails) - module Etlify class << self def config @@ -36,3 +33,7 @@ def configure end end end + +require_relative "../app/jobs/etlify/sync_job" +require_relative "etlify/railtie" if defined?(Rails) +require_relative "etlify/engine" if defined?(Rails) diff --git a/lib/etlify/adapters/null_adapter.rb b/lib/etlify/adapters/null_adapter.rb index 4a3de80..96a4624 100644 --- a/lib/etlify/adapters/null_adapter.rb +++ b/lib/etlify/adapters/null_adapter.rb @@ -4,8 +4,8 @@ module Etlify module Adapters # Adapter no-op pour dev/test class NullAdapter - def upsert!(payload:, object_type:) - payload.fetch(:id, SecureRandom.uuid).to_s + def upsert!(payload:, object_type:, id_property:) + payload.fetch(id_property, SecureRandom.uuid).to_s end def delete!(crm_id:, object_type:) diff --git a/lib/etlify/deleter.rb b/lib/etlify/deleter.rb index efff049..0f7527f 100644 --- a/lib/etlify/deleter.rb +++ b/lib/etlify/deleter.rb @@ -1,21 +1,43 @@ module Etlify class Deleter - def self.call(record) - new(record).call + attr_accessor( + :adapter, + :conf, + :crm_name, + :resource + ) + + # @param resource [ActiveRecord::Base] + # @param crm [Symbol,String] + def self.call(resource, crm_name:) + new(resource, crm_name: crm_name).call end - def initialize(record) - @record = record + def initialize(resource, crm_name:) + @resource = resource + @crm_name = crm_name.to_sym + @conf = resource.class.etlify_crms.fetch(@crm_name) + @adapter = @conf[:adapter].new end def call - sync_line = @record.crm_synchronisation - return :noop unless sync_line&.crm_id.present? + line = sync_line + return :noop unless line&.crm_id.present? - Etlify.config.crm_adapter.delete!(crm_id: sync_line.crm_id) + @adapter.delete!( + crm_id: line.crm_id, + object_type: conf[:crm_object_type], + id_property: conf[:id_property] + ) :deleted - rescue StandardError => e - raise Etlify::Errors::SyncError, e.message + rescue => e + raise Etlify::SyncError, e.message + end + + private + + def sync_line + resource.crm_synchronisations.find_by(crm_name: crm_name) end end end diff --git a/lib/etlify/engine.rb b/lib/etlify/engine.rb index 023b9b4..8cc95a8 100644 --- a/lib/etlify/engine.rb +++ b/lib/etlify/engine.rb @@ -11,7 +11,7 @@ class Engine < ::Rails::Engine CrmSynchronisation.table_exists? && !CrmSynchronisation.column_names.include?("crm_name") raise( - Etlify::Errors::MissingColumnError, + Etlify::MissingColumnError, <<~MSG.squish Missing column "crm_name" on table "crm_synchronisations". Please generate a migration with: diff --git a/lib/etlify/errors.rb b/lib/etlify/error.rb similarity index 94% rename from lib/etlify/errors.rb rename to lib/etlify/error.rb index 4dc6d25..f7c42af 100644 --- a/lib/etlify/errors.rb +++ b/lib/etlify/error.rb @@ -33,15 +33,14 @@ class TransportError < Error; end # HTTP errors class ApiError < Error; end - class Unauthorized < ApiError; end # 401/403 - class NotFound < ApiError; end # 404 - class RateLimited < ApiError; end # 429 - class ValidationFailed < ApiError; end # 409/422 + # Internal errors + class SyncError < StandardError; end + # Configuration error (update) class MissingColumnError < StandardError; end end diff --git a/lib/etlify/model.rb b/lib/etlify/model.rb index 4ddf855..2036549 100644 --- a/lib/etlify/model.rb +++ b/lib/etlify/model.rb @@ -103,52 +103,52 @@ def crm_synced?(crm: nil) crm_synchronisation.present? end - def build_crm_payload(crm:) - raise_unless_crm_is_configured(crm) + def build_crm_payload(crm_name:) + raise_unless_crm_is_configured(crm_name) - conf = self.class.etlify_crms.fetch(crm.to_sym) + conf = self.class.etlify_crms.fetch(crm_name.to_sym) conf[:serializer].new(self).as_crm_payload end # @param crm [Symbol] which CRM to use # @param async [Boolean] whether to enqueue or run inline # @param job_class [Class,String,nil] explicit override - def crm_sync!(crm:, async: true, job_class: nil) - return false unless allow_sync_for?(crm) + def crm_sync!(crm_name:, async: true, job_class: nil) + return false unless allow_sync_for?(crm_name) if async - jc = resolve_job_class_for(crm, override: job_class) + jc = resolve_job_class_for(crm_name, override: job_class) if jc.respond_to?(:perform_later) - jc.perform_later(self.class.name, id, crm.to_s) + jc.perform_later(self.class.name, id, crm_name.to_s) elsif jc.respond_to?(:perform_async) - jc.perform_async(self.class.name, id, crm.to_s) + jc.perform_async(self.class.name, id, crm_name.to_s) else raise ArgumentError, "No job class available for CRM sync" end else - Etlify::Synchronizer.call(self, crm: crm) + Etlify::Synchronizer.call(self, crm_name: crm_name) end end - def crm_delete!(crm:) - Etlify::Deleter.call(self, crm: crm) + def crm_delete!(crm_name:) + Etlify::Deleter.call(self, crm_name: crm_name) end private # Guard evaluation per CRM - def allow_sync_for?(crm) - conf = self.class.etlify_crms[crm.to_sym] + def allow_sync_for?(crm_name) + conf = self.class.etlify_crms[crm_name.to_sym] return false unless conf guard = conf[:guard] guard ? guard.call(self) : true end - def resolve_job_class_for(crm, override:) + def resolve_job_class_for(crm_name, override:) return constantize_if_needed(override) if override - conf = self.class.etlify_crms.fetch(crm.to_sym) + conf = self.class.etlify_crms.fetch(crm_name.to_sym) given = conf[:job_class] return constantize_if_needed(given) if given @@ -162,9 +162,9 @@ def constantize_if_needed(klass_or_name) klass_or_name.constantize end - def raise_unless_crm_is_configured(crm) - unless self.class.etlify_crms && self.class.etlify_crms[crm.to_sym] - raise ArgumentError, "crm not configured for #{crm}" + def raise_unless_crm_is_configured(crm_name) + unless self.class.etlify_crms && self.class.etlify_crms[crm_name.to_sym] + raise ArgumentError, "crm not configured for #{crm_name}" end end end diff --git a/lib/etlify/stale_records/batch_sync.rb b/lib/etlify/stale_records/batch_sync.rb index 58c2c8c..d44c430 100644 --- a/lib/etlify/stale_records/batch_sync.rb +++ b/lib/etlify/stale_records/batch_sync.rb @@ -9,46 +9,47 @@ class BatchSync # Public: Run a batch sync over all stale records. # # models: Optional Array to restrict scanned models. + # crm_name: Optional Symbol/String; restrict processing to this CRM. # async: true => enqueue jobs; false => perform inline. # batch_size: # of ids per batch. - # throttle: Optional Float (seconds) to sleep between processed records. - # dry_run: If true, compute counts but do not enqueue/perform. - # logger: IO-like logger; defaults to Etlify.config.logger. # # Returns a Hash with :total, :per_model, :errors. def self.call(models: nil, - async: true, - batch_size: DEFAULT_BATCH_SIZE, - throttle: nil, - dry_run: false, - logger: Etlify.config.logger) + crm_name: nil, + async: true, + batch_size: DEFAULT_BATCH_SIZE) new( models: models, + crm_name: crm_name, async: async, - batch_size: batch_size, - throttle: throttle, - dry_run: dry_run, - logger: logger + batch_size: batch_size ).call end - def initialize(models:, async:, batch_size:, throttle:, dry_run:, logger:) + def initialize(models:, crm_name:, async:, batch_size:) @models = models - @async = async + @crm_name = crm_name&.to_sym + @async = !!async @batch_size = Integer(batch_size) - @throttle = throttle - @dry_run = !!dry_run - @logger = logger || Etlify.config.logger end def call - stats = { total: 0, per_model: {}, errors: 0 } + stats = {total: 0, per_model: {}, errors: 0} - Finder.call(models: @models).each do |model, rel| - processed = process_model(model, rel) - stats[:per_model][model.name] = processed[:count] - stats[:total] += processed[:count] - stats[:errors] += processed[:errors] + # Finder returns: { ModelClass => { crm_sym => relation(ids-only) } } + Finder.call(models: @models, crm_name: @crm_name).each do |model, per_crm| + model_count = 0 + model_errors = 0 + + per_crm.each do |crm, relation| + processed = process_model(model, relation, crm_name: crm) + model_count += processed[:count] + model_errors += processed[:errors] + end + + stats[:per_model][model.name] = model_count + stats[:total] += model_count + stats[:errors] += model_errors end stats @@ -56,66 +57,39 @@ def call private - # Process one model's stale relation (ids-only relation). - def process_model(model, relation) + # Process one model's stale relation (ids-only relation) for a given CRM. + def process_model(model, relation, crm_name:) count = 0 errors = 0 pk = model.primary_key.to_sym - # Pull ids in batches to avoid loading full records in async mode. relation.in_batches(of: @batch_size) do |batch_rel| ids = batch_rel.pluck(pk) next if ids.empty? - if @dry_run - count += ids.size - next - end - if @async - enqueue_async(model, ids) + enqueue_async(model, ids, crm_name: crm_name) count += ids.size else # Load full records only when performing inline. model.where(pk => ids).find_each(batch_size: @batch_size) do |rec| - begin - Etlify::Synchronizer.call(rec) - count += 1 - sleep(@throttle) if @throttle - rescue StandardError => e - log_error(model, rec.id, e) - errors += 1 - end + Etlify::Synchronizer.call(rec, crm_name: crm_name) + count += 1 + rescue + # Count and continue; no logging by design. + errors += 1 end end end - { count: count, errors: errors } + {count: count, errors: errors} end # Enqueue one job per id without loading the records. - def enqueue_async(model, ids) - job_klass = resolve_job_class + def enqueue_async(model, ids, crm_name:) ids.each do |id| - job_klass.perform_later(model.name, id) - sleep(@throttle) if @throttle + Etlify::SyncJob.perform_later(model.name, id, crm_name.to_s) end - rescue StandardError => e - # If enqueue fails at the batch level, log and re-raise for visibility. - @logger.error("[Etlify] enqueue failure for #{model.name}: #{e.message}") - raise - end - - def resolve_job_class - klass = Etlify.config.sync_job_class - klass.is_a?(String) ? klass.constantize : klass - end - - def log_error(model, id, error) - @logger.error( - "[Etlify] sync failure #{model.name}(id=#{id}): #{error.class} " \ - "#{error.message}" - ) end end end diff --git a/lib/etlify/stale_records/finder.rb b/lib/etlify/stale_records/finder.rb index 3a43cd2..bed0a1b 100644 --- a/lib/etlify/stale_records/finder.rb +++ b/lib/etlify/stale_records/finder.rb @@ -2,60 +2,97 @@ module Etlify module StaleRecords class Finder class << self - # Public: Build a Hash of { ModelClass => ActiveRecord::Relation (ids only) } - # models - Optional array of model classes to restrict the search. + # Public: Build a nested Hash of + # { ModelClass => { crm_sym => ActiveRecord::Relation(ids only) } } + # models - Optional Array of model classes to restrict the search. + # crm_name - Optional Symbol/String to target a single CRM. # Returns a Hash. - def call(models: nil) - targets = models || etlified_models - targets.each_with_object({}) do |model, h| + def call(models: nil, crm_name: nil) + targets = models || etlified_models(crm_name: crm_name) + targets.each_with_object({}) do |model, out| next unless model.table_exists? - h[model] = stale_relation_for(model) + + crms = configured_crm_names_for(model, crm_name: crm_name) + next if crms.empty? + + out[model] = crms.each_with_object({}) do |crm, per_crm| + per_crm[crm] = stale_relation_for(model, crm_name: crm) + end end end private - # Detect models that actually called `etlified_with`. - def etlified_models + # Detect models that included Etlify::Model and have at least one CRM + # configured (optionally filtered by crm_name). + def etlified_models(crm_name: nil) ActiveRecord::Base.descendants.select do |m| next false unless m.respond_to?(:table_exists?) && m.table_exists? - m.respond_to?(:etlify_crm_object_type) && - m.etlify_crm_object_type.present? + next false unless m.respond_to?(:etlify_crms) && m.etlify_crms.present? + + if crm_name + m.etlify_crms.key?(crm_name.to_sym) + else + m.etlify_crms.any? + end end end - # Build the relation returning only PKs for stale/missing crm sync rows. - def stale_relation_for(model) - conn = model.connection - epoch = epoch_literal(conn) + # List configured CRM names for a model (optionally filtered). + def configured_crm_names_for(model, crm_name: nil) + return [] unless model.respond_to?(:etlify_crms) && model.etlify_crms.present? + return [crm_name.to_sym] if crm_name && model.etlify_crms.key?(crm_name.to_sym) + + model.etlify_crms.keys + end + + # Build the relation returning only PKs for stale/missing sync rows + # for the given CRM. The JOIN is scoped by crm_name. + def stale_relation_for(model, crm_name:) + conn = model.connection + owner_tbl = model.table_name + owner_pk = model.primary_key + crm_tbl = CrmSynchronisation.table_name + epoch = epoch_literal(conn) + + threshold_sql = latest_timestamp_sql(model, epoch, crm_name: crm_name) - threshold_sql = latest_timestamp_sql(model, epoch) + # Scope the LEFT OUTER JOIN to the specific crm_name and owner row. + join_on = [ + "#{quoted(crm_tbl, 'resource_type', conn)} = #{conn.quote(model.name)}", + "#{quoted(crm_tbl, 'resource_id', conn)} = " \ + "#{quoted(owner_tbl, owner_pk, conn)}", + "#{quoted(crm_tbl, 'crm_name', conn)} = #{conn.quote(crm_name.to_s)}", + ].join(" AND ") - crm_tbl = CrmSynchronisation.table_name - crm_last_synced = - "COALESCE(#{quoted(crm_tbl, 'last_synced_at', conn)}, #{epoch})" + last_synced = "COALESCE(#{quoted(crm_tbl, 'last_synced_at', conn)}, #{epoch})" where_sql = <<-SQL.squish - #{quoted(crm_tbl, 'id', conn)} IS NULL OR - #{crm_last_synced} < (#{threshold_sql}) + #{quoted(crm_tbl, 'id', conn)} IS NULL + OR #{last_synced} < (#{threshold_sql}) SQL model - .left_outer_joins(:crm_synchronisation) + .joins("LEFT OUTER JOIN #{conn.quote_table_name(crm_tbl)} ON #{join_on}") .where(Arel.sql(where_sql)) - .select(model.arel_table[model.primary_key]) + .select(model.arel_table[owner_pk]) end - # Build SQL for the "latest updated_at" across record and its dependencies. - def latest_timestamp_sql(model, epoch) - conn = model.connection + # Build SQL for "latest updated_at" across record and its CRM-specific deps. + def latest_timestamp_sql(model, epoch, crm_name:) + conn = model.connection owner_tbl = model.table_name parts = ["COALESCE(#{quoted(owner_tbl, 'updated_at', conn)}, #{epoch})"] - Array(model.try(:etlify_dependencies)).each do |dep_name| + deps = Array( + model.etlify_crms.dig(crm_name.to_sym, :dependencies) + ).map(&:to_sym) + + deps.each do |dep_name| reflection = model.reflect_on_association(dep_name) next unless reflection + parts << dependency_max_timestamp_sql(model, reflection, epoch) end @@ -75,7 +112,7 @@ def dependency_max_timestamp_sql(model, reflection, epoch) # Non-through associations. def direct_dependency_timestamp_sql(model, reflection, epoch) - conn = model.connection + conn = model.connection owner_tbl = model.table_name case reflection.macro @@ -92,7 +129,6 @@ def direct_dependency_timestamp_sql(model, reflection, epoch) LIMIT 1 SQL "COALESCE((#{sub}), #{epoch})" - when :has_one, :has_many dep_tbl = reflection.klass.table_name fk = reflection.foreign_key @@ -112,14 +148,13 @@ def direct_dependency_timestamp_sql(model, reflection, epoch) WHERE #{preds.map { |p| "(#{p})" }.join(' AND ')} SQL "COALESCE((#{sub}), #{epoch})" - else # Unknown macro: safely ignore with epoch fallback. epoch end end - # has_* :through => build a correlated subquery joining through->source. + # has_* :through => correlated subquery from through -> source. def through_dependency_timestamp_sql(model, reflection, epoch) conn = model.connection through = reflection.through_reflection @@ -130,7 +165,6 @@ def through_dependency_timestamp_sql(model, reflection, epoch) source_pk = reflection.klass.primary_key owner_tbl = model.table_name - # Filter through rows that point to the owner. preds = [] preds << "#{quoted(through_tbl, through.foreign_key, conn)} = " \ "#{quoted(owner_tbl, model.primary_key, conn)}" @@ -139,7 +173,6 @@ def through_dependency_timestamp_sql(model, reflection, epoch) "#{conn.quote(model.name)}" end - # Join through -> source via the source reflection (usually belongs_to). join_on = "#{quoted(source_tbl, source_pk, conn)} = " \ "#{quoted(through_tbl, source.foreign_key, conn)}" @@ -154,8 +187,7 @@ def through_dependency_timestamp_sql(model, reflection, epoch) "COALESCE((#{sub}), #{epoch})" end - # belongs_to polymorphic: enumerate concrete types found in data, and - # pick the greatest updated_at among the matching target row. + # belongs_to polymorphic: enumerate concrete types and pick greatest ts. def polymorphic_belongs_to_timestamp_sql(model, reflection, epoch) conn = model.connection owner_tbl = model.table_name @@ -183,13 +215,13 @@ def polymorphic_belongs_to_timestamp_sql(model, reflection, epoch) end return epoch if parts.empty? + greatest(parts, conn) end # Adapter portability helpers. def greatest(parts, conn) - # If there is only one expression, return it as-is to avoid SQLite - # interpreting MAX(expr) as an aggregate in WHERE. + # With a single expression, return as-is (SQLite aggregate quirk). return parts.first if parts.size == 1 fn = greatest_function_name(conn) diff --git a/lib/etlify/synchronizer.rb b/lib/etlify/synchronizer.rb index 3471106..90c551c 100644 --- a/lib/etlify/synchronizer.rb +++ b/lib/etlify/synchronizer.rb @@ -1,25 +1,36 @@ module Etlify class Synchronizer - # main entry point - # @param record [ActiveRecord::Base] - def self.call(record) - new(record).call + attr_accessor( + :adapter, + :conf, + :crm_name, + :resource + ) + # main entry point (CRM-aware) + # @param resource [ActiveRecord::Base] + # @param crm [Symbol,String] + def self.call(resource, crm_name:) + new(resource, crm_name: crm_name).call end - def initialize(record) - @record = record + def initialize(resource, crm_name:) + @resource = resource + @crm_name = crm_name.to_sym + @conf = resource.class.etlify_crms.fetch(@crm_name) + @adapter = @conf[:adapter].new end def call - @record.with_lock do + resource.with_lock do if sync_line.stale?(digest) - crm_id = Etlify.config.crm_adapter.upsert!( + crm_id = adapter.upsert!( payload: payload, - id_property: @record.etlify_id_property, - object_type: @record.etlify_crm_object_type + id_property: conf[:id_property], + object_type: conf[:crm_object_type] ) sync_line.update!( + crm_name: crm_name, crm_id: crm_id.presence, last_digest: digest, last_synced_at: Time.current, @@ -32,22 +43,29 @@ def call :not_modified end end - rescue StandardError => e + rescue => e sync_line.update!(last_error: e.message) + + :error end private + # Compute once to keep idempotency inside the lock def digest - Etlify.config.digest_strategy.call(payload) + @digest ||= Etlify.config.digest_strategy.call(payload) end def payload - @__payload ||= @record.build_crm_payload + @payload ||= resource.build_crm_payload(crm_name: crm_name) end + # Select or build the per-CRM sync line. + # If you still have has_one, this keeps working but won't handle multi-CRM. def sync_line - @record.crm_synchronisation || @record.build_crm_synchronisation + resource.crm_synchronisations.find_or_initialize_by( + crm_name: crm_name + ) end end end diff --git a/lib/generators/etlify/install/templates/initializer.rb b/lib/generators/etlify/install/templates/initializer.rb deleted file mode 100644 index b690762..0000000 --- a/lib/generators/etlify/install/templates/initializer.rb +++ /dev/null @@ -1,14 +0,0 @@ -require "etlify" - -Etlify.configure do |config| - # CRM adapter (must respond to #upsert!(payload:) and #delete!(crm_id:)) - # Check in lib/etlify/adapters for available adapters (e.g. Etlify::Adapters::Hubspot) - config.crm_adapter = MyCrmAdapter.new - - # @crm_adapter = Etlify::Adapters::NullAdapter.new - # @digest_strategy = Etlify::Digest.method(:stable_sha256) - # @job_queue_name = "low" - # @sync_job_class = "Etlify::SyncJob" - # @logger = defined?(Rails) ? Rails.logger : Logger.new($stdout) - # @cache_store = defined?(Rails) ? Rails.cache : ActiveSupport::Cache::MemoryStore.new -end diff --git a/lib/generators/etlify/install/templates/initializer.rb.tt b/lib/generators/etlify/install/templates/initializer.rb.tt new file mode 100644 index 0000000..cec6a10 --- /dev/null +++ b/lib/generators/etlify/install/templates/initializer.rb.tt @@ -0,0 +1,24 @@ +require "etlify" + +Etlify.configure do |config| + # Register CRM likewise: + Etlify::CRM.register( + :hubspot, adapter: Etlify::Adapters::HubspotV3Adapter, + options: { job_class: Etlify::SyncJob } + ) + # will provide DSL below for models + # hubspot_etlified_with(...) + + # Etlify::CRM.register( + # :another_crm, adapter: Etlify::Adapters::AnotherAdapter, + # options: { job_class: Etlify::SyncJob } + # ) + # will provide DSL below for models + # another_crm_etlified_with(...) + + # overridable defaults config + # @job_queue_name = "low" + # @digest_strategy = Etlify::Digest.method(:stable_sha256) + # @logger = defined?(Rails) ? Rails.logger : Logger.new($stdout) + # @cache_store = Rails.cache +end diff --git a/lib/generators/etlify/migration/migration_generator.rb b/lib/generators/etlify/migration/migration_generator.rb index 6096007..1ae8f46 100644 --- a/lib/generators/etlify/migration/migration_generator.rb +++ b/lib/generators/etlify/migration/migration_generator.rb @@ -1,4 +1,3 @@ -# lib/generators/etlify/migration/migration_generator.rb require "rails/generators" require "rails/generators/active_record" diff --git a/lib/generators/etlify/migration/templates/create_crm_synchronisations.rb.tt b/lib/generators/etlify/migration/templates/create_crm_synchronisations.rb.tt index ea68e75..fe1584b 100644 --- a/lib/generators/etlify/migration/templates/create_crm_synchronisations.rb.tt +++ b/lib/generators/etlify/migration/templates/create_crm_synchronisations.rb.tt @@ -20,7 +20,8 @@ class <%= file_name.camelize %> < ActiveRecord::Migration[<%= ActiveRecord::VERS add_index ( :crm_synchronisations, [:crm_name, :resource_type, :resource_id], - name: "idx_crm_sync_on_crm_and_resource" + unique: true, + name: "idx_unique_crm_sync_resource_crm" ) end end diff --git a/spec/adapters/null_adapter_spec.rb b/spec/adapters/null_adapter_spec.rb index 38dfc94..3bd0ef1 100644 --- a/spec/adapters/null_adapter_spec.rb +++ b/spec/adapters/null_adapter_spec.rb @@ -1,13 +1,14 @@ require "rails_helper" RSpec.describe Etlify::Adapters::NullAdapter do - let(:payload) { { id: 1, any: "data" } } + let(:payload) { {id: 1, any: "data"} } it "returns an id for upsert!" do expect( described_class.new.upsert!( payload: payload, - object_type: "contacts" + object_type: "contacts", + id_property: "id" ) ).to be_a(String) end diff --git a/spec/etlify/config_spec.rb b/spec/etlify/config_spec.rb new file mode 100644 index 0000000..165cfc3 --- /dev/null +++ b/spec/etlify/config_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Etlify::Config do + it "sets sane defaults and is mutable" do + conf = described_class.new + + # Defaults: digest strategy, queue name, logger, cache store. + expect(conf.digest_strategy).to be_a(Method) + expect(conf.job_queue_name).to eq("low") + expect(conf.logger).to be_a(Logger) + expect(conf.cache_store).to respond_to(:read) + + # Mutability check + new_logger = Logger.new(nil) + conf.logger = new_logger + conf.job_queue_name = "default" + expect(conf.logger).to be(new_logger) + expect(conf.job_queue_name).to eq("default") + end + + it "is exposed via Etlify.config and configurable via .configure" do + expect(Etlify.config).to be_a(described_class) + + Etlify.configure do |c| + c.job_queue_name = "critical" + end + expect(Etlify.config.job_queue_name).to eq("critical") + end +end diff --git a/spec/etlify/crm_registry_spec.rb b/spec/etlify/crm_registry_spec.rb new file mode 100644 index 0000000..6524ded --- /dev/null +++ b/spec/etlify/crm_registry_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Etlify::CRM do + before do + # Reset registry to keep tests isolated. + Etlify::CRM.registry.clear + end + + it "registers CRMs and exposes names + fetch" do + # Ensure the install hook is called on the Model DSL. + allow(Etlify::Model).to receive(:install_dsl_for_crm) + + Etlify::CRM.register( + :hubspot, + adapter: Etlify::Adapters::NullAdapter, + options: {job_class: "DummyJob"} + ) + + item = Etlify::CRM.fetch(:hubspot) + expect(item.name).to eq(:hubspot) + expect(item.adapter).to eq(Etlify::Adapters::NullAdapter) + expect(item.options).to eq({job_class: "DummyJob"}) + expect(Etlify::CRM.names).to contain_exactly(:hubspot) + + expect(Etlify::Model).to have_received(:install_dsl_for_crm).with(:hubspot) + end + + it "normalizes registry keys to symbols" do + Etlify::CRM.register("custom", adapter: Etlify::Adapters::NullAdapter) + expect(Etlify::CRM.fetch(:custom).name).to eq(:custom) + end +end diff --git a/spec/etlify/deleter_spec.rb b/spec/etlify/deleter_spec.rb new file mode 100644 index 0000000..ff5df15 --- /dev/null +++ b/spec/etlify/deleter_spec.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Etlify::Deleter do + let(:company) { Company.create!(name: "CapSens", domain: "capsens.eu") } + let(:user) do + User.create!( + email: "dev@capsens.eu", full_name: "Emo-gilles", company_id: company.id + ) + end + + def create_line(resource, crm_name:, crm_id:) + CrmSynchronisation.create!( + resource: resource, crm_name: crm_name, crm_id: crm_id + ) + end + + context "when no sync line exists" do + it "returns :noop and does not call adapter.delete!" do + adapter = instance_double(Etlify::Adapters::NullAdapter) + allow(Etlify::Adapters::NullAdapter).to receive(:new).and_return(adapter) + expect(adapter).not_to receive(:delete!) + + res = described_class.call(user, crm_name: :hubspot) + expect(res).to eq(:noop) + end + end + + context "when sync line exists without crm_id" do + it "returns :noop and does not call adapter.delete!" do + create_line(user, crm_name: "hubspot", crm_id: nil) + + adapter = instance_double(Etlify::Adapters::NullAdapter) + allow(Etlify::Adapters::NullAdapter).to receive(:new).and_return(adapter) + expect(adapter).not_to receive(:delete!) + + res = described_class.call(user, crm_name: :hubspot) + expect(res).to eq(:noop) + end + end + + context "when sync line exists with crm_id" do + it "calls adapter.delete! with params and returns :deleted" do + line = create_line(user, crm_name: "hubspot", crm_id: "crm-123") + + calls = [] + adapter = Class.new do + # Capture arguments to verify them later + define_method(:delete!) do |crm_id:, object_type:, id_property:| + ObjectSpace.each_object(Array).find do |arr| + arr.equal?(calls = arr) # no-op to silence linter + end + true + end + end + + # Replace adapter for this example to observe args + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: adapter, + id_property: "id", + crm_object_type: "contacts", + }, + } + ) + + # Spy on a real instance to check args + instance = adapter.new + allow(adapter).to receive(:new).and_return(instance) + expect(instance).to receive(:delete!).with( + crm_id: "crm-123", + object_type: "contacts", + id_property: "id" + ).and_return(true) + + res = described_class.call(user, crm_name: :hubspot) + expect(res).to eq(:deleted) + + # Ensure sync line not altered by the deleter itself. + expect(line.reload.crm_id).to eq("crm-123") + end + end + + context "when adapter.delete! raises" do + class FailingDeleteAdapter + def delete!(crm_id:, object_type:, id_property:) + raise "remote failure" + end + end + + it "wraps the error into Etlify::SyncError" do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: FailingDeleteAdapter, + id_property: "id", + crm_object_type: "contacts", + }, + } + ) + + create_line(user, crm_name: "hubspot", crm_id: "crm-err") + + expect do + described_class.call(user, crm_name: :hubspot) + end.to raise_error(Etlify::SyncError, /remote failure/) + end + end +end diff --git a/spec/etlify/digest_spec.rb b/spec/etlify/digest_spec.rb new file mode 100644 index 0000000..1326302 --- /dev/null +++ b/spec/etlify/digest_spec.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Etlify::Digest do + describe ".normalize" do + it "sorts hash keys recursively and preserves array order", + :aggregate_failures do + # Hash keys (including nested) are sorted; arrays keep their order. + input = { + b: 2, + a: { + z: [3, {k: 1}], + c: 1, + }, + } + + out = described_class.normalize(input) + + expect(out.keys).to eq(%i[a b]) + expect(out[:a].keys).to eq(%i[c z]) + expect(out[:a][:z]).to eq([3, {k: 1}]) + end + + it "recursively normalizes arrays of mixed types", + :aggregate_failures do + input = [ + {b: 1, a: 2}, + 3, + [ + {d: 4, c: 5}, + 6, + ], + ] + + out = described_class.normalize(input) + + expect(out).to eq( + [ + {a: 2, b: 1}, + 3, + [ + {c: 5, d: 4}, + 6, + ], + ] + ) + end + + it "returns scalars unchanged (String, Numeric, booleans, nil)", + :aggregate_failures do + expect(described_class.normalize("x")).to eq("x") + expect(described_class.normalize(42)).to eq(42) + expect(described_class.normalize(true)).to eq(true) + expect(described_class.normalize(false)).to eq(false) + expect(described_class.normalize(nil)).to be_nil + end + end + + describe ".stable_sha256" do + it "returns a deterministic 64-char lowercase hex digest", + :aggregate_failures do + payload = { + a: 1, + b: [ + 2, + 3, + {c: 4}, + ], + } + + d1 = described_class.stable_sha256(payload) + d2 = described_class.stable_sha256(payload) + + expect(d1).to match(/\A\h{64}\z/) + expect(d1).to eq(d2) + end + + it "is insensitive to hash key ordering (shallow and nested)", + :aggregate_failures do + p1 = {a: 1, b: 2} + p2 = {b: 2, a: 1} + + expect(described_class.stable_sha256(p1)) + .to eq(described_class.stable_sha256(p2)) + + n1 = { + x: {m: 1, n: 2}, + y: [3, 4], + } + n2 = { + x: {n: 2, m: 1}, + y: [3, 4], + } + + expect(described_class.stable_sha256(n1)) + .to eq(described_class.stable_sha256(n2)) + end + + it "changes when a value changes", + :aggregate_failures do + p1 = { + a: 1, + b: [2, 3], + } + p2 = { + a: 1, + b: [2, 4], + } + + expect(described_class.stable_sha256(p1)) + .not_to eq(described_class.stable_sha256(p2)) + end + + it "is sensitive to array element order", + :aggregate_failures do + p1 = {a: [1, 2, 3]} + p2 = {a: [3, 2, 1]} + + expect(described_class.stable_sha256(p1)) + .not_to eq(described_class.stable_sha256(p2)) + end + + it "handles primitive inputs (String, Numeric, booleans, nil)", + :aggregate_failures do + expect(described_class.stable_sha256("hello")).to match(/\A\h{64}\z/) + expect(described_class.stable_sha256(123)).to match(/\A\h{64}\z/) + expect(described_class.stable_sha256(true)).to match(/\A\h{64}\z/) + expect(described_class.stable_sha256(false)).to match(/\A\h{64}\z/) + expect(described_class.stable_sha256(nil)).to match(/\A\h{64}\z/) + end + + it "matches for deeply equivalent structures regardless of key order", + :aggregate_failures do + p1 = { + a: [ + {z: 1, y: 2}, + {c: [3, {b: 4, a: 5}]}, + ], + k: 9, + } + p2 = { + k: 9, + a: [ + {y: 2, z: 1}, + {c: [3, {a: 5, b: 4}]}, + ], + } + + expect(described_class.stable_sha256(p1)) + .to eq(described_class.stable_sha256(p2)) + end + end +end diff --git a/spec/etlify/engine_initializer_spec.rb b/spec/etlify/engine_initializer_spec.rb new file mode 100644 index 0000000..d7b4fe1 --- /dev/null +++ b/spec/etlify/engine_initializer_spec.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Etlify::Engine do + def run_initializer + ActiveSupport.run_load_hooks(:active_record, ActiveRecord::Base) + end + + context "when the required column is present" do + it "does not raise" do + # L'initializer a déjà été exécuté au chargement de la gem, + # son on_load est donc enregistré : on peut juste déclencher le hook. + expect { run_initializer }.not_to raise_error + end + end + + context "when the required column is missing" do + before do + # Drop + recreate the table without the crm_name column. + ActiveRecord::Base.connection.execute( + "DROP TABLE IF EXISTS crm_synchronisations" + ) + ActiveRecord::Schema.define do + create_table :crm_synchronisations, force: true do |t| + # crm_name intentionally omitted + t.string :crm_id + t.string :last_digest + t.datetime :last_synced_at + t.text :last_error + t.string :resource_type, null: false + t.integer :resource_id, null: false + t.timestamps + end + end + CrmSynchronisation.reset_column_information + end + + after do + # Restore correct schema for other specs. + ActiveRecord::Base.connection.execute( + "DROP TABLE IF EXISTS crm_synchronisations" + ) + ActiveRecord::Schema.define do + create_table :crm_synchronisations, force: true do |t| + t.string :crm_name, null: false + t.string :crm_id + t.string :last_digest + t.datetime :last_synced_at + t.text :last_error + t.string :resource_type, null: false + t.integer :resource_id, null: false + t.timestamps + end + add_index :crm_synchronisations, + %i[resource_type resource_id crm_name], + unique: true, + name: "idx_sync_polymorphic_unique" + end + CrmSynchronisation.reset_column_information + end + + it "raises a helpful Etlify::MissingColumnError" do + # Find the initializer and run it INSIDE the expectation. + init = Etlify::Engine.initializers.find do |i| + i.name == "etlify.check_crm_name_column" + end + + expect { init.run(Etlify::Engine.instance) }.to raise_error( + Etlify::MissingColumnError, + /Missing column "crm_name" on table "crm_synchronisations"/ + ) + end + end + + context "when DB is not ready yet" do + it "ignores ActiveRecord::NoDatabaseError" do + allow(ActiveRecord::Base).to receive(:connection) + .and_raise(ActiveRecord::NoDatabaseError) + + expect { ActiveSupport.run_load_hooks(:active_record, nil) } + .not_to raise_error + end + + it "ignores ActiveRecord::StatementInvalid" do + allow(CrmSynchronisation).to receive(:table_exists?) + .and_raise(ActiveRecord::StatementInvalid.new("boom")) + + expect { run_initializer }.not_to raise_error + end + end +end diff --git a/spec/etlify/error_spec.rb b/spec/etlify/error_spec.rb new file mode 100644 index 0000000..6c61aa8 --- /dev/null +++ b/spec/etlify/error_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe "Etlify errors" do + it "carry rich context attributes" do + err = Etlify::ApiError.new( + "bad", + status: 422, + code: "unprocessable", + category: "validation", + correlation_id: "cid-123", + details: {field: "email"}, + raw: {http: "body"} + ) + + expect(err.message).to eq("bad") + expect(err.status).to eq(422) + expect(err.code).to eq("unprocessable") + expect(err.category).to eq("validation") + expect(err.correlation_id).to eq("cid-123") + expect(err.details).to eq({field: "email"}) + expect(err.raw).to eq({http: "body"}) + end + + it "provides subclasses for transport, HTTP and config" do + expect(Etlify::TransportError.new("x", status: 0)).to be_a(Etlify::Error) + expect(Etlify::Unauthorized.new("x", status: 401)).to be_a(Etlify::ApiError) + expect(Etlify::NotFound.new("x", status: 404)).to be_a(Etlify::ApiError) + expect(Etlify::RateLimited.new("x", status: 429)).to be_a(Etlify::ApiError) + expect( + Etlify::ValidationFailed.new("x", status: 422) + ).to be_a(Etlify::ApiError) + expect(Etlify::MissingColumnError).to be < StandardError + end +end diff --git a/spec/etlify/model_spec.rb b/spec/etlify/model_spec.rb new file mode 100644 index 0000000..ea81db1 --- /dev/null +++ b/spec/etlify/model_spec.rb @@ -0,0 +1,275 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Etlify::Model do + # -- Test doubles ----------------------------------------------------------- + class TestAdapter + def initialize(*) + end + end + + class TestSerializer + def initialize(record) + @record = record + end + + def as_crm_payload + {id: @record.id, kind: @record.class.name} + end + end + + class AltJob + class << self + attr_accessor :calls + def perform_later(*args) + (self.calls ||= []) << args + end + + def reset! + self.calls = [] + end + end + end + + # -- Registry isolation ----------------------------------------------------- + before do + @registry_backup = Etlify::CRM.registry.dup + Etlify::CRM.registry.clear + end + + after do + Etlify::CRM.registry.clear + Etlify::CRM.registry.merge!(@registry_backup) + end + + # Build a fresh anonymous model class and include the concern each time. + def new_model_class + Class.new do + include Etlify::Model + attr_reader :id + def initialize(id:) + @id = id + end + end + end + + # Helper to register a CRM named :alpha. + def register_alpha + Etlify::CRM.register( + :alpha, + adapter: TestAdapter, + options: {job_class: AltJob} + ) + end + + # Helper to apply the DSL on a given class for :alpha. + def dsl_apply(klass) + klass.alpha_etlified_with( + serializer: TestSerializer, + crm_object_type: "contacts", + id_property: "id", + dependencies: %i[name email], + sync_if: ->(r) { r.id.odd? }, + job_class: nil + ) + end + + describe "inclusion and DSL installation" do + it "installs DSL on include when a CRM is already registered", + :aggregate_failures do + register_alpha + klass = new_model_class + + expect(klass).to respond_to(:alpha_etlified_with) + inst = klass.new(id: 1) + expect(inst).to respond_to(:alpha_build_payload) + expect(inst).to respond_to(:alpha_sync!) + expect(inst).to respond_to(:alpha_delete!) + end + + it "installs DSL on classes already including the concern when a CRM " \ + "is registered afterwards", :aggregate_failures do + klass = new_model_class + expect(klass).not_to respond_to(:alpha_etlified_with) + + register_alpha + + expect(klass).to respond_to(:alpha_etlified_with) + inst = klass.new(id: 1) + expect(inst).to respond_to(:alpha_build_payload) + expect(inst).to respond_to(:alpha_sync!) + expect(inst).to respond_to(:alpha_delete!) + end + + it "tracks including classes in __included_klasses__", + :aggregate_failures do + before_list = Etlify::Model.__included_klasses__.dup + klass = new_model_class + + expect(Etlify::Model.__included_klasses__).to include(klass) + expect(Etlify::Model.__included_klasses__.size) + .to eq(before_list.size + 1) + end + end + + describe "DSL method _etlified_with" do + it "stores per-CRM config on class.etlify_crms and symbolized deps", + :aggregate_failures do + register_alpha + klass = new_model_class + + dsl_apply(klass) + + conf = klass.etlify_crms[:alpha] + expect(conf[:serializer]).to eq(TestSerializer) + expect(conf[:crm_object_type]).to eq("contacts") + expect(conf[:id_property]).to eq("id") + expect(conf[:dependencies]).to eq(%i[name email]) + expect(conf[:adapter]).to eq(TestAdapter) + # job_class from registry when nil in DSL + expect(conf[:job_class]).to eq(AltJob) + # guard should be installed + expect(conf[:guard]).to be_a(Proc) + end + end + + describe "instance helpers creation and delegation" do + it "defines _build_payload / _sync! / _delete!", + :aggregate_failures do + register_alpha + klass = new_model_class + dsl_apply(klass) + inst = klass.new(id: 7) + + expect(inst).to respond_to(:alpha_build_payload) + expect(inst).to respond_to(:alpha_sync!) + expect(inst).to respond_to(:alpha_delete!) + end + + it "delegates to build_crm_payload / crm_sync! / crm_delete! with " \ + "the CRM name", :aggregate_failures do + register_alpha + klass = Class.new do + include Etlify::Model + attr_reader :id + def initialize(id:) + @id = id + end + + # The generated helper calls build_crm_payload(crm: ...) + def build_crm_payload(crm:) + [:payload_called, crm] + end + + def crm_sync!(crm:, async:, job_class:) + [:sync_called, crm, async] + end + + def crm_delete!(crm:) + [:delete_called, crm] + end + end + + dsl_apply(klass) + inst = klass.new(id: 3) + + expect(inst.alpha_build_payload).to eq([:payload_called, :alpha]) + expect(inst.alpha_sync!(async: false)) + .to eq([:sync_called, :alpha, false]) + expect(inst.alpha_delete!).to eq([:delete_called, :alpha]) + end + end + + describe "#build_crm_payload via configured serializer" do + it "uses serializer.as_crm_payload(record)", :aggregate_failures do + register_alpha + klass = new_model_class + dsl_apply(klass) + inst = klass.new(id: 11) + + out = inst.build_crm_payload(crm_name: :alpha) + expect(out).to eq({id: 11, kind: klass.name}) + end + end + + describe "#crm_sync! job dispatch and guards" do + it "returns false when guard denies synchronization", + :aggregate_failures do + register_alpha + klass = new_model_class + # Guard false for even ids + klass.alpha_etlified_with( + serializer: TestSerializer, + crm_object_type: "contacts", + id_property: "id", + dependencies: [], + sync_if: ->(r) { r.id.odd? }, + job_class: nil + ) + + even = klass.new(id: 2) + expect(even.crm_sync!(crm_name: :alpha, async: true)).to eq(false) + end + + it "uses override job_class when provided", :aggregate_failures do + register_alpha + klass = new_model_class + dsl_apply(klass) + inst = klass.new(id: 5) + + AltJob.reset! + class CustomJob + class << self + attr_accessor :args + def perform_later(*a) + self.args = a + end + end + end + + inst.crm_sync!( + crm_name: :alpha, + async: true, + job_class: CustomJob + ) + + expect(CustomJob.args).to eq([klass.name, 5, "alpha"]) + end + + it "falls back to registry job_class when no override is provided", + :aggregate_failures do + register_alpha + klass = new_model_class + dsl_apply(klass) + inst = klass.new(id: 7) + + AltJob.reset! + inst.crm_sync!( + crm_name: :alpha, + async: true + ) + + expect(AltJob.calls).to eq([[klass.name, 7, "alpha"]]) + end + + it "runs inline when async: false by calling Synchronizer", + :aggregate_failures do + register_alpha + klass = new_model_class + dsl_apply(klass) + inst = klass.new(id: 9) + + expect(Etlify::Synchronizer).to receive(:call).with( + inst, + crm_name: :alpha + ).and_return(:synced) + + res = inst.crm_sync!( + crm_name: :alpha, + async: false + ) + expect(res).to eq(:synced) + end + end +end diff --git a/spec/etlify/stale_records/batch_sync_spec.rb b/spec/etlify/stale_records/batch_sync_spec.rb new file mode 100644 index 0000000..74f8f5c --- /dev/null +++ b/spec/etlify/stale_records/batch_sync_spec.rb @@ -0,0 +1,275 @@ +# spec/etlify/stale_records/batch_sync_spec.rb +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Etlify::StaleRecords::BatchSync do + include AJTestAdapterHelpers + + before do + aj_set_test_adapter! + aj_clear_jobs + # Clear enqueue locks between examples to avoid cross-test interference + Etlify.config.cache_store.clear + end + + let!(:company) { Company.create!(name: "CapSens", domain: "capsens.eu") } + + def create_user!(idx:) + User.create!( + email: "user#{idx}@example.com", + full_name: "User #{idx}", + company: company + ) + end + + describe ".call in async mode" do + it "enqueues one job per stale id for all CRMs when no filter is given" do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "email", + crm_object_type: "contacts", + }, + salesforce: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "email", + crm_object_type: "contacts", + }, + } + ) + + u1 = create_user!(idx: 1) + u2 = create_user!(idx: 2) + + stats = described_class.call(async: true, batch_size: 10) + + # Stats count per CRM (2 users × 2 CRMs) + expect(stats[:total]).to eq(4) + expect(stats[:errors]).to eq(0) + expect(stats[:per_model]["User"]).to eq(4) + + # Enqueue lock collapses multi-CRM into one job per record + jobs = aj_enqueued_jobs + expect(jobs.size).to eq(2) + + jobs.each do |j| + model, id, crm = j[:args] + expect(model).to eq("User") + expect([u1.id, u2.id]).to include(id) + # First CRM wins because enqueue_lock ignores crm_name + expect(crm).to eq("hubspot") + end + end + + it "filters by crm_name when provided (only that CRM is enqueued)" do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "email", + crm_object_type: "contacts", + }, + salesforce: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "email", + crm_object_type: "contacts", + }, + } + ) + + u1 = create_user!(idx: 1) + u2 = create_user!(idx: 2) + + stats = described_class.call( + async: true, + batch_size: 10, + crm_name: :hubspot + ) + + expect(stats[:total]).to eq(2) + expect(stats[:errors]).to eq(0) + expect(stats[:per_model]["User"]).to eq(2) + + jobs = aj_enqueued_jobs + expect(jobs.size).to eq(2) + jobs.each do |j| + model, id, crm = j[:args] + expect(model).to eq("User") + expect([u1.id, u2.id]).to include(id) + expect(crm).to eq("hubspot") + end + end + + it "honors batch_size while enqueueing all ids once" do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "email", + crm_object_type: "contacts", + }, + } + ) + + u1 = create_user!(idx: 1) + u2 = create_user!(idx: 2) + u3 = create_user!(idx: 3) + + stats = described_class.call(async: true, batch_size: 2) + + expect(stats[:total]).to eq(3) + expect(stats[:errors]).to eq(0) + expect(stats[:per_model]["User"]).to eq(3) + + ids = aj_enqueued_jobs.map { |j| j[:args][1] } + expect(ids.sort).to eq([u1.id, u2.id, u3.id].sort) + end + + it "returns zeros when there is nothing to sync" do + allow(User).to receive(:etlify_crms).and_return({}) + + stats = described_class.call(async: true, batch_size: 10) + + expect(stats[:total]).to eq(0) + expect(stats[:errors]).to eq(0) + expect(stats[:per_model]).to eq({}) + expect(aj_enqueued_jobs).to be_empty + end + end + + describe ".call in sync mode (inline)" do + before do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "email", + crm_object_type: "contacts", + }, + } + ) + end + + it "invokes Synchronizer with the proper crm_name for each record" do + u1 = create_user!(idx: 1) + u2 = create_user!(idx: 2) + + calls = [] + allow(Etlify::Synchronizer).to receive(:call) do |rec, crm_name:| + calls << [rec.class.name, rec.id, crm_name] + end + + stats = described_class.call(async: false, batch_size: 10) + + expect(stats[:total]).to eq(2) + expect(stats[:errors]).to eq(0) + expect(stats[:per_model]["User"]).to eq(2) + + expect(Etlify::Synchronizer).to have_received(:call).twice + expect(calls).to match_array( + [ + ["User", u1.id, :hubspot], + ["User", u2.id, :hubspot], + ] + ) + end + + it "counts errors but continues processing other records" do + create_user!(idx: 1) + u2 = create_user!(idx: 2) + create_user!(idx: 3) + + allow(Etlify::Synchronizer).to receive(:call) do |rec, crm_name:| + raise "boom" if rec.id == u2.id + + true + end + + stats = described_class.call(async: false, batch_size: 10) + + expect(stats[:total]).to eq(2) # 2 successes + expect(stats[:errors]).to eq(1) + expect(stats[:per_model]["User"]).to eq(2) + + expect(Etlify::Synchronizer).to have_received(:call).exactly(3).times + end + + it "restricts to the provided crm_name when passed" do + # Pretend model has two CRMs; we filter on :hubspot in the call. + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "email", + crm_object_type: "contacts", + }, + salesforce: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "email", + crm_object_type: "contacts", + }, + } + ) + + create_user!(idx: 1) + create_user!(idx: 2) + + calls = [] + allow(Etlify::Synchronizer).to receive(:call) do |rec, crm_name:| + calls << crm_name + true + end + + stats = described_class.call( + async: false, + batch_size: 10, + crm_name: :hubspot + ) + + expect(stats[:total]).to eq(2) + expect(stats[:errors]).to eq(0) + expect(stats[:per_model]["User"]).to eq(2) + expect(calls).to all(eq(:hubspot)) + end + end + + describe "multiple models in async mode" do + it "aggregates per_model counts across models" do + crm_conf = { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "email", + crm_object_type: "contacts", + }, + salesforce: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "email", + crm_object_type: "contacts", + }, + } + allow(User).to receive(:etlify_crms).and_return(crm_conf) + allow(Company).to receive(:etlify_crms).and_return(crm_conf) + + create_user!(idx: 1) + # company already created by let! + + stats = described_class.call(async: true, batch_size: 10) + + # Stats are per CRM: 2 models × 1 record each × 2 CRMs = 4 + expect(stats[:total]).to eq(4) + expect(stats[:errors]).to eq(0) + expect(stats[:per_model]["User"]).to eq(2) + expect(stats[:per_model]["Company"]).to eq(2) + + # Enqueue lock => one job per record (2 jobs total), crm = "hubspot" + expect(aj_enqueued_jobs.size).to eq(2) + aj_enqueued_jobs.each do |j| + model, _id, crm = j[:args] + expect(%w[User Company]).to include(model) + expect(crm).to eq("hubspot") + end + end + end +end diff --git a/spec/etlify/stale_records/finder_spec.rb b/spec/etlify/stale_records/finder_spec.rb new file mode 100644 index 0000000..a88b87e --- /dev/null +++ b/spec/etlify/stale_records/finder_spec.rb @@ -0,0 +1,702 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Etlify::StaleRecords::Finder do + # Build extra schema for dependency scenarios + before(:all) do + ActiveRecord::Schema.define do + create_table :profiles, force: true do |t| + t.integer :user_id + t.timestamps null: true + end + + create_table :notes, force: true do |t| + t.integer :user_id + t.string :body + t.timestamps null: true + end + + create_table :projects, force: true do |t| + t.string :name + t.timestamps null: true + end + + create_table :memberships, force: true do |t| + t.integer :user_id + t.integer :project_id + t.timestamps null: true + end + + create_table :uploads, force: true do |t| + t.string :owner_type + t.integer :owner_id + t.string :path + t.timestamps null: true + end + + create_table :activities, force: true do |t| + t.string :subject_type + t.integer :subject_id + t.timestamps null: true + end + + # Polymorphic belongs_to on users (owner side) + add_column :users, :avatarable_type, :string + add_column :users, :avatarable_id, :integer + + # Concrete targets for avatarable + create_table :photos, force: true do |t| + t.timestamps null: true + end + + create_table :documents, force: true do |t| + t.timestamps null: true + end + + # HABTM to cover "unknown macro" branch + create_table :tags, force: true do |t| + t.string :name + t.timestamps null: true + end + + create_table :tags_users, id: false, force: true do |t| + t.integer :tag_id + t.integer :user_id + end + + create_table :linkages, force: true do |t| + t.string :owner_type + t.integer :owner_id + t.integer :project_id + t.timestamps null: true + end + end + + User.reset_column_information + + stub_models! + end + + # ----------------- Helpers to define models/constants ----------------- + + # Define a real constant for an AR model (no rspec-mocks). + def define_model_const(name) + Object.send(:remove_const, name) if Object.const_defined?(name) + klass = Class.new(ApplicationRecord) + klass.table_name = name.to_s.underscore.pluralize + yield klass if block_given? + Object.const_set(name, klass) + end + + def stub_models! + define_model_const("Profile") do |k| + k.belongs_to :user, optional: true + end + + define_model_const("Note") do |k| + k.belongs_to :user, optional: true + end + + define_model_const("Project") do |k| + k.has_many :memberships, dependent: :destroy + k.has_many :users, through: :memberships + end + + define_model_const("Membership") do |k| + k.belongs_to :user + k.belongs_to :project + end + + define_model_const("Upload") do |k| + k.belongs_to :owner, polymorphic: true, optional: true + end + + define_model_const("Activity") do |k| + k.belongs_to :subject, polymorphic: true, optional: true + end + + define_model_const("Linkage") do |k| + k.belongs_to :owner, polymorphic: true + k.belongs_to :project + end + + define_model_const("Photo") + define_model_const("Document") + define_model_const("Tag") + + # Reopen User to add associations used by tests + User.class_eval do + has_one :profile, dependent: :destroy + has_many :notes, dependent: :destroy + has_many :memberships, dependent: :destroy + has_many :projects, through: :memberships + has_many :uploads, as: :owner, dependent: :destroy + has_many :activities, as: :subject, dependent: :destroy + belongs_to :avatarable, polymorphic: true, optional: true + has_and_belongs_to_many :tags, join_table: "tags_users" + has_many :linkages, as: :owner, dependent: :destroy + has_many :poly_projects, through: :linkages, source: :project + end + end + + # ------------------------------ Helpers ------------------------------ + + def create_sync!(resource, crm:, last_synced_at:) + CrmSynchronisation.create!( + crm_name: crm.to_s, + resource_type: resource.class.name, + resource_id: resource.id, + last_synced_at: last_synced_at + ) + end + + def now + Time.now + end + + # Default multi-CRM configuration for User in these specs + before do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "id", + crm_object_type: "contacts", + dependencies: [ + :company, :notes, :profile, :projects, :uploads, :activities + ] + }, + salesforce: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "Id", + crm_object_type: "Lead", + dependencies: [:company] + } + } + ) + end + + # ---------------- A. Model discovery / filtering ---------------- + + describe ".call model discovery" do + it "includes AR descendants with config and existing table" do + u = User.create!(email: "a@b.c") + res = described_class.call + expect(res.keys).to include(User) + expect(res[User].keys).to include(:hubspot, :salesforce) + expect(res[User][:hubspot].arel.projections.size).to eq(1) + expect(res[User][:salesforce].arel.projections.size).to eq(1) + expect(u.id).to be_a(Integer) + end + + it "when crm_name is given, keeps only models configured for it" do + res = described_class.call(crm_name: :hubspot) + expect(res.keys).to include(User) + expect(res[User].keys).to eq([:hubspot]) + end + + it "when models: is given, restricts to that subset" do + res = described_class.call(models: [User]) + expect(res.keys).to eq([User]) + end + end + + # ----------------------- B. Return shape ----------------------- + + describe ".call return shape" do + it "returns { Model => { crm => relation } } for single CRM" do + res = described_class.call(crm_name: :hubspot) + expect(res).to be_a(Hash) + expect(res[User]).to be_a(Hash) + expect(res[User][:hubspot]).to be_a(ActiveRecord::Relation) + end + + it "includes one entry per CRM when multiple configured" do + res = described_class.call + expect(res[User].keys).to contain_exactly(:hubspot, :salesforce) + end + + it "relations select only primary key" do + rel = described_class.call[User][:hubspot] + cols = rel.arel.projections + expect(cols.size).to eq(1) + end + end + + # --------------- C. Join scoping to crm_name ---------------- + + describe "JOIN scoped to crm_name" do + it "treats missing row for given crm as stale" do + u = User.create!(email: "x@x.x") + create_sync!(u, crm: :salesforce, last_synced_at: now) + res = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(res.pluck(:id)).to include(u.id) + end + + it "stale only for the outdated CRM" do + u = User.create!(email: "x@x.x") + create_sync!(u, crm: :hubspot, last_synced_at: now - 3600) + create_sync!(u, crm: :salesforce, last_synced_at: now + 3600) + res_all = described_class.call + expect(res_all[User][:hubspot].pluck(:id)).to include(u.id) + expect(res_all[User][:salesforce].pluck(:id)).not_to include(u.id) + end + + it "fresh for both CRMs yields no ids" do + u = User.create!(email: "x@x.x", updated_at: now - 10) + create_sync!(u, crm: :hubspot, last_synced_at: now) + create_sync!(u, crm: :salesforce, last_synced_at: now) + res = described_class.call + expect(res[User][:hubspot].pluck(:id)).not_to include(u.id) + expect(res[User][:salesforce].pluck(:id)).not_to include(u.id) + end + end + + # -------------------- D. Staleness logic -------------------- + + describe "staleness threshold" do + it "missing crm_synchronisation row => stale" do + u = User.create!(email: "x@x.x") + ids = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(ids.pluck(:id)).to include(u.id) + end + + it "NULL last_synced_at acts like epoch and becomes stale" do + u = User.create!(email: "x@x.x") + CrmSynchronisation.create!( + crm_name: "hubspot", resource_type: "User", + resource_id: u.id, last_synced_at: nil + ) + ids = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(ids.pluck(:id)).to include(u.id) + end + + it "compares strictly: < stale, == not stale, > not stale" do + t0 = now + u = User.create!(email: "x@x.x", updated_at: t0) + create_sync!(u, crm: :hubspot, last_synced_at: t0 - 1) + expect(described_class.call(crm_name: :hubspot)[User][:hubspot] + .pluck(:id)).to include(u.id) + + CrmSynchronisation.where( + resource_id: u.id, crm_name: "hubspot" + ).update_all(last_synced_at: t0) + expect(described_class.call(crm_name: :hubspot)[User][:hubspot] + .pluck(:id)).not_to include(u.id) + + CrmSynchronisation.where( + resource_id: u.id, crm_name: "hubspot" + ).update_all(last_synced_at: t0 + 1) + expect(described_class.call(crm_name: :hubspot)[User][:hubspot] + .pluck(:id)).not_to include(u.id) + end + + it "no dependencies => threshold is owner's updated_at" do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "id", + crm_object_type: "contacts", + dependencies: [] + } + } + ) + u = User.create!(email: "x@x.x", updated_at: now) + create_sync!(u, crm: :hubspot, last_synced_at: now - 1) + res = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(res.pluck(:id)).to include(u.id) + end + end + + # ----------------- E. Direct dependencies ----------------- + + describe "dependencies direct associations" do + it "belongs_to: updating company makes user stale" do + c = Company.create!(name: "ACME") + u = User.create!(email: "u@x.x", company: c) + create_sync!(u, crm: :hubspot, last_synced_at: now) + c.update!(updated_at: now + 10) + res = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(res.pluck(:id)).to include(u.id) + end + + it "belongs_to missing target falls back to epoch, not crashing" do + u = User.create!(email: "u@x.x", company: nil) + create_sync!(u, crm: :hubspot, last_synced_at: now + 10) + res = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(res.pluck(:id)).not_to include(u.id) + end + + it "has_one: updating profile makes user stale" do + u = User.create!(email: "u@x.x") + p = u.create_profile! + create_sync!(u, crm: :hubspot, last_synced_at: now) + p.update!(updated_at: now + 10) + res = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(res.pluck(:id)).to include(u.id) + end + + it "has_many: newest note updated makes user stale" do + u = User.create!(email: "u@x.x") + u.notes.create!(body: "a", updated_at: now) + u.notes.create!(body: "b", updated_at: now + 20) + create_sync!(u, crm: :hubspot, last_synced_at: now + 5) + res = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(res.pluck(:id)).to include(u.id) + end + + it "polymorphic has_many via :as ignores unrelated rows" do + u1 = User.create!(email: "u1@x.x") + u2 = User.create!(email: "u2@x.x") + u1.uploads.create!(path: "p1", updated_at: now) + u2.uploads.create!(path: "p2", updated_at: now + 60) + create_sync!(u1, crm: :hubspot, last_synced_at: now + 10) + res = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(res.pluck(:id)).not_to include(u1.id) + end + end + + # -------- F. Through / polymorphic belongs_to (child side) -------- + + describe "through and polymorphic belongs_to" do + it "has_many :through: source newer marks user stale" do + u = User.create!(email: "u@x.x") + p = Project.create!(name: "P") + Membership.create!(user: u, project: p) + create_sync!(u, crm: :hubspot, last_synced_at: now) + p.update!(updated_at: now + 30) + res = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(res.pluck(:id)).to include(u.id) + end + + it "polymorphic child: newest concrete subject wins" do + u = User.create!(email: "u@x.x") + act = Activity.create!(subject: u, updated_at: now) + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "id", + crm_object_type: "contacts", + dependencies: [:activities] + } + } + ) + create_sync!(u, crm: :hubspot, last_synced_at: now + 1) + expect(described_class.call(crm_name: :hubspot)[User][:hubspot] + .pluck(:id)).not_to include(u.id) + act.update!(updated_at: now + 10) + expect(described_class.call(crm_name: :hubspot)[User][:hubspot] + .pluck(:id)).to include(u.id) + end + + it "polymorphic with non-constantizable type is ignored safely" do + u = User.create!(email: "u@x.x") + ts = now.utc.strftime("%Y-%m-%d %H:%M:%S") + Activity.connection.execute( + "INSERT INTO activities (subject_type, subject_id, created_at," \ + " updated_at) VALUES ('Nope::Missing', 123, '#{ts}', '#{ts}')" + ) + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "id", + crm_object_type: "contacts", + dependencies: [:activities] + } + } + ) + create_sync!(u, crm: :hubspot, last_synced_at: now + 5) + res = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(res.pluck(:id)).not_to include(u.id) + end + + it "has_many :through with polymorphic through (as:) adds type predicate" do + # Only track the through association that uses as: :owner + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "id", + crm_object_type: "contacts", + dependencies: [:poly_projects] + } + } + ) + + u = User.create!(email: "t@x.x") + p = Project.create!(name: "P", updated_at: now) + Linkage.create!(owner: u, project: p) + + create_sync!(u, crm: :hubspot, last_synced_at: now + 1) + + # Older source => not stale + expect(described_class.call(crm_name: :hubspot)[User][:hubspot] + .pluck(:id)).not_to include(u.id) + + # Make source (projects) newer => stale via through with as: predicate + p.update!(updated_at: now + 20) + expect(described_class.call(crm_name: :hubspot)[User][:hubspot] + .pluck(:id)).to include(u.id) + end + end + + # --------- NEW: owner belongs_to polymorphic (avatarable) ---------- + + describe "owner belongs_to polymorphic dependency" do + it "uses concrete target updated_at when avatarable is set" do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "id", + crm_object_type: "contacts", + dependencies: [:avatarable] + } + } + ) + u = User.create!(email: "p@x.x") + p = Photo.create!(updated_at: now) + + # 🔧 change here: set the association via its writer, then save + u.avatarable = p + u.updated_at = now + u.save! + + create_sync!(u, crm: :hubspot, last_synced_at: now + 1) + expect(described_class.call(crm_name: :hubspot)[User][:hubspot] + .pluck(:id)).not_to include(u.id) + + p.update!(updated_at: now + 20) + expect(described_class.call(crm_name: :hubspot)[User][:hubspot] + .pluck(:id)).to include(u.id) + end + + it "returns epoch when no concrete types exist (parts empty)" do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "id", + crm_object_type: "contacts", + dependencies: [:avatarable] + } + } + ) + u = User.create!(email: "q@x.x") + create_sync!(u, crm: :hubspot, last_synced_at: now + 10) + expect(described_class.call(crm_name: :hubspot)[User][:hubspot] + .pluck(:id)).not_to include(u.id) + end + end + + # ------------- NEW: unknown macro branch (HABTM) ------------- + + describe "unknown macro branch coverage" do + it "ignores HABTM dependency (epoch fallback, no crash)" do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "id", + crm_object_type: "contacts", + dependencies: [:tags] + } + } + ) + u = User.create!(email: "habtm@x.x", updated_at: now) + t = Tag.create!(name: "x", updated_at: now + 60) + u.tags << t + create_sync!(u, crm: :hubspot, last_synced_at: now + 10) + expect(described_class.call(crm_name: :hubspot)[User][:hubspot] + .pluck(:id)).not_to include(u.id) + end + end + + # --------------- G. Timestamp edge cases ---------------- + + describe "timestamp edge cases" do + it "NULL updated_at are treated as epoch (no crash)" do + u = User.create!(email: "u@x.x") + n = u.notes.create!(body: "n") + Note.where(id: n.id).update_all(updated_at: nil) + create_sync!(u, crm: :hubspot, last_synced_at: now + 10) + res = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(res.pluck(:id)).not_to include(u.id) + end + + it "children NULL updated_at does not mark stale unless owner newer" do + u = User.create!(email: "u@x.x", updated_at: now) + n = u.notes.create!(body: "n") + Note.where(id: n.id).update_all(updated_at: nil) + create_sync!(u, crm: :hubspot, last_synced_at: now + 5) + res = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(res.pluck(:id)).not_to include(u.id) + end + end + + # ------------- H. Adapter portability (unit-level) ------------- + + describe "adapter portability helpers" do + it "uses GREATEST on Postgres, MAX on SQLite" do + pg = double("Conn", adapter_name: "PostgreSQL") + sq = double("Conn", adapter_name: "SQLite") + fn_pg = described_class.send(:greatest_function_name, pg) + fn_sq = described_class.send(:greatest_function_name, sq) + expect(fn_pg).to eq("GREATEST") + expect(fn_sq).to eq("MAX") + end + + it "epoch literal differs by adapter" do + pg = double("Conn", adapter_name: "PostgreSQL") + sq = double("Conn", adapter_name: "SQLite") + e_pg = described_class.send(:epoch_literal, pg) + e_sq = described_class.send(:epoch_literal, sq) + expect(e_pg).to include("TIMESTAMP") + expect(e_sq).to include("DATETIME") + end + + it "greatest returns single part as-is to avoid SQLite quirk" do + conn = double("Conn", adapter_name: "SQLite") + res = described_class.send(:greatest, ["A_ONLY"], conn) + expect(res).to eq("A_ONLY") + end + end + + # ----------- I. CRM-specific dependencies isolation ----------- + + describe "CRM-specific dependencies isolation" do + it "changing a dep for CRM A does not mark CRM B stale" do + u = User.create!(email: "a@x.x") + c = Company.create!(name: "ACME") + u.update!(company: c) + create_sync!(u, crm: :hubspot, last_synced_at: now) + create_sync!(u, crm: :salesforce, last_synced_at: now) + u.notes.create!(body: "x", updated_at: now + 30) + res = described_class.call + expect(res[User][:hubspot].pluck(:id)).to include(u.id) + expect(res[User][:salesforce].pluck(:id)).not_to include(u.id) + end + + it "changing a dep for CRM B marks stale only for CRM B" do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "id", + crm_object_type: "contacts", + dependencies: [:notes] + }, + salesforce: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "Id", + crm_object_type: "Lead", + dependencies: [:company] + } + } + ) + u = User.create!(email: "b@x.x") + c = Company.create!(name: "ACME") + u.update!(company: c) + create_sync!(u, crm: :hubspot, last_synced_at: now + 30) + create_sync!(u, crm: :salesforce, last_synced_at: now) + c.update!(updated_at: now + 60) + res = described_class.call + expect(res[User][:salesforce].pluck(:id)).to include(u.id) + expect(res[User][:hubspot].pluck(:id)).not_to include(u.id) + end + end + + # ------------- J. Empty results / absent CRM ---------------- + + describe "empty and absent CRM cases" do + it "omits models not configured for targeted crm_name" do + allow(User).to receive(:etlify_crms).and_return( + { hubspot: User.etlify_crms[:hubspot] } + ) + res = described_class.call(crm_name: :salesforce) + expect(res).to eq({}) + end + + it "returns {} when no model qualifies" do + klass = Class.new(ApplicationRecord) do + self.table_name = "projects" + def self.etlify_crms = {} + end + Object.const_set("NopeModel", klass) + res = described_class.call(models: [NopeModel]) + expect(res).to eq({}) + ensure + Object.send(:remove_const, "NopeModel") if + Object.const_defined?("NopeModel") + end + + it "relation exists but may be empty when nothing is stale" do + u = User.create!(email: "ok@x.x", updated_at: now - 1) + create_sync!(u, crm: :hubspot, last_synced_at: now) + rel = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(rel).to be_a(ActiveRecord::Relation) + expect(rel.pluck(:id)).to be_empty + end + end + + # ----------------- K. Robustness + helpers ----------------- + + describe "robustness" do + it "ignores unknown dependency names" do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: User.etlify_crms[:hubspot].merge( + dependencies: [:does_not_exist] + ) + } + ) + u = User.create!(email: "u@x.x", updated_at: now) + create_sync!(u, crm: :hubspot, last_synced_at: now + 10) + res = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(res.pluck(:id)).not_to include(u.id) + end + + it "uses a single LEFT OUTER JOIN per CRM and selects id only" do + rel = described_class.call(crm_name: :hubspot)[User][:hubspot] + sql = rel.to_sql + expect(sql.scan(/LEFT OUTER JOIN/i).size).to eq(1) + expect(sql).to include('SELECT "users"."id"') + end + + it "quotes names safely to avoid crashes with reserved words" do + rel = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect { rel.to_a }.not_to raise_error + end + end + + describe "private helpers direct calls" do + it "builds MAX/GREATEST SQL for multiple parts" do + sq = double("Conn", adapter_name: "SQLite") + pg = double("Conn", adapter_name: "PostgreSQL") + expect(described_class.send(:greatest, ["A", "B"], sq)) + .to eq("MAX(A, B)") + expect(described_class.send(:greatest, ["A", "B"], pg)) + .to eq("GREATEST(A, B)") + end + + it "quotes table/column names" do + conn = ActiveRecord::Base.connection + q = described_class.send(:quoted, "users", "id", conn) + expect(q).to match(/"users"\."id"/) + end + + it "etlified_models excludes models without etlify_crms" do + klass = Class.new(ApplicationRecord) { self.table_name = "projects" } + Object.const_set("NoCrmModel", klass) + res = described_class.send(:etlified_models) + expect(res).not_to include(NoCrmModel) + ensure + Object.send(:remove_const, "NoCrmModel") if + Object.const_defined?("NoCrmModel") + end + end +end diff --git a/spec/etlify/synchronizer_spec.rb b/spec/etlify/synchronizer_spec.rb new file mode 100644 index 0000000..ceb61a9 --- /dev/null +++ b/spec/etlify/synchronizer_spec.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Etlify::Synchronizer do + let(:company) { Company.create!(name: "CapSens", domain: "capsens.eu") } + let(:user) do + User.create!( + email: "dev@capsens.eu", full_name: "Emo-gilles", company_id: company.id + ) + end + + def sync_lines_for(resource) + CrmSynchronisation.where(resource: resource) + end + + before do + # Assure a stable digest strategy for deterministic tests (and overrideable). + Etlify.configure do |c| + c.digest_strategy = Etlify::Digest.method(:stable_sha256) + end + end + + context "when payload has changed (stale digest)" do + it "upserts, updates line and returns :synced", :aggregate_failures do + result = described_class.call(user, crm_name: :hubspot) + expect(result).to eq(:synced) + + line = sync_lines_for(user).find_by(crm_name: "hubspot") + expect(line).to be_present + expect(line.crm_id).to be_present + expect(line.last_digest).to be_present + expect(line.last_error).to be_nil + expect(line.last_synced_at).to be_within(2).of(Time.current) + end + end + + context "when payload is not modified" do + it "only touches timestamp and returns :not_modified", :aggregate_failures do + # First sync creates the line. + first = described_class.call(user, crm_name: :hubspot) + expect(first).to eq(:synced) + + travel_to(Time.current + 60) do + # Force stale? false by setting last_digest to current digest. + line = sync_lines_for(user).find_by(crm_name: "hubspot") + digest = Etlify.config.digest_strategy.call( + user.build_crm_payload(crm_name: :hubspot) + ) + line.update!(last_digest: digest) + + res = described_class.call(user, crm_name: :hubspot) + expect(res).to eq(:not_modified) + + line.reload + expect(line.last_error).to be_nil + expect(line.last_synced_at).to be_within(1).of(Time.current) + end + end + end + + context "argument passing to adapter" do + # We assert that Synchronizer passes the correct keywords to adapter.upsert! + it "passes payload, id_property and object_type", :aggregate_failures do + adapter_instance = instance_double(Etlify::Adapters::NullAdapter) + + # Ensure the adapter instance used is our spy + allow(Etlify::Adapters::NullAdapter).to receive(:new) + .and_return(adapter_instance) + + # Build the expected payload + expected_payload = Etlify::Serializers::UserSerializer + .new(user).as_crm_payload + + expect(adapter_instance).to receive(:upsert!).with( + payload: expected_payload, + id_property: "id", + object_type: "contacts" + ).and_return("crm-xyz") + + result = described_class.call(user, crm_name: :hubspot) + line = sync_lines_for(user).find_by(crm_name: "hubspot") + + expect(result).to eq(:synced) + expect(line.crm_id).to eq("crm-xyz") + end + end + + context "memoization" do + it "computes digest only once per call", :aggregate_failures do + calls = 0 + begin + Etlify.configure do |c| + c.digest_strategy = lambda do |payload| + calls += 1 + Etlify::Digest.stable_sha256(payload) + end + end + + # First call (stale) should invoke the strategy once even if digest + # is used twice (stale? + update!). + res = described_class.call(user, crm_name: :hubspot) + expect(res).to eq(:synced) + expect(calls).to eq(1) + + # Second call with same digest: we make stale? false by aligning last_digest + line = sync_lines_for(user).find_by(crm_name: "hubspot") + digest = Etlify.config.digest_strategy.call( + user.build_crm_payload(crm_name: :hubspot) + ) + line.update!(last_digest: digest) + + calls = 0 + res2 = described_class.call(user, crm_name: :hubspot) + expect(res2).to eq(:not_modified) + # stale? computes digest once + expect(calls).to eq(1) + ensure + # Restore the default stable strategy (already set in before block, + # but ensure no leak if the example fails mid-way). + Etlify.configure do |c| + c.digest_strategy = Etlify::Digest.method(:stable_sha256) + end + end + end + + it "builds payload only once per call" do + allow(user).to receive(:build_crm_payload).and_call_original + + res = described_class.call(user, crm_name: :hubspot) + expect(res).to eq(:synced) + expect(user).to have_received(:build_crm_payload).once + end + end + + context "when adapter raises" do + class FailingAdapter + def upsert!(payload:, id_property:, object_type:) + raise "boom" + end + end + + it "records last_error and returns :error", :aggregate_failures do + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: FailingAdapter, + id_property: "id", + crm_object_type: "contacts", + }, + } + ) + + result = described_class.call(user, crm_name: :hubspot) + line = sync_lines_for(user).find_by(crm_name: "hubspot") + + expect(result).to eq(:error) + expect(line.last_error).to eq("boom") + expect(line.last_synced_at).to be_nil + expect(line.last_digest).to be_nil + end + end +end diff --git a/spec/factories.rb b/spec/factories.rb deleted file mode 100644 index d0a9965..0000000 --- a/spec/factories.rb +++ /dev/null @@ -1,24 +0,0 @@ -RSpec.shared_context "with companies and users" do - let!(:company) do - Company.create!( - name: "Capsens", - domain: "capsens.eu" - ) - end - - let!(:user) do - User.create!( - email: "john@capsens.eu", - full_name: "John Doe", - company: company - ) - end -end - -def create_sync_for!(record, last_synced_at:) - CrmSynchronisation.create!( - resource_type: record.class.name, - resource_id: record.id, - last_synced_at: last_synced_at - ) -end diff --git a/spec/generators/etlify/install_generator_spec.rb b/spec/generators/etlify/install_generator_spec.rb new file mode 100644 index 0000000..544a135 --- /dev/null +++ b/spec/generators/etlify/install_generator_spec.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require "rails_helper" +require "fileutils" +require "tmpdir" + +RSpec.describe Etlify::Generators::InstallGenerator, type: :generator do + around do |example| + Dir.mktmpdir do |dir| + @tmp = dir + Dir.chdir(@tmp) { example.run } + end + end + + def build_generator + described_class.new( + [], # args + {}, # options + destination_root: @tmp # thor config + ) + end + + def generated_initializer_path + File.join(@tmp, "config/initializers/etlify.rb") + end + + it "creates config/initializers/etlify.rb", + :aggregate_failures do + gen = build_generator + gen.invoke_all + + expect(File.exist?(generated_initializer_path)).to be(true) + + content = File.read(generated_initializer_path) + expect(content).to include("Etlify.configure do |config|") + expect(content).to include("Etlify::CRM.register(") + expect(content).to include("adapter: Etlify::Adapters::HubspotV3Adapter") + expect(content).to include("options: { job_class: Etlify::SyncJob }") + expect(content).to include("# @job_queue_name = \"low\"") + expect(content).to include( + "# @digest_strategy = Etlify::Digest.method(:stable_sha256)" + ) + expect(content).to include("# @cache_store = Rails.cache") + end + + it "the generated initializer configures default HubSpot mapping by " \ + "calling Etlify::CRM.register", + :aggregate_failures do + gen = build_generator + gen.invoke_all + + # Préserve et restaure le registre pour ne pas polluer d'autres specs + begin + original = Etlify::CRM.registry.dup + + expect(Etlify::CRM).to receive(:register).with( + :hubspot, + adapter: Etlify::Adapters::HubspotV3Adapter, + options: {job_class: Etlify::SyncJob} + ) + + # Charge le fichier généré (exécute Etlify.configure + register) + load generated_initializer_path + ensure + Etlify::CRM.registry.clear + Etlify::CRM.registry.merge!(original) + end + end + + it "est chargeable plusieurs fois sans exploser (idempotent à l'exécution)", + :aggregate_failures do + gen = build_generator + gen.invoke_all + + # On autorise plusieurs register identiques ; on stub pour le vérifier + allow(Etlify::CRM).to receive(:register) + + expect do + load generated_initializer_path + load generated_initializer_path + end.not_to raise_error + + expect(Etlify::CRM).to have_received(:register).twice + end +end diff --git a/spec/generators/etlify/migration_generator_spec.rb b/spec/generators/etlify/migration_generator_spec.rb new file mode 100644 index 0000000..c97c9ad --- /dev/null +++ b/spec/generators/etlify/migration_generator_spec.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +require "rails_helper" +require "fileutils" +require "tmpdir" + +RSpec.describe Etlify::Generators::MigrationGenerator, type: :generator do + # Build a fresh generator instance targeting a temp destination. + def build_generator(args) + described_class.new( + args, + {}, + destination_root: @tmp_dir + ) + end + + # Find the single generated migration file by a suffix pattern. + def find_migration_by_suffix(suffix) + Dir[File.join(@tmp_dir, "db/migrate/*_#{suffix}")].first + end + + around do |example| + Dir.mktmpdir do |dir| + @tmp_dir = dir + Dir.chdir(@tmp_dir) do + FileUtils.mkdir_p("db/migrate") + example.run + end + end + end + + describe "#copy_migration (default filename)" do + it "creates a timestamped migration using the default name", + :aggregate_failures do + # Thor requires an argument; pass "" to trigger presence ⇒ nil + gen = build_generator([""]) + gen.invoke_all + + path = find_migration_by_suffix("create_crm_synchronisations.rb") + expect(path).to be_a(String) + expect(File.exist?(path)).to eq(true) + + content = File.read(path) + expect(content).to match( + /class CreateCrmSynchronisations < ActiveRecord::Migration\[\d+\.\d+\]/ + ) + major = ActiveRecord::VERSION::MAJOR + minor = ActiveRecord::VERSION::MINOR + expect(content).to include( + "ActiveRecord::Migration[#{major}.#{minor}]" + ) + end + end + + describe "#copy_migration (custom filename)" do + it "creates a timestamped migration using the provided name", + :aggregate_failures do + gen = build_generator(["add_foo_bar"]) + gen.invoke_all + + path = find_migration_by_suffix("add_foo_bar.rb") + expect(path).to be_a(String) + expect(File.exist?(path)).to eq(true) + + content = File.read(path) + expect(content).to include( + "class AddFooBar < ActiveRecord::Migration" + ) + end + end + + describe "template content" do + it "contains the expected columns and indexes", + :aggregate_failures do + gen = build_generator([""]) + gen.invoke_all + + path = find_migration_by_suffix("create_crm_synchronisations.rb") + content = File.read(path) + + # Columns + expect(content).to include("t.string :crm_id") + expect(content).to include( + "t.string :resource_type, null: false" + ) + expect(content).to include( + "t.bigint :resource_id, null: false" + ) + expect(content).to include("t.string :last_digest") + expect(content).to include("t.datetime :last_synced_at") + expect(content).to include("t.string :last_error") + expect(content).to include("t.string :crm") + + # Indexes (assert literal content from template) + expect(content).to include( + "add_index :crm_synchronisations, :crm_id, unique: true" + ) + expect(content).to include( + "add_index :crm_synchronisations, %i[resource_type resource_id], " \ + "unique: true, name: \"idx_crm_sync_on_resource\"" + ) + expect(content).to include( + "add_index :crm_synchronisations, :last_synced_at" + ) + expect(content).to include( + "add_index :crm_synchronisations, :resource_type" + ) + expect(content).to include( + "add_index :crm_synchronisations, :resource_id" + ) + # Composite index literal (note: template uses :crm_name here) + expect(content).to include( + "[:crm_name, :resource_type, :resource_id]," + ) + expect(content).to include( + "name: \"idx_unique_crm_sync_resource_crm\"" + ) + end + end + + describe "private helpers" do + it "file_name returns default when name is blank (\"\")", + :aggregate_failures do + gen = build_generator([""]) + expect(gen.send(:file_name)).to eq( + described_class::DEFAULT_MIGRATION_FILENAME + ) + end + + it "file_name underscores a provided CamelCase name", + :aggregate_failures do + gen = build_generator(["AddFooBar"]) + expect(gen.send(:file_name)).to eq("add_foo_bar") + end + end + + describe ".next_migration_number" do + it "returns a UTC timestamp (YYYYMMDDHHMMSS) and is deterministic " \ + "for the same time", :aggregate_failures do + fixed = Time.utc(2025, 8, 22, 9, 45, 12) + allow(Time).to receive(:now).and_return(fixed) + + n1 = described_class.next_migration_number("ignored") + n2 = described_class.next_migration_number("ignored") + + expect(n1).to match(/\A\d{14}\z/) + expect(n1).to eq("20250822094512") + expect(n2).to eq(n1) + end + end +end diff --git a/spec/generators/install_generator_spec.rb b/spec/generators/install_generator_spec.rb deleted file mode 100644 index 0bf815d..0000000 --- a/spec/generators/install_generator_spec.rb +++ /dev/null @@ -1,11 +0,0 @@ -require "rails_helper" - -RSpec.describe Etlify::Generators::InstallGenerator, type: :generator do - it "defines an existing initializer template" do - path = File.expand_path( - "../../lib/generators/etlify/install/templates/initializer.rb", - __dir__ - ) - expect(File).to exist(path) - end -end diff --git a/spec/generators/migration_generator_spec.rb b/spec/generators/migration_generator_spec.rb deleted file mode 100644 index 80458fc..0000000 --- a/spec/generators/migration_generator_spec.rb +++ /dev/null @@ -1,140 +0,0 @@ -# spec/generators/migration_generator_spec.rb -require "rails_helper" -require "tmpdir" - -RSpec.describe Etlify::Generators::MigrationGenerator, type: :generator do - # Run a generator instance in a temporary destination. - def run_generator_in(dir, args = nil, at: nil) - # Thor requires a positional `name` argument. - # Use empty string to trigger your DEFAULT_MIGRATION_FILENAME. - argv = args.nil? ? [""] : args - gen = described_class.new(argv, {}, destination_root: dir) - if at - Timecop.freeze(at) { gen.invoke_all } - else - gen.invoke_all - end - end - - def first_file(glob) - Dir.glob(glob).first - end - - describe "template presence" do - it "exposes a migration template" do - path = File.expand_path( - "../../lib/generators/etlify/migration/templates/" \ - "create_crm_synchronisations.rb.tt", - __dir__ - ) - expect(File).to exist(path) - end - end - - describe "running the generator" do - it "creates a migration with the default filename", :aggregate_failures do - Dir.mktmpdir do |dir| - run_generator_in( - dir, - nil, - at: Time.utc(2025, 8, 19, 12, 34, 56) - ) - - files = Dir.glob( - File.join( - dir, - "db/migrate/*_create_crm_synchronisations.rb" - ) - ) - expect(files.size).to eq(1) - - basename = File.basename(files.first) - expect(basename).to match(/\A20250819123456_create_crm_/) - - content = File.read(files.first) - expect(content).to include("create_table :crm_synchronisations") - expect(content).to include("t.string :crm_id") - expect(content).to include("t.string :resource_type, null: false") - expect(content).to include("t.bigint :resource_id, null: false") - expect(content).to include("t.string :last_digest") - expect(content).to include("t.datetime :last_synced_at") - expect(content).to include("t.string :last_error") - expect(content).to include( - "add_index :crm_synchronisations, :crm_id, unique: true" - ) - expect(content).to include('name: "idx_crm_sync_on_resource"') - expect(content).to include( - "add_index :crm_synchronisations, :last_synced_at" - ) - end - end - - it "accepts a custom migration name and camelizes the class", :aggregate_failures do - Dir.mktmpdir do |dir| - run_generator_in( - dir, - ["init_crm_sync"], - at: Time.utc(2025, 8, 19, 13, 0, 0) - ) - - files = Dir.glob(File.join(dir, "db/migrate/*_init_crm_sync.rb")) - expect(files.size).to eq(1) - - content = File.read(files.first) - expect(content).to match( - /class InitCrmSync < ActiveRecord::Migration/ - ) - expect(content).to include("create_table :crm_synchronisations") - expect(content).to include( - "add_index :crm_synchronisations, :crm_id, unique: true" - ) - end - end - - it "targets the current AR major.minor in migration superclass", :aggregate_failures do - Dir.mktmpdir do |dir| - run_generator_in(dir, [""]) - - path = first_file( - File.join( - dir, - "db/migrate/*_create_crm_synchronisations.rb" - ) - ) - content = File.read(path) - - expected = "#{ActiveRecord::VERSION::MAJOR}." \ - "#{ActiveRecord::VERSION::MINOR}" - - expect(content).to match( - /< ActiveRecord::Migration\[#{Regexp.escape(expected)}\]/ - ) - end - end - - it "generates unique timestamps on multiple runs", :aggregate_failures do - Dir.mktmpdir do |dir| - t1 = Time.utc(2025, 8, 19, 14, 0, 0) - t2 = t1 + 1 - - run_generator_in(dir, [""], at: t1) - run_generator_in(dir, ["second_one"], at: t2) - - files = Dir.glob(File.join(dir, "db/migrate/*.rb")) - expect(files.size).to eq(2) - - stamps = files.map { |f| File.basename(f).split("_").first } - expect(stamps.uniq.size).to eq(2) - end - end - end - - describe ".next_migration_number" do - it "formats the timestamp as UTC YYYYMMDDHHMMSS" do - Timecop.freeze(Time.utc(2031, 1, 2, 3, 4, 5)) do - num = described_class.next_migration_number(nil) - expect(num).to eq("20310102030405") - end - end - end -end diff --git a/spec/generators/serializer_generator_spec.rb b/spec/generators/serializer_generator_spec.rb deleted file mode 100644 index e75e304..0000000 --- a/spec/generators/serializer_generator_spec.rb +++ /dev/null @@ -1,76 +0,0 @@ -# spec/generators/serializer_generator_spec.rb -require "rails_helper" -require "tmpdir" - -RSpec.describe Etlify::Generators::SerializerGenerator, type: :generator do - # Run a generator instance in a temp destination. - def run_generator_in(dir, name) - gen = described_class.new([name], {}, destination_root: dir) - gen.invoke_all - end - - def read(path) - File.read(path) - end - - describe "internal helper" do - it "builds the expected serializer_class_name" do - Dir.mktmpdir do |dir| - gen = described_class.new(["admin/user"], {}, destination_root: dir) - expect(gen.send(:serializer_class_name)) - .to eq("Admin::UserSerializer") - end - end - end - - describe "template presence" do - it "exposes a serializer template" do - path = File.expand_path( - "../../lib/generators/etlify/serializer/templates/serializer.rb.tt", - __dir__ - ) - expect(File).to exist(path) - end - end - - describe "generation" do - it "creates a basic serializer under app/serializers/etlify", :aggregate_failures do - Dir.mktmpdir do |dir| - run_generator_in(dir, "user") - - path = File.join( - dir, - "app/serializers/etlify/user_serializer.rb" - ) - expect(File).to exist(path) - - content = read(path) - expect(content).to include("module Etlify") - expect(content).to include("class UserSerializer") - expect(content).to include("attr_reader :record") - expect(content).to include("def as_crm_payload") - expect(content).to include("id: record.id") - end - end - - it "respects namespaces in class_path and file layout", :aggregate_failures do - Dir.mktmpdir do |dir| - run_generator_in(dir, "admin/user") - - path = File.join( - dir, - "app/serializers/etlify/admin/user_serializer.rb" - ) - expect(File).to exist(path) - - content = read(path) - expect(content).to include("module Etlify") - # NamedBase gives class_name "Admin::User" - expect(content).to include("class Admin::UserSerializer") - # still has the contract of BaseSerializer skeleton - expect(content).to include("attr_reader :record") - expect(content).to include("id: record.id") - end - end - end -end diff --git a/spec/jobs/config_job_class_spec.rb b/spec/jobs/config_job_class_spec.rb deleted file mode 100644 index e4d6d20..0000000 --- a/spec/jobs/config_job_class_spec.rb +++ /dev/null @@ -1,37 +0,0 @@ -require "rails_helper" - -RSpec.describe "Etlify sync_job_class config" do - context "when given sync_job_class is a class" do - before do - class DummyJob < Etlify::SyncJob; end - Etlify.configure { |c| c.sync_job_class = DummyJob } - end - - let(:user) do - User.create!(full_name: "Test User", email: "test101@example.com") - end - - it "uses the configured job class to enqueue" do - expect(DummyJob).to receive(:perform_later).with("User", user.id) - - user.crm_sync!(async: true) - end - end - - context "when given sync_job_class is a string" do - before do - class DummyJob < Etlify::SyncJob; end - Etlify.configure { |c| c.sync_job_class = "DummyJob" } - end - - let(:user) do - User.create!(full_name: "Test User", email: "test101@example.com") - end - - it "uses the configured job class to enqueue" do - expect(DummyJob).to receive(:perform_later).with("User", user.id) - - user.crm_sync!(async: true) - end - end -end diff --git a/spec/jobs/etlify/sync_job_spec.rb b/spec/jobs/etlify/sync_job_spec.rb new file mode 100644 index 0000000..3bf7480 --- /dev/null +++ b/spec/jobs/etlify/sync_job_spec.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Etlify::SyncJob do + let(:company) { Company.create!(name: "CapSens", domain: "capsens.eu") } + let(:user) do + User.create!( + email: "dev@capsens.eu", + full_name: "Emo-gilles", + company_id: company.id + ) + end + + let(:crm_name) { "hubspot" } + let(:queue_name) { Etlify.config.job_queue_name } + let(:cache) { Etlify.config.cache_store } + + before do + # Use the test adapter without ActiveJob::TestHelper / Minitest + aj_set_test_adapter! + aj_clear_jobs + # Clear cache to avoid stale enqueue locks across examples + cache.clear if cache.respond_to?(:clear) + end + + def lock_key_for(klass_name, id) + "etlify:jobs:sync:#{klass_name}:#{id}" + end + + it "enqueues on the configured queue and dedupes with a cache lock", + :aggregate_failures do + key = lock_key_for("User", user.id) + cache.delete(key) + + expect do + described_class.perform_later("User", user.id, crm_name) + # Second enqueue should be dropped by around_enqueue lock. + described_class.perform_later("User", user.id, crm_name) + end.to change { aj_enqueued_jobs.size }.by(1) + + job = aj_enqueued_jobs.first + expect(job[:job]).to eq(described_class) + expect(job[:args]).to eq(["User", user.id, crm_name]) + expect(job[:queue]).to eq(queue_name) + expect(cache.exist?(key)).to be(true) + end + + it "clears the enqueue lock after perform (even on success)", + :aggregate_failures do + key = lock_key_for("User", user.id) + cache.delete(key) + + described_class.perform_later("User", user.id, crm_name) + expect(cache.exist?(key)).to be(true) + + # Perform only immediate jobs (scheduled ones stay queued) + aj_perform_enqueued_jobs + + expect(cache.exist?(key)).to be(false) + end + + it "does nothing when the record cannot be found" do + expect(Etlify::Synchronizer).not_to receive(:call) + + described_class.perform_later("User", -999_999, crm_name) + aj_perform_enqueued_jobs + + expect(aj_enqueued_jobs.size).to eq(0) + end + + it "calls Synchronizer with the record and crm_name keyword", + :aggregate_failures do + expect(Etlify::Synchronizer).to receive(:call).with( + user, + crm_name: :hubspot + ).and_return(:synced) + + described_class.perform_later("User", user.id, "hubspot") + aj_perform_enqueued_jobs + + expect(aj_enqueued_jobs).to be_empty + expect( + Etlify.config.cache_store.exist?(lock_key_for("User", user.id)) + ).to be(false) + end + + it "retries on StandardError and leaves a scheduled retry, while " \ + "keeping a fresh lock for that retry", :aggregate_failures do + allow(Etlify::Synchronizer).to receive(:call).and_raise(StandardError) + + key = lock_key_for("User", user.id) + cache.delete(key) + + described_class.perform_later("User", user.id, crm_name) + expect(cache.exist?(key)).to be(true) + + # Perform immediate job; the perform will fail, retry_on schedules a retry. + # around_perform clears the lock for the *initial* run, then + # around_enqueue of the retry sets it again. + aj_perform_enqueued_jobs + + # A retry should be scheduled (with :at) and the lock should be present + # for that scheduled retry. + scheduled = aj_enqueued_jobs.select { |j| j[:job] == described_class } + expect(scheduled.size).to eq(1) + expect(scheduled.first[:args]).to eq(["User", user.id, crm_name]) + expect(scheduled.first[:at]).to be_a(Numeric) + + # The lock remains because the retry was enqueued and around_enqueue ran. + expect(cache.exist?(key)).to be(true) + end + + it "re-enqueues after TTL expiry", :aggregate_failures do + key = lock_key_for("User", user.id) + cache.delete(key) + + described_class.perform_later("User", user.id, "hubspot") + expect(aj_enqueued_jobs.size).to eq(1) + + # Attempt before TTL expiry -> dropped + described_class.perform_later("User", user.id, "hubspot") + expect(aj_enqueued_jobs.size).to eq(1) + + # After TTL -> allowed + travel 16.minutes do + described_class.perform_later("User", user.id, "hubspot") + end + expect(aj_enqueued_jobs.size).to eq(2) + end +end diff --git a/spec/jobs/sync_jobs_spec.rb b/spec/jobs/sync_jobs_spec.rb deleted file mode 100644 index c23ef9c..0000000 --- a/spec/jobs/sync_jobs_spec.rb +++ /dev/null @@ -1,69 +0,0 @@ -require "rails_helper" -require "active_job/test_helper" -require "active_support/cache" - -RSpec.describe Etlify::SyncJob do - include ActiveJob::TestHelper - - before do - ActiveJob::Base.queue_adapter = :test - clear_enqueued_jobs - clear_performed_jobs - - Etlify.config.cache_store = ActiveSupport::Cache::MemoryStore.new - Etlify.config.job_queue_name = "low" - - stub_const("User", Class.new do - def self.name = "User" - def self.to_s = name - def self.find_by(id:) = (id == 1 ? Object.new : nil) - end) - end - - let(:lock_key) { "etlify:jobs:sync:User:1" } - - describe "enqueue deduplication via cache lock" do - it( - "enqueues only once while the lock exists, then allows again after perform", - :aggregate_failures - ) do - expect { described_class.perform_later("User", 1) } - .to change { enqueued_jobs.size }.by(1) - expect(Etlify.config.cache_store.read(lock_key)).to eq(1) - expect { described_class.perform_later("User", 1) } - .not_to change { enqueued_jobs.size } - allow(Etlify::Synchronizer).to receive(:call).and_return(:synced) - described_class.perform_now("User", 1) - expect(Etlify.config.cache_store.read(lock_key)).to be_nil - expect { described_class.perform_later("User", 1) } - .to change { enqueued_jobs.size }.by(1) - end - end - - describe "queue name" do - it "uses the configured queue name" do - expect(described_class.new.queue_name).to eq("low") - end - end - - describe "#perform" do - it "calls the synchronizer when the record exists" do - expect(Etlify::Synchronizer).to( - receive(:call).with(instance_of(Object)).and_return(:synced) - ) - described_class.perform_now("User", 1) - end - - it "does nothing when the record does not exist" do - expect(Etlify::Synchronizer).not_to receive(:call) - described_class.perform_now("User", -1) - end - end - - describe "lock TTL safety" do - it "sets a finite TTL on the enqueue lock to avoid stale keys" do - described_class.perform_later("User", 1) - expect(Etlify.config.cache_store.read(lock_key)).to eq(1) - end - end -end diff --git a/spec/lib/stale_records/batch_sync_spec.rb b/spec/lib/stale_records/batch_sync_spec.rb deleted file mode 100644 index ddee0f1..0000000 --- a/spec/lib/stale_records/batch_sync_spec.rb +++ /dev/null @@ -1,206 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -RSpec.describe Etlify::StaleRecords::BatchSync do - before do - ActiveJob::Base.queue_adapter = :test - ActiveJob::Base.queue_adapter.enqueued_jobs.clear - ActiveJob::Base.queue_adapter.performed_jobs.clear - end - - let(:logger_io) { StringIO.new } - let(:logger) { Logger.new(logger_io) } - - # Build a plain ids-only relation for a model. - def ids_relation(model, ids) - model.where(id: ids).select(model.primary_key) - end - - # Stub Finder.call to return a mapping, honoring the `models:` filter. - def stub_finder(mapping) - allow(Etlify::StaleRecords::Finder) - .to receive(:call) { |**kw| - models = kw[:models] - models ? mapping.slice(*models) : mapping - } - end - - describe ".call" do - it "returns aggregated stats from the instance call", - :aggregate_failures do - allow_any_instance_of(described_class) - .to receive(:call) - .and_return({ total: 0, per_model: {}, errors: 0 }) - - stats = described_class.call - expect(stats).to eq(total: 0, per_model: {}, errors: 0) - end - end - - describe "#call (integration over Finder mapping)" do - context "when dry_run is true" do - it "counts stale ids without enqueuing or performing", - :aggregate_failures do - users = 3.times.map do |i| - User.create!(email: "u#{i}@ex.com", full_name: "User #{i}") - end - stub_finder(User => ids_relation(User, users.map(&:id))) - - allow(Etlify::Synchronizer).to receive(:call) - expect do - described_class.call( - async: true, - dry_run: true, - batch_size: 2, - logger: logger - ) - end.not_to change { - ActiveJob::Base.queue_adapter.enqueued_jobs.size - } - - stats = described_class.call( - async: true, - dry_run: true, - batch_size: 2, - logger: logger - ) - - expect(stats[:total]).to eq(3) - expect(stats[:errors]).to eq(0) - expect(stats[:per_model]).to eq("User" => 3) - expect(Etlify::Synchronizer).not_to have_received(:call) - end - end - - context "when async is true" do - it "enqueues one job per id and returns counts", - :aggregate_failures do - users = 5.times.map do |i| - User.create!(email: "u#{i}@ex.com", full_name: "User #{i}") - end - stub_finder(User => ids_relation(User, users.map(&:id))) - - expect do - stats = described_class.call( - async: true, - batch_size: 2, - logger: logger - ) - expect(stats[:total]).to eq(5) - expect(stats[:errors]).to eq(0) - expect(stats[:per_model]).to eq("User" => 5) - end.to change { - ActiveJob::Base.queue_adapter.enqueued_jobs.size - }.by(5) - - job = Etlify.config.sync_job_class - enq = ActiveJob::Base.queue_adapter.enqueued_jobs - - expect(enq.map { |j| j[:job].to_s }.uniq).to include(job) - - args = enq.map { |j| j[:args] } - expect(args).to all( - match([a_string_matching("User"), a_kind_of(Integer)]) - ) - end - - it "logs and re-raises when enqueue fails at batch level", - :aggregate_failures do - user = User.create!(email: "x@ex.com", full_name: "X") - stub_finder(User => ids_relation(User, [user.id])) - - stub_const("DummyJob", Class.new) - allow(DummyJob).to receive(:perform_later) - .and_raise(StandardError, "boom") - Etlify.config.sync_job_class = "DummyJob" - - expect do - described_class.call(async: true, logger: logger) - end.to raise_error(StandardError, "boom") - - expect(logger_io.string) - .to include("[Etlify] enqueue failure for User: boom") - ensure - Etlify.config.sync_job_class = "Etlify::SyncJob" - end - end - - context "when async is false (inline)" do - it "calls Synchronizer for each record and returns counts", - :aggregate_failures do - users = 4.times.map do |i| - User.create!(email: "u#{i}@ex.com", full_name: "User #{i}") - end - stub_finder(User => ids_relation(User, users.map(&:id))) - - allow(Etlify::Synchronizer).to receive(:call) - allow_any_instance_of(described_class).to receive(:sleep) - - stats = described_class.call( - async: false, - batch_size: 3, - throttle: 0.01, - logger: logger - ) - - expect(Etlify::Synchronizer).to have_received(:call).exactly(4).times - expect(stats[:total]).to eq(4) - expect(stats[:errors]).to eq(0) - expect(stats[:per_model]).to eq("User" => 4) - end - - it "counts errors without incrementing success count", - :aggregate_failures do - users = 3.times.map do |i| - User.create!(email: "u#{i}@ex.com", full_name: "User #{i}") - end - stub_finder(User => ids_relation(User, users.map(&:id))) - - failing_id = users.first.id - call_stub = lambda do |rec| - raise(StandardError, "sync failed") if rec.id == failing_id - end - allow(Etlify::Synchronizer).to receive(:call) { |rec| call_stub.call(rec) } - allow_any_instance_of(described_class).to receive(:sleep) - - stats = described_class.call( - async: false, - batch_size: 10, - logger: logger - ) - - expect(stats[:total]).to eq(2) - expect(stats[:errors]).to eq(1) - expect(stats[:per_model]).to eq("User" => 2) - expect(logger_io.string) - .to include("[Etlify] sync failure User(id=#{failing_id}):") - end - end - - context "with multiple models and model filtering" do - it "aggregates per model and honors the models filter", - :aggregate_failures do - c1 = Company.create!(name: "C1", domain: "c1.example") - c2 = Company.create!(name: "C2", domain: "c2.example") - u1 = User.create!(email: "u1@ex.com", full_name: "U1", company: c1) - u2 = User.create!(email: "u2@ex.com", full_name: "U2", company: c2) - - full_map = { - Company => ids_relation(Company, [c1.id, c2.id]), - User => ids_relation(User, [u1.id, u2.id]) - } - stub_finder(full_map) - - expect do - stats = described_class.call(async: true, batch_size: 2, logger: logger) - expect(stats[:total]).to eq(4) - expect(stats[:errors]).to eq(0) - expect(stats[:per_model]).to eq("Company" => 2, "User" => 2) - end.to change { - ActiveJob::Base.queue_adapter.enqueued_jobs.size - }.by(4) - end - end - end -end diff --git a/spec/lib/stale_records/finder_spec.rb b/spec/lib/stale_records/finder_spec.rb deleted file mode 100644 index c633546..0000000 --- a/spec/lib/stale_records/finder_spec.rb +++ /dev/null @@ -1,212 +0,0 @@ -require "rails_helper" - -RSpec.describe Etlify::StaleRecords::Finder do - include_context "with companies and users" - - def ids_for(rel) - rel.pluck(rel.klass.primary_key) - end - - describe ".call" do - it "returns a Hash of { ModelClass => Relation }", :aggregate_failures do - result = described_class.call - expect(result).to be_a(Hash) - expect(result.keys).to include(Company, User) - expect(result[Company]).to be_a(ActiveRecord::Relation) - expect(result[User]).to be_a(ActiveRecord::Relation) - end - - it "lists records with no crm_synchronisation as stale", :aggregate_failures do - result = described_class.call - expect(ids_for(result[Company])).to contain_exactly(company.id) - expect(ids_for(result[User])).to contain_exactly(user.id) - end - - it "excludes records whose sync is up to date vs self updated_at" do - Timecop.freeze do - synced_at = Time.current - create_sync_for!(company, last_synced_at: synced_at) - create_sync_for!(user, last_synced_at: synced_at) - - result = described_class.call - - aggregate_failures do - expect(ids_for(result[Company])).to be_empty - expect(ids_for(result[User])).to be_empty - end - end - end - - it "includes records when self updated_at is newer than last_synced_at" do - Timecop.freeze do - create_sync_for!(company, last_synced_at: Time.current) - create_sync_for!(user, last_synced_at: 2.hours.ago) - - user.touch - - result = described_class.call - - aggregate_failures do - expect(ids_for(result[Company])).to be_empty - expect(ids_for(result[User])).to contain_exactly(user.id) - end - end - end - - it "includes records when a dependency updated_at is newer (User depends on Company)" do - Timecop.freeze do - synced_at = 2.hours.ago - create_sync_for!(company, last_synced_at: synced_at) - create_sync_for!(user, last_synced_at: synced_at) - - company.update!(name: "CapSens Updated") - - result = described_class.call - expect(ids_for(result[Company])).to contain_exactly(company.id) - expect(ids_for(result[User])).to contain_exactly(user.id) - end - end - - it "selects only the id column in relations (memory efficient)" do - result = described_class.call - sql = result[User].to_sql - expect(sql).to match(/\ASELECT\s+"users"\."id"\s+FROM\s+"users"/i) - end - - it "respects models: option to restrict searched models" do - result = described_class.call(models: [User]) - expect(result.keys).to contain_exactly(User) - end - - context "when a record has a recent sync but dependency changes later" do - it "marks it stale based on the greatest updated_at among deps" do - Timecop.freeze do - create_sync_for!(user, last_synced_at: 1.hour.ago) - create_sync_for!(company, last_synced_at: 1.hour.ago) - - company.touch - - result = described_class.call - expect(ids_for(result[User])).to contain_exactly(user.id) - end - end - end - - context "when a record regains freshness after sync" do - it "excludes it after last_synced_at catches up" do - Timecop.freeze do - expect(ids_for(described_class.call[User])).to contain_exactly(user.id) - - create_sync_for!(user, last_synced_at: Time.current) - - result = described_class.call - expect(ids_for(result[User])).to be_empty - end - end - end - - # --- discovery: only etlified models are listed ----------------------------- - it "discovers only models that were etlified", :aggregate_failures do - class Widget < ActiveRecord::Base; end rescue nil - # Widget is not defined in schema; we only assert that not all descendants - # are returned, but known etlified models are. - result = described_class.call - expect(result.keys).to include(User, Company, Task) - # Should not contain arbitrary non-etlified classes - expect(result.keys.grep(Class).all? { |k| k.respond_to?(:etlify_crm_object_type) }).to be true - end - - # --- has_many direct dependency (Company depends on users) ------------------- - it "marks Company stale when a dependent user updates (direct has_many)" do - Timecop.freeze do - # Fresh syncs for both - create_sync_for!(company, last_synced_at: Time.current) - create_sync_for!(user, last_synced_at: Time.current) - Timecop.travel(1.second.from_now) - - # Update a user only -> company should become stale due to deps [:users] - user.update!(full_name: "John V2") - result = described_class.call - expect(ids_for(result[Company])).to contain_exactly(company.id) - end - end - - # --- has_many :through dependency (User depends on teams) -------------------- - it "marks User stale when a through dependency changes (teams via memberships)" do - Timecop.freeze do - team = Team.create!(name: "Core") - Membership.create!(user: user, team: team) - create_sync_for!(user, last_synced_at: Time.current) - Timecop.travel(1.second.from_now) - - # Change the through target (team) -> should stale the user - team.update!(name: "Core v2") - - result = described_class.call - expect(ids_for(result[User])).to contain_exactly(user.id) - end - end - - # --- polymorphic belongs_to dependency (Task depends on owner: Company/User) - - it "marks Task stale when its polymorphic owner changes (owner_type Company)" do - Timecop.freeze do - task = Task.create!(title: "T1", owner: company) - create_sync_for!(task, last_synced_at: Time.current) - Timecop.travel(1.second.from_now) - - # Change company -> task becomes stale due to polymorphic belongs_to - company.touch - result = described_class.call - expect(ids_for(result[Task])).to contain_exactly(task.id) - end - end - - it "polymorphic owner types are handled per concrete class (owner_type User)" do - Timecop.freeze do - task = Task.create!(title: "T2", owner: user) - create_sync_for!(task, last_synced_at: Time.current) - Timecop.travel(1.second.from_now) - - user.touch - result = described_class.call - expect(ids_for(result[Task])).to include(task.id) - end - end - - # --- id-only selection also for Company ------------------------------------- - it "selects only id for Company as well" do - result = described_class.call - sql = result[Company].to_sql - expect(sql).to match(/\ASELECT\s+"companies"\."id"\s+FROM\s+"companies"/i) - end - - # --- adapter portability helpers (greatest / epoch_literal) ------------------ - # We hit the private class methods via .send and a tiny connection double. - class ConnDouble - def initialize(name); @name = name; end - def adapter_name; @name; end - def quote_table_name(x); %("#{x}"); end - def quote_column_name(x); %("#{x}"); end - end - - it "uses MAX on non-Postgres adapters and GREATEST on Postgres", :aggregate_failures do - parts = ["1", "2"] - expect(described_class.send(:greatest, parts, ConnDouble.new("SQLite"))).to eq("MAX(1, 2)") - expect(described_class.send(:greatest, parts, ConnDouble.new("PostgreSQL"))) - .to eq("GREATEST(1, 2)") - end - - it "returns the single part unwrapped to avoid SQLite aggregate misuse" do - single = ["42"] - expect(described_class.send(:greatest, single, ConnDouble.new("SQLite"))) - .to eq("42") - end - - it "uses adapter-specific epoch literals", :aggregate_failures do - expect(described_class.send(:epoch_literal, ConnDouble.new("PostgreSQL"))) - .to eq("TIMESTAMP '1970-01-01 00:00:00'") - expect(described_class.send(:epoch_literal, ConnDouble.new("SQLite"))) - .to eq("DATETIME('1970-01-01 00:00:00')") - end - end -end diff --git a/spec/models/crm_synchronisation_spec.rb b/spec/models/crm_synchronisation_spec.rb index ec65ec3..21a76ad 100644 --- a/spec/models/crm_synchronisation_spec.rb +++ b/spec/models/crm_synchronisation_spec.rb @@ -1,9 +1,156 @@ +# frozen_string_literal: true + require "rails_helper" RSpec.describe CrmSynchronisation, type: :model do - it "is stale when the digest differs", :aggregate_failures do - line = described_class.new(last_digest: "abc") - expect(line.stale?("xyz")).to be true - expect(line.stale?("abc")).to be false + let(:company) do + Company.create!( + name: "CapSens", + domain: "capsens.eu" + ) + end + + let(:user) do + User.create!( + email: "dev@capsens.eu", + full_name: "Emo-gilles", + company: company + ) + end + + describe "associations" do + it "belongs to a polymorphic resource" do + sync = described_class.create!( + resource: user, + crm_name: "hubspot", + crm_id: "crm-1" + ) + expect(sync.resource).to eq(user) + expect(sync.resource_type).to eq("User") + expect(sync.resource_id).to eq(user.id) + end + end + + describe "validations" do + it "requires resource_type and resource_id" do + sync = described_class.new( + crm_name: "hubspot" + ) + expect(sync).not_to be_valid + expect(sync.errors[:resource_type]).to be_present + expect(sync.errors[:resource_id]).to be_present + end + + it "enforces crm_id uniqueness but allows nil" do + # Unicité de crm_id (valeur non nulle) + described_class.create!( + resource: user, + crm_name: "hubspot", + crm_id: "dup-1" + ) + + dup_val = described_class.new( + resource: company, + crm_name: "hubspot", + crm_id: "dup-1" + ) + expect(dup_val).not_to be_valid + expect(dup_val.errors[:crm_id]).to be_present + + # Nil est autorisé sur crm_id (avec des resources différentes) + user2 = User.create!( + email: "other@capsens.eu", + full_name: "Autre", + company: company + ) + company2 = Company.create!( + name: "OtherCo", + domain: "other.tld" + ) + + a = described_class.new( + resource: user2, + crm_name: "hubspot", + crm_id: nil + ) + b = described_class.new( + resource: company2, + crm_name: "hubspot", + crm_id: nil + ) + expect(a).to be_valid + expect(b).to be_valid + end + + it "enforces resource_id uniqueness scoped to resource_type" do + described_class.create!( + resource: user, + crm_name: "hubspot", + crm_id: "u-1" + ) + + dup_same_resource = described_class.new( + resource: user, + crm_name: "hubspot", + crm_id: "u-2" + ) + expect(dup_same_resource).not_to be_valid + expect(dup_same_resource.errors[:resource_id]).to be_present + + # Même id numérique mais type différent : OK + # Pour éviter une collision de PK, on crée d'abord un user SANS company, + # puis on crée une company avec le même id que ce user. + lonely_user = User.create!( + email: "lonely@capsens.eu", + full_name: "Lonely", + company: nil + ) + + other_company = Company.create!( + id: lonely_user.id, + name: "Other", + domain: "other.tld" + ) + + ok = described_class.new( + resource: other_company, + crm_name: "hubspot", + crm_id: "c-1" + ) + expect(ok).to be_valid + end + end + + describe "#stale?" do + it "returns true when digests differ, false when equal" do + sync = described_class.create!( + resource: user, + crm_name: "hubspot", + crm_id: "sync-1", + last_digest: "OLD" + ) + expect(sync.stale?("NEW")).to eq(true) + expect(sync.stale?("OLD")).to eq(false) + end + end + + describe "scopes" do + it ".with_error returns only rows with last_error and " \ + ".without_error the inverse" do + ok = described_class.create!( + resource: user, + crm_name: "hubspot", + crm_id: "ok-1", + last_error: nil + ) + bad = described_class.create!( + resource: company, + crm_name: "hubspot", + crm_id: "ko-1", + last_error: "boom" + ) + expect(described_class.with_error).to eq([bad]) + expect(described_class.without_error).to eq([ok]) + end end end diff --git a/spec/models/etlify_model_spec.rb b/spec/models/etlify_model_spec.rb deleted file mode 100644 index 0e2c4d6..0000000 --- a/spec/models/etlify_model_spec.rb +++ /dev/null @@ -1,149 +0,0 @@ -require "rails_helper" - -RSpec.describe Etlify::Model do - include ActiveJob::TestHelper - - before do - ActiveJob::Base.queue_adapter = :test - clear_enqueued_jobs - clear_performed_jobs - end - - module Etlify - module Serializers - class TestUserSerializer < BaseSerializer - def as_crm_payload - { id: record.id, email: record.email } - end - end - end - end - - class TestUser < ActiveRecord::Base - self.table_name = "users" - include Etlify::Model - belongs_to :company, optional: true - - etlified_with( - serializer: Etlify::Serializers::TestUserSerializer, - crm_object_type: "contacts", - id_property: :id, - sync_if: ->(u) { u.email.present? } - ) - - def crm_object_type - "contacts" - end - end - - class GuardedUser < ActiveRecord::Base - self.table_name = "users" - include Etlify::Model - - etlified_with( - serializer: Etlify::Serializers::TestUserSerializer, - crm_object_type: "contacts", - id_property: :id, - sync_if: ->(_u) { false } - ) - - def crm_object_type - "contacts" - end - end - - let!(:user) { TestUser.create!(email: "john@example.com", full_name: "John") } - - describe ".crm_synced" do - it "declares the class_attributes and the has_one association correctly" do - expect(TestUser.respond_to?(:etlify_serializer)).to be true - expect(TestUser.etlify_serializer).to eq(Etlify::Serializers::TestUserSerializer) - - expect(TestUser.respond_to?(:etlify_guard)).to be true - expect(TestUser.etlify_guard).to be_a(Proc) - - reflection = TestUser.reflect_on_association(:crm_synchronisation) - expect(reflection.macro).to eq(:has_one) - expect(reflection.options[:as]).to eq(:resource) - expect(reflection.options[:dependent]).to eq(:destroy) - expect(reflection.options[:class_name]).to eq("CrmSynchronisation") - end - end - - describe "#crm_synced?" do - it "returns false without a sync record, then true after creation" do - expect(user.crm_synced?).to be false - - CrmSynchronisation.create!( - resource_type: "TestUser", - resource_id: user.id - ) - - expect(user.reload.crm_synced?).to be true - end - end - - describe "#build_crm_payload" do - it "uses the configured serializer and returns a stable Hash" do - payload = user.build_crm_payload - expect(payload).to include(id: user.id, email: "john@example.com") - end - - it "raises an error if crm_synced is not configured (documentation test)", :aggregate_failures do - klass = Class.new(ActiveRecord::Base) do - self.table_name = "users" - include Etlify::Model - # Note: no call to crm_synced here - end - - rec = klass.create!(email: "nope@example.com", full_name: "Nope") - - expect { - rec.build_crm_payload - }.to raise_error(ArgumentError, /crm_synced not configured/) - end - end - - describe "#crm_sync!" do - it "enqueues a job when async=true (default)", :aggregate_failures do - expect { - user.crm_sync! # async defaults to true - }.to change { ActiveJob::Base.queue_adapter.enqueued_jobs.size }.by(1) - - job = ActiveJob::Base.queue_adapter.enqueued_jobs.last - expect(job[:job].to_s).to eq(Etlify.config.sync_job_class.to_s) - expect(job[:args]).to include("TestUser", user.id) - end - - it "calls the synchronizer inline when async=false" do - allow(Etlify::Synchronizer).to receive(:call).and_return(:synced) - result = user.crm_sync!(async: false) - expect(result).to eq(:synced) - expect(Etlify::Synchronizer).to have_received(:call).with(instance_of(TestUser)) - end - - it "respects the guard and does not sync if sync_if returns false" do - guarded = GuardedUser.create!(email: "guarded@example.com", full_name: "G") - allow(Etlify::Synchronizer).to receive(:call) - - expect(guarded.crm_sync!(async: false)).to be false - expect(Etlify::Synchronizer).not_to have_received(:call) - expect { - guarded.crm_sync! # async true - }.not_to change { ActiveJob::Base.queue_adapter.enqueued_jobs.size } - end - end - - describe "#crm_delete!" do - it "delegates to Etlify::Deleter" do - unless defined?(Etlify::Deleter) - stub_const("Etlify::Deleter", Class.new do - def self.call(_); :deleted; end - end) - end - - expect(Etlify::Deleter).to receive(:call).with(instance_of(TestUser)).and_return(:deleted) - expect(user.crm_delete!).to eq(:deleted) - end - end -end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index ecd5c6e..8357d31 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -1,128 +1,45 @@ +# frozen_string_literal: true + require "simplecov" -SimpleCov.start +SimpleCov.start "rails" + +require "bundler/setup" -require "spec_helper" +require "rails" require "active_record" require "active_job" -require "rspec" -require "timecop" +require "logger" +require "active_support" +require "active_support/time" # for Time.current / time zone +require "support/time_helpers" +require "support/aj_test_adapter_helpers" -require_relative "../lib/etlify" +require "etlify" class ApplicationRecord < ActiveRecord::Base self.abstract_class = true end require_relative "../app/models/crm_synchronisation" -require_relative "../app/jobs/etlify/sync_job" -require_relative "../lib/etlify/serializers/user_serializer" -require_relative "../lib/etlify/serializers/company_serializer" - -require_relative "./factories" - -# DB en mémoire -ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: ":memory:") -ActiveRecord::Schema.verbose = false - -# Schéma minimal pour tests -ActiveRecord::Schema.define do - create_table :companies, force: true do |t| - t.string :name - t.string :domain - t.references :user, foreign_key: true - t.timestamps - end - - create_table :users, force: true do |t| - t.string :email - t.string :full_name - t.references :company - t.timestamps - end - - create_table :crm_synchronisations, force: true do |t| - t.string :crm_id - t.string :resource_type, null: false - t.bigint :resource_id, null: false - t.string :last_digest - t.datetime :last_synced_at - t.string :last_error - t.timestamps - end +require "etlify/serializers/base_serializer" +require "etlify/serializers/user_serializer" +require "etlify/serializers/company_serializer" - create_table :teams, force: true do |t| - t.string :name - t.timestamps - end - - create_table :memberships, force: true do |t| - t.references :user, null: false - t.references :team, null: false - t.timestamps - end - - create_table :tasks, force: true do |t| - t.string :title - t.string :owner_type - t.bigint :owner_id - t.timestamps +RSpec.configure do |config| + config.include RSpecTimeHelpers + config.include AJTestAdapterHelpers + config.order = :random + Kernel.srand config.seed + + # Use transactions for a clean state + config.around(:each) do |example| + ActiveRecord::Base.connection.transaction do + example.run + raise ActiveRecord::Rollback + end end - add_index :crm_synchronisations, :crm_id, unique: true - add_index :crm_synchronisations, %i[resource_type resource_id], unique: true - add_index :crm_synchronisations, :last_digest - add_index :crm_synchronisations, :last_synced_at -end - -# Dummy models -class Company < ActiveRecord::Base - has_many :users, dependent: :nullify - include Etlify::Model - etlified_with( - serializer: Etlify::Serializers::CompanySerializer, - crm_object_type: "companies", - id_property: :id, - dependencies: [:users] - ) -end - -class User < ActiveRecord::Base - include Etlify::Model - belongs_to :company, optional: true - has_many :memberships, dependent: :destroy - has_many :teams, through: :memberships - - etlified_with( - serializer: Etlify::Serializers::UserSerializer, - crm_object_type: "contacts", - id_property: :id, - dependencies: [:company, :teams] - ) -end - -class Team < ActiveRecord::Base - has_many :memberships, dependent: :destroy - has_many :users, through: :memberships -end - -class Membership < ActiveRecord::Base - belongs_to :user - belongs_to :team -end - -class Task < ActiveRecord::Base - belongs_to :owner, polymorphic: true, optional: true - - include Etlify::Model - etlified_with( - serializer: Etlify::Serializers::BaseSerializer, # not used by specs - crm_object_type: "tasks", - id_property: :id, - dependencies: [:owner] # <-- triggers polymorphic branch - ) -end - -RSpec.configure do |config| + # suppress ActiveJob and Thor output ActiveJob::Base.logger = Logger.new(nil) config.before(type: :generator) do allow_any_instance_of(Thor::Shell::Basic).to( @@ -132,12 +49,62 @@ class Task < ActiveRecord::Base receive(:say) ) end - config.before(:each) do - CrmSynchronisation.delete_all - Membership.delete_all - Team.delete_all - Task.delete_all - User.delete_all - Company.delete_all + + # Setup in-memory SQLite once + config.before(:suite) do + ActiveRecord::Base.establish_connection( + adapter: "sqlite3", + database: ":memory:" + ) + ActiveRecord::Migration.verbose = false + + ActiveRecord::Schema.define do + create_table :crm_synchronisations, force: true do |t| + t.string :crm_name, null: false + t.string :crm_id + t.string :last_digest + t.datetime :last_synced_at + t.text :last_error + t.string :resource_type, null: false + t.integer :resource_id, null: false + t.timestamps + end + + create_table :companies, force: true do |t| + t.string :name + t.string :domain + t.timestamps + end + + create_table :users, force: true do |t| + t.string :email + t.string :full_name + t.integer :company_id + t.timestamps + end + end + + class Company < ApplicationRecord + has_many :crm_synchronisations, as: :resource, dependent: :destroy + end + + class User < ApplicationRecord + belongs_to :company, optional: true + has_many :crm_synchronisations, as: :resource, dependent: :destroy + + def build_crm_payload(crm_name:) + Etlify::Serializers::UserSerializer.new(self).as_crm_payload + end + + def self.etlify_crms + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "id", + crm_object_type: "contacts", + }, + } + end + end end end diff --git a/spec/serializers/base_serializer_spec.rb b/spec/serializers/base_serializer_spec.rb new file mode 100644 index 0000000..7ba183d --- /dev/null +++ b/spec/serializers/base_serializer_spec.rb @@ -0,0 +1,8 @@ +require "rails_helper" + +RSpec.describe Etlify::Serializers::BaseSerializer do + it "requires subclass to implement as_crm_payload" do + dummy = Class.new(described_class).new(double("rec")) + expect { dummy.as_crm_payload }.to raise_error(NotImplementedError) + end +end diff --git a/spec/services/deleter_spec.rb b/spec/services/deleter_spec.rb deleted file mode 100644 index 2c86a9a..0000000 --- a/spec/services/deleter_spec.rb +++ /dev/null @@ -1,23 +0,0 @@ -require "rails_helper" - -RSpec.describe Etlify::Deleter do - include_context "with companies and users" - - it "deletes on the CRM side if crm_id is present" do - sync = user.create_crm_synchronisation!( - crm_id: "crm-42", - resource_type: "User", - resource_id: user.id - ) - adapter = instance_double("Adapter") - expect(adapter).to receive(:delete!).with(crm_id: "crm-42") - allow(Etlify.config).to receive(:crm_adapter).and_return(adapter) - - expect(described_class.call(user)).to eq(:deleted) - end - - it "noop if no crm_id" do - user.create_crm_synchronisation!(resource_type: "User", resource_id: user.id) - expect(described_class.call(user)).to eq(:noop) - end -end diff --git a/spec/services/synchronizer_spec.rb b/spec/services/synchronizer_spec.rb deleted file mode 100644 index f995dbb..0000000 --- a/spec/services/synchronizer_spec.rb +++ /dev/null @@ -1,112 +0,0 @@ -require "rails_helper" - -RSpec.describe Etlify::Synchronizer do - include_context "with companies and users" - - it "creates the row and updates the digest", :aggregate_failures do - adapter = instance_double("Adapter", upsert!: "crm-123") - allow(Etlify.config).to receive(:crm_adapter).and_return(adapter) - - expect { described_class.call(user) } - .to change { CrmSynchronisation.count }.by(1) - - sync = user.crm_synchronisation - expect(sync.crm_id).to eq("crm-123") - expect(sync.last_digest).to be_present - expect(sync.last_synced_at).to be_present - end - - it "is idempotent if the digest hasn't changed", :aggregate_failures do - frozen_time = Time.current - - Timecop.freeze(frozen_time) do - adapter = instance_double("Adapter") - allow(adapter).to receive(:upsert!).and_return("crm-456") - allow(Etlify.config).to receive(:crm_adapter).and_return(adapter) - first = described_class.call(user) - - expect(first).to eq(:synced) - sync = user.crm_synchronisation - expect(sync.last_synced_at).to eq(frozen_time) - end - - Timecop.freeze(frozen_time + 1.second) do - second = described_class.call(user) - sync = user.crm_synchronisation - expect(second).to eq(:not_modified) - expect(sync.last_synced_at).to eq(frozen_time + 1.second) - end - end - - it "records the error on the sync_line when the adapter fails", :aggregate_failures do - # Adapter raises an API-level error - adapter = instance_double("Adapter") - allow(adapter).to receive(:upsert!).and_raise( - Etlify::ApiError.new("Upsert failed", status: 500) - ) - allow(Etlify.config).to receive(:crm_adapter).and_return(adapter) - - # It should not raise; it should create the sync row and persist the error - expect { - described_class.call(user) - }.not_to raise_error - - sync = user.reload.crm_synchronisation - expect(sync).to be_present - expect(sync.last_error).to eq("Upsert failed") - expect(sync.crm_id).to be_nil - expect(sync.last_synced_at).to be_nil - expect(sync.last_digest).to be_nil - end - - it "purges any previous error when succeeding", :aggregate_failures do - adapter = instance_double("Adapter", upsert!: "crm-789") - allow(Etlify.config).to receive(:crm_adapter).and_return(adapter) - - user.create_crm_synchronisation! - user.crm_synchronisation.update!( - last_error: "Previous error", - ) - - expect(user.crm_synchronisation.last_error).to eq("Previous error") - described_class.call(user) - expect(user.reload.crm_synchronisation.last_error).to be_nil - end - - it( - "does not overwrite previous successful state, only updates last_error on failure", - :aggregate_failures - ) do - # 1) First run succeeds - ok_adapter = instance_double("Adapter", upsert!: "crm-xyz") - allow(Etlify.config).to receive(:crm_adapter).and_return(ok_adapter) - - expect { described_class.call(user) } - .to change { CrmSynchronisation.count }.by(1) - - sync_before = user.reload.crm_synchronisation - expect(sync_before.crm_id).to eq("crm-xyz") - expect(sync_before.last_digest).to be_present - expect(sync_before.last_synced_at).to be_present - expect(sync_before.last_error).to be_nil - - # 2) Second run fails - failing_adapter = instance_double("Adapter") - allow(failing_adapter).to receive(:upsert!).and_raise( - Etlify::ApiError.new("Network hiccup", status: 429) - ) - allow(Etlify.config).to receive(:crm_adapter).and_return(failing_adapter) - - user.update!(full_name: "John Doe 2") - - expect { - described_class.call(user) - }.not_to raise_error - - sync_after = user.reload.crm_synchronisation - expect(sync_after.crm_id).to eq("crm-xyz") - expect(sync_after.last_digest).to eq(sync_before.last_digest) - expect(sync_after.last_synced_at).to eq(sync_before.last_synced_at) - expect(sync_after.last_error).to eq("Network hiccup") - end -end diff --git a/spec/support/aj_test_adapter_helpers.rb b/spec/support/aj_test_adapter_helpers.rb new file mode 100644 index 0000000..9c01729 --- /dev/null +++ b/spec/support/aj_test_adapter_helpers.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module AJTestAdapterHelpers + def aj_set_test_adapter! + ActiveJob::Base.queue_adapter = + ActiveJob::QueueAdapters::TestAdapter.new + end + + def aj_enqueued_jobs + ActiveJob::Base.queue_adapter.enqueued_jobs + end + + def aj_performed_jobs + ActiveJob::Base.queue_adapter.performed_jobs + end + + def aj_clear_jobs + aj_enqueued_jobs.clear + aj_performed_jobs.clear + end + + # Perform only immediate jobs; scheduled (with :at) are left in queue. + def aj_perform_enqueued_jobs + jobs = aj_enqueued_jobs.dup + aj_enqueued_jobs.clear + jobs.each do |j| + next if j[:at] + + j[:job].perform_now(*j[:args]) + end + end +end diff --git a/spec/support/time_helpers.rb b/spec/support/time_helpers.rb new file mode 100644 index 0000000..1d1f7c1 --- /dev/null +++ b/spec/support/time_helpers.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module RSpecTimeHelpers + # Internal stack to support nested freeze/travel + def __time_stack + @__rspec_time_stack ||= [] + end + + # Freeze to a specific moment. Works with or without a block. + def freeze_time(moment = Time.now, &blk) + travel_to(moment, &blk) + end + + # Travel to a specific moment. If block given, auto-restore afterwards. + def travel_to(moment) + push_time_stub(moment) + if block_given? + begin + yield + ensure + travel_back + end + end + end + + # Travel by a duration (Numeric seconds or ActiveSupport::Duration). + # If block given, auto-restore afterwards; else, stays until travel_back. + def travel(duration, &blk) + travel_to(Time.now + duration, &blk) + end + + # Undo one level of travel/freeze. Restore previous or original behavior. + def travel_back + __time_stack.pop + if (prev = __time_stack.last) + apply_time_stub(prev) + else + remove_time_stub + end + end + + private + + def push_time_stub(moment) + __time_stack << moment + apply_time_stub(moment) + end + + def apply_time_stub(moment) + # Stub Time.now + allow(Time).to receive(:now).and_return(moment) + + # Stub Time.current + allow(Time).to receive(:current).and_return( + (moment.respond_to?(:in_time_zone) ? moment.in_time_zone : moment) + ) + end + + def remove_time_stub + allow(Time).to receive(:now).and_call_original + allow(Time).to receive(:current).and_call_original + end +end From 6f15d9ef1cc7e3609fd4a7599bf3e6c98eb829d5 Mon Sep 17 00:00:00 2001 From: Ismael Boukhars Date: Tue, 26 Aug 2025 11:00:56 +0200 Subject: [PATCH 3/6] [handle-multi-crm] fix: bump coverage --- lib/etlify/model.rb | 30 +- spec/etlify/model_spec.rb | 652 ++++++++++++++++++++++++++------------ 2 files changed, 475 insertions(+), 207 deletions(-) diff --git a/lib/etlify/model.rb b/lib/etlify/model.rb index 2036549..265dd49 100644 --- a/lib/etlify/model.rb +++ b/lib/etlify/model.rb @@ -69,7 +69,8 @@ def define_crm_dsl_on(klass, crm_name) end end - # Define instance helpers: "_build_payload", "_sync!", "_delete!" + # Define instance helpers: "_build_payload", "_sync!", + # "_delete!" def define_crm_instance_helpers_on(klass, crm_name) payload_m = "#{crm_name}_build_payload" sync_m = "#{crm_name}_sync!" @@ -77,19 +78,19 @@ def define_crm_instance_helpers_on(klass, crm_name) unless klass.method_defined?(payload_m) klass.define_method(payload_m) do - build_crm_payload(crm: crm_name) + build_crm_payload(crm_name: crm_name) end end unless klass.method_defined?(sync_m) klass.define_method(sync_m) do |async: true, job_class: nil| - crm_sync!(crm: crm_name, async: async, job_class: job_class) + crm_sync!(crm_name: crm_name, async: async, job_class: job_class) end end unless klass.method_defined?(delete_m) klass.define_method(delete_m) do - crm_delete!(crm: crm_name) + crm_delete!(crm_name: crm_name) end end end @@ -103,17 +104,26 @@ def crm_synced?(crm: nil) crm_synchronisation.present? end - def build_crm_payload(crm_name:) + # Accept both crm_name: and crm: for backward compatibility. + def build_crm_payload(crm_name: nil, crm: nil) + crm_name ||= crm + raise ArgumentError, "crm_name is required" if crm_name.nil? + raise_unless_crm_is_configured(crm_name) conf = self.class.etlify_crms.fetch(crm_name.to_sym) conf[:serializer].new(self).as_crm_payload end - # @param crm [Symbol] which CRM to use + # @param crm_name [Symbol] which CRM to use # @param async [Boolean] whether to enqueue or run inline # @param job_class [Class,String,nil] explicit override - def crm_sync!(crm_name:, async: true, job_class: nil) + # + # Accept both crm_name: and crm: for backward compatibility. + def crm_sync!(crm_name: nil, async: true, job_class: nil, crm: nil) + crm_name ||= crm + raise ArgumentError, "crm_name is required" if crm_name.nil? + return false unless allow_sync_for?(crm_name) if async @@ -130,7 +140,11 @@ def crm_sync!(crm_name:, async: true, job_class: nil) end end - def crm_delete!(crm_name:) + # Accept both crm_name: and crm: for backward compatibility. + def crm_delete!(crm_name: nil, crm: nil) + crm_name ||= crm + raise ArgumentError, "crm_name is required" if crm_name.nil? + Etlify::Deleter.call(self, crm_name: crm_name) end diff --git a/spec/etlify/model_spec.rb b/spec/etlify/model_spec.rb index ea81db1..7dfe991 100644 --- a/spec/etlify/model_spec.rb +++ b/spec/etlify/model_spec.rb @@ -1,275 +1,529 @@ -# frozen_string_literal: true - require "rails_helper" RSpec.describe Etlify::Model do - # -- Test doubles ----------------------------------------------------------- - class TestAdapter - def initialize(*) - end - end - - class TestSerializer - def initialize(record) - @record = record - end - - def as_crm_payload - {id: @record.id, kind: @record.class.name} - end + # Simple fake adapter used by the registry + let(:dummy_adapter) do + Class.new end - class AltJob - class << self - attr_accessor :calls - def perform_later(*args) - (self.calls ||= []) << args + # Minimal serializer returning a hash payload + let(:dummy_serializer) do + Class.new do + def initialize(record) + @record = record end - def reset! - self.calls = [] + def as_crm_payload + {ok: true, id: (@record.respond_to?(:id) ? @record.id : nil)} end end end - # -- Registry isolation ----------------------------------------------------- before do - @registry_backup = Etlify::CRM.registry.dup - Etlify::CRM.registry.clear - end - - after do - Etlify::CRM.registry.clear - Etlify::CRM.registry.merge!(@registry_backup) + # Stub CRM registry for deterministic behavior in all examples + reg_item = Etlify::CRM::RegistryItem.new( + name: :hubspot, + adapter: dummy_adapter, + options: {job_class: "DefaultJobFromRegistry"} + ) + allow(Etlify::CRM).to receive(:fetch).with(:hubspot).and_return(reg_item) + allow(Etlify::CRM).to receive(:names).and_return([:hubspot]) end - # Build a fresh anonymous model class and include the concern each time. - def new_model_class + # Helper: build a plain class including the concern + def build_including_class(&blk) Class.new do include Etlify::Model - attr_reader :id - def initialize(id:) - @id = id - end + class_eval(&blk) if blk end end - # Helper to register a CRM named :alpha. - def register_alpha - Etlify::CRM.register( - :alpha, - adapter: TestAdapter, - options: {job_class: AltJob} - ) + describe "included hook" do + it "tracks including classes and defines class_attribute" do + klass = build_including_class + expect(Etlify::Model.__included_klasses__).to include(klass) + expect(klass.respond_to?(:etlify_crms)).to be true + expect(klass.etlify_crms).to eq({}) + end + + it "defines instance helpers for already-registered CRMs" do + klass = build_including_class + expect(klass.instance_methods).to include(:hubspot_build_payload) + expect(klass.instance_methods).to include(:hubspot_sync!) + expect(klass.instance_methods).to include(:hubspot_delete!) + end end - # Helper to apply the DSL on a given class for :alpha. - def dsl_apply(klass) - klass.alpha_etlified_with( - serializer: TestSerializer, - crm_object_type: "contacts", - id_property: "id", - dependencies: %i[name email], - sync_if: ->(r) { r.id.odd? }, - job_class: nil - ) + describe ". __included_klasses__" do + it "returns the same memoized array across calls" do + arr1 = described_class.__included_klasses__ + arr2 = described_class.__included_klasses__ + expect(arr1.object_id).to eq(arr2.object_id) + end end - describe "inclusion and DSL installation" do - it "installs DSL on include when a CRM is already registered", - :aggregate_failures do - register_alpha - klass = new_model_class + describe ".install_dsl_for_crm" do + it "reinstalls DSL and helpers on all previous classes" do + klass = build_including_class - expect(klass).to respond_to(:alpha_etlified_with) - inst = klass.new(id: 1) - expect(inst).to respond_to(:alpha_build_payload) - expect(inst).to respond_to(:alpha_sync!) - expect(inst).to respond_to(:alpha_delete!) + # Make the call deterministic: only our klass is considered + allow(described_class).to receive(:__included_klasses__) + .and_return([klass]) + + # We expect the installer to call the two Model methods, even if + # they early-return because definitions already exist. + expect(described_class).to receive(:define_crm_dsl_on) + .with(klass, :hubspot).and_call_original + expect(described_class).to receive(:define_crm_instance_helpers_on) + .with(klass, :hubspot).and_call_original + + described_class.install_dsl_for_crm(:hubspot) + + # Methods should still be present after reinstall. + expect(klass.respond_to?(:hubspot_etlified_with)).to be true + expect(klass.instance_methods).to include(:hubspot_build_payload) end + end - it "installs DSL on classes already including the concern when a CRM " \ - "is registered afterwards", :aggregate_failures do - klass = new_model_class - expect(klass).not_to respond_to(:alpha_etlified_with) + describe ".define_crm_dsl_on" do + it "is a no-op if the DSL method already exists" do + klass = Class.new do + def self.hubspot_etlified_with(**) + end + end + expect do + described_class.define_crm_dsl_on(klass, :hubspot) + end.not_to change { + klass.singleton_methods.include?(:hubspot_etlified_with) + } + end - register_alpha + it "defines _etlified_with and stores full configuration" do + klass = build_including_class + described_class.define_crm_dsl_on(klass, :hubspot) + + klass.hubspot_etlified_with( + serializer: dummy_serializer, + crm_object_type: :contact, + id_property: :external_id, + dependencies: %w[company owner], + sync_if: ->(r) { r.respond_to?(:active?) ? r.active? : true }, + job_class: "OverrideJob" + ) - expect(klass).to respond_to(:alpha_etlified_with) - inst = klass.new(id: 1) - expect(inst).to respond_to(:alpha_build_payload) - expect(inst).to respond_to(:alpha_sync!) - expect(inst).to respond_to(:alpha_delete!) + conf = klass.etlify_crms[:hubspot] + expect(conf[:serializer]).to eq(dummy_serializer) + expect(conf[:guard]).to be_a(Proc) + expect(conf[:crm_object_type]).to eq(:contact) + expect(conf[:id_property]).to eq(:external_id) + expect(conf[:dependencies]).to eq(%i[company owner]) + expect(conf[:adapter]).to eq(dummy_adapter) + expect(conf[:job_class]).to eq("OverrideJob") end - it "tracks including classes in __included_klasses__", - :aggregate_failures do - before_list = Etlify::Model.__included_klasses__.dup - klass = new_model_class + it "defaults sync_if to a proc returning true when not provided" do + klass = build_including_class + described_class.define_crm_dsl_on(klass, :hubspot) - expect(Etlify::Model.__included_klasses__).to include(klass) - expect(Etlify::Model.__included_klasses__.size) - .to eq(before_list.size + 1) + # Call DSL without sync_if keyword + klass.hubspot_etlified_with( + serializer: dummy_serializer, + crm_object_type: :contact, + id_property: :external_id + ) + + conf = klass.etlify_crms[:hubspot] + expect(conf[:guard]).to be_a(Proc) + + # By default, it should always return true + expect(conf[:guard].call(double("record"))).to be true + expect(klass.new.send(:allow_sync_for?, :hubspot)).to be true + end + + it "does not clobber other CRM entries in etlify_crms" do + klass = build_including_class + # Use the real writer so the attribute can be updated by the DSL + klass.etlify_crms = {salesforce: {anything: 1}} + + described_class.define_crm_dsl_on(klass, :hubspot) + klass.hubspot_etlified_with( + serializer: dummy_serializer, + crm_object_type: :contact, + id_property: :external_id + ) + + expect(klass.etlify_crms.keys).to include(:salesforce, :hubspot) + end + + it "propagates errors from Etlify::CRM.fetch" do + klass = build_including_class + allow(Etlify::CRM).to receive(:fetch).and_raise("boom") + described_class.define_crm_dsl_on(klass, :hubspot) + expect do + klass.hubspot_etlified_with( + serializer: dummy_serializer, + crm_object_type: :contact, + id_property: :external_id + ) + end.to raise_error(RuntimeError, "boom") end end - describe "DSL method _etlified_with" do - it "stores per-CRM config on class.etlify_crms and symbolized deps", - :aggregate_failures do - register_alpha - klass = new_model_class - - dsl_apply(klass) - - conf = klass.etlify_crms[:alpha] - expect(conf[:serializer]).to eq(TestSerializer) - expect(conf[:crm_object_type]).to eq("contacts") - expect(conf[:id_property]).to eq("id") - expect(conf[:dependencies]).to eq(%i[name email]) - expect(conf[:adapter]).to eq(TestAdapter) - # job_class from registry when nil in DSL - expect(conf[:job_class]).to eq(AltJob) - # guard should be installed - expect(conf[:guard]).to be_a(Proc) + describe ".define_crm_instance_helpers_on" do + it "creates helpers only if missing (idempotent)" do + klass = build_including_class + methods_before = klass.instance_methods.grep(/hubspot_/) + described_class.define_crm_instance_helpers_on(klass, :hubspot) + methods_after = klass.instance_methods.grep(/hubspot_/) + expect(methods_after).to include(*methods_before) + end + + it "delegates payload helper with crm_name keyword" do + # Capture the keywords passed to build_crm_payload + klass = build_including_class do + attr_reader :seen_kw + def build_crm_payload(**kw) + @seen_kw = kw + :ok + end + end + inst = klass.new + expect(inst.hubspot_build_payload).to eq(:ok) + expect(inst.seen_kw).to eq({crm_name: :hubspot}) + end + + it "delegates sync helper with crm_name, async, job_class" do + klass = build_including_class do + attr_reader :seen_sync + def crm_sync!(**kw) + @seen_sync = kw + :done + end + end + inst = klass.new + expect( + inst.hubspot_sync!(async: false, job_class: "X") + ).to eq(:done) + expect(inst.seen_sync).to eq( + {crm_name: :hubspot, async: false, job_class: "X"} + ) + end + + it "delegates delete helper with crm_name keyword" do + klass = build_including_class do + attr_reader :seen_del + def crm_delete!(**kw) + @seen_del = kw + :deleted + end + end + inst = klass.new + expect(inst.hubspot_delete!).to eq(:deleted) + expect(inst.seen_del).to eq({crm_name: :hubspot}) end end - describe "instance helpers creation and delegation" do - it "defines _build_payload / _sync! / _delete!", - :aggregate_failures do - register_alpha - klass = new_model_class - dsl_apply(klass) - inst = klass.new(id: 7) + describe "#crm_synced?" do + it "returns true when crm_synchronisation is present" do + klass = build_including_class do + def crm_synchronisation + :something + end + end + expect(klass.new.crm_synced?(crm: :hubspot)).to be true + end + + it "returns false when crm_synchronisation is nil" do + klass = build_including_class do + def crm_synchronisation + nil + end + end + expect(klass.new.crm_synced?(crm: :hubspot)).to be false + end - expect(inst).to respond_to(:alpha_build_payload) - expect(inst).to respond_to(:alpha_sync!) - expect(inst).to respond_to(:alpha_delete!) + it "ignores the crm: argument (compat quirk)" do + klass = build_including_class do + def crm_synchronisation + :x + end + end + expect(klass.new.crm_synced?(crm: :other)).to be true end + end - it "delegates to build_crm_payload / crm_sync! / crm_delete! with " \ - "the CRM name", :aggregate_failures do - register_alpha - klass = Class.new do - include Etlify::Model - attr_reader :id - def initialize(id:) - @id = id + describe "#build_crm_payload" do + it "raises when CRM is not configured" do + klass = build_including_class + inst = klass.new + expect do + inst.build_crm_payload(crm_name: :hubspot) + end.to raise_error(ArgumentError, /crm not configured/) + end + + it "works with crm_name: and returns serializer payload" do + klass = build_including_class do + def self.etlify_crms + { + hubspot: { + serializer: Class.new do + def initialize(_) + end + + def as_crm_payload + {email: "x@y", ok: true} + end + end, + guard: ->(_r) { true }, + crm_object_type: :contact, + id_property: :external_id, + adapter: Class.new, + }, + } end + end + inst = klass.new + expect(inst.build_crm_payload(crm_name: :hubspot)) + .to eq(email: "x@y", ok: true) + end - # The generated helper calls build_crm_payload(crm: ...) - def build_crm_payload(crm:) - [:payload_called, crm] + it "also accepts legacy crm: keyword" do + klass = build_including_class do + def self.etlify_crms + { + hubspot: { + serializer: Class.new do + def initialize(_) + end + + def as_crm_payload + {legacy: true} + end + end, + guard: ->(_r) { true }, + crm_object_type: :contact, + id_property: :external_id, + adapter: Class.new, + }, + } end + end + inst = klass.new + expect(inst.build_crm_payload(crm: :hubspot)).to eq(legacy: true) + end + end - def crm_sync!(crm:, async:, job_class:) - [:sync_called, crm, async] + describe "#crm_sync!" do + let(:klass) do + build_including_class do + attr_reader :id + def initialize + @id = 42 end - def crm_delete!(crm:) - [:delete_called, crm] + def self.etlify_crms + { + hubspot: { + serializer: Class.new do + def initialize(*) + end + + def as_crm_payload + {} + end +end, + guard: ->(_r) { true }, + crm_object_type: :contact, + id_property: :external_id, + adapter: Class.new, + }, + } end end + end - dsl_apply(klass) - inst = klass.new(id: 3) + it "returns false when guard forbids sync" do + inst = klass.new + allow(inst).to receive(:allow_sync_for?).with(:hubspot).and_return(false) + expect(inst.crm_sync!(crm_name: :hubspot)).to be false + end - expect(inst.alpha_build_payload).to eq([:payload_called, :alpha]) - expect(inst.alpha_sync!(async: false)) - .to eq([:sync_called, :alpha, false]) - expect(inst.alpha_delete!).to eq([:delete_called, :alpha]) + it "enqueues with perform_later when available" do + job = Class.new do + def self.perform_later(*) + end +end + inst = klass.new + allow(inst).to receive(:allow_sync_for?).and_return(true) + allow(inst).to receive(:resolve_job_class_for).and_return(job) + expect(job).to receive(:perform_later) + .with(klass.name, 42, "hubspot") + inst.crm_sync!(crm_name: :hubspot, async: true) end + + it "enqueues with perform_async when available" do + job = Class.new do + def self.perform_async(*) end +end + inst = klass.new + allow(inst).to receive(:allow_sync_for?).and_return(true) + allow(inst).to receive(:resolve_job_class_for).and_return(job) + expect(job).to receive(:perform_async) + .with(klass.name, 42, "hubspot") + inst.crm_sync!(crm_name: :hubspot, async: true) + end - describe "#build_crm_payload via configured serializer" do - it "uses serializer.as_crm_payload(record)", :aggregate_failures do - register_alpha - klass = new_model_class - dsl_apply(klass) - inst = klass.new(id: 11) + it "raises when no job API is available" do + job = Class.new + inst = klass.new + allow(inst).to receive(:allow_sync_for?).and_return(true) + allow(inst).to receive(:resolve_job_class_for).and_return(job) + expect do + inst.crm_sync!(crm_name: :hubspot, async: true) + end.to raise_error(ArgumentError, /No job class available/) + end - out = inst.build_crm_payload(crm_name: :alpha) - expect(out).to eq({id: 11, kind: klass.name}) + it "runs inline with Synchronizer when async: false" do + inst = klass.new + allow(inst).to receive(:allow_sync_for?).and_return(true) + expect(Etlify::Synchronizer).to receive(:call) + .with(inst, crm_name: :hubspot) + inst.crm_sync!(crm_name: :hubspot, async: false) end + + it "accepts job_class override as String and constantizes it" do + job = Class.new do + def self.perform_later(*) end +end + stub_const("MyInlineJob", job) + inst = klass.new + allow(inst).to receive(:allow_sync_for?).and_return(true) + expect(job).to receive(:perform_later) + inst.crm_sync!(crm_name: :hubspot, async: true, job_class: "MyInlineJob") + end - describe "#crm_sync! job dispatch and guards" do - it "returns false when guard denies synchronization", - :aggregate_failures do - register_alpha - klass = new_model_class - # Guard false for even ids - klass.alpha_etlified_with( - serializer: TestSerializer, - crm_object_type: "contacts", - id_property: "id", - dependencies: [], - sync_if: ->(r) { r.id.odd? }, - job_class: nil - ) + it "also accepts legacy crm: keyword" do + job = Class.new do + def self.perform_later(*) + end +end + inst = klass.new + allow(inst).to receive(:allow_sync_for?).and_return(true) + allow(inst).to receive(:resolve_job_class_for).and_return(job) + expect(job).to receive(:perform_later) + .with(klass.name, 42, "hubspot") + inst.crm_sync!(crm: :hubspot, async: true) + end + end - even = klass.new(id: 2) - expect(even.crm_sync!(crm_name: :alpha, async: true)).to eq(false) + describe "#crm_delete!" do + it "delegates to Etlify::Deleter.call" do + klass = build_including_class + inst = klass.new + expect(Etlify::Deleter).to receive(:call).with(inst, crm_name: :hubspot) + inst.crm_delete!(crm_name: :hubspot) end - it "uses override job_class when provided", :aggregate_failures do - register_alpha - klass = new_model_class - dsl_apply(klass) - inst = klass.new(id: 5) + it "also accepts legacy crm: keyword" do + klass = build_including_class + inst = klass.new + expect(Etlify::Deleter).to receive(:call).with(inst, crm_name: :hubspot) + inst.crm_delete!(crm: :hubspot) + end + end - AltJob.reset! - class CustomJob - class << self - attr_accessor :args - def perform_later(*a) - self.args = a - end + describe "#allow_sync_for?" do + it "returns false when CRM conf is missing" do + klass = build_including_class + expect(klass.new.send(:allow_sync_for?, :hubspot)).to be false + end + + it "returns true when guard is nil" do + klass = build_including_class do + def self.etlify_crms + {hubspot: {guard: nil}} end end + expect(klass.new.send(:allow_sync_for?, :hubspot)).to be true + end - inst.crm_sync!( - crm_name: :alpha, - async: true, - job_class: CustomJob - ) + it "evaluates guard proc with the instance" do + klass = build_including_class + outer = klass + klass.define_singleton_method(:etlify_crms) do + {hubspot: {guard: ->(r) { r.is_a?(outer) }}} + end + expect(klass.new.send(:allow_sync_for?, :hubspot)).to be true + end - expect(CustomJob.args).to eq([klass.name, 5, "alpha"]) + it "returns false when guard returns false" do + klass = build_including_class do + def self.etlify_crms + {hubspot: {guard: ->(_r) { false }}} + end + end + expect(klass.new.send(:allow_sync_for?, :hubspot)).to be false end + end - it "falls back to registry job_class when no override is provided", - :aggregate_failures do - register_alpha - klass = new_model_class - dsl_apply(klass) - inst = klass.new(id: 7) + describe "#resolve_job_class_for and #constantize_if_needed" do + it "returns override class as-is" do + klass = build_including_class + job = Class.new + out = klass.new.send(:resolve_job_class_for, :hubspot, override: job) + expect(out).to eq(job) + end - AltJob.reset! - inst.crm_sync!( - crm_name: :alpha, - async: true - ) + it "falls back to Etlify::SyncJob when no override or conf job_class is given" do + klass = build_including_class do + def self.etlify_crms + {hubspot: {}} + end + end + stub_const("Etlify::SyncJob", Class.new) + out = klass.new.send(:resolve_job_class_for, :hubspot, override: nil) + expect(out).to eq(Etlify::SyncJob) + end - expect(AltJob.calls).to eq([[klass.name, 7, "alpha"]]) + it "constantizes override String" do + klass = build_including_class + stub_const("MyConstJob", Class.new) + out = klass.new.send(:resolve_job_class_for, :hubspot, + override: "MyConstJob") + expect(out).to eq(MyConstJob) end - it "runs inline when async: false by calling Synchronizer", - :aggregate_failures do - register_alpha - klass = new_model_class - dsl_apply(klass) - inst = klass.new(id: 9) + it "uses conf job_class String, else falls back to SyncJob" do + klass = build_including_class do + def self.etlify_crms + {hubspot: {job_class: "Etlify::SyncJob"}} + end + end + stub_const("Etlify::SyncJob", Class.new) + out = klass.new.send(:resolve_job_class_for, :hubspot, override: nil) + expect(out).to eq(Etlify::SyncJob) + end + end - expect(Etlify::Synchronizer).to receive(:call).with( - inst, - crm_name: :alpha - ).and_return(:synced) + describe "#raise_unless_crm_is_configured" do + it "raises with informative message when not configured" do + klass = build_including_class + expect do + klass.new.send(:raise_unless_crm_is_configured, :hubspot) + end.to raise_error(ArgumentError, /crm not configured for hubspot/) + end - res = inst.crm_sync!( - crm_name: :alpha, - async: false - ) - expect(res).to eq(:synced) + it "does not raise when configuration is present" do + klass = build_including_class do + def self.etlify_crms + {hubspot: {}} + end + end + expect do + klass.new.send(:raise_unless_crm_is_configured, :hubspot) + end.not_to raise_error end end end From 20357b1bd34499174556e31454b495212b241f22 Mon Sep 17 00:00:00 2001 From: Ismael Boukhars Date: Wed, 27 Aug 2025 08:59:34 +0200 Subject: [PATCH 4/6] [handle-multi-crm] fix: allow to launch migration --- lib/etlify/engine.rb | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/lib/etlify/engine.rb b/lib/etlify/engine.rb index 8cc95a8..44a2aba 100644 --- a/lib/etlify/engine.rb +++ b/lib/etlify/engine.rb @@ -7,9 +7,21 @@ class Engine < ::Rails::Engine initializer "etlify.check_crm_name_column" do ActiveSupport.on_load(:active_record) do - if defined?(CrmSynchronisation) && - CrmSynchronisation.table_exists? && - !CrmSynchronisation.column_names.include?("crm_name") + # Skip check during DB tasks or if explicitly disabled. + next if Etlify.db_task_running? || ENV["SKIP_ETLIFY_DB_CHECK"] == "1" + + begin + # Ensure model and table exist before checking the column. + next unless defined?(CrmSynchronisation) + + conn = ActiveRecord::Base.connection + table = "crm_synchronisations" + + next unless conn.data_source_exists?(table) + + has_column = conn.column_exists?(table, "crm_name") + next if has_column + raise( Etlify::MissingColumnError, <<~MSG.squish @@ -22,11 +34,25 @@ class Engine < ::Rails::Engine Then run: rails db:migrate MSG ) + rescue ActiveRecord::NoDatabaseError, ActiveRecord::StatementInvalid + # Happens during `db:create`, before schema is loaded, etc. + # Silently ignore; check will run again once DB is ready. end - rescue ActiveRecord::NoDatabaseError, ActiveRecord::StatementInvalid - # Happens during `rails db:create` or before schema is loaded. - # Silently ignore; check will run again once DB is ready. end end end + + # Detect if a database-related rake task is running. + def self.db_task_running? + # Rake may not be loaded outside tasks. + return false unless defined?(Rake) + + tasks = Rake.application.top_level_tasks + tasks.any? do |t| + t.start_with?("db:") || t.start_with?("app:db:") + end + rescue + # Be conservative: if unsure, do not block boot. + false + end end From eee751dbe77cfa3e52685a7574cd4528e56df20c Mon Sep 17 00:00:00 2001 From: Ismael Boukhars Date: Thu, 28 Aug 2025 09:13:35 +0200 Subject: [PATCH 5/6] [handle-multi-crm] feat: Beta1 --- Gemfile | 5 +- Gemfile.lock | 9 +- README.md | 160 ++++++++++-------- app/models/crm_synchronisation.rb | 2 +- etlify.gemspec | 2 +- lib/etlify.rb | 4 +- lib/etlify/config.rb | 8 +- lib/etlify/deleter.rb | 8 +- lib/etlify/engine.rb | 39 +---- lib/etlify/model.rb | 72 +++++--- lib/etlify/stale_records/batch_sync.rb | 8 +- lib/etlify/stale_records/finder.rb | 17 +- lib/etlify/synchronizer.rb | 4 +- lib/etlify/version.rb | 2 +- .../install/templates/initializer.rb.tt | 6 +- spec/etlify/deleter_spec.rb | 21 +-- spec/etlify/digest_spec.rb | 70 ++++---- spec/etlify/engine_initializer_spec.rb | 68 +++----- spec/etlify/model_spec.rb | 29 ---- spec/etlify/stale_records/finder_spec.rb | 53 +++++- spec/etlify/synchronizer_spec.rb | 2 +- .../etlify/install_generator_spec.rb | 31 ++-- spec/rails_helper.rb | 2 +- 23 files changed, 312 insertions(+), 310 deletions(-) diff --git a/Gemfile b/Gemfile index be8102f..4264658 100644 --- a/Gemfile +++ b/Gemfile @@ -2,10 +2,7 @@ source "https://rubygems.org" gemspec -gem "rails", "~> 7.1" -gem "activerecord", ">= 7.0" -gem "activesupport", ">= 7.0" -gem "activejob", ">= 7.0" +gem "rails", "~> 7.2" # Dev group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index 520adc0..a64e6f9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,8 +1,8 @@ PATH remote: . specs: - etlify (0.6.0) - rails (>= 7.2, < 8) + etlify (1.0.0.beta1) + rails (>= 7.0) GEM remote: https://rubygems.org/ @@ -277,11 +277,8 @@ PLATFORMS x86_64-linux-musl DEPENDENCIES - activejob (>= 7.0) - activerecord (>= 7.0) - activesupport (>= 7.0) etlify! - rails (~> 7.1) + rails (~> 7.2) rspec (~> 3.13) rspec-rails (~> 6.1) rubocop (~> 1.79) diff --git a/README.md b/README.md index aa98845..aebd49f 100644 --- a/README.md +++ b/README.md @@ -75,17 +75,29 @@ Create `config/initializers/etlify.rb`: ```ruby # config/initializers/etlify.rb +require "etlify" + Etlify.configure do |config| - # Choose the CRM adapter (default is a NullAdapter; be sure to change it) - config.crm_adapter = Etlify::Adapters::HubspotV3Adapter.new( - access_token: ENV["HUBSPOT_PRIVATE_APP_TOKEN"] + Etlify::CRM.register( + :hubspot, + adapter: Etlify::Adapters::HubspotV3Adapter.new( + access_token: ENV["HUBSPOT_PRIVATE_APP_TOKEN"] + ), + options: {job_class: "Etlify::SyncObjectWorker"} ) - - # Optional settings (shown with defaults) - config.digest_strategy = Etlify::Digest.method(:stable_sha256) # -> String - config.logger = Rails.logger - config.job_class = "Etlify::SyncJob" - config.job_queue_name = "low" + # will provide DSL below for models + # hubspot_etlified_with(...) + + # Etlify::CRM.register( + # :another_crm, adapter: Etlify::Adapters::AnotherAdapter, + # options: { job_class: Etlify::SyncJob } + # ) + # will provide DSL below for models + # another_crm_etlified_with(...) + + # @digest_strategy = Etlify::Digest.method(:stable_sha256) + # @job_queue_name = "low" + @logger = Rails.logger end ``` @@ -98,13 +110,13 @@ class User < ApplicationRecord has_many :investments, dependent: :destroy - etlified_with( + hubspot_etlified_with( serializer: UserSerializer, crm_object_type: "contacts", id_property: :id, # Only sync when an email exists sync_if: ->(user) { user.email.present? }, - # useful if your serialization include dependencies + # useful if your object serialization includes dependencies dependencies: [:investments] ) end @@ -114,9 +126,16 @@ end ```ruby # app/serializers/etlify/user_serializer.rb -class UserSerializer < Etlify::Serializers::BaseSerializer +class UserSerializer + attr_accessor :user + + # your serializer must implement #intiialize(object) #and as_crm_payload + def initialize(user) + @user = user + end + # Must return a Hash that matches your CRM field names - def as_crm_payload(user) + def as_crm_payload { email: user.email, firstname: user.first_name, @@ -135,8 +154,9 @@ end ```ruby user = User.find(1) -# Async by default (enqueues Etlify.config.sync_job_class (default: "Etlify::SyncJob") on the configured queue) -user.crm_sync! +# Async by default (enqueues an Etlify::SyncJob by default) +# The job class can be overriden when registering the CRM +user.hubspot_crm_sync! # or user.#{registered_crm_name}_crm_sync! # Run inline (no job) user.crm_sync!(async: false) @@ -146,37 +166,36 @@ user.crm_sync!(async: false) ```ruby # Inline delete (not enqueued) -user.crm_delete! +user.hubspot_crm_delete! # or user.#{registered_crm_name}_crm_delete! ``` ### Custom serializer example ```ruby # app/serializers/etlify/company_serializer.rb -class CompanySerializer < Etlify::Serializers::BaseSerializer +class CompanySerializer + attr_accessor :company + + def initialize(company) + @company = company + end + # Keep serialisation small and predictable - def as_crm_payload(company) + def as_crm_payload { name: company.name, domain: company.domain, - hs_lead_status: company.lead_status # Example custom property + hs_lead_status: company.lead_status } end end ``` -### Swapping adapters - -```ruby -# Switch to a different adapter at runtime (for a test, a rake task, etc.) -Etlify.config.crm_adapter = MyCrmAdapter.new(api_key: ENV["MYCRM_API_KEY"]) -``` - --- ## Batch synchronisation -Beyond single-record sync, Etlify provides a **batch resynchronisation API** that targets **all “stale” records** (those whose data has changed in Rails since the last CRM sync). This is useful for: +Beyond single-record sync, Etlify provides a **batch resynchronisation API** that targets **all “stale” records** (those whose data has changed since the last CRM sync). This is useful for: - recovering from CRM or worker outages, - triggering periodic re-syncs (cron jobs), @@ -191,21 +210,20 @@ Etlify::StaleRecords::BatchSync.call # Restrict to specific models Etlify::StaleRecords::BatchSync.call(models: [User, Company]) -# Run inline (no jobs), useful for scripts/maintenance +# Restrict to specifics CRM +Etlify::StaleRecords::BatchSync.call(crm_name: :hubspot) + +# Or both +Etlify::StaleRecords::BatchSync.call( + crm_name: :hubspot, + models: [User, Company] +) + +# Run inline (no jobs), useful for scripts/maintenance or testing Etlify::StaleRecords::BatchSync.call(async: false) # Adjust SQL batch size (number of IDs per batch) Etlify::StaleRecords::BatchSync.call(batch_size: 1_000) - -# Throttle (pause in seconds) between processed records -Etlify::StaleRecords::BatchSync.call(async: false, throttle: 0.01) - -# Dry-run: count without enqueuing or syncing -Etlify::StaleRecords::BatchSync.call(dry_run: true) - -# Custom logger (IO-like) -logger = Logger.new($stdout) -Etlify::StaleRecords::BatchSync.call(logger: logger) ``` **Return value** @@ -219,34 +237,26 @@ The method returns a stats Hash: } ``` -> By default, jobs are enqueued via `Etlify.config.sync_job_class` -> (e.g. `"Etlify::SyncJob"`) and executed by your ActiveJob backend. +> By default, jobs are enqueued via `"Etlify::SyncJob"` and executed by your +> ActiveJob backend. It can be overriden per CRM when registering it +> It is very usefull to handle custom throttling rules -### Examples - -```ruby -# 1) Resync the entire app by enqueuing (production-friendly) -stats = Etlify::StaleRecords::BatchSync.call -# => { total: 1532, per_model: { "User"=>920, "Company"=>612 }, errors: 0 } - -# 2) Maintenance script, inline execution with slight throttle -stats = Etlify::StaleRecords::BatchSync.call( - async: false, - batch_size: 500, - throttle: 0.005 +``` +Etlify::CRM.register( + :hubspot, + adapter: Etlify::Adapters::HubspotV3Adapter.new( + access_token: ENV["HUBSPOT_PRIVATE_APP_TOKEN"] + ), + options: {job_class: "Etlify::SyncObjectWorker"} ) -# 3) Dry-run to estimate scope before the actual run -preview = Etlify::StaleRecords::BatchSync.call(dry_run: true) - -# 4) Targeted run (e.g. only Users) for testing -only_users = Etlify::StaleRecords::BatchSync.call(models: [User]) +> the chosen class must implement .perform_later(record_class, id, crm_name) ``` ### How it works - `Etlify::StaleRecords::Finder` scans all **etlified models** - (those that called `etlified_with`) and builds, for each, + (those that called `#{crm_name}_etlified_with`) and builds, for each, a **SQL relation selecting only the PKs** of stale records. - A record is considered stale if: - it **has no** `crm_synchronisation` row, **or** @@ -261,8 +271,6 @@ only_users = Etlify::StaleRecords::BatchSync.call(models: [User]) - in **async: false** mode: load each record and pass it to `Etlify::Synchronizer.call(record)` **inline** (errors are logged and counted without interrupting the batch). -- The optional `throttle` adds a **short pause** between processed records - (useful to protect third-party APIs or the DB when running inline). ## Best practices @@ -272,7 +280,6 @@ only_users = Etlify::StaleRecords::BatchSync.call(models: [User]) benefit from **idempotence**. - **Dependencies**: declare `dependencies:` accurately in `etlified_with` so indirect changes trigger resyncs. -- **Dry-run** before a large run to estimate load (`dry_run: true`). - **Batch size**: adjust `batch_size` to your DB to balance throughput and memory. --- @@ -305,8 +312,12 @@ Etlify ships with `Etlify::Adapters::HubspotV3Adapter`. It supports native objec ```ruby Etlify.configure do |config| - config.crm_adapter = Etlify::Adapters::HubspotV3Adapter.new( - access_token: ENV["HUBSPOT_PRIVATE_APP_TOKEN"] + Etlify::CRM.register( + :hubspot, + adapter: Etlify::Adapters::HubspotV3Adapter.new( + access_token: ENV["HUBSPOT_PRIVATE_APP_TOKEN"] + ), + options: {job_class: "Etlify::SyncObjectWorker"} ) end ``` @@ -323,16 +334,16 @@ end class User < ApplicationRecord include Etlify::Model - etlified_with( + hubspot_etlified_with( serializer: UserSerializer, crm_object_type: "contacts", - id_property: :cs_capsens_id, + id_property: :email, sync_if: ->(user) { user.email.present? } ) end # Later -user.crm_sync! # Adapter performs an upsert +user.hubspot_crm_sync! # Adapter performs an upsert ``` ### Example: Custom object @@ -341,7 +352,7 @@ user.crm_sync! # Adapter performs an upsert class Subscription < ApplicationRecord include Etlify::Model - etlified_with( + hubspot_etlified_with( serializer: SubscriptionSerializer, crm_object_type: "p1234567_subscription" # Custom object API name, id_propery: :id, @@ -429,21 +440,28 @@ bundle exec rspec ```ruby # In your spec fake_adapter = instance_double("Adapter") -allow(Etlify.config).to receive(:crm_adapter).and_return(fake_adapter) allow(fake_adapter).to receive(:upsert!).and_return("crm_123") allow(fake_adapter).to receive(:delete!).and_return(true) +# Override the registry for this CRM (ex: :hubspot) +Etlify::CRM.register( + :hubspot, + adapter: fake_adapter, + options: {} +) + user = create(:user, email: "someone@example.com") -user.crm_sync! + +# Enqueue or perform a sync for this CRM +user.hubspot_sync!(async: false) expect(fake_adapter).to have_received(:upsert!).with( object_type: "contacts", payload: hash_including(email: "someone@example.com"), id_property: anything ) -``` -> For end-to-end tests, use VCR or WebMock around your adapter, but prefer unit-level tests against your serialisers and model logic. +``` --- diff --git a/app/models/crm_synchronisation.rb b/app/models/crm_synchronisation.rb index aba48b0..d38e831 100644 --- a/app/models/crm_synchronisation.rb +++ b/app/models/crm_synchronisation.rb @@ -1,4 +1,4 @@ -class CrmSynchronisation < ApplicationRecord +class CrmSynchronisation < ActiveRecord::Base self.table_name = "crm_synchronisations" belongs_to :resource, polymorphic: true diff --git a/etlify.gemspec b/etlify.gemspec index 93eb74f..7e30680 100644 --- a/etlify.gemspec +++ b/etlify.gemspec @@ -16,7 +16,7 @@ Gem::Specification.new do |spec| ).select { |f| File.file?(f) } + %w[README.md] spec.require_paths = ["lib"] - spec.add_dependency "rails", ">= 7.2", "< 8" + spec.add_dependency "rails", ">= 7.0" spec.add_development_dependency "rspec", "~> 3.13" spec.add_development_dependency "rspec-rails", "~> 6.1" spec.add_development_dependency "sqlite3", "~> 2.7" diff --git a/lib/etlify.rb b/lib/etlify.rb index f2abc6a..f029241 100644 --- a/lib/etlify.rb +++ b/lib/etlify.rb @@ -35,5 +35,5 @@ def configure end require_relative "../app/jobs/etlify/sync_job" -require_relative "etlify/railtie" if defined?(Rails) -require_relative "etlify/engine" if defined?(Rails) +require_relative "etlify/railtie" +require_relative "etlify/engine" diff --git a/lib/etlify/config.rb b/lib/etlify/config.rb index 700ecdd..1b0ea27 100644 --- a/lib/etlify/config.rb +++ b/lib/etlify/config.rb @@ -10,12 +10,8 @@ class Config def initialize @digest_strategy = Etlify::Digest.method(:stable_sha256) @job_queue_name = "low" - - rails_logger = (defined?(Rails) && Rails.respond_to?(:logger)) ? Rails.logger : nil - @logger = rails_logger || Logger.new($stdout) - - rails_cache = (defined?(Rails) && Rails.respond_to?(:cache)) ? Rails.cache : nil - @cache_store = rails_cache || ActiveSupport::Cache::MemoryStore.new + @logger = Rails.logger || Logger.new($stdout) + @cache_store = Rails.cache || ActiveSupport::Cache::MemoryStore.new end end end diff --git a/lib/etlify/deleter.rb b/lib/etlify/deleter.rb index 0f7527f..8dbdfd2 100644 --- a/lib/etlify/deleter.rb +++ b/lib/etlify/deleter.rb @@ -8,7 +8,7 @@ class Deleter ) # @param resource [ActiveRecord::Base] - # @param crm [Symbol,String] + # @param crm_name [Symbol,String] def self.call(resource, crm_name:) new(resource, crm_name: crm_name).call end @@ -17,7 +17,7 @@ def initialize(resource, crm_name:) @resource = resource @crm_name = crm_name.to_sym @conf = resource.class.etlify_crms.fetch(@crm_name) - @adapter = @conf[:adapter].new + @adapter = @conf[:adapter] end def call @@ -31,7 +31,9 @@ def call ) :deleted rescue => e - raise Etlify::SyncError, e.message + error = Etlify::SyncError.new(e.message) + error.set_backtrace(e.backtrace) + raise error end private diff --git a/lib/etlify/engine.rb b/lib/etlify/engine.rb index 44a2aba..0fbb3a2 100644 --- a/lib/etlify/engine.rb +++ b/lib/etlify/engine.rb @@ -7,23 +7,9 @@ class Engine < ::Rails::Engine initializer "etlify.check_crm_name_column" do ActiveSupport.on_load(:active_record) do - # Skip check during DB tasks or if explicitly disabled. - next if Etlify.db_task_running? || ENV["SKIP_ETLIFY_DB_CHECK"] == "1" + if Etlify::Engine.should_display_crm_name_warning? - begin - # Ensure model and table exist before checking the column. - next unless defined?(CrmSynchronisation) - - conn = ActiveRecord::Base.connection - table = "crm_synchronisations" - - next unless conn.data_source_exists?(table) - - has_column = conn.column_exists?(table, "crm_name") - next if has_column - - raise( - Etlify::MissingColumnError, + Etlify.config.logger.error( <<~MSG.squish Missing column "crm_name" on table "crm_synchronisations". Please generate a migration with: @@ -34,25 +20,18 @@ class Engine < ::Rails::Engine Then run: rails db:migrate MSG ) - rescue ActiveRecord::NoDatabaseError, ActiveRecord::StatementInvalid - # Happens during `db:create`, before schema is loaded, etc. - # Silently ignore; check will run again once DB is ready. end + rescue ActiveRecord::NoDatabaseError, ActiveRecord::StatementInvalid + # Happens during `rails db:create` or before schema is loaded. + # Silently ignore; check will run again once DB is ready. end end - end - # Detect if a database-related rake task is running. - def self.db_task_running? - # Rake may not be loaded outside tasks. - return false unless defined?(Rake) + def self.should_display_crm_name_warning? + return unless defined?(CrmSynchronisation) + return unless CrmSynchronisation.table_exists? - tasks = Rake.application.top_level_tasks - tasks.any? do |t| - t.start_with?("db:") || t.start_with?("app:db:") + !CrmSynchronisation.column_names.include?("crm_name") end - rescue - # Be conservative: if unsure, do not block boot. - false end end diff --git a/lib/etlify/model.rb b/lib/etlify/model.rb index 265dd49..afaffb2 100644 --- a/lib/etlify/model.rb +++ b/lib/etlify/model.rb @@ -1,14 +1,31 @@ +# lib/etlify/model.rb module Etlify module Model extend ActiveSupport::Concern included do - # Track classes that included this concern to backfill DSL on register. + # Track including classes for DSL backfill. Etlify::Model.__included_klasses__ << self - # Hash keyed by CRM name, with config per CRM + # Per-CRM configuration storage. class_attribute :etlify_crms, instance_writer: false, default: {} + # Declare associations only for ActiveRecord models. + if defined?(ActiveRecord::Base) && self < ActiveRecord::Base + if respond_to?(:reflect_on_association) && + !reflect_on_association(:crm_synchronisations) + has_many( + :crm_synchronisations, + -> { order(id: :asc) }, + class_name: "CrmSynchronisation", + as: :resource, + dependent: :destroy, + inverse_of: :resource + ) + end + end + + # Install DSL and instance helpers for already-registered CRMs. Etlify::CRM.names.each do |crm_name| Etlify::Model.define_crm_dsl_on(self, crm_name) Etlify::Model.define_crm_instance_helpers_on(self, crm_name) @@ -16,12 +33,10 @@ module Model end class << self - # Internal: store all including classes def __included_klasses__ @__included_klasses__ ||= [] end - # Called by Etlify::CRM.register to (re)install DSL on all classes def install_dsl_for_crm(crm_name) __included_klasses__.each do |klass| define_crm_dsl_on(klass, crm_name) @@ -29,11 +44,8 @@ def install_dsl_for_crm(crm_name) end end - # Define the class-level DSL method: "_etlified_with" def define_crm_dsl_on(klass, crm_name) dsl_name = "#{crm_name}_etlified_with" - - # Avoid redefining if already defined return if klass.respond_to?(dsl_name) klass.define_singleton_method(dsl_name) do | @@ -44,10 +56,8 @@ def define_crm_dsl_on(klass, crm_name) sync_if: ->(_r) { true }, job_class: nil | - # Fetch registered CRM (adapter, options) reg = Etlify::CRM.fetch(crm_name) - # Merge model-level config for this CRM conf = { serializer: serializer, guard: sync_if, @@ -55,22 +65,17 @@ def define_crm_dsl_on(klass, crm_name) id_property: id_property, dependencies: Array(dependencies).map(&:to_sym), adapter: reg.adapter, - # Job class priority: method arg > registry options > nil job_class: job_class || reg.options[:job_class], } - # Store into class attribute hash new_hash = (etlify_crms || {}).dup new_hash[crm_name.to_sym] = conf self.etlify_crms = new_hash - # Ensure instance helpers exist Etlify::Model.define_crm_instance_helpers_on(self, crm_name) end end - # Define instance helpers: "_build_payload", "_sync!", - # "_delete!" def define_crm_instance_helpers_on(klass, crm_name) payload_m = "#{crm_name}_build_payload" sync_m = "#{crm_name}_sync!" @@ -93,18 +98,38 @@ def define_crm_instance_helpers_on(klass, crm_name) crm_delete!(crm_name: crm_name) end end + + unless klass.method_defined?("registered_crms") + klass.define_method("registered_crms") do + self.class.etlify_crms.keys.map(&:to_s) + end + end + + # Define filtered has_one only for AR models. + if defined?(ActiveRecord::Base) && klass < ActiveRecord::Base + assoc_name = :"#{crm_name}_crm_synchronisation" + if klass.respond_to?(:reflect_on_association) && + !klass.reflect_on_association(assoc_name) + klass.has_one( + assoc_name, + -> { where(crm_name: crm_name.to_s) }, + class_name: "CrmSynchronisation", + as: :resource, + dependent: :destroy, + inverse_of: :resource + ) + end + end end end - # ---------- Public generic API (now CRM-aware) ---------- + # ---------- Public generic API (CRM-aware) ---------- def crm_synced?(crm: nil) - # If you have per-CRM synchronisation records, adapt accordingly. - # For now keep a single association; adjust when your schema changes. - crm_synchronisation.present? + # Keep backward-compatible single-sync check if association exists. + respond_to?(:crm_synchronisation) && crm_synchronisation.present? end - # Accept both crm_name: and crm: for backward compatibility. def build_crm_payload(crm_name: nil, crm: nil) crm_name ||= crm raise ArgumentError, "crm_name is required" if crm_name.nil? @@ -115,15 +140,9 @@ def build_crm_payload(crm_name: nil, crm: nil) conf[:serializer].new(self).as_crm_payload end - # @param crm_name [Symbol] which CRM to use - # @param async [Boolean] whether to enqueue or run inline - # @param job_class [Class,String,nil] explicit override - # - # Accept both crm_name: and crm: for backward compatibility. def crm_sync!(crm_name: nil, async: true, job_class: nil, crm: nil) crm_name ||= crm raise ArgumentError, "crm_name is required" if crm_name.nil? - return false unless allow_sync_for?(crm_name) if async @@ -140,7 +159,6 @@ def crm_sync!(crm_name: nil, async: true, job_class: nil, crm: nil) end end - # Accept both crm_name: and crm: for backward compatibility. def crm_delete!(crm_name: nil, crm: nil) crm_name ||= crm raise ArgumentError, "crm_name is required" if crm_name.nil? @@ -150,7 +168,6 @@ def crm_delete!(crm_name: nil, crm: nil) private - # Guard evaluation per CRM def allow_sync_for?(crm_name) conf = self.class.etlify_crms[crm_name.to_sym] return false unless conf @@ -166,7 +183,6 @@ def resolve_job_class_for(crm_name, override:) given = conf[:job_class] return constantize_if_needed(given) if given - # Fallback to default sync job name if you want one constantize_if_needed("Etlify::SyncJob") end diff --git a/lib/etlify/stale_records/batch_sync.rb b/lib/etlify/stale_records/batch_sync.rb index d44c430..78df130 100644 --- a/lib/etlify/stale_records/batch_sync.rb +++ b/lib/etlify/stale_records/batch_sync.rb @@ -14,10 +14,12 @@ class BatchSync # batch_size: # of ids per batch. # # Returns a Hash with :total, :per_model, :errors. - def self.call(models: nil, + def self.call( + models: nil, crm_name: nil, async: true, - batch_size: DEFAULT_BATCH_SIZE) + batch_size: DEFAULT_BATCH_SIZE + ) new( models: models, crm_name: crm_name, @@ -75,7 +77,7 @@ def process_model(model, relation, crm_name:) model.where(pk => ids).find_each(batch_size: @batch_size) do |rec| Etlify::Synchronizer.call(rec, crm_name: crm_name) count += 1 - rescue + rescue StandardError # Count and continue; no logging by design. errors += 1 end diff --git a/lib/etlify/stale_records/finder.rb b/lib/etlify/stale_records/finder.rb index bed0a1b..0e394ca 100644 --- a/lib/etlify/stale_records/finder.rb +++ b/lib/etlify/stale_records/finder.rb @@ -167,14 +167,23 @@ def through_dependency_timestamp_sql(model, reflection, epoch) preds = [] preds << "#{quoted(through_tbl, through.foreign_key, conn)} = " \ - "#{quoted(owner_tbl, model.primary_key, conn)}" + "#{quoted(owner_tbl, model.primary_key, conn)}" if (as = through.options[:as]) preds << "#{quoted(through_tbl, "#{as}_type", conn)} = " \ - "#{conn.quote(model.name)}" + "#{conn.quote(model.name)}" end - join_on = "#{quoted(source_tbl, source_pk, conn)} = " \ - "#{quoted(through_tbl, source.foreign_key, conn)}" + # 🔧 Join orientation fix: + # If source is belongs_to, FK lives on the through table. + # Else (has_many/has_one), FK lives on the source table. + join_on = + if source.macro == :belongs_to + "#{quoted(source_tbl, source_pk, conn)} = " \ + "#{quoted(through_tbl, source.foreign_key, conn)}" + else + "#{quoted(source_tbl, source.foreign_key, conn)} = " \ + "#{quoted(through_tbl, through.klass.primary_key, conn)}" + end sub = <<-SQL.squish SELECT MAX(#{quoted(source_tbl, 'updated_at', conn)}) diff --git a/lib/etlify/synchronizer.rb b/lib/etlify/synchronizer.rb index 90c551c..caeabae 100644 --- a/lib/etlify/synchronizer.rb +++ b/lib/etlify/synchronizer.rb @@ -17,7 +17,7 @@ def initialize(resource, crm_name:) @resource = resource @crm_name = crm_name.to_sym @conf = resource.class.etlify_crms.fetch(@crm_name) - @adapter = @conf[:adapter].new + @adapter = @conf[:adapter] end def call @@ -43,7 +43,7 @@ def call :not_modified end end - rescue => e + rescue StandardError => e sync_line.update!(last_error: e.message) :error diff --git a/lib/etlify/version.rb b/lib/etlify/version.rb index 1f1103b..f000071 100644 --- a/lib/etlify/version.rb +++ b/lib/etlify/version.rb @@ -1,3 +1,3 @@ module Etlify - VERSION = "0.6.0" + VERSION = "1.0.0.beta1" end diff --git a/lib/generators/etlify/install/templates/initializer.rb.tt b/lib/generators/etlify/install/templates/initializer.rb.tt index cec6a10..d8ccdd6 100644 --- a/lib/generators/etlify/install/templates/initializer.rb.tt +++ b/lib/generators/etlify/install/templates/initializer.rb.tt @@ -17,8 +17,8 @@ Etlify.configure do |config| # another_crm_etlified_with(...) # overridable defaults config - # @job_queue_name = "low" # @digest_strategy = Etlify::Digest.method(:stable_sha256) - # @logger = defined?(Rails) ? Rails.logger : Logger.new($stdout) - # @cache_store = Rails.cache + # @job_queue_name = "low" + # @logger = Rails.logger || Logger.new($stdout) + # @cache_store = Rails.cache || ActiveSupport::Cache::MemoryStore.new end diff --git a/spec/etlify/deleter_spec.rb b/spec/etlify/deleter_spec.rb index ff5df15..13c8a8c 100644 --- a/spec/etlify/deleter_spec.rb +++ b/spec/etlify/deleter_spec.rb @@ -44,32 +44,25 @@ def create_line(resource, crm_name:, crm_id:) it "calls adapter.delete! with params and returns :deleted" do line = create_line(user, crm_name: "hubspot", crm_id: "crm-123") - calls = [] - adapter = Class.new do - # Capture arguments to verify them later + adapter_class = Class.new do + # Keep English comments and ≤85 chars per line define_method(:delete!) do |crm_id:, object_type:, id_property:| - ObjectSpace.each_object(Array).find do |arr| - arr.equal?(calls = arr) # no-op to silence linter - end true end end + adapter_instance = adapter_class.new - # Replace adapter for this example to observe args allow(User).to receive(:etlify_crms).and_return( { hubspot: { - adapter: adapter, + adapter: adapter_instance, id_property: "id", crm_object_type: "contacts", }, } ) - # Spy on a real instance to check args - instance = adapter.new - allow(adapter).to receive(:new).and_return(instance) - expect(instance).to receive(:delete!).with( + expect(adapter_instance).to receive(:delete!).with( crm_id: "crm-123", object_type: "contacts", id_property: "id" @@ -77,8 +70,6 @@ def create_line(resource, crm_name:, crm_id:) res = described_class.call(user, crm_name: :hubspot) expect(res).to eq(:deleted) - - # Ensure sync line not altered by the deleter itself. expect(line.reload.crm_id).to eq("crm-123") end end @@ -94,7 +85,7 @@ def delete!(crm_id:, object_type:, id_property:) allow(User).to receive(:etlify_crms).and_return( { hubspot: { - adapter: FailingDeleteAdapter, + adapter: FailingDeleteAdapter.new, id_property: "id", crm_object_type: "contacts", }, diff --git a/spec/etlify/digest_spec.rb b/spec/etlify/digest_spec.rb index 1326302..b7c51a1 100644 --- a/spec/etlify/digest_spec.rb +++ b/spec/etlify/digest_spec.rb @@ -1,11 +1,8 @@ -# frozen_string_literal: true - require "rails_helper" RSpec.describe Etlify::Digest do describe ".normalize" do - it "sorts hash keys recursively and preserves array order", - :aggregate_failures do + it "sorts hash keys recursively and preserves array order" do # Hash keys (including nested) are sorted; arrays keep their order. input = { b: 2, @@ -17,13 +14,14 @@ out = described_class.normalize(input) - expect(out.keys).to eq(%i[a b]) - expect(out[:a].keys).to eq(%i[c z]) - expect(out[:a][:z]).to eq([3, {k: 1}]) + aggregate_failures do + expect(out.keys).to eq(%i[a b]) + expect(out[:a].keys).to eq(%i[c z]) + expect(out[:a][:z]).to eq([3, {k: 1}]) + end end - it "recursively normalizes arrays of mixed types", - :aggregate_failures do + it "recursively normalizes arrays of mixed types" do input = [ {b: 1, a: 2}, 3, @@ -47,19 +45,19 @@ ) end - it "returns scalars unchanged (String, Numeric, booleans, nil)", - :aggregate_failures do - expect(described_class.normalize("x")).to eq("x") - expect(described_class.normalize(42)).to eq(42) - expect(described_class.normalize(true)).to eq(true) - expect(described_class.normalize(false)).to eq(false) - expect(described_class.normalize(nil)).to be_nil + it "returns scalars unchanged (String, Numeric, booleans, nil)" do + aggregate_failures do + expect(described_class.normalize("x")).to eq("x") + expect(described_class.normalize(42)).to eq(42) + expect(described_class.normalize(true)).to eq(true) + expect(described_class.normalize(false)).to eq(false) + expect(described_class.normalize(nil)).to be_nil + end end end describe ".stable_sha256" do - it "returns a deterministic 64-char lowercase hex digest", - :aggregate_failures do + it "returns a deterministic 64-char lowercase hex digest" do payload = { a: 1, b: [ @@ -72,12 +70,16 @@ d1 = described_class.stable_sha256(payload) d2 = described_class.stable_sha256(payload) - expect(d1).to match(/\A\h{64}\z/) - expect(d1).to eq(d2) + aggregate_failures do + expect(d1).to match(/\A\h{64}\z/) + expect(d1).to eq(d2) + end end - it "is insensitive to hash key ordering (shallow and nested)", - :aggregate_failures do + it( + "is insensitive to hash key ordering (shallow and nested)", + :aggregate_failures + ) do p1 = {a: 1, b: 2} p2 = {b: 2, a: 1} @@ -97,8 +99,7 @@ .to eq(described_class.stable_sha256(n2)) end - it "changes when a value changes", - :aggregate_failures do + it "changes when a value changes" do p1 = { a: 1, b: [2, 3], @@ -112,8 +113,7 @@ .not_to eq(described_class.stable_sha256(p2)) end - it "is sensitive to array element order", - :aggregate_failures do + it "is sensitive to array element order" do p1 = {a: [1, 2, 3]} p2 = {a: [3, 2, 1]} @@ -121,17 +121,17 @@ .not_to eq(described_class.stable_sha256(p2)) end - it "handles primitive inputs (String, Numeric, booleans, nil)", - :aggregate_failures do - expect(described_class.stable_sha256("hello")).to match(/\A\h{64}\z/) - expect(described_class.stable_sha256(123)).to match(/\A\h{64}\z/) - expect(described_class.stable_sha256(true)).to match(/\A\h{64}\z/) - expect(described_class.stable_sha256(false)).to match(/\A\h{64}\z/) - expect(described_class.stable_sha256(nil)).to match(/\A\h{64}\z/) + it "handles primitive inputs (String, Numeric, booleans, nil)" do + aggregate_failures do + expect(described_class.stable_sha256("hello")).to match(/\A\h{64}\z/) + expect(described_class.stable_sha256(123)).to match(/\A\h{64}\z/) + expect(described_class.stable_sha256(true)).to match(/\A\h{64}\z/) + expect(described_class.stable_sha256(false)).to match(/\A\h{64}\z/) + expect(described_class.stable_sha256(nil)).to match(/\A\h{64}\z/) + end end - it "matches for deeply equivalent structures regardless of key order", - :aggregate_failures do + it "matches for deeply equivalent structures regardless of key order" do p1 = { a: [ {z: 1, y: 2}, diff --git a/spec/etlify/engine_initializer_spec.rb b/spec/etlify/engine_initializer_spec.rb index d7b4fe1..ee09961 100644 --- a/spec/etlify/engine_initializer_spec.rb +++ b/spec/etlify/engine_initializer_spec.rb @@ -4,72 +4,46 @@ RSpec.describe Etlify::Engine do def run_initializer + # Triggers the on_load(:active_record) hooks ActiveSupport.run_load_hooks(:active_record, ActiveRecord::Base) end context "when the required column is present" do it "does not raise" do - # L'initializer a déjà été exécuté au chargement de la gem, - # son on_load est donc enregistré : on peut juste déclencher le hook. expect { run_initializer }.not_to raise_error end end - context "when the required column is missing" do + context "when the required column is missing (logging mode)" do before do - # Drop + recreate the table without the crm_name column. - ActiveRecord::Base.connection.execute( - "DROP TABLE IF EXISTS crm_synchronisations" + # Do not mutate schema: stub what the initializer reads + allow(CrmSynchronisation).to receive(:table_exists?).and_return(true) + allow(CrmSynchronisation).to receive(:column_names).and_return( + %w[ + id crm_id last_digest last_synced_at last_error + resource_type resource_id created_at updated_at + ] ) - ActiveRecord::Schema.define do - create_table :crm_synchronisations, force: true do |t| - # crm_name intentionally omitted - t.string :crm_id - t.string :last_digest - t.datetime :last_synced_at - t.text :last_error - t.string :resource_type, null: false - t.integer :resource_id, null: false - t.timestamps - end - end CrmSynchronisation.reset_column_information end - after do - # Restore correct schema for other specs. - ActiveRecord::Base.connection.execute( - "DROP TABLE IF EXISTS crm_synchronisations" - ) - ActiveRecord::Schema.define do - create_table :crm_synchronisations, force: true do |t| - t.string :crm_name, null: false - t.string :crm_id - t.string :last_digest - t.datetime :last_synced_at - t.text :last_error - t.string :resource_type, null: false - t.integer :resource_id, null: false - t.timestamps - end - add_index :crm_synchronisations, - %i[resource_type resource_id crm_name], - unique: true, - name: "idx_sync_polymorphic_unique" - end - CrmSynchronisation.reset_column_information - end + it "logs a helpful error" do + # Use a test logger spy without printing to STDOUT/STDERR + test_logger = instance_double(Logger) + allow(test_logger).to receive(:error) + allow(Etlify.config).to receive(:logger).and_return(test_logger) - it "raises a helpful Etlify::MissingColumnError" do - # Find the initializer and run it INSIDE the expectation. + # Ensure the initializer is registered, then trigger the hook init = Etlify::Engine.initializers.find do |i| i.name == "etlify.check_crm_name_column" end + init.run(Etlify::Engine.instance) - expect { init.run(Etlify::Engine.instance) }.to raise_error( - Etlify::MissingColumnError, - /Missing column "crm_name" on table "crm_synchronisations"/ - ) + run_initializer + + expect(test_logger).to have_received(:error).with( + match(/Missing column "crm_name" on table "crm_synchronisations"/) + ).at_least(:once) end end diff --git a/spec/etlify/model_spec.rb b/spec/etlify/model_spec.rb index 7dfe991..b660fba 100644 --- a/spec/etlify/model_spec.rb +++ b/spec/etlify/model_spec.rb @@ -223,35 +223,6 @@ def crm_delete!(**kw) end end - describe "#crm_synced?" do - it "returns true when crm_synchronisation is present" do - klass = build_including_class do - def crm_synchronisation - :something - end - end - expect(klass.new.crm_synced?(crm: :hubspot)).to be true - end - - it "returns false when crm_synchronisation is nil" do - klass = build_including_class do - def crm_synchronisation - nil - end - end - expect(klass.new.crm_synced?(crm: :hubspot)).to be false - end - - it "ignores the crm: argument (compat quirk)" do - klass = build_including_class do - def crm_synchronisation - :x - end - end - expect(klass.new.crm_synced?(crm: :other)).to be true - end - end - describe "#build_crm_payload" do it "raises when CRM is not configured" do klass = build_including_class diff --git a/spec/etlify/stale_records/finder_spec.rb b/spec/etlify/stale_records/finder_spec.rb index a88b87e..0bc4c0b 100644 --- a/spec/etlify/stale_records/finder_spec.rb +++ b/spec/etlify/stale_records/finder_spec.rb @@ -71,6 +71,12 @@ t.integer :project_id t.timestamps null: true end + + create_table :subscriptions, force: true do |t| + # FK lives on the source table -> references profiles.id + t.integer :users_profile_id + t.timestamps null: true + end end User.reset_column_information @@ -125,6 +131,20 @@ def stub_models! define_model_const("Document") define_model_const("Tag") + define_model_const("Subscription") do |k| + # Source holds the FK to through table (profiles) + k.belongs_to :profile, + foreign_key: "users_profile_id", + optional: true + end + + Profile.class_eval do + has_many :subscriptions, + class_name: "Subscription", + foreign_key: "users_profile_id", + dependent: :destroy + end + # Reopen User to add associations used by tests User.class_eval do has_one :profile, dependent: :destroy @@ -137,6 +157,7 @@ def stub_models! has_and_belongs_to_many :tags, join_table: "tags_users" has_many :linkages, as: :owner, dependent: :destroy has_many :poly_projects, through: :linkages, source: :project + has_many :subscriptions, through: :profile end end @@ -442,9 +463,39 @@ def now expect(described_class.call(crm_name: :hubspot)[User][:hubspot] .pluck(:id)).to include(u.id) end + + describe "has_many :through where FK lives on source table" do + it "marks owner stale when a source row becomes newer" do + # Configure Finder to track :subscriptions for this CRM + allow(User).to receive(:etlify_crms).and_return( + { + hubspot: { + adapter: Etlify::Adapters::NullAdapter, + id_property: "id", + crm_object_type: "contacts", + dependencies: [:subscriptions] + } + } + ) + + u = User.create!(email: "x@x.x") + p = Profile.create!(user: u, updated_at: now) + s = Subscription.create!(users_profile_id: p.id, updated_at: now) + + # Fresh sync -> not stale + create_sync!(u, crm: :hubspot, last_synced_at: now + 1) + rel = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(rel.pluck(:id)).not_to include(u.id) + + # Make the source newer -> becomes stale + s.update!(updated_at: now + 20) + rel = described_class.call(crm_name: :hubspot)[User][:hubspot] + expect(rel.pluck(:id)).to include(u.id) + end + end end - # --------- NEW: owner belongs_to polymorphic (avatarable) ---------- + # --------- owner belongs_to polymorphic (avatarable) ---------- describe "owner belongs_to polymorphic dependency" do it "uses concrete target updated_at when avatarable is set" do diff --git a/spec/etlify/synchronizer_spec.rb b/spec/etlify/synchronizer_spec.rb index ceb61a9..2ad5c6c 100644 --- a/spec/etlify/synchronizer_spec.rb +++ b/spec/etlify/synchronizer_spec.rb @@ -144,7 +144,7 @@ def upsert!(payload:, id_property:, object_type:) allow(User).to receive(:etlify_crms).and_return( { hubspot: { - adapter: FailingAdapter, + adapter: FailingAdapter.new, id_property: "id", crm_object_type: "contacts", }, diff --git a/spec/generators/etlify/install_generator_spec.rb b/spec/generators/etlify/install_generator_spec.rb index 544a135..aec7ea0 100644 --- a/spec/generators/etlify/install_generator_spec.rb +++ b/spec/generators/etlify/install_generator_spec.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - require "rails_helper" require "fileutils" require "tmpdir" @@ -24,28 +22,27 @@ def generated_initializer_path File.join(@tmp, "config/initializers/etlify.rb") end - it "creates config/initializers/etlify.rb", - :aggregate_failures do + it( + "creates config/initializers/etlify.rb with a valid skeleton", + :aggregate_failures + ) do gen = build_generator gen.invoke_all expect(File.exist?(generated_initializer_path)).to be(true) content = File.read(generated_initializer_path) + + expect(content).to include(%(require "etlify")) expect(content).to include("Etlify.configure do |config|") expect(content).to include("Etlify::CRM.register(") - expect(content).to include("adapter: Etlify::Adapters::HubspotV3Adapter") - expect(content).to include("options: { job_class: Etlify::SyncJob }") - expect(content).to include("# @job_queue_name = \"low\"") - expect(content).to include( - "# @digest_strategy = Etlify::Digest.method(:stable_sha256)" - ) - expect(content).to include("# @cache_store = Rails.cache") end - it "the generated initializer configures default HubSpot mapping by " \ - "calling Etlify::CRM.register", - :aggregate_failures do + it( + "the generated initializer configures default HubSpot mapping by " \ + "calling Etlify::CRM.register", + :aggregate_failures + ) do gen = build_generator gen.invoke_all @@ -67,8 +64,10 @@ def generated_initializer_path end end - it "est chargeable plusieurs fois sans exploser (idempotent à l'exécution)", - :aggregate_failures do + it( + "is safe to load multiple times (idempotent at runtime)", + :aggregate_failures + ) do gen = build_generator gen.invoke_all diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 8357d31..939f490 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -99,7 +99,7 @@ def build_crm_payload(crm_name:) def self.etlify_crms { hubspot: { - adapter: Etlify::Adapters::NullAdapter, + adapter: Etlify::Adapters::NullAdapter.new, id_property: "id", crm_object_type: "contacts", }, From 022de8c4c68ab6317794d46c379d467315a72be3 Mon Sep 17 00:00:00 2001 From: Ismael Boukhars Date: Tue, 26 Aug 2025 10:25:51 +0200 Subject: [PATCH 6/6] [airtable-adapter] feat: Add new Airtable Adapter --- lib/etlify.rb | 1 + lib/etlify/adapters/airtable_v0_adapter.rb | 264 +++++++++++ spec/adapters/airtable_v0_adapter_spec.rb | 508 +++++++++++++++++++++ 3 files changed, 773 insertions(+) create mode 100644 lib/etlify/adapters/airtable_v0_adapter.rb create mode 100644 spec/adapters/airtable_v0_adapter_spec.rb diff --git a/lib/etlify.rb b/lib/etlify.rb index f029241..1b1011a 100644 --- a/lib/etlify.rb +++ b/lib/etlify.rb @@ -17,6 +17,7 @@ require_relative "etlify/stale_records/batch_sync" require_relative "etlify/adapters/null_adapter" require_relative "etlify/adapters/hubspot_v3_adapter" +require_relative "etlify/adapters/airtable_v0_adapter" require_relative "etlify/serializers/base_serializer" require_relative "generators/etlify/install/install_generator" require_relative "generators/etlify/migration/migration_generator" diff --git a/lib/etlify/adapters/airtable_v0_adapter.rb b/lib/etlify/adapters/airtable_v0_adapter.rb new file mode 100644 index 0000000..a55418c --- /dev/null +++ b/lib/etlify/adapters/airtable_v0_adapter.rb @@ -0,0 +1,264 @@ +require "json" +require "uri" +require "net/http" + +module Etlify + module Adapters + # Airtable REST adapter (public HTTP API, endpoint namespace v0). + # + # This mirrors HubspotV3Adapter's public surface so it can be dropped in + # as-is by Etlify. It purposely keeps zero runtime deps and allows DI of + # an HTTP client for testing. + # + # Notes + # - Upsert strategy: optional lookup on `id_property` using filterByFormula + # then `PATCH` on hit, else `POST` to create. + # - Delete: returns true on 2xx, false on 404, raises otherwise. + # - Errors: maps common HTTP statuses to Etlify exceptions. + # - Transport errors: wrapped into Etlify::TransportError. + class AirtableV0Adapter + API_BASE = "https://api.airtable.com/v0" + + # @param api_key [String] Airtable personal access token + # @param base_id [String] Airtable Base ID (e.g., "app...") + # @param table [String, nil] Optional default table name + # @param http_client [#request] Optional injected HTTP client + def initialize(api_key:, base_id:, table: nil, http_client: nil) + @api_key = api_key + @base_id = base_id + @default_table = table + @http = http_client || DefaultHttp.new + end + + # Upsert a record into `object_type` (table). If `id_property` is given + # and present in payload, we try to find the record and PATCH it. If not + # found (or no id_property), we POST to create. + # + # @return [String, nil] Airtable record id (e.g., "rec...") or nil + def upsert!(payload:, object_type: nil, id_property: nil) + table = resolve_table!(object_type) + raise ArgumentError, "payload must be a Hash" unless payload.is_a?(Hash) + + fields = payload.dup + unique_value = nil + + if id_property + key_str = id_property.to_s + key_sym = key_str.to_sym + unique_value = fields.delete(key_str) || fields.delete(key_sym) + end + + record_id = if id_property && unique_value + find_record_id_by_field(table, id_property, unique_value) + end + + if record_id + update_record(table, record_id, fields) + record_id.to_s + else + create_record(table, fields, id_property, unique_value) + end + end + + # Delete a record by Airtable record id (rec...). + # @return [Boolean] true on 2xx, false on 404 + def delete!(crm_id:, object_type: nil) + table = resolve_table!(object_type) + raise ArgumentError, "crm_id must be provided" if crm_id.to_s.empty? + + path = "/#{@base_id}/#{enc(table)}/#{crm_id}" + resp = request(:delete, path) + return true if resp[:status].between?(200, 299) + return false if resp[:status] == 404 + + raise_for_error!(resp, path: path) + end + + private + + # Simple Net::HTTP client for dependency-free default. + class DefaultHttp + def request(method, url, headers: {}, body: nil) + uri = URI(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = uri.scheme == "https" + + klass = { + get: Net::HTTP::Get, + post: Net::HTTP::Post, + patch: Net::HTTP::Patch, + delete: Net::HTTP::Delete, + }.fetch(method) { raise ArgumentError, "Unsupported method: #{method}" } + + req = klass.new(uri.request_uri, headers) + req.body = body if body + + res = http.request(req) + {status: res.code.to_i, body: res.body} + end + end + + def resolve_table!(object_type) + table = object_type || @default_table + if !table || table.to_s.empty? + raise ArgumentError, "object_type (table) must be provided" + end + + table + end + + def request(method, path, body: nil, query: {}) + url = API_BASE + path + url += "?#{URI.encode_www_form(query)}" unless query.empty? + + headers = { + "Authorization" => "Bearer #{@api_key}", + "Content-Type" => "application/json", + "Accept" => "application/json", + } + + raw_body = body && JSON.dump(body) + + begin + res = @http.request(method, url, headers: headers, body: raw_body) + rescue => e + raise Etlify::TransportError.new( + "HTTP transport error: #{e.class}: #{e.message}", + status: 0, + raw: nil + ) + end + + res[:json] = parse_json_safe(res[:body]) + res + end + + def raise_for_error!(resp, path:) + status = resp[:status].to_i + return if status.between?(200, 299) + + payload = resp[:json].is_a?(Hash) ? resp[:json] : {} + err = payload["error"] if payload + + message = if err.is_a?(Hash) + err["message"] || "Airtable API request failed" + else + payload["message"] || "Airtable API request failed" + end + + type = err["type"] if err.is_a?(Hash) + full = "#{message} (status=#{status}, path=#{path}" + full << ", type=#{type}" if type + full << ")" + + klass = case status + when 401, 403 then Etlify::Unauthorized + when 404 then Etlify::NotFound + when 409, 422 then Etlify::ValidationFailed + when 429 then Etlify::RateLimited + else Etlify::ApiError + end + + raise klass.new( + full, + status: status, + code: type, + category: type, + correlation_id: nil, + details: err, + raw: resp[:body] + ) + end + + def parse_json_safe(str) + return nil if str.nil? || str.empty? + + JSON.parse(str) + rescue JSON::ParserError + nil + end + + def find_record_id_by_field(table, field, value) + formula = build_equality_formula(field.to_s, value) + path = "/#{@base_id}/#{enc(table)}" + query = {filterByFormula: formula, maxRecords: 1, pageSize: 1} + + resp = request(:get, path, query: query) + + if resp[:status] == 200 && resp[:json].is_a?(Hash) + recs = resp[:json]["records"] + return recs.first["id"] if recs.is_a?(Array) && recs.any? + + return nil + end + + return nil if resp[:status] == 404 + + raise_for_error!(resp, path: path) + end + + def update_record(table, record_id, fields) + path = "/#{@base_id}/#{enc(table)}/#{record_id}" + body = {fields: stringify_keys(fields)} + resp = request(:patch, path, body: body) + raise_for_error!(resp, path: path) + true + end + + def create_record(table, fields, id_property, unique_value) + path = "/#{@base_id}/#{enc(table)}" + fs = stringify_keys(fields) + if id_property && unique_value && !fs.key?(id_property.to_s) + fs[id_property.to_s] = unique_value + end + + resp = request(:post, path, body: {fields: fs}) + + if resp[:status].between?(200, 299) && + resp[:json].is_a?(Hash) && + resp[:json]["id"] + return resp[:json]["id"].to_s + elsif resp[:status].between?(200, 299) + # 2xx sans id => cas inattendu => ApiError + raise Etlify::ApiError.new( + "Airtable create returned 2xx without id (status=#{resp[:status]})", + status: resp[:status], + code: nil, + category: nil, + correlation_id: nil, + details: resp[:json], + raw: resp[:body] + ) + end + + raise_for_error!(resp, path: path) + end + + def stringify_keys(hash) + hash.each_with_object({}) { |(k, v), h| h[k.to_s] = v } + end + + def build_equality_formula(field, value) + lhs = "{" + field.to_s.gsub("}", ")") + "}" + rhs = case value + when String + escaped = value.to_s.each_char.map { |ch| (ch == "'") ? "'" : ch }.join + "'" + escaped + "'" + when TrueClass, FalseClass + value ? "TRUE()" : "FALSE()" + when Numeric + value.to_s + else + json = JSON.dump(value) + escaped = json.each_char.map { |ch| (ch == "'") ? "'" : ch }.join + "'" + escaped + "'" + end + "#{lhs} = #{rhs}" + end + + def enc(str) + URI.encode_www_form_component(str) + end + end + end +end diff --git a/spec/adapters/airtable_v0_adapter_spec.rb b/spec/adapters/airtable_v0_adapter_spec.rb new file mode 100644 index 0000000..ae4584d --- /dev/null +++ b/spec/adapters/airtable_v0_adapter_spec.rb @@ -0,0 +1,508 @@ +require "rails_helper" +require "json" + +RSpec.describe Etlify::Adapters::AirtableV0Adapter do + let(:api_key) { "key_test" } + let(:base_id) { "app123" } + let(:table) { "Contacts" } + + # Minimal fake HTTP client with a queue of canned responses + let(:http) do + Class.new do + attr_reader :calls + def initialize + @calls = [] + @queue = [] + @next = nil + end + def request(method, url, headers:, body: nil) + @calls << { method: method, url: url, headers: headers, body: body } + return @queue.shift if @queue.any? + return @next if @next + { status: 200, body: "{}" } + end + def queue=(arr) + @queue = arr.dup + end + def next=(resp) + @next = resp + end + end.new + end + + subject(:adapter) do + described_class.new(api_key: api_key, base_id: base_id, table: table, + http_client: http) + end + + describe "#upsert!" do + it "creates when no match is found" do + http.queue = [ + { status: 200, body: { records: [] }.to_json }, + { status: 200, body: { id: "recNEW" }.to_json } + ] + + id = adapter.upsert!( + object_type: "Contacts", + payload: { email: "a@b.com", first_name: "A" }, + id_property: "email" + ) + + expect(id).to eq("recNEW") + get_call, post_call = http.calls + expect(get_call[:method]).to eq(:get) + expect(get_call[:url]).to include("filterByFormula=") + expect(post_call[:method]).to eq(:post) + fields = JSON.parse(post_call[:body])["fields"] + expect(fields).to eq({ "email" => "a@b.com", "first_name" => "A" }) + end + + it "updates when a match is found via id_property" do + http.queue = [ + { status: 200, body: { records: [{ id: "recABC" }] }.to_json }, + { status: 200, body: { id: "recABC" }.to_json } + ] + + id = adapter.upsert!( + payload: { email: "a@b.com", first_name: "A" }, + id_property: :email + ) + + expect(id).to eq("recABC") + _, patch_call = http.calls + expect(patch_call[:method]).to eq(:patch) + expect(patch_call[:url]).to include("/v0/app123/Contacts/recABC") + expect(JSON.parse(patch_call[:body])["fields"]).to eq({ + "first_name" => "A" + }) + end + + it "injects id_property back on create when removed for search" do + http.queue = [ + { status: 200, body: { records: [] }.to_json }, + { status: 200, body: { id: "recZ" }.to_json } + ] + + id = adapter.upsert!( + payload: { email: "a@b.com", first_name: "A" }, + id_property: "email" + ) + + expect(id).to eq("recZ") + post_call = http.calls.last + fields = JSON.parse(post_call[:body])["fields"] + expect(fields["email"]).to eq("a@b.com") + end + + it "creates when id_property has no value in payload" do + http.queue = [ + { status: 200, body: { id: "recN" }.to_json } + ] + + id = adapter.upsert!( + payload: { first_name: "A" }, + id_property: :email + ) + + expect(id).to eq("recN") + post_call = http.calls.last + fields = JSON.parse(post_call[:body])["fields"] + expect(fields.key?("email")).to be(false) + end + + it "accepts object_type override over default table" do + http.queue = [ + { status: 200, body: { records: [] }.to_json }, + { status: 200, body: { id: "recNEW" }.to_json } + ] + + adapter.upsert!( + object_type: "Another", + payload: { email: "x@x" }, + id_property: :email + ) + + expect(http.calls.first[:url]).to include("/v0/app123/Another?") + end + + it "raises on non-Hash payload" do + expect do + adapter.upsert!(payload: "oops") + end.to raise_error(ArgumentError) + end + + it "raises Etlify errors on search failure (401)" do + http.next = { + status: 401, + body: { error: { type: "AUTH", message: "no" } }.to_json + } + + expect do + adapter.upsert!(payload: { email: "x@y" }, id_property: :email) + end.to raise_error(Etlify::Unauthorized) + end + + it "raises Etlify::ApiError on unexpected status (500)" do + http.next = { + status: 500, + body: { error: { type: "SERVER", message: "boom" } }.to_json + } + + expect do + adapter.upsert!(payload: { email: "x@y" }, id_property: :email) + end.to raise_error(Etlify::ApiError) + end + + it "wraps transport errors into Etlify::TransportError" do + exploding = Class.new do + def request(*) = raise IOError, "socket closed" + end.new + + bad = described_class.new(api_key: api_key, base_id: base_id, + table: table, http_client: exploding) + + expect do + bad.upsert!(payload: { email: "x@y" }, id_property: :email) + end.to raise_error(Etlify::TransportError) + end + + it "returns id as String even if JSON has non-string id" do + http.queue = [ + { status: 200, body: { records: [] }.to_json }, + { status: 200, body: { id: 123 }.to_json } + ] + + id = adapter.upsert!( + payload: { email: "a@b.com" }, + id_property: :email + ) + + expect(id).to eq("123") + end + end + + describe "#delete!" do + it "returns true on success" do + http.next = { status: 200, body: { deleted: true }.to_json } + expect(adapter.delete!(crm_id: "rec123")).to be(true) + end + + it "returns false on 404" do + http.next = { status: 404, body: {}.to_json } + expect(adapter.delete!(crm_id: "rec404")).to be(false) + end + + it "raises mapped error on 429" do + http.next = { + status: 429, + body: { error: { type: "RATE", message: "slow" } }.to_json + } + + expect do + adapter.delete!(crm_id: "rec1") + end.to raise_error(Etlify::RateLimited) + end + + it "raises mapped error on 422" do + http.next = { status: 422, body: { error: {} }.to_json } + + expect do + adapter.delete!(crm_id: "rec1") + end.to raise_error(Etlify::ValidationFailed) + end + + it "raises ArgumentError when crm_id missing" do + expect do + adapter.delete!(crm_id: "") + end.to raise_error(ArgumentError) + end + + it "accepts object_type override" do + http.next = { status: 200, body: { deleted: true }.to_json } + adapter.delete!(object_type: "Companies", crm_id: "rec1") + expect(http.calls.first[:url]).to include("/v0/app123/Companies/rec1") + end + end + + describe "table resolution" do + it "raises when neither object_type nor default is set" do + a = described_class.new(api_key: api_key, base_id: base_id, + table: nil, http_client: http) + expect do + a.upsert!(payload: {}) + end.to raise_error(ArgumentError) + end + end + + describe "#request (private)" do + it "builds URL with query string and sets headers" do + http.next = { status: 200, body: "{}" } + resp = adapter.send( + :request, + :get, + "/#{base_id}/#{table}", + query: { filterByFormula: "{Email} = 'a'" } + ) + + expect(resp[:status]).to eq(200) + call = http.calls.last + expect(call[:url]).to include("?filterByFormula=") + expect(call[:headers]["Authorization"]).to eq("Bearer #{api_key}") + expect(call[:headers]["Accept"]).to eq("application/json") + end + + it "serializes body as JSON when provided" do + http.next = { status: 200, body: "{}" } + adapter.send( + :request, + :post, + "/#{base_id}/#{table}", + body: { fields: { a: 1 } } + ) + + body = http.calls.last[:body] + expect(body).to be_a(String) + expect(JSON.parse(body)).to eq({ "fields" => { "a" => 1 } }) + end + + it "wraps transport error" do + exploding = Class.new do + def request(*) = raise IOError, "boom" + end.new + + a = described_class.new(api_key: api_key, base_id: base_id, + table: table, http_client: exploding) + + expect do + a.send(:request, :get, "/x") + end.to raise_error(Etlify::TransportError) + end + end + + describe "#raise_for_error! (private)" do + it "maps 401/403 to Unauthorized" do + resp = { status: 401, json: { error: { type: "AUTH", message: "x" } } } + expect do + adapter.send(:raise_for_error!, resp, path: "/x") + end.to raise_error(Etlify::Unauthorized) + + resp = { status: 403, json: { error: { message: "x" } } } + expect do + adapter.send(:raise_for_error!, resp, path: "/x") + end.to raise_error(Etlify::Unauthorized) + end + + it "maps 404 to NotFound" do + resp = { status: 404, json: { error: { type: "NF" } } } + expect do + adapter.send(:raise_for_error!, resp, path: "/x") + end.to raise_error(Etlify::NotFound) + end + + it "maps 409/422 to ValidationFailed" do + [409, 422].each do |s| + resp = { status: s, json: { error: {} } } + expect do + adapter.send(:raise_for_error!, resp, path: "/x") + end.to raise_error(Etlify::ValidationFailed) + end + end + + it "maps 429 to RateLimited" do + resp = { status: 429, json: { error: {} } } + expect do + adapter.send(:raise_for_error!, resp, path: "/x") + end.to raise_error(Etlify::RateLimited) + end + + it "maps 500 to ApiError" do + resp = { status: 500, json: { message: "boom" } } + expect do + adapter.send(:raise_for_error!, resp, path: "/x") + end.to raise_error(Etlify::ApiError) + end + + it "handles nil/invalid json payloads gracefully" do + resp = { status: 500, json: nil } + expect do + adapter.send(:raise_for_error!, resp, path: "/x") + end.to raise_error(Etlify::ApiError) + end + end + + describe "#find_record_id_by_field (private)" do + it "returns nil on empty records" do + http.next = { status: 200, body: { records: [] }.to_json } + id = adapter.send(:find_record_id_by_field, "Contacts", "email", "a") + expect(id).to be_nil + end + + it "returns id when present" do + http.next = { status: 200, body: { records: [{ id: "recX" }] }.to_json } + id = adapter.send(:find_record_id_by_field, "Contacts", "email", "a") + expect(id).to eq("recX") + end + + it "returns nil on 404" do + http.next = { status: 404, body: {}.to_json } + id = adapter.send(:find_record_id_by_field, "Contacts", "email", "a") + expect(id).to be_nil + end + + it "raises on 500+" do + http.next = { status: 500, body: { error: {} }.to_json } + expect do + adapter.send(:find_record_id_by_field, "Contacts", "email", "a") + end.to raise_error(Etlify::ApiError) + end + + it "sets filterByFormula, maxRecords and pageSize" do + http.next = { status: 200, body: { records: [] }.to_json } + adapter.send(:find_record_id_by_field, "Contacts", "email", "a") + url = http.calls.last[:url] + expect(url).to include("filterByFormula=") + expect(url).to include("maxRecords=1") + expect(url).to include("pageSize=1") + end + end + + describe "#update_record (private)" do + it "returns true on success and stringifies keys" do + http.next = { status: 200, body: { id: "rec1" }.to_json } + ok = adapter.send(:update_record, "Contacts", "rec1", { a: 1 }) + expect(ok).to be(true) + body = JSON.parse(http.calls.last[:body]) + expect(body).to eq({ "fields" => { "a" => 1 } }) + end + + it "raises mapped error on 422" do + http.next = { status: 422, body: { error: {} }.to_json } + expect do + adapter.send(:update_record, "Contacts", "rec1", { a: 1 }) + end.to raise_error(Etlify::ValidationFailed) + end + end + + describe "#create_record (private)" do + it "returns id on success" do + http.next = { status: 200, body: { id: "rec99" }.to_json } + id = adapter.send(:create_record, "Contacts", { a: 1 }, nil, nil) + expect(id).to eq("rec99") + end + + it "raises ApiError when 2xx without id" do + # simulate 200 with no id and then ensure raise mapping occurs + http.next = { status: 200, body: {}.to_json } + expect do + adapter.send(:create_record, "Contacts", { a: 1 }, nil, nil) + end.to raise_error(Etlify::ApiError) + end + + it "raises mapped error (403)" do + http.next = { status: 403, body: { error: {} }.to_json } + expect do + adapter.send(:create_record, "Contacts", { a: 1 }, nil, nil) + end.to raise_error(Etlify::Unauthorized) + end + + it "injects id_property when missing in fields" do + http.next = { status: 200, body: { id: "rec1" }.to_json } + adapter.send(:create_record, "Contacts", { a: 1 }, :email, "a@b") + body = JSON.parse(http.calls.last[:body]) + expect(body["fields"]).to include({ "email" => "a@b" }) + end + end + + describe "helpers (private)" do + it "builds equality formula for strings with quotes" do + f = adapter.send(:build_equality_formula, "Name", "O'Hara") + expect(f).to eq("{Name} = 'O\'Hara'") + end + + it "builds equality formula for booleans" do + expect(adapter.send(:build_equality_formula, "Active", true)) + .to eq("{Active} = TRUE()") + expect(adapter.send(:build_equality_formula, "Active", false)) + .to eq("{Active} = FALSE()") + end + + it "builds equality formula for numbers" do + expect(adapter.send(:build_equality_formula, "Age", 12)) + .to eq("{Age} = 12") + end + + it "builds equality formula for objects (JSON)" do + f = adapter.send(:build_equality_formula, "Meta", { a: 1 }) + expect(f).to eq("{Meta} = '{\"a\":1}'") + end + + it "escapes } in field name" do + f = adapter.send(:build_equality_formula, "A}B", "x") + expect(f).to start_with("{A)B}") + end + + it "stringify_keys works" do + out = adapter.send(:stringify_keys, { a: 1, "b" => 2 }) + expect(out).to eq({ "a" => 1, "b" => 2 }) + end + + it "parse_json_safe returns nil on invalid JSON and on empty" do + expect(adapter.send(:parse_json_safe, "not json")).to be_nil + expect(adapter.send(:parse_json_safe, "")).to be_nil + end + + it "enc encodes special characters" do + expect(adapter.send(:enc, "A B/C")).to eq("A+B%2FC") + end + end +end + +# Cover DefaultHttp paths explicitly to hit lines inside that class +RSpec.describe Etlify::Adapters::AirtableV0Adapter::DefaultHttp do + subject(:http) { described_class.new } + + it "raises on unsupported method" do + expect do + http.request(:put, "https://x", headers: {}) + end.to raise_error(ArgumentError, /Unsupported method/) + end + + it "returns status and body on success (with Net::HTTP stubbed)" do + # Build a fake Net::HTTP ecosystem + response = Struct.new(:code, :body) + fake_res = response.new("200", "{}") + + fake_req_class = Class.new do + def initialize(path, headers) + @path = path + @headers = headers + end + attr_accessor :body + end + + fake_http = Class.new do + def initialize(*); end + def use_ssl=(v); @ssl = v; end + def request(req); @req = req; @res; end + attr_writer :res + end + + stub_const("Net::HTTP::Get", fake_req_class) + + instance = fake_http.new + instance.res = fake_res + + allow(Net::HTTP).to receive(:new).and_return(instance) + + out = http.request(:get, "https://api.airtable.com/v0", headers: {}) + expect(out).to eq({ status: 200, body: "{}" }) + end + + it "re-raises transport errors (rescued then raised)" do + allow(Net::HTTP).to receive(:new).and_raise(Errno::ECONNREFUSED) + + expect { + http.request(:get, "https://x", headers: {}) + }.to raise_error(Errno::ECONNREFUSED) + end +end