From f7ea03bcf63dd89040182607d4868ebce99fec86 Mon Sep 17 00:00:00 2001 From: Perry Date: Fri, 26 Jan 2018 12:55:56 -0800 Subject: [PATCH 1/9] Log queries in DEBUG level --- lib/lhm/chunker.rb | 1 + lib/lhm/entangler.rb | 1 + lib/lhm/migrator.rb | 1 + 3 files changed, 3 insertions(+) diff --git a/lib/lhm/chunker.rb b/lib/lhm/chunker.rb index 0691462c..b8365f37 100644 --- a/lib/lhm/chunker.rb +++ b/lib/lhm/chunker.rb @@ -29,6 +29,7 @@ def execute @next_to_insert = @start while @next_to_insert < @limit || (@start == @limit) stride = @throttler.stride + Lhm.logger.debug(copy(bottom, top(stride))) affected_rows = @connection.update(copy(bottom, top(stride))) if @throttler && affected_rows > 0 diff --git a/lib/lhm/entangler.rb b/lib/lhm/entangler.rb index c73bf2be..2d31764f 100644 --- a/lib/lhm/entangler.rb +++ b/lib/lhm/entangler.rb @@ -79,6 +79,7 @@ def validate def before entangle.each do |stmt| + Lhm.logger.debug(tagged(stmt)) @connection.execute(tagged(stmt)) end end diff --git a/lib/lhm/migrator.rb b/lib/lhm/migrator.rb index b0ee6608..de84fcd9 100644 --- a/lib/lhm/migrator.rb +++ b/lib/lhm/migrator.rb @@ -210,6 +210,7 @@ def destination_create original = %{CREATE TABLE `#{ @origin.name }`} replacement = %{CREATE TABLE `#{ @origin.destination_name }`} stmt = @origin.ddl.gsub(original, replacement) + Lhm.logger.debug(tagged(stmt)) @connection.execute(tagged(stmt)) end From f953a6151b1338cf6f0c106f69a67b8331ce9552 Mon Sep 17 00:00:00 2001 From: Perry Date: Fri, 26 Jan 2018 13:30:59 -0800 Subject: [PATCH 2/9] Allow different naming strategy --- lib/lhm/table.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/lhm/table.rb b/lib/lhm/table.rb index 7ff30fd2..861710fd 100644 --- a/lib/lhm/table.rb +++ b/lib/lhm/table.rb @@ -6,6 +6,8 @@ module Lhm class Table attr_reader :name, :columns, :indices, :pk, :ddl + @@naming_strategy = nil + @@default_naming_strategy = lambda { |name| "lhmn_#{ @name }" } def initialize(name, pk = 'id', ddl = nil) @name = name @@ -15,13 +17,18 @@ def initialize(name, pk = 'id', ddl = nil) @ddl = ddl end + def self.naming_strategy=(naming_strategy) + @@naming_strategy = naming_strategy + end + def satisfies_id_column_requirement? !!((id = columns['id']) && id[:type] =~ /(bigint|int)\(\d+\)/) end def destination_name - "lhmn_#{ @name }" + naming_strategy = @@naming_strategy || @@default_naming_strategy + naming_strategy.call(@name) end def self.parse(table_name, connection) From 604a9da750e7ba04fb7666e6939ee4febf49a4b9 Mon Sep 17 00:00:00 2001 From: Perry Date: Fri, 26 Jan 2018 14:20:34 -0800 Subject: [PATCH 3/9] Add sync_table --- lib/lhm.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/lhm.rb b/lib/lhm.rb index 8436e926..b322601d 100644 --- a/lib/lhm.rb +++ b/lib/lhm.rb @@ -50,6 +50,14 @@ def change_table(table_name, options = {}, &block) true end + def sync_table(table_name, sync_table, options = {}, &block) + origin = Table.parse(table_name, connection) + invoker = Invoker.new(origin, connection) + block.call(invoker.migrator) + invoker.run(options) + true + end + # Cleanup tables and triggers # # @param [Boolean] run execute now or just display information From 0a192db5cc67891b03d1a66032abcdf5df3d1b18 Mon Sep 17 00:00:00 2001 From: Perry Date: Fri, 26 Jan 2018 14:21:44 -0800 Subject: [PATCH 4/9] Override naming_strategy for sync_table --- lib/lhm.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/lhm.rb b/lib/lhm.rb index b322601d..9232f2b4 100644 --- a/lib/lhm.rb +++ b/lib/lhm.rb @@ -51,10 +51,13 @@ def change_table(table_name, options = {}, &block) end def sync_table(table_name, sync_table, options = {}, &block) + Lhm::Table.naming_strategy = lambda{|x| sync_table.to_s } origin = Table.parse(table_name, connection) invoker = Invoker.new(origin, connection) block.call(invoker.migrator) invoker.run(options) + ensure + Lhm::Table.naming_strategy = nil true end From 28b35e513383c1663be55eed711fe12a9adbce8f Mon Sep 17 00:00:00 2001 From: Perry Date: Fri, 26 Jan 2018 14:22:39 -0800 Subject: [PATCH 5/9] Allow disable switcher --- lib/lhm.rb | 1 + lib/lhm/invoker.rb | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/lhm.rb b/lib/lhm.rb index 9232f2b4..575bd791 100644 --- a/lib/lhm.rb +++ b/lib/lhm.rb @@ -55,6 +55,7 @@ def sync_table(table_name, sync_table, options = {}, &block) origin = Table.parse(table_name, connection) invoker = Invoker.new(origin, connection) block.call(invoker.migrator) + options.merge!({disable_switcher: true}) invoker.run(options) ensure Lhm::Table.naming_strategy = nil diff --git a/lib/lhm/invoker.rb b/lib/lhm/invoker.rb index 72baac1f..49d35446 100644 --- a/lib/lhm/invoker.rb +++ b/lib/lhm/invoker.rb @@ -44,10 +44,14 @@ def run(options = {}) Entangler.new(migration, @connection).run do Chunker.new(migration, @connection, options).run - if options[:atomic_switch] - AtomicSwitcher.new(migration, @connection).run + if options[:disable_switcher] + Lhm.logger.debug 'switcher is disabled' else - LockedSwitcher.new(migration, @connection).run + if options[:atomic_switch] + AtomicSwitcher.new(migration, @connection).run + else + LockedSwitcher.new(migration, @connection).run + end end end end From d605c737792c8c4d8a39f0d45ec9de3a4c8cf319 Mon Sep 17 00:00:00 2001 From: Perry Date: Fri, 26 Jan 2018 14:23:11 -0800 Subject: [PATCH 6/9] Allow not delete triggers --- lib/lhm.rb | 2 +- lib/lhm/entangler.rb | 6 ++++-- lib/lhm/invoker.rb | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/lhm.rb b/lib/lhm.rb index 575bd791..33d01053 100644 --- a/lib/lhm.rb +++ b/lib/lhm.rb @@ -55,7 +55,7 @@ def sync_table(table_name, sync_table, options = {}, &block) origin = Table.parse(table_name, connection) invoker = Invoker.new(origin, connection) block.call(invoker.migrator) - options.merge!({disable_switcher: true}) + options.merge!({disable_switcher: true, delete_trigger_after: false}) invoker.run(options) ensure Lhm::Table.naming_strategy = nil diff --git a/lib/lhm/entangler.rb b/lib/lhm/entangler.rb index 2d31764f..09196e9f 100644 --- a/lib/lhm/entangler.rb +++ b/lib/lhm/entangler.rb @@ -9,15 +9,16 @@ class Entangler include Command include SqlHelper - attr_reader :connection + attr_reader :connection, :delete_trigger_after # Creates entanglement between two tables. All creates, updates and deletes # to origin will be repeated on the destination table. - def initialize(migration, connection = nil) + def initialize(migration, connection = nil, delete_trigger_after = true) @intersection = migration.intersection @origin = migration.origin @destination = migration.destination @connection = connection + @delete_trigger_after = delete_trigger_after end def entangle @@ -85,6 +86,7 @@ def before end def after + return unless @delete_trigger_after untangle.each do |stmt| @connection.execute(tagged(stmt)) end diff --git a/lib/lhm/invoker.rb b/lib/lhm/invoker.rb index 49d35446..48649903 100644 --- a/lib/lhm/invoker.rb +++ b/lib/lhm/invoker.rb @@ -42,7 +42,7 @@ def run(options = {}) set_session_lock_wait_timeouts migration = @migrator.run - Entangler.new(migration, @connection).run do + Entangler.new(migration, @connection, options[:delete_trigger_after] && true).run do Chunker.new(migration, @connection, options).run if options[:disable_switcher] Lhm.logger.debug 'switcher is disabled' From 1c529fb91453d5f3bbd363c342e7f38b1eacc850 Mon Sep 17 00:00:00 2001 From: Perry Date: Fri, 26 Jan 2018 16:10:07 -0800 Subject: [PATCH 7/9] Add SyncInvoker and SyncENtangler --- lib/lhm/entangler.rb | 6 ++++++ lib/lhm/invoker.rb | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/lib/lhm/entangler.rb b/lib/lhm/entangler.rb index 09196e9f..28eb9de1 100644 --- a/lib/lhm/entangler.rb +++ b/lib/lhm/entangler.rb @@ -102,4 +102,10 @@ def strip(sql) sql.strip.gsub(/\n */, "\n") end end + + class SyncEntangler < Entangler + def after + # do nothing + end + end end diff --git a/lib/lhm/invoker.rb b/lib/lhm/invoker.rb index 48649903..037868ed 100644 --- a/lib/lhm/invoker.rb +++ b/lib/lhm/invoker.rb @@ -82,4 +82,19 @@ def normalize_options(options) raise end end + + class SyncInvoker < Invoker + def run(options = {}) + normalize_options(options) + set_session_lock_wait_timeouts + migration = @migrator.run + + SyncEntangler.new(migration, @connection).run do + Chunker.new(migration, @connection, options).run + end + rescue => e + revert + raise + end + end end From 122b5ff2d2067c68106d53020ab1a2003e7d0686 Mon Sep 17 00:00:00 2001 From: Perry Date: Fri, 26 Jan 2018 16:11:03 -0800 Subject: [PATCH 8/9] Replace SyncInvoker when syncing table --- lib/lhm.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lhm.rb b/lib/lhm.rb index 33d01053..d6409e63 100644 --- a/lib/lhm.rb +++ b/lib/lhm.rb @@ -53,7 +53,7 @@ def change_table(table_name, options = {}, &block) def sync_table(table_name, sync_table, options = {}, &block) Lhm::Table.naming_strategy = lambda{|x| sync_table.to_s } origin = Table.parse(table_name, connection) - invoker = Invoker.new(origin, connection) + invoker = SyncInvoker.new(origin, connection) block.call(invoker.migrator) options.merge!({disable_switcher: true, delete_trigger_after: false}) invoker.run(options) From ff23c4d228c6558cb1e797b7de1d23b7e568de76 Mon Sep 17 00:00:00 2001 From: Perry Date: Fri, 26 Jan 2018 16:11:49 -0800 Subject: [PATCH 9/9] Remove disable_switcher flag --- lib/lhm.rb | 1 - lib/lhm/entangler.rb | 6 ++---- lib/lhm/invoker.rb | 10 +++------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/lhm.rb b/lib/lhm.rb index d6409e63..2bbbc78b 100644 --- a/lib/lhm.rb +++ b/lib/lhm.rb @@ -55,7 +55,6 @@ def sync_table(table_name, sync_table, options = {}, &block) origin = Table.parse(table_name, connection) invoker = SyncInvoker.new(origin, connection) block.call(invoker.migrator) - options.merge!({disable_switcher: true, delete_trigger_after: false}) invoker.run(options) ensure Lhm::Table.naming_strategy = nil diff --git a/lib/lhm/entangler.rb b/lib/lhm/entangler.rb index 28eb9de1..57502918 100644 --- a/lib/lhm/entangler.rb +++ b/lib/lhm/entangler.rb @@ -9,16 +9,15 @@ class Entangler include Command include SqlHelper - attr_reader :connection, :delete_trigger_after + attr_reader :connection # Creates entanglement between two tables. All creates, updates and deletes # to origin will be repeated on the destination table. - def initialize(migration, connection = nil, delete_trigger_after = true) + def initialize(migration, connection = nil) @intersection = migration.intersection @origin = migration.origin @destination = migration.destination @connection = connection - @delete_trigger_after = delete_trigger_after end def entangle @@ -86,7 +85,6 @@ def before end def after - return unless @delete_trigger_after untangle.each do |stmt| @connection.execute(tagged(stmt)) end diff --git a/lib/lhm/invoker.rb b/lib/lhm/invoker.rb index 037868ed..e3e44516 100644 --- a/lib/lhm/invoker.rb +++ b/lib/lhm/invoker.rb @@ -44,14 +44,10 @@ def run(options = {}) Entangler.new(migration, @connection, options[:delete_trigger_after] && true).run do Chunker.new(migration, @connection, options).run - if options[:disable_switcher] - Lhm.logger.debug 'switcher is disabled' + if options[:atomic_switch] + AtomicSwitcher.new(migration, @connection).run else - if options[:atomic_switch] - AtomicSwitcher.new(migration, @connection).run - else - LockedSwitcher.new(migration, @connection).run - end + LockedSwitcher.new(migration, @connection).run end end end