diff --git a/Gemfile b/Gemfile index 1ed161f..52fc08d 100644 --- a/Gemfile +++ b/Gemfile @@ -73,6 +73,7 @@ group :development, :test do gem 'ruby-lsp-rspec', require: false gem 'simplecov' gem 'simplecov-cobertura' + gem 'sqlite3', '~> 2.9' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index cb6a151..e2cdf0d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -565,6 +565,10 @@ GEM actionpack (>= 6.1) activesupport (>= 6.1) sprockets (>= 3.0.0) + sqlite3 (2.9.1) + mini_portile2 (~> 2.8.0) + sqlite3 (2.9.1-aarch64-linux-gnu) + sqlite3 (2.9.1-x86_64-linux-gnu) stringio (3.2.0) thor (1.5.0) timeout (0.6.0) @@ -630,6 +634,7 @@ DEPENDENCIES simplecov simplecov-cobertura sprockets-rails + sqlite3 (~> 2.9) CHECKSUMS actioncable (7.2.3) sha256=e15d17b245f1dfe7cafdda4a0c6f7ba8ebaab1af33884415e09cfef4e93ad4f9 @@ -823,6 +828,9 @@ CHECKSUMS simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428 sprockets (4.2.2) sha256=761e5a49f1c288704763f73139763564c845a8f856d52fba013458f8af1b59b1 sprockets-rails (3.5.2) sha256=a9e88e6ce9f8c912d349aa5401509165ec42326baf9e942a85de4b76dbc4119e + sqlite3 (2.9.1) sha256=f6ddc2ec850434ac14498944da9d768fe154dbcd4163fc9e173a524d95e2f887 + sqlite3 (2.9.1-aarch64-linux-gnu) sha256=85535ddf1c37f116ebebe0330bbbffc2ccb55d09f69717a565f8cfb35142f136 + sqlite3 (2.9.1-x86_64-linux-gnu) sha256=1cbb644204ed143e5c96f6d59b5c571ba6f18b18a9dc5aa11c101187ff227afd stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 thor (1.5.0) sha256=e3a9e55fe857e44859ce104a84675ab6e8cd59c650a49106a05f55f136425e73 timeout (0.6.0) sha256=6d722ad619f96ee383a0c557ec6eb8c4ecb08af3af62098a0be5057bf00de1af diff --git a/lib/tasks/sqlite.rake b/lib/tasks/sqlite.rake new file mode 100644 index 0000000..7efe4cb --- /dev/null +++ b/lib/tasks/sqlite.rake @@ -0,0 +1,250 @@ +# frozen_string_literal: true + +namespace :sqlite do + desc 'Create a sqlite3 database for configured tables.' + task create: :environment do + require 'sqlite3' + require 'json' + require 'zlib' + + target_models = [ + Card, + CardCycle, + CardPool, + CardPoolCardCycle, + CardPoolCardSet, + CardSet, + CardSetType, + CardSubtype, + CardType, + Faction, + Format, + Illustrator, + Printing, + Restriction, + RestrictionCardBanned, + RestrictionCardGlobalPenalty, + RestrictionCardPoints, + RestrictionCardRestricted, + RestrictionCardSubtypeBanned, + RestrictionCardUniversalFactionCost, + Side, + Snapshot + ] + + puts 'Loading model table definitions and data from Postgres...' + schema_definitions = {} + data_cache = {} + + target_models.each do |model| + # Parse schema definitions + schema_definitions[model] = model.columns.map do |col| + type = col.type + options = { + limit: col.limit, + precision: col.precision, + scale: col.scale, + default: col.default, + null: col.null + } + + if col.respond_to?(:array) && col.array + type = :text + options[:limit] = nil + options[:precision] = nil + options[:scale] = nil + end + + { + name: col.name, + type: type, + options: options + } + end + + data_cache[model] = model.all.map(&:attributes) + puts "Loaded #{data_cache[model].size} records for #{model.name}" + end + + db_file = Rails.root.join('db', 'netrunnerdb.sqlite3') + File.delete(db_file) if File.exist?(db_file) + + puts "Creating tables in SQLite db at #{db_file}..." + ActiveRecord::Base.establish_connection( + adapter: 'sqlite3', + database: db_file.to_s + ) + connection = ActiveRecord::Base.connection + + puts 'Creating tables...' + target_models.each do |model| + table_name = model.table_name + columns = schema_definitions[model] + primary_key = model.primary_key + + puts "Creating table #{table_name}..." + connection.create_table(table_name, id: false, force: true) do |t| + columns.each do |col_def| + options = col_def[:options] + options[:primary_key] = true if col_def[:name] == primary_key + t.column col_def[:name], col_def[:type], **options + end + end + end + + puts 'Inserting data into SQLite db...' + target_models.each do |model| + rows = data_cache[model] + next if rows.empty? + + # Convert array values to JSON strings for SQLite compatibility + rows.each do |row| + row.each do |key, value| + row[key] = value.to_json if value.is_a?(Array) + end + end + + # Force a reload of backing model information to force using new SQLite schema. + model.reset_column_information + puts " Inserting #{rows.size} records into #{model.table_name}..." + rows.each_slice(100) { |batch| model.insert_all(batch) } # rubocop:disable Rails/SkipsModelValidations + end + + puts 'Verifying data...' + target_models.each do |model| + puts " Verifying #{model.name}..." + original_records = data_cache[model] + + if model.primary_key + verify_by_primary_key(original_records, model) + else + verify_without_primary_key(original_records, model) + end + end + + compress_db(db_file) + + puts 'Done.' + end + + def verify_by_primary_key(original_records, model) + original_records.each do |original_attrs| + pk = model.primary_key + id = original_attrs[pk] + + # Reload explicitly from the new connection + sqlite_record = model.find_by(pk => id) + + unless sqlite_record + puts "MISSING RECORD: #{model.name} #{id}" + next + end + + original_attrs.each do |col, original_val| + sqlite_val = sqlite_record[col] + + # Simple normalization for comparison + match = if original_val.is_a?(Time) && sqlite_val.is_a?(Time) + # Compare up to seconds to allow for precision loss + original_val.to_i == sqlite_val.to_i + else + lhs = original_val + rhs = sqlite_val + + # Handle Array comparison (Postgres Array vs SQLite JSON/String) + if lhs.is_a?(Array) + rhs = begin + JSON.parse(rhs) + rescue StandardError + rhs + end + + # Sort values first if both are arrays + if rhs.is_a?(Array) + lhs = lhs.sort + rhs = rhs.sort + end + end + + lhs == rhs + end + + unless match + puts "MISMATCH: #{model.name} #{id} [#{col}] - PG: #{original_val.inspect} vs SQL: #{sqlite_val.inspect}" + end + end + end + end + + def verify_without_primary_key(original_records, model) + # Verification without Primary Key (Full Table Sort & Compare) + puts " No primary key for #{model.name}, performing full table comparison..." + + # Helper to normalize records for comparison + normalize_for_sort = lambda do |record| + # Transform hash values for consistent sorting + record.transform_values do |val| + case val + when Time + val.to_i + when String + if val.strip.start_with?('[') + begin + JSON.parse(val) + rescue StandardError + val + end + else + val + end + when Array + val.sort + else + val + end + end + end + + # Capture SQLite records + sqlite_records = model.all.map(&:attributes) + + if original_records.size != sqlite_records.size + puts "COUNT MISMATCH: PG: #{original_records.size} vs SQL: #{sqlite_records.size}" + end + + # Sort key generator: use JSON representation to ensure deterministic order + sort_key_gen = ->(r) { r.sort.to_h.to_json } + + normalized_original = original_records.map(&normalize_for_sort).sort_by(&sort_key_gen) + normalized_sqlite = sqlite_records.map(&normalize_for_sort).sort_by(&sort_key_gen) + + normalized_original.each_with_index do |orig_record, idx| + sqlite_record = normalized_sqlite[idx] + + next unless orig_record != sqlite_record + + puts " MISMATCH at sorted index #{idx}:" + puts " PG: #{orig_record.inspect}" + puts " SQL: #{sqlite_record.inspect}" + + orig_record.each do |k, v| + puts " Diff [#{k}]: #{v.inspect} != #{sqlite_record[k].inspect}" if sqlite_record[k] != v + end + end + end + + def compress_db(db_file) + puts 'Compressing database...' + timestamp = Time.now.to_i + gzip_path = "#{db_file}.#{timestamp}.gz" + + Zlib::GzipWriter.open(gzip_path) do |gz| + File.open(db_file.to_s, 'rb') do |file| + while (chunk = file.read(1024 * 1024)) + gz.write(chunk) + end + end + end + puts "Database compressed to #{gzip_path}" + end +end