Task to create a SQLite DB for the card data

Gemfile

@@ -73,6 +73,7 @@ group :development, :test do
   gem 'ruby-lsp-rspec', require: false
   gem 'simplecov'
   gem 'simplecov-cobertura'
+  gem 'sqlite3', '~> 2.9'
 end
 
 group :development do

Gemfile.lock

@@ -565,6 +565,10 @@ GEM
       actionpack (>= 6.1)
       activesupport (>= 6.1)
       sprockets (>= 3.0.0)
+    sqlite3 (2.9.1)
+      mini_portile2 (~> 2.8.0)
+    sqlite3 (2.9.1-aarch64-linux-gnu)
+    sqlite3 (2.9.1-x86_64-linux-gnu)
     stringio (3.2.0)
     thor (1.5.0)
     timeout (0.6.0)
@@ -630,6 +634,7 @@ DEPENDENCIES
   simplecov
   simplecov-cobertura
   sprockets-rails
+  sqlite3 (~> 2.9)
 
 CHECKSUMS
   actioncable (7.2.3) sha256=e15d17b245f1dfe7cafdda4a0c6f7ba8ebaab1af33884415e09cfef4e93ad4f9
@@ -823,6 +828,9 @@ CHECKSUMS
   simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428
   sprockets (4.2.2) sha256=761e5a49f1c288704763f73139763564c845a8f856d52fba013458f8af1b59b1
   sprockets-rails (3.5.2) sha256=a9e88e6ce9f8c912d349aa5401509165ec42326baf9e942a85de4b76dbc4119e
+  sqlite3 (2.9.1) sha256=f6ddc2ec850434ac14498944da9d768fe154dbcd4163fc9e173a524d95e2f887
+  sqlite3 (2.9.1-aarch64-linux-gnu) sha256=85535ddf1c37f116ebebe0330bbbffc2ccb55d09f69717a565f8cfb35142f136
+  sqlite3 (2.9.1-x86_64-linux-gnu) sha256=1cbb644204ed143e5c96f6d59b5c571ba6f18b18a9dc5aa11c101187ff227afd
   stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1
   thor (1.5.0) sha256=e3a9e55fe857e44859ce104a84675ab6e8cd59c650a49106a05f55f136425e73
   timeout (0.6.0) sha256=6d722ad619f96ee383a0c557ec6eb8c4ecb08af3af62098a0be5057bf00de1af

lib/tasks/sqlite.rake

@@ -0,0 +1,250 @@
+# frozen_string_literal: true
+
+namespace :sqlite do
+  desc 'Create a sqlite3 database for configured tables.'
+  task create: :environment do
+    require 'sqlite3'
+    require 'json'
+    require 'zlib'
+
+    target_models = [
+      Card,
+      CardCycle,
+      CardPool,
+      CardPoolCardCycle,
+      CardPoolCardSet,
+      CardSet,
+      CardSetType,
+      CardSubtype,
+      CardType,
+      Faction,
+      Format,
+      Illustrator,
+      Printing,
+      Restriction,
+      RestrictionCardBanned,
+      RestrictionCardGlobalPenalty,
+      RestrictionCardPoints,
+      RestrictionCardRestricted,
+      RestrictionCardSubtypeBanned,
+      RestrictionCardUniversalFactionCost,
+      Side,
+      Snapshot
+    ]
+
+    puts 'Loading model table definitions and data from Postgres...'
+    schema_definitions = {}
+    data_cache = {}
+
+    target_models.each do |model|
+      # Parse schema definitions
+      schema_definitions[model] = model.columns.map do |col|
+        type = col.type
+        options = {
+          limit: col.limit,
+          precision: col.precision,
+          scale: col.scale,
+          default: col.default,
+          null: col.null
+        }
+
+        if col.respond_to?(:array) && col.array
+          type = :text
+          options[:limit] = nil
+          options[:precision] = nil
+          options[:scale] = nil
+        end
+
+        {
+          name: col.name,
+          type: type,
+          options: options
+        }
+      end
+
+      data_cache[model] = model.all.map(&:attributes)
+      puts "Loaded #{data_cache[model].size} records for #{model.name}"
+    end
+
+    db_file = Rails.root.join('db', 'netrunnerdb.sqlite3')
+    File.delete(db_file) if File.exist?(db_file)
+
+    puts "Creating tables in SQLite db at #{db_file}..."
+    ActiveRecord::Base.establish_connection(
+      adapter: 'sqlite3',
+      database: db_file.to_s
+    )
+    connection = ActiveRecord::Base.connection
+
+    puts 'Creating tables...'
+    target_models.each do |model|
+      table_name = model.table_name
+      columns = schema_definitions[model]
+      primary_key = model.primary_key
+
+      puts "Creating table #{table_name}..."
+      connection.create_table(table_name, id: false, force: true) do |t|
+        columns.each do |col_def|
+          options = col_def[:options]
+          options[:primary_key] = true if col_def[:name] == primary_key
+          t.column col_def[:name], col_def[:type], **options
+        end
+      end
+    end
+
+    puts 'Inserting data into SQLite db...'
+    target_models.each do |model|
+      rows = data_cache[model]
+      next if rows.empty?
+
+      # Convert array values to JSON strings for SQLite compatibility
+      rows.each do |row|
+        row.each do |key, value|
+          row[key] = value.to_json if value.is_a?(Array)
+        end
+      end
+
+      # Force a reload of backing model information to force using new SQLite schema.
+      model.reset_column_information
+      puts "  Inserting #{rows.size} records into #{model.table_name}..."
+      rows.each_slice(100) { |batch| model.insert_all(batch) } # rubocop:disable Rails/SkipsModelValidations
+    end
+
+    puts 'Verifying data...'
+    target_models.each do |model|
+      puts "  Verifying #{model.name}..."
+      original_records = data_cache[model]
+
+      if model.primary_key
+        verify_by_primary_key(original_records, model)
+      else
+        verify_without_primary_key(original_records, model)
+      end
+    end
+
+    compress_db(db_file)
+
+    puts 'Done.'
+  end
+
+  def verify_by_primary_key(original_records, model)
+    original_records.each do |original_attrs|
+      pk = model.primary_key
+      id = original_attrs[pk]
+
+      # Reload explicitly from the new connection
+      sqlite_record = model.find_by(pk => id)
+
+      unless sqlite_record
+        puts "MISSING RECORD: #{model.name} #{id}"
+        next
+      end
+
+      original_attrs.each do |col, original_val|
+        sqlite_val = sqlite_record[col]
+
+        # Simple normalization for comparison
+        match = if original_val.is_a?(Time) && sqlite_val.is_a?(Time)
+                  # Compare up to seconds to allow for precision loss
+                  original_val.to_i == sqlite_val.to_i
+                else
+                  lhs = original_val
+                  rhs = sqlite_val
+
+                  # Handle Array comparison (Postgres Array vs SQLite JSON/String)
+                  if lhs.is_a?(Array)
+                    rhs = begin
+                      JSON.parse(rhs)
+                    rescue StandardError
+                      rhs
+                    end
+
+                    # Sort values first if both are arrays
+                    if rhs.is_a?(Array)
+                      lhs = lhs.sort
+                      rhs = rhs.sort
+                    end
+                  end
+
+                  lhs == rhs
+                end
+
+        unless match
+          puts "MISMATCH: #{model.name} #{id} [#{col}] - PG: #{original_val.inspect} vs SQL: #{sqlite_val.inspect}"
+        end
+      end
+    end
+  end
+
+  def verify_without_primary_key(original_records, model)
+    # Verification without Primary Key (Full Table Sort & Compare)
+    puts "    No primary key for #{model.name}, performing full table comparison..."
+
+    # Helper to normalize records for comparison
+    normalize_for_sort = lambda do |record|
+      # Transform hash values for consistent sorting
+      record.transform_values do |val|
+        case val
+        when Time
+          val.to_i
+        when String
+          if val.strip.start_with?('[')
+            begin
+              JSON.parse(val)
+            rescue StandardError
+              val
+            end
+          else
+            val
+          end
+        when Array
+          val.sort
+        else
+          val
+        end
+      end
+    end
+
+    # Capture SQLite records
+    sqlite_records = model.all.map(&:attributes)
+
+    if original_records.size != sqlite_records.size
+      puts "COUNT MISMATCH: PG: #{original_records.size} vs SQL: #{sqlite_records.size}"
+    end
+
+    # Sort key generator: use JSON representation to ensure deterministic order
+    sort_key_gen = ->(r) { r.sort.to_h.to_json }
+
+    normalized_original = original_records.map(&normalize_for_sort).sort_by(&sort_key_gen)
+    normalized_sqlite = sqlite_records.map(&normalize_for_sort).sort_by(&sort_key_gen)
+
+    normalized_original.each_with_index do |orig_record, idx|
+      sqlite_record = normalized_sqlite[idx]
+
+      next unless orig_record != sqlite_record
+
+      puts "    MISMATCH at sorted index #{idx}:"
+      puts "      PG: #{orig_record.inspect}"
+      puts "      SQL: #{sqlite_record.inspect}"
+
+      orig_record.each do |k, v|
+        puts "    Diff [#{k}]: #{v.inspect} != #{sqlite_record[k].inspect}" if sqlite_record[k] != v
+      end
+    end
+  end
+
+  def compress_db(db_file)
+    puts 'Compressing database...'
+    timestamp = Time.now.to_i
+    gzip_path = "#{db_file}.#{timestamp}.gz"
+
+    Zlib::GzipWriter.open(gzip_path) do |gz|
+      File.open(db_file.to_s, 'rb') do |file|
+        while (chunk = file.read(1024 * 1024))
+          gz.write(chunk)
+        end
+      end
+    end
+    puts "Database compressed to #{gzip_path}"
+  end
+end