diff --git a/Gemfile b/Gemfile index b14a6d3..44359d4 100644 --- a/Gemfile +++ b/Gemfile @@ -1,16 +1,13 @@ -source 'http://rubygems.org' +source 'https://rubygems.org' -# To use debugger -# gem 'debugger' - -gem 'rails', '>=3.2' +gem 'rails', '>=6.0' +gem 'nokogiri' group :test do - # Pretty printed test output gem 'sqlite3' - gem 'shoulda' + gem 'shoulda-context' gem 'shoulda-matchers' gem 'minitest' - gem 'minitest-reporters' # Rubymine needs this - gem 'mocha', :require => false # gives us stubs and mocks -end \ No newline at end of file + gem 'minitest-reporters' + gem 'mocha', require: false +end diff --git a/Gemfile.lock b/Gemfile.lock index 6bafdef..6dc5258 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,126 +1,248 @@ GEM - remote: http://rubygems.org/ + remote: https://rubygems.org/ specs: - actionmailer (4.2.5) - actionpack (= 4.2.5) - actionview (= 4.2.5) - activejob (= 4.2.5) - mail (~> 2.5, >= 2.5.4) - rails-dom-testing (~> 1.0, >= 1.0.5) - actionpack (4.2.5) - actionview (= 4.2.5) - activesupport (= 4.2.5) - rack (~> 1.6) - rack-test (~> 0.6.2) - rails-dom-testing (~> 1.0, >= 1.0.5) - rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (4.2.5) - activesupport (= 4.2.5) + action_text-trix (2.1.16) + railties + actioncable (8.1.2) + actionpack (= 8.1.2) + activesupport (= 8.1.2) + nio4r (~> 2.0) + websocket-driver (>= 0.6.1) + zeitwerk (~> 2.6) + actionmailbox (8.1.2) + actionpack (= 8.1.2) + activejob (= 8.1.2) + activerecord (= 8.1.2) + activestorage (= 8.1.2) + activesupport (= 8.1.2) + mail (>= 2.8.0) + actionmailer (8.1.2) + actionpack (= 8.1.2) + actionview (= 8.1.2) + activejob (= 8.1.2) + activesupport (= 8.1.2) + mail (>= 2.8.0) + rails-dom-testing (~> 2.2) + actionpack (8.1.2) + actionview (= 8.1.2) + activesupport (= 8.1.2) + nokogiri (>= 1.8.5) + rack (>= 2.2.4) + rack-session (>= 1.0.1) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + useragent (~> 0.16) + actiontext (8.1.2) + action_text-trix (~> 2.1.15) + actionpack (= 8.1.2) + activerecord (= 8.1.2) + activestorage (= 8.1.2) + activesupport (= 8.1.2) + globalid (>= 0.6.0) + nokogiri (>= 1.8.5) + actionview (8.1.2) + activesupport (= 8.1.2) builder (~> 3.1) - erubis (~> 2.7.0) - rails-dom-testing (~> 1.0, >= 1.0.5) - rails-html-sanitizer (~> 1.0, >= 1.0.2) - activejob (4.2.5) - activesupport (= 4.2.5) - globalid (>= 0.3.0) - activemodel (4.2.5) - activesupport (= 4.2.5) - builder (~> 3.1) - activerecord (4.2.5) - activemodel (= 4.2.5) - activesupport (= 4.2.5) - arel (~> 6.0) - activesupport (4.2.5) - i18n (~> 0.7) - json (~> 1.7, >= 1.7.7) - minitest (~> 5.1) - thread_safe (~> 0.3, >= 0.3.4) - tzinfo (~> 1.1) + erubi (~> 1.11) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + activejob (8.1.2) + activesupport (= 8.1.2) + globalid (>= 0.3.6) + activemodel (8.1.2) + activesupport (= 8.1.2) + activerecord (8.1.2) + activemodel (= 8.1.2) + activesupport (= 8.1.2) + timeout (>= 0.4.0) + activestorage (8.1.2) + actionpack (= 8.1.2) + activejob (= 8.1.2) + activerecord (= 8.1.2) + activesupport (= 8.1.2) + marcel (~> 1.0) + activesupport (8.1.2) + base64 + bigdecimal + concurrent-ruby (~> 1.0, >= 1.3.1) + connection_pool (>= 2.2.5) + drb + i18n (>= 1.6, < 2) + json + logger (>= 1.4.2) + minitest (>= 5.1) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) ansi (1.5.0) - arel (6.0.3) - builder (3.2.2) - concurrent-ruby (1.0.0) - erubis (2.7.0) - globalid (0.3.6) - activesupport (>= 4.1.0) - i18n (0.7.0) - json (1.8.3) - loofah (2.0.3) - nokogiri (>= 1.5.9) - mail (2.6.3) - mime-types (>= 1.16, < 3) - metaclass (0.0.4) - mime-types (2.99) - mini_portile2 (2.0.0) - minitest (5.8.3) - minitest-reporters (1.1.7) + base64 (0.3.0) + bigdecimal (4.0.1) + builder (3.3.0) + concurrent-ruby (1.3.6) + connection_pool (3.0.2) + crass (1.0.6) + date (3.5.1) + drb (2.2.3) + erb (6.0.1) + erubi (1.13.1) + globalid (1.3.0) + activesupport (>= 6.1) + i18n (1.14.8) + concurrent-ruby (~> 1.0) + io-console (0.8.2) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + json (2.18.0) + logger (1.7.0) + loofah (2.25.0) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + mail (2.9.0) + logger + mini_mime (>= 0.1.1) + net-imap + net-pop + net-smtp + marcel (1.1.0) + mini_mime (1.1.5) + minitest (6.0.1) + prism (~> 1.5) + minitest-reporters (1.7.1) ansi builder minitest (>= 5.0) ruby-progressbar - mocha (1.1.0) - metaclass (~> 0.0.1) - nokogiri (1.6.7.1) - mini_portile2 (~> 2.0.0.rc2) - rack (1.6.4) - rack-test (0.6.3) - rack (>= 1.0) - rails (4.2.5) - actionmailer (= 4.2.5) - actionpack (= 4.2.5) - actionview (= 4.2.5) - activejob (= 4.2.5) - activemodel (= 4.2.5) - activerecord (= 4.2.5) - activesupport (= 4.2.5) - bundler (>= 1.3.0, < 2.0) - railties (= 4.2.5) - sprockets-rails - rails-deprecated_sanitizer (1.0.3) - activesupport (>= 4.2.0.alpha) - rails-dom-testing (1.0.7) - activesupport (>= 4.2.0.beta, < 5.0) - nokogiri (~> 1.6.0) - rails-deprecated_sanitizer (>= 1.0.1) - rails-html-sanitizer (1.0.2) - loofah (~> 2.0) - railties (4.2.5) - actionpack (= 4.2.5) - activesupport (= 4.2.5) - rake (>= 0.8.7) - thor (>= 0.18.1, < 2.0) - rake (10.4.2) - ruby-progressbar (1.7.5) - shoulda (3.5.0) - shoulda-context (~> 1.0, >= 1.0.1) - shoulda-matchers (>= 1.4.1, < 3.0) - shoulda-context (1.2.1) - shoulda-matchers (2.8.0) - activesupport (>= 3.0.0) - sprockets (3.5.2) + mocha (3.0.1) + ruby2_keywords (>= 0.0.5) + net-imap (0.6.2) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout + net-smtp (0.5.1) + net-protocol + nio4r (2.7.5) + nokogiri (1.19.0-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-aarch64-linux-musl) + racc (~> 1.4) + nokogiri (1.19.0-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm-linux-musl) + racc (~> 1.4) + nokogiri (1.19.0-arm64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-linux-musl) + racc (~> 1.4) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.9.0) + psych (5.3.1) + date + stringio + racc (1.8.1) + rack (3.2.4) + rack-session (2.1.1) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.2.0) + rack (>= 1.3) + rackup (2.3.1) + rack (>= 3) + rails (8.1.2) + actioncable (= 8.1.2) + actionmailbox (= 8.1.2) + actionmailer (= 8.1.2) + actionpack (= 8.1.2) + actiontext (= 8.1.2) + actionview (= 8.1.2) + activejob (= 8.1.2) + activemodel (= 8.1.2) + activerecord (= 8.1.2) + activestorage (= 8.1.2) + activesupport (= 8.1.2) + bundler (>= 1.15.0) + railties (= 8.1.2) + rails-dom-testing (2.3.0) + activesupport (>= 5.0.0) + minitest + nokogiri (>= 1.6) + rails-html-sanitizer (1.6.2) + loofah (~> 2.21) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (8.1.2) + actionpack (= 8.1.2) + activesupport (= 8.1.2) + irb (~> 1.13) + rackup (>= 1.0.0) + rake (>= 12.2) + thor (~> 1.0, >= 1.2.2) + tsort (>= 0.2) + zeitwerk (~> 2.6) + rake (13.3.1) + rdoc (7.1.0) + erb + psych (>= 4.0.0) + tsort + reline (0.6.3) + io-console (~> 0.5) + ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) + securerandom (0.4.1) + shoulda-context (2.0.0) + shoulda-matchers (7.0.1) + activesupport (>= 7.1) + sqlite3 (2.9.0-aarch64-linux-gnu) + sqlite3 (2.9.0-aarch64-linux-musl) + sqlite3 (2.9.0-arm-linux-gnu) + sqlite3 (2.9.0-arm-linux-musl) + sqlite3 (2.9.0-arm64-darwin) + sqlite3 (2.9.0-x86_64-darwin) + sqlite3 (2.9.0-x86_64-linux-gnu) + sqlite3 (2.9.0-x86_64-linux-musl) + stringio (3.2.0) + thor (1.5.0) + timeout (0.6.0) + tsort (0.2.0) + tzinfo (2.0.6) concurrent-ruby (~> 1.0) - rack (> 1, < 3) - sprockets-rails (3.0.0) - actionpack (>= 4.0) - activesupport (>= 4.0) - sprockets (>= 3.0.0) - sqlite3 (1.3.11) - thor (0.19.1) - thread_safe (0.3.5) - tzinfo (1.2.2) - thread_safe (~> 0.1) + uri (1.1.1) + useragent (0.16.11) + websocket-driver (0.8.0) + base64 + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.5) + zeitwerk (2.7.4) PLATFORMS - ruby + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl + arm64-darwin + x86_64-darwin + x86_64-linux-gnu + x86_64-linux-musl DEPENDENCIES minitest minitest-reporters mocha - rails (>= 3.2) - shoulda + nokogiri + rails (>= 6.0) + shoulda-context shoulda-matchers sqlite3 BUNDLED WITH - 1.11.2 + 2.6.9 diff --git a/History.txt b/History.txt index 59d0c8c..c99bea8 100644 --- a/History.txt +++ b/History.txt @@ -1,3 +1,19 @@ +== 3.1.0 2026-01-31 + +* Switch from PIP to XML format for SDN data + OFAC discontinued the pipe-delimited (.pip) files on September 25, 2023. + The gem now downloads and parses the XML format from: + https://www.treasury.gov/ofac/downloads/sdn.xml + +* Add Nokogiri dependency for XML parsing + +* Update test infrastructure for Ruby 3.4 and Rails 8 compatibility + - Use shoulda-context instead of deprecated shoulda gem + - Update migrations to use versioned format + - Fix deprecated method calls (File.exists? -> File.exist?) + +* Improve HTTP download with redirect handling + == 3.0.0 2014-07-17 * Complete overhaul: diff --git a/README.rdoc b/README.rdoc index 18f29cf..0925e6f 100644 --- a/README.rdoc +++ b/README.rdoc @@ -4,6 +4,7 @@ * http://www.drexel-labs.com * https://www.treasury.gov/resource-center/sanctions/SDN-List/Pages/default.aspx +* Data source: https://www.treasury.gov/ofac/downloads/sdn.xml == DESCRIPTION: @@ -102,7 +103,8 @@ Acceptable hash keys and their weighting in score calculation: == REQUIREMENTS: -* Rails 3.2.0 or greater +* Rails 6.0 or greater +* Nokogiri gem (for XML parsing) == INSTALL: @@ -112,7 +114,7 @@ Acceptable hash keys and their weighting in score calculation: bin/rake ofac_engine:install:migrations * To add the gem to your Rails project: ===== Add the gem to your Gemfile: - gem "ofac", "~> 3.0.0" + gem "ofac", "~> 3.1.0" ===== Run the Bundler install command bundle install * To load your table with the current OFAC data, from the command line, run: diff --git a/app/models/ofac_sdn_individual_loader.rb b/app/models/ofac_sdn_individual_loader.rb index 85d9027..c043fbc 100644 --- a/app/models/ofac_sdn_individual_loader.rb +++ b/app/models/ofac_sdn_individual_loader.rb @@ -1,6 +1,7 @@ require 'net/http' require 'active_record' require 'tempfile' +require 'nokogiri' begin require 'active_record/connection_adapters/mysql2_adapter' rescue Gem::LoadError, LoadError @@ -12,301 +13,238 @@ end class OfacSdnIndividualLoader - #Loads the most recent file from https://www.treasury.gov/resource-center/sanctions/SDN-List/Pages/default.aspx + SDN_XML_URL = 'https://www.treasury.gov/ofac/downloads/sdn.xml'.freeze + + # Loads the most recent SDN file from OFAC + # The XML format replaced the PIP format which was discontinued on September 25, 2023 def self.load_current_sdn_file puts "Reloading OFAC sdn data" - puts "Downloading OFAC data from https://www.treasury.gov/resource-center/sanctions/SDN-List/Pages/default.aspx" - yield "Downloading OFAC data from https://www.treasury.gov/resource-center/sanctions/SDN-List/Pages/default.aspx" if block_given? - #get the 3 data files - sdn = Tempfile.new('sdn') - uri = URI.parse('https://www.treasury.gov/ofac/downloads/sdn.pip') - proxy_addr, proxy_port = ENV['http_proxy'].gsub("http://", "").split(/:/) if ENV['http_proxy'] - - bytes = sdn.write(Net::HTTP::Proxy(proxy_addr, proxy_port).get(uri)) - sdn.rewind - if bytes == 0 || convert_line_to_array(sdn.readline).size != 12 - puts "Trouble downloading file. The url may have changed." - yield "Trouble downloading file. The url may have changed." if block_given? - return - else - sdn.rewind - end - address = Tempfile.new('sdn') - address.write(Net::HTTP::Proxy(proxy_addr, proxy_port).get(URI.parse('https://www.treasury.gov/ofac/downloads/add.pip'))) - address.rewind - alt = Tempfile.new('sdn') - alt.write(Net::HTTP::Proxy(proxy_addr, proxy_port).get(URI.parse('https://www.treasury.gov/ofac/downloads/alt.pip'))) - alt.rewind - - if (defined?(ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter) && OfacSdnIndividual.connection.kind_of?(ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter)) || (defined?(ActiveRecord::ConnectionAdapters::JdbcAdapter) && OfacSdnIndividual.connection.kind_of?(ActiveRecord::ConnectionAdapters::JdbcAdapter)) - puts "Converting file to csv format for Mysql import. This could take several minutes." - yield "Converting file to csv format for Mysql import. This could take several minutes." if block_given? - - csv_file = convert_to_flattened_csv(sdn, address, alt) do |status| - yield status if block_given? - end + puts "Downloading OFAC data from #{SDN_XML_URL}" + yield "Downloading OFAC data from #{SDN_XML_URL}" if block_given? + + xml_file = download_sdn_xml + return unless xml_file + + records = parse_sdn_xml(xml_file) + puts "Parsed #{records.size} individual records from XML" + yield "Parsed #{records.size} individual records from XML" if block_given? + if mysql_adapter? + puts "Converting to CSV format for MySQL import..." + yield "Converting to CSV format for MySQL import..." if block_given? + + csv_file = convert_records_to_csv(records) begin bulk_mysql_update(csv_file) do |status| yield status if block_given? end rescue ActiveRecord::StatementInvalid - sdn.rewind # due to this being read as a side effect of `convert_to_flattened_csv` - - active_record_file_load(sdn, address, alt) do |status| + active_record_load(records) do |status| yield status if block_given? end end - else - active_record_file_load(sdn, address, alt) do |status| + active_record_load(records) do |status| yield status if block_given? end end - sdn.close - @address.close - @alt.close + xml_file.close + puts "OFAC data reload complete" + yield "OFAC data reload complete" if block_given? end + # Load from an XML file (used by tests) + def self.load_from_xml_file(xml_file) + records = parse_sdn_xml(xml_file) + active_record_load(records) + end private - #convert the file's null value to an empty string - #and removes " chars. - def self.clean_file_string(line) - line.gsub!(/-0-(\s)?/, '') - line.gsub!(/[\n\r"]/, '') - line + def self.mysql_adapter? + (defined?(ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter) && + OfacSdnIndividual.connection.kind_of?(ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter)) || + (defined?(ActiveRecord::ConnectionAdapters::JdbcAdapter) && + OfacSdnIndividual.connection.kind_of?(ActiveRecord::ConnectionAdapters::JdbcAdapter)) end - #split the line into an array - def self.convert_line_to_array(line) - clean_file_string(line).split('|', -1) unless line.nil? - end + def self.download_sdn_xml + xml_file = Tempfile.new('sdn_xml') + uri = URI.parse(SDN_XML_URL) + + begin + response = fetch_with_redirects(uri) + bytes = xml_file.write(response) + xml_file.rewind - #return an 2 arrays of the records matching the sdn primary key - #1 array of address records and one array of alt records - def self.foreign_key_records(sdn_id) - address_records = [] - alt_records = [] - - #the first element in each array is the primary and foreign keys - #we are denormalizing the data - loop do - break if @current_address_hash.blank? || @current_address_hash[:id].to_i > sdn_id.to_i - if @current_address_hash && @current_address_hash[:id] == sdn_id - address_records << @current_address_hash + if bytes == 0 || !response.include?(' e + puts "Error downloading SDN XML: #{e.message}" + return nil end - loop do - break if @current_alt_hash.blank? || @current_alt_hash[:id].to_i > sdn_id.to_i - if @current_alt_hash && @current_alt_hash[:id] == sdn_id - alt_records << @current_alt_hash - end - @current_alt_hash = alt_text_to_hash(@alt.gets) + xml_file + end + + def self.fetch_with_redirects(uri, limit = 10) + raise 'Too many redirects' if limit == 0 + + proxy_addr, proxy_port = nil, nil + if ENV['http_proxy'] + proxy_addr, proxy_port = ENV['http_proxy'].gsub("http://", "").split(/:/) + end + + http = Net::HTTP.new(uri.host, uri.port, proxy_addr, proxy_port) + http.use_ssl = (uri.scheme == 'https') + http.open_timeout = 30 + http.read_timeout = 120 + + request = Net::HTTP::Get.new(uri.request_uri) + response = http.request(request) + + case response + when Net::HTTPSuccess + response.body + when Net::HTTPRedirection + location = response['location'] + new_uri = URI.parse(location) + # Handle relative redirects + new_uri = URI.join(uri.to_s, location) unless new_uri.host + fetch_with_redirects(new_uri, limit - 1) + else + raise "HTTP Error: #{response.code} #{response.message}" end - return address_records.uniq, alt_records.uniq end - def self.sdn_text_to_hash(line) - if line.present? - value_array = convert_line_to_array(line) - if value_array[2] == 'individual' # sdn_type - last_name, first_name = value_array[1].to_s.split(',') - last_name.try(:gsub!, /[[:punct:]]/, '') - first_name1, first_name2, first_name3, first_name4, first_name5, first_name6, first_name7, first_name8 = first_name.try(:gsub, /[[:punct:]]/, '').try(:split, ' ') - {id: value_array[0], - last_name: last_name.try(:upcase), - first_name_1: first_name1.try(:upcase), - first_name_2: first_name2.try(:upcase), - first_name_3: first_name3.try(:upcase), - first_name_4: first_name4.try(:upcase), - first_name_5: first_name5.try(:upcase), - first_name_6: first_name6.try(:upcase), - first_name_7: first_name7.try(:upcase), - first_name_8: Array(first_name8).first.try(:upcase) # in case the first name has more than 8 parts, only take up to 8. - } + # Parse the XML file and return an array of record hashes + # Each hash contains the denormalized data for one database record + def self.parse_sdn_xml(xml_file) + doc = Nokogiri::XML(xml_file) + doc.remove_namespaces! # OFAC XML uses a namespace that complicates XPath queries + records = [] + + doc.xpath('//sdnEntry').each do |entry| + sdn_type = entry.at_xpath('sdnType')&.text + next unless sdn_type&.downcase == 'individual' + + sdn_attributes = extract_sdn_attributes(entry) + addresses = extract_addresses(entry) + aliases = extract_aliases(entry) + + # Denormalize: create a record for each address/alias combination + # This matches the behavior of the old PIP loader + if addresses.empty? && aliases.empty? + records << sdn_attributes.merge(empty_address).merge(empty_alias) + elsif addresses.empty? + aliases.each do |alt| + records << sdn_attributes.merge(empty_address).merge(alt) + end + elsif aliases.empty? + addresses.each do |addr| + records << sdn_attributes.merge(addr).merge(empty_alias) + end + else + addresses.each do |addr| + aliases.each do |alt| + records << sdn_attributes.merge(addr).merge(alt) + end + end end end + + records end - def self.address_text_to_hash(line) - unless line.nil? - value_array = convert_line_to_array(line) - {:id => value_array[0], - :address => value_array[2].try(:upcase), - :city => value_array[3].try(:upcase) + def self.extract_sdn_attributes(entry) + last_name = clean_name(entry.at_xpath('lastName')&.text) + first_name = entry.at_xpath('firstName')&.text || '' + first_names = split_first_name(first_name) + + { + last_name: last_name, + first_name_1: first_names[0], + first_name_2: first_names[1], + first_name_3: first_names[2], + first_name_4: first_names[3], + first_name_5: first_names[4], + first_name_6: first_names[5], + first_name_7: first_names[6], + first_name_8: first_names[7] + } + end + + def self.extract_addresses(entry) + addresses = [] + entry.xpath('.//addressList/address').each do |addr| + addresses << { + address: addr.at_xpath('address1')&.text&.upcase, + city: addr.at_xpath('city')&.text&.upcase } end + addresses.uniq end - def self.alt_text_to_hash(line) - unless line.nil? - value_array = convert_line_to_array(line) - alternate_last_name, alternate_first_name = value_array[3].to_s.split(',') - alternate_last_name.try(:gsub!, /[[:punct:]]/, '') - alternate_first_name1, alternate_first_name2, alternate_first_name3, alternate_first_name4, alternate_first_name5, alternate_first_name6, alternate_first_name7, alternate_first_name8 = alternate_first_name.try(:gsub, /[[:punct:]]/, '').try(:split, ' ') - {:id => value_array[0], - alternate_last_name: alternate_last_name.try(:upcase), - alternate_first_name_1: alternate_first_name1.try(:upcase), - alternate_first_name_2: alternate_first_name2.try(:upcase), - alternate_first_name_3: alternate_first_name3.try(:upcase), - alternate_first_name_4: alternate_first_name4.try(:upcase), - alternate_first_name_5: alternate_first_name5.try(:upcase), - alternate_first_name_6: alternate_first_name6.try(:upcase), - alternate_first_name_7: alternate_first_name7.try(:upcase), - alternate_first_name_8: Array(alternate_first_name8).first.try(:upcase) # in case the alternate_first name has more than 8 parts, only take up to 8. + def self.extract_aliases(entry) + aliases = [] + entry.xpath('.//akaList/aka').each do |aka| + last_name = clean_name(aka.at_xpath('lastName')&.text) + first_name = aka.at_xpath('firstName')&.text || '' + first_names = split_first_name(first_name) + + aliases << { + alternate_last_name: last_name, + alternate_first_name_1: first_names[0], + alternate_first_name_2: first_names[1], + alternate_first_name_3: first_names[2], + alternate_first_name_4: first_names[3], + alternate_first_name_5: first_names[4], + alternate_first_name_6: first_names[5], + alternate_first_name_7: first_names[6], + alternate_first_name_8: first_names[7] } end + aliases.uniq end - def self.convert_hash_to_mysql_import_string(record_hash) - new_line = - "`#{record_hash[:last_name]}`|" + - "`#{record_hash[:first_name_1]}`|" + - "`#{record_hash[:first_name_2]}`|" + - "`#{record_hash[:first_name_3]}`|" + - "`#{record_hash[:first_name_4]}`|" + - "`#{record_hash[:first_name_5]}`|" + - "`#{record_hash[:first_name_6]}`|" + - "`#{record_hash[:first_name_7]}`|" + - "`#{record_hash[:first_name_8]}`|" + - "`#{record_hash[:alternate_last_name]}`|" + - "`#{record_hash[:alternate_first_name_1]}`|" + - "`#{record_hash[:alternate_first_name_2]}`|" + - "`#{record_hash[:alternate_first_name_3]}`|" + - "`#{record_hash[:alternate_first_name_4]}`|" + - "`#{record_hash[:alternate_first_name_5]}`|" + - "`#{record_hash[:alternate_first_name_6]}`|" + - "`#{record_hash[:alternate_first_name_7]}`|" + - "`#{record_hash[:alternate_first_name_8]}`|" + - "`#{record_hash[:address]}`|" + - "`#{record_hash[:city]}`|" + - #:created_at - "`#{@db_time}`|" + - # updated_at - "`#{@db_time}`" + "\n" - new_line + def self.clean_name(name) + return nil if name.nil? || name.strip.empty? + name.gsub(/[[:punct:]]/, '').upcase end - def self.convert_to_flattened_csv(sdn_file, address_file, alt_file) - @address = address_file - @alt = alt_file - - @db_time = Time.now.to_s(:db) + def self.split_first_name(first_name) + return Array.new(8) if first_name.nil? || first_name.strip.empty? + parts = first_name.gsub(/[[:punct:]]/, '').upcase.split(/\s+/) + # Only take first 8 parts + (0..7).map { |i| parts[i] } + end - csv_file = Tempfile.new("ofac") # create temp file for converted csv format. - #get the first line from the address and alt files - @current_address_hash = address_text_to_hash(@address.gets) - @current_alt_hash = alt_text_to_hash(@alt.gets) - - start = Time.now - sdn_file.each_with_index do |line, i| - - #initialize the address and alt atributes to empty strings - address_attributes = address_text_to_hash("|||||") - alt_attributes = alt_text_to_hash("||||") - - sdn_attributes = sdn_text_to_hash(line) - - if sdn_attributes.present? - #get the foreign key records for this sdn - address_records, alt_records = foreign_key_records(sdn_attributes[:id]) - - if address_records.empty? - #no matching address records, so initialized blank values will be used. - if alt_records.empty? - #no matching address records, so initialized blank values will be used. - record = convert_hash_to_mysql_import_string(sdn_attributes.merge(address_attributes).merge(alt_attributes)) - csv_file.syswrite(record) if record - else - alt_records.each do |alt| - record = convert_hash_to_mysql_import_string(sdn_attributes.merge(address_attributes).merge(alt)) - csv_file.syswrite(record) if record - end - end - else - address_records.each do |address| - if alt_records.empty? - #no matching address records, so initialized blank values will be used. - record = convert_hash_to_mysql_import_string(sdn_attributes.merge(address).merge(alt_attributes)) - csv_file.syswrite(record) if record - else - alt_records.each do |alt| - record = convert_hash_to_mysql_import_string(sdn_attributes.merge(address).merge(alt)) - csv_file.syswrite(record) if record - end - end - end - end - end - if (i % 1000 == 0) && (i > 0) - puts "#{i} records processed." - yield "#{i} records processed." if block_given? - end - end - puts "File conversion ran for #{(Time.now - start) / 60} minutes." - yield "File conversion ran for #{(Time.now - start) / 60} minutes." if block_given? - return csv_file + def self.empty_address + { address: nil, city: nil } end - def self.active_record_file_load(sdn_file, address_file, alt_file) - @address = address_file - @alt = alt_file + def self.empty_alias + { + alternate_last_name: nil, + alternate_first_name_1: nil, + alternate_first_name_2: nil, + alternate_first_name_3: nil, + alternate_first_name_4: nil, + alternate_first_name_5: nil, + alternate_first_name_6: nil, + alternate_first_name_7: nil, + alternate_first_name_8: nil + } + end - #OFAC data is a complete list, so we have to dump and load + def self.active_record_load(records) OfacSdnIndividual.delete_all - #get the first line from the address and alt files - @current_address_hash = address_text_to_hash(@address.gets) - @current_alt_hash = alt_text_to_hash(@alt.gets) - attributes = {} - sdn_file.each_with_index do |line, i| - - #initialize the address and alt atributes to empty strings - address_attributes = address_text_to_hash("|||||") - alt_attributes = alt_text_to_hash("||||") - - sdn_attributes = sdn_text_to_hash(line) - - if sdn_attributes.present? - #get the foreign key records for this sdn - address_records, alt_records = foreign_key_records(sdn_attributes[:id]) - - if address_records.empty? - #no matching address records, so initialized blank values will be used. - if alt_records.empty? - #no matching address records, so initialized blank values will be used. - attributes = sdn_attributes.merge(address_attributes).merge(alt_attributes) - attributes.delete(:id) - OfacSdnIndividual.create(attributes) - else - alt_records.each do |alt| - attributes = sdn_attributes.merge(address_attributes).merge(alt) - attributes.delete(:id) - OfacSdnIndividual.create(attributes) - end - end - else - address_records.each do |address| - if alt_records.empty? - #no matching address records, so initialized blank values will be used. - attributes = sdn_attributes.merge(address).merge(alt_attributes) - attributes.delete(:id) - OfacSdnIndividual.create(attributes) - else - alt_records.each do |alt| - attributes = sdn_attributes.merge(address).merge(alt) - attributes.delete(:id) - OfacSdnIndividual.create(attributes) - end - end - end - end - end + records.each_with_index do |record, i| + OfacSdnIndividual.create(record) if (i % 5000 == 0) && (i > 0) puts "#{i} records processed." yield "#{i} records processed." if block_given? @@ -314,15 +252,47 @@ def self.active_record_file_load(sdn_file, address_file, alt_file) end end - # For mysql, use: - # LOAD DATA LOCAL INFILE 'ssdm1.csv' INTO TABLE death_master_files FIELDS TERMINATED BY '|' ENCLOSED BY "`" LINES TERMINATED BY '\n'; - # This is a much faster way of loading large amounts of data into mysql. For information on the LOAD DATA command - # see http://dev.mysql.com/doc/refman/5.1/en/load-data.html + def self.convert_records_to_csv(records) + @db_time = Time.now.to_s(:db) + csv_file = Tempfile.new("ofac") + + records.each do |record| + csv_file.syswrite(convert_hash_to_mysql_import_string(record)) + end + + csv_file.rewind + csv_file + end + + def self.convert_hash_to_mysql_import_string(record_hash) + "`#{record_hash[:last_name]}`|" + + "`#{record_hash[:first_name_1]}`|" + + "`#{record_hash[:first_name_2]}`|" + + "`#{record_hash[:first_name_3]}`|" + + "`#{record_hash[:first_name_4]}`|" + + "`#{record_hash[:first_name_5]}`|" + + "`#{record_hash[:first_name_6]}`|" + + "`#{record_hash[:first_name_7]}`|" + + "`#{record_hash[:first_name_8]}`|" + + "`#{record_hash[:alternate_last_name]}`|" + + "`#{record_hash[:alternate_first_name_1]}`|" + + "`#{record_hash[:alternate_first_name_2]}`|" + + "`#{record_hash[:alternate_first_name_3]}`|" + + "`#{record_hash[:alternate_first_name_4]}`|" + + "`#{record_hash[:alternate_first_name_5]}`|" + + "`#{record_hash[:alternate_first_name_6]}`|" + + "`#{record_hash[:alternate_first_name_7]}`|" + + "`#{record_hash[:alternate_first_name_8]}`|" + + "`#{record_hash[:address]}`|" + + "`#{record_hash[:city]}`|" + + "`#{@db_time}`|" + + "`#{@db_time}`\n" + end + def self.bulk_mysql_update(csv_file) puts "Deleting all records in ofac_sdn_individuals..." yield "Deleting all records in ofac_sdn_individuals..." if block_given? - #OFAC data is a complete list, so we have to dump and load OfacSdnIndividual.connection.execute("TRUNCATE ofac_sdn_individuals;") puts "Importing into Mysql..." @@ -335,7 +305,5 @@ def self.bulk_mysql_update(csv_file) OfacSdnIndividual.connection.execute(mysql_command) puts "Mysql import complete." yield "Mysql import complete." if block_given? - end - end diff --git a/db/migrate/20140715173012_create_ofac_sdn_individuals.rb b/db/migrate/20140715173012_create_ofac_sdn_individuals.rb index fb3f0ed..a3d17f0 100644 --- a/db/migrate/20140715173012_create_ofac_sdn_individuals.rb +++ b/db/migrate/20140715173012_create_ofac_sdn_individuals.rb @@ -1,4 +1,4 @@ -class CreateOfacSdnIndividuals < ActiveRecord::Migration +class CreateOfacSdnIndividuals < ActiveRecord::Migration[5.0] def change create_table :ofac_sdn_individuals do |t| t.string :last_name, limit: 50 diff --git a/db/migrate/20140715173240_drop_ofac_sdns.rb b/db/migrate/20140715173240_drop_ofac_sdns.rb index e5fe981..6be3379 100644 --- a/db/migrate/20140715173240_drop_ofac_sdns.rb +++ b/db/migrate/20140715173240_drop_ofac_sdns.rb @@ -1,4 +1,4 @@ -class DropOfacSdns < ActiveRecord::Migration +class DropOfacSdns < ActiveRecord::Migration[5.0] def change drop_table :ofac_sdns if ActiveRecord::Base.connection.table_exists? 'ofac_sdns' end diff --git a/lib/ofac/version.rb b/lib/ofac/version.rb index 8a5a7ea..a126964 100644 --- a/lib/ofac/version.rb +++ b/lib/ofac/version.rb @@ -1,3 +1,3 @@ module Ofac - VERSION = '3.0.1' + VERSION = '3.1.0' end diff --git a/ofac.gemspec b/ofac.gemspec index 8fe0d87..a0dccef 100644 --- a/ofac.gemspec +++ b/ofac.gemspec @@ -18,6 +18,7 @@ Gem::Specification.new do |s| s.test_files = Dir["test/**/*"] s.add_dependency "rails", ">= 3.2" + s.add_dependency "nokogiri" s.add_development_dependency 'sqlite3' diff --git a/test/dummy/config/application.rb b/test/dummy/config/application.rb index fc038b3..2316dc3 100644 --- a/test/dummy/config/application.rb +++ b/test/dummy/config/application.rb @@ -11,12 +11,7 @@ class Application < Rails::Application # Application configuration should go into files in config/initializers # -- all .rb files in that directory are automatically loaded. - # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone. - # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC. - # config.time_zone = 'Central Time (US & Canada)' - - # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded. - # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s] - # config.i18n.default_locale = :de + # Don't check migrations in test environment (using in-memory DB) + config.active_record.maintain_test_schema = false end -end \ No newline at end of file +end diff --git a/test/dummy/config/boot.rb b/test/dummy/config/boot.rb index ef36047..6266cfc 100644 --- a/test/dummy/config/boot.rb +++ b/test/dummy/config/boot.rb @@ -1,5 +1,5 @@ # Set up gems listed in the Gemfile. ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../../Gemfile', __FILE__) -require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE']) +require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE']) $LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__) diff --git a/test/dummy/config/database.yml b/test/dummy/config/database.yml index 51a4dd4..018b8bd 100644 --- a/test/dummy/config/database.yml +++ b/test/dummy/config/database.yml @@ -1,20 +1,15 @@ # SQLite version 3.x -# gem install sqlite3 -# -# Ensure the SQLite 3 gem is defined in your Gemfile -# gem 'sqlite3' +# Use in-memory database for tests to avoid schema environment issues + development: adapter: sqlite3 database: db/development.sqlite3 pool: 5 timeout: 5000 -# Warning: The database defined as "test" will be erased and -# re-generated from your development database when you run "rake". -# Do not set this db to the same as development or production. test: adapter: sqlite3 - database: db/test.sqlite3 + database: ":memory:" pool: 5 timeout: 5000 diff --git a/test/dummy/config/environments/development.rb b/test/dummy/config/environments/development.rb index 9d26e12..00251cc 100644 --- a/test/dummy/config/environments/development.rb +++ b/test/dummy/config/environments/development.rb @@ -25,5 +25,7 @@ # Debug mode disables concatenation and preprocessing of assets. # This option may cause significant delays in view rendering with a large # number of complex assets. - config.assets.debug = true + if config.respond_to?(:assets) + config.assets.debug = true + end end diff --git a/test/dummy/config/environments/test.rb b/test/dummy/config/environments/test.rb index afbc0ae..15e7973 100644 --- a/test/dummy/config/environments/test.rb +++ b/test/dummy/config/environments/test.rb @@ -1,27 +1,26 @@ +require 'active_support/core_ext/integer/time' + Dummy::Application.configure do # Settings specified here will take precedence over those in config/application.rb. - # The test environment is used exclusively to run your application's - # test suite. You never need to work with it otherwise. Remember that - # your test database is "scratch space" for the test suite and is wiped - # and recreated between test runs. Don't rely on the data there! - config.cache_classes = true + # While tests run files are not watched, reloading is not necessary. + config.enable_reloading = false - # Do not eager load code on boot. This avoids loading your whole application - # just for the purpose of running a single test. If you are using a tool that - # preloads Rails for running tests, you may have to set it to true. - config.eager_load = false + # Eager loading loads your entire application. When running a single test locally, + # this is usually not necessary, and can slow down your test suite. However, it's + # recommended that you enable it in continuous integration systems to ensure eager + # loading is working properly before deploying your code. + config.eager_load = ENV["CI"].present? - # Configure static asset server for tests with Cache-Control for performance. - config.serve_static_assets = true - config.static_cache_control = "public, max-age=3600" + # Configure public file server for tests with cache-control for performance. + config.public_file_server.headers = { "cache-control" => "public, max-age=3600" } - # Show full error reports and disable caching. - config.consider_all_requests_local = true - config.action_controller.perform_caching = false + # Show full error reports. + config.consider_all_requests_local = true + config.cache_store = :null_store - # Raise exceptions instead of rendering exception templates. - config.action_dispatch.show_exceptions = false + # Render exception templates for rescuable exceptions and raise for other exceptions. + config.action_dispatch.show_exceptions = :rescuable # Disable request forgery protection in test environment. config.action_controller.allow_forgery_protection = false @@ -31,6 +30,18 @@ # ActionMailer::Base.deliveries array. config.action_mailer.delivery_method = :test + # Set host to be used by links generated in mailer templates. + config.action_mailer.default_url_options = { host: "localhost", port: 3000 } + # Print deprecation notices to the stderr. config.active_support.deprecation = :stderr + + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + + # Disable pending migration check + config.active_record.migration_error = false end diff --git a/test/dummy/db/schema.rb b/test/dummy/db/schema.rb index 323fcfe..95019eb 100644 --- a/test/dummy/db/schema.rb +++ b/test/dummy/db/schema.rb @@ -1,19 +1,17 @@ -# encoding: UTF-8 # This file is auto-generated from the current state of the database. Instead # of editing this file, please use the migrations feature of Active Record to # incrementally modify your database, and then regenerate this schema definition. # -# Note that this schema.rb definition is the authoritative source for your -# database schema. If you need to create the application database on another -# system, you should be using db:schema:load, not running all the migrations -# from scratch. The latter is a flawed and unsustainable approach (the more migrations -# you'll amass, the slower it'll run and the greater likelihood for issues). +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20140715173240) do - - create_table "ofac_sdn_individuals", force: true do |t| +ActiveRecord::Schema[7.0].define(version: 2014_07_15_173240) do + create_table "ofac_sdn_individuals", force: :cascade do |t| t.string "last_name", limit: 50 t.string "first_name_1", limit: 25 t.string "first_name_2", limit: 25 @@ -36,8 +34,7 @@ t.string "city" t.datetime "created_at" t.datetime "updated_at" - end - - add_index "ofac_sdn_individuals", ["last_name", "first_name_1", "first_name_2", "first_name_3", "first_name_4", "first_name_5", "first_name_6", "first_name_7", "first_name_8", "alternate_last_name", "alternate_first_name_1", "alternate_first_name_2", "alternate_first_name_3", "alternate_first_name_4", "alternate_first_name_5", "alternate_first_name_6", "alternate_first_name_7", "alternate_first_name_8"], name: "ofac_sdn_individuals_names" + t.index ["last_name", "first_name_1", "first_name_2", "first_name_3", "first_name_4", "first_name_5", "first_name_6", "first_name_7", "first_name_8", "alternate_last_name", "alternate_first_name_1", "alternate_first_name_2", "alternate_first_name_3", "alternate_first_name_4", "alternate_first_name_5", "alternate_first_name_6", "alternate_first_name_7", "alternate_first_name_8"], name: "ofac_sdn_individuals_names" + end end diff --git a/test/dummy/db/test.sqlite3 b/test/dummy/db/test.sqlite3 index 48914ac..1537a46 100644 Binary files a/test/dummy/db/test.sqlite3 and b/test/dummy/db/test.sqlite3 differ diff --git a/test/files/test_sdn_data_load.xml b/test/files/test_sdn_data_load.xml new file mode 100644 index 0000000..15fadf9 --- /dev/null +++ b/test/files/test_sdn_data_load.xml @@ -0,0 +1,170 @@ + + + + 01/30/2026 + 9 + + + + + 10 + ABASTECEDORA NAVAL Y INDUSTRIAL, s.a. + Entity + + CUBA + + + + + + 15 + A Really Very Extraordinary Incredibly Long First Name Extra + ABDELNUR + Individual + + CUBA + + +
+ 12 + Panama +
+ + + + 14 + a.k.a. + strong + A Really Very Extraordinary Incredibly Long First Name Extra + VIAJES + + + + + + + 22 + Oscar + HERNANDEZ + Individual + + CUBA + + +
+ 14 + 123 Somewhere Ln + Clearwater + United States +
+ + + + 15 + a.k.a. + strong + Oscar Grouch + HERNANDEZ + + + 16 + a.k.a. + strong + Alternate + Name + + + + + + + 24 + Luis Eduardo + LOPEZ MENDEZ + Individual + + CUBA + + + + + + 25 + ACEFROSTY SHIPPING CO., LTD. + Entity + + CUBA + + + + + + 36 + AEROCARIBBEAN AIRLINES + Entity + + CUBA + + + + + + 39 + AEROTAXI EJECUTIVO, S.A. + Entity + + CUBA + + + + + + 41 + AGENCIA DE VIAJES GUAMA + Entity + + CUBA + + + + + + 66 + al-Rahman + AGUIAR + Individual + + CUBA + + +
+ 111 + Italy +
+
+ 117 + Panama +
+
+ 125 + 1840 West 49th Street + United States +
+ + + + 223 + a.k.a. + strong + BNC + + + 224 + a.k.a. + strong + NATIONAL BANK OF CUBA + + + + + diff --git a/test/models/ofac_sdn_individual_loader_test.rb b/test/models/ofac_sdn_individual_loader_test.rb index 23f9bb9..c1dee00 100644 --- a/test/models/ofac_sdn_individual_loader_test.rb +++ b/test/models/ofac_sdn_individual_loader_test.rb @@ -6,7 +6,13 @@ class OfacSdnIndividualLoaderTest < ActiveSupport::TestCase should "load table from files multiple times and always have the same record count with only individual sdn_type records" do assert_equal(0, OfacSdnIndividual.count) load_test_sdn_file - # there are 19 records total but only 8 unique individual records + # there are 9 entries total in the XML but only 4 individuals + # After denormalization (addresses x aliases): + # - ABDELNUR: 1 address x 1 alias = 1 record + # - HERNANDEZ: 1 address x 2 aliases = 2 records + # - LOPEZ MENDEZ: 0 address x 0 alias = 1 record (empty defaults) + # - AGUIAR: 2 unique addresses x 2 aliases = 4 records + # Total: 8 records assert_equal(8, OfacSdnIndividual.count) load_test_sdn_file assert_equal(8, OfacSdnIndividual.count) @@ -40,27 +46,30 @@ class OfacSdnIndividualLoaderTest < ActiveSupport::TestCase # original value of first name is al-Rahman assert_equal 'ALRAHMAN', test.first_name_1 end - should "create flattened_csv_file_for_mysql_import" do - #since, I'm using sqlight3 for it's in memory db, I can't test the mysql load - #but I can test the csv file creation. - sdn = File.new(File.dirname(__FILE__) + '/../files/test_sdn_data_load.pip') - address = File.new(File.dirname(__FILE__) + '/../files/test_address_data_load.pip') - alt = File.new(File.dirname(__FILE__) + '/../files/test_alt_data_load.pip') - csv = OfacSdnIndividualLoader.send(:convert_to_flattened_csv, sdn, address, alt) - correctly_formatted_csv = File.open(File.dirname(__FILE__) + '/../files/valid_flattened_file.csv') + should "parse XML and extract addresses correctly" do + load_test_sdn_file + # HERNANDEZ has an address + test = OfacSdnIndividual.find_by(last_name: 'HERNANDEZ') + assert_equal '123 SOMEWHERE LN', test.address + assert_equal 'CLEARWATER', test.city + end + + should "parse XML and extract aliases correctly" do + load_test_sdn_file + # HERNANDEZ has alias "Alternate Name" + test = OfacSdnIndividual.find_by(last_name: 'HERNANDEZ', alternate_last_name: 'NAME') + assert_equal 'NAME', test.alternate_last_name + assert_equal 'ALTERNATE', test.alternate_first_name_1 + end - csv.rewind - generated_file = csv.readlines - #compare the values of each csv line, with the correctly formated "control file" - correctly_formatted_csv.each_with_index do |line, i| - csv_line = generated_file[i] - correctly_formatted_record_array = line.split('|') - csv_record_array = csv_line.split('|') - (0..9).each do |i| #skip indices 10 and 11, they are the created_at and updated_at fields, they will never match. - assert_equal correctly_formatted_record_array[i], csv_record_array[i] - end - end + should "handle individuals with no address or alias" do + load_test_sdn_file + test = OfacSdnIndividual.find_by(last_name: 'LOPEZ MENDEZ') + assert_not_nil test + assert_nil test.address + assert_nil test.city + assert_nil test.alternate_last_name end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 331a5d5..a242498 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -4,19 +4,15 @@ require File.expand_path('../dummy/config/environment.rb', __FILE__) require 'rails/test_help' -# for RubyMine -require 'minitest/reporters' -MiniTest::Reporters.use! MiniTest::Reporters::SpecReporter.new +# Load schema for in-memory database +ActiveRecord::Schema.verbose = false +schema_file = File.expand_path('../dummy/db/schema.rb', __FILE__) +load(schema_file) class ActiveSupport::TestCase def load_test_sdn_file - sdn = File.new(File.dirname(__FILE__) + '/files/test_sdn_data_load.pip') - address = File.new(File.dirname(__FILE__) + '/files/test_address_data_load.pip') - alt = File.new(File.dirname(__FILE__) + '/files/test_alt_data_load.pip') - OfacSdnIndividualLoader.active_record_file_load(sdn, address, alt) - sdn.close - address.close - alt.close + xml_file = File.new(File.dirname(__FILE__) + '/files/test_sdn_data_load.xml') + OfacSdnIndividualLoader.load_from_xml_file(xml_file) + xml_file.close end end -