diff --git a/app/controllers/admin/registrars_controller.rb b/app/controllers/admin/registrars_controller.rb index 7aa5809d49..7ca79a4502 100644 --- a/app/controllers/admin/registrars_controller.rb +++ b/app/controllers/admin/registrars_controller.rb @@ -1,4 +1,6 @@ require 'net/http' +require 'securerandom' +require 'stringio' module Admin class RegistrarsController < BaseController # rubocop:disable Metrics/ClassLength @@ -13,8 +15,25 @@ class RegistrarsController < BaseController # rubocop:disable Metrics/ClassLengt def index registrars = filter_by_status @q = registrars.ransack(params[:q]) - @registrars = @q.result(distinct: true).page(params[:page]) + @registrars_scope = @q.result(distinct: true) + @registrars = @registrars_scope.page(params[:page]) @registrars = @registrars.per(params[:results_per_page]) if paginate? + + respond_to do |format| + format.html + format.csv do + export_scope = selected_export_scope(@registrars_scope) + send_data( + CsvSync::Exporter.call( + model_class: Registrar, + records: export_scope, + fields: csv_export_fields + ), + filename: "registrars-#{Time.zone.today}.csv", + type: 'text/csv' + ) + end + end end def new @@ -64,6 +83,65 @@ def destroy end end + def import + @csv_sync_fields = Registrar.csv_sync_default_import_fields + @selected_fields = @csv_sync_fields + @csv_sync_field_groups = csv_sync_field_groups + end + + def import_preview + set_import_form_defaults + @result = empty_import_result + + if params[:file].blank? + flash.now[:alert] = t('admin.registrars.import_preview.file_required') + return render :import + end + + @result = CsvSync::Importer.preview( + model_class: Registrar, + file: params[:file], + fields: @selected_fields + ) + @row_results = @result.row_results + @import_token = cache_import_file(params[:file]) + render :import_preview + rescue StandardError => e + @result = empty_import_result(errors: 1, row_results: [{ line_number: '-', action: :error, key_values: {}, changes: [], error: e.message }]) + flash.now[:alert] = t('admin.registrars.import_preview.failed') + render :import + end + + def import_apply + set_import_form_defaults + @result = empty_import_result + file = file_from_apply_params + + if file.blank? + flash[:alert] = t('admin.registrars.import_apply.file_not_found') + return redirect_to import_admin_registrars_path + end + + @result = CsvSync::Importer.apply( + model_class: Registrar, + file: file, + fields: @selected_fields + ) + clear_cached_import_file(params[:import_token]) + + flash[:notice] = t( + 'admin.registrars.import_apply.completed', + created: @result.created, + updated: @result.updated, + unchanged: @result.unchanged, + errors: @result.errors + ) + redirect_to admin_registrars_path + rescue StandardError => e + flash[:alert] = "#{t('admin.registrars.import_apply.failed')}: #{e.message}" + redirect_to import_admin_registrars_path + end + private def filter_by_status @@ -134,5 +212,65 @@ def allowed_method(records_param) allowed_methods = %w[api_users white_ips] records_param if allowed_methods.include?(records_param) end + + def csv_export_fields + params.fetch(:csv_fields, []).reject(&:blank?) + end + + def selected_export_scope(scope) + return scope unless params[:export_selected].present? + + selected_ids = params.fetch(:registrar_ids, []).reject(&:blank?) + return scope.none if selected_ids.empty? + + scope.where(id: selected_ids) + end + + def set_import_form_defaults + @csv_sync_fields = Registrar.csv_sync_default_import_fields + @selected_fields = params.fetch(:fields, []).reject(&:blank?) + @selected_fields = @csv_sync_fields if @selected_fields.empty? + @csv_sync_field_groups = csv_sync_field_groups + end + + def csv_sync_field_groups + { t('admin.csv_sync.field_checkboxes.default_group') => Registrar.csv_sync_default_import_fields } + end + + def empty_import_result(errors: 0, row_results: []) + CsvSync::Importer::Result.new( + created: 0, + updated: 0, + unchanged: 0, + errors: errors, + row_results: row_results + ) + end + + def cache_import_file(file) + token = SecureRandom.uuid + io = file.respond_to?(:tempfile) ? file.tempfile : file + io.rewind if io.respond_to?(:rewind) + csv_payload = io.read + session[:registrars_csv_imports] ||= {} + session[:registrars_csv_imports][token] = csv_payload + token + end + + def file_from_apply_params + return params[:file] if params[:file].present? + return nil if params[:import_token].blank? + + csv_payload = session.fetch(:registrars_csv_imports, {})[params[:import_token]] + return nil if csv_payload.blank? + + StringIO.new(csv_payload) + end + + def clear_cached_import_file(token) + return if token.blank? + + session[:registrars_csv_imports]&.delete(token) + end end end diff --git a/app/models/concerns/csv_sync/model_config.rb b/app/models/concerns/csv_sync/model_config.rb new file mode 100644 index 0000000000..4b3a0759fc --- /dev/null +++ b/app/models/concerns/csv_sync/model_config.rb @@ -0,0 +1,104 @@ +module CsvSync::ModelConfig + extend ActiveSupport::Concern + + BOOLEAN_TRUE_VALUES = %w[1 true t yes y on].freeze + BOOLEAN_FALSE_VALUES = %w[0 false f no n off].freeze + + class_methods do + def csv_sync_field_definitions + const_get(:FIELD_DEFINITIONS) + end + + def csv_sync_fields + csv_sync_field_definitions.keys + end + + def csv_sync_key_fields + csv_sync_field_definitions.filter_map { |field, config| field if config[:key] } + end + + def csv_sync_default_export_fields + csv_sync_field_definitions.filter_map { |field, config| field if config[:default_export] } + end + + def csv_sync_default_import_fields + csv_sync_field_definitions.filter_map { |field, config| field if config[:default_import] } + end + + def csv_sync_type_for(field) + csv_sync_field_definitions.fetch(field.to_sym).fetch(:type) + end + + def csv_sync_import_value(field, raw_value) + csv_sync_deserialize(raw_value, type: csv_sync_type_for(field)) + end + + def csv_sync_export_value(field, value) + csv_sync_serialize(value, type: csv_sync_type_for(field)) + end + + private + + def csv_sync_serialize(value, type:) + return nil if value.nil? + + case type + when :boolean + value ? 'true' : 'false' + when :decimal + value.to_s + when :datetime + value.respond_to?(:iso8601) ? value.iso8601 : value.to_s + when :json + value.to_json + else + value.to_s + end + end + + def csv_sync_deserialize(raw_value, type:) + value = raw_value.is_a?(String) ? raw_value.strip : raw_value + return nil if value.blank? + + case type + when :boolean + parse_csv_sync_boolean(value) + when :decimal + BigDecimal(value.to_s) + when :datetime + Time.zone.parse(value.to_s) + when :json + parse_csv_sync_json(value) + else + value + end + end + + def parse_csv_sync_boolean(value) + normalized = value.to_s.strip.downcase + return true if BOOLEAN_TRUE_VALUES.include?(normalized) + return false if BOOLEAN_FALSE_VALUES.include?(normalized) + + raise ArgumentError, "Invalid boolean value: #{value.inspect}" + end + + def parse_csv_sync_json(value) + case value + when Hash + value + else + JSON.parse(value.to_s) + end + rescue JSON::ParserError => e + raise ArgumentError, "Invalid JSON value: #{e.message}" + end + end + + def csv_sync_export_value(field) + self.class.csv_sync_export_value(field, public_send(field)) + end + + def csv_sync_import_value(field, raw_value) + self.class.csv_sync_import_value(field, raw_value) + end +end diff --git a/app/models/concerns/registrar/csv_sync.rb b/app/models/concerns/registrar/csv_sync.rb new file mode 100644 index 0000000000..232b5406c7 --- /dev/null +++ b/app/models/concerns/registrar/csv_sync.rb @@ -0,0 +1,38 @@ +module Registrar::CsvSync + extend ActiveSupport::Concern + include CsvSync::ModelConfig + + FIELD_DEFINITIONS = { + code: { + key: true, + default_export: true, + default_import: false, + type: :string, + }, + name: { default_export: true, default_import: true, type: :string }, + reg_no: { default_export: true, default_import: true, type: :string }, + email: { default_export: true, default_import: true, type: :string }, + billing_email: { default_export: true, default_import: true, type: :string }, + phone: { default_export: true, default_import: true, type: :string }, + website: { default_export: true, default_import: true, type: :string }, + language: { default_export: true, default_import: true, type: :string }, + test_registrar: { default_export: true, default_import: false, type: :boolean }, + address_street: { default_export: true, default_import: true, type: :string }, + address_zip: { default_export: true, default_import: true, type: :string }, + address_city: { default_export: true, default_import: true, type: :string }, + address_state: { default_export: true, default_import: true, type: :string }, + address_country_code: { default_export: true, default_import: true, type: :string }, + vat_no: { default_export: true, default_import: true, type: :string }, + vat_rate: { default_export: true, default_import: true, type: :decimal }, + iban: { default_export: true, default_import: true, type: :string }, + accounting_customer_code: { default_export: true, default_import: false, type: :string }, + reference_no: { default_export: true, default_import: false, type: :string }, + legaldoc_optout: { default_export: true, default_import: true, type: :boolean }, + legaldoc_optout_comment: { default_export: true, default_import: true, type: :string }, + accept_pdf_invoices: { default_export: true, default_import: true, type: :boolean }, + settings: { default_export: true, default_import: true, type: :json }, + accreditation_date: { default_export: true, default_import: false, type: :datetime }, + accreditation_expire_date: { default_export: true, default_import: false, type: :datetime }, + }.freeze + +end diff --git a/app/models/registrar.rb b/app/models/registrar.rb index afe304242f..7c7ad26777 100644 --- a/app/models/registrar.rb +++ b/app/models/registrar.rb @@ -1,6 +1,7 @@ class Registrar < ApplicationRecord # rubocop:disable Metrics/ClassLength include Versions # version/registrar_version.rb include Registrar::BookKeeping + include Registrar::CsvSync include EmailVerifable include Registrar::LegalDoc diff --git a/app/services/csv_sync/exporter.rb b/app/services/csv_sync/exporter.rb new file mode 100644 index 0000000000..ec269a07e7 --- /dev/null +++ b/app/services/csv_sync/exporter.rb @@ -0,0 +1,38 @@ +require 'csv' + +class CsvSync::Exporter + def self.call(model_class:, records:, fields: nil) + new(model_class: model_class, records: records, fields: fields).to_csv + end + + def initialize(model_class:, records:, fields: nil) + @model_class = model_class + @records = records + @fields = normalize_fields(fields) + end + + def to_csv + CSV.generate do |csv| + csv << headers + records.find_each do |record| + csv << fields.map { |field| model_class.csv_sync_export_value(field, record.public_send(field)) } + end + end + end + + private + + attr_reader :model_class, :records, :fields + + def normalize_fields(fields) + allowed_fields = model_class.csv_sync_fields + selected_fields = Array(fields).map(&:to_sym).select { |field| allowed_fields.include?(field) } + selected_fields = model_class.csv_sync_default_export_fields if selected_fields.empty? + + (model_class.csv_sync_key_fields + selected_fields).uniq + end + + def headers + fields.map(&:to_s) + end +end diff --git a/app/services/csv_sync/importer.rb b/app/services/csv_sync/importer.rb new file mode 100644 index 0000000000..5b4777fccd --- /dev/null +++ b/app/services/csv_sync/importer.rb @@ -0,0 +1,226 @@ +require 'csv' + +class CsvSync::Importer + Result = Struct.new(:created, :updated, :unchanged, :errors, :row_results, keyword_init: true) do + def total + created + updated + unchanged + errors + end + end + + def self.preview(model_class:, file:, fields: nil) + new(model_class: model_class, file: file, fields: fields).preview + end + + def self.apply(model_class:, file:, fields: nil) + new(model_class: model_class, file: file, fields: fields).apply + end + + def initialize(model_class:, file:, fields: nil) + @model_class = model_class + @file = file + @fields = normalize_fields(fields) + end + + def preview + process_rows(persist: false) + end + + def apply + process_rows(persist: true) + end + + private + + attr_reader :model_class, :file, :fields + + def process_rows(persist:) + rows = parse_rows + ensure_key_headers!(rows.headers) + + row_results = rows.map.with_index(2) do |row, line_number| + process_row(row, line_number: line_number, persist: persist) + rescue StandardError => e + build_error_result(line_number: line_number, error: e.message) + end + + summarize(row_results) + end + + def process_row(row, line_number:, persist:) + key_values = parse_key_values(row) + record = find_record(key_values) + attrs = parse_import_attrs(row) + + if record + diff_attrs = changed_attrs(record, attrs) + return build_row_result(line_number: line_number, action: :unchanged) if diff_attrs.empty? + + if persist + apply_update!(record, diff_attrs) + else + record.assign_attributes(diff_attrs) + raise ActiveRecord::RecordInvalid, record unless record.valid? + end + + build_row_result( + line_number: line_number, + action: :updated, + key_values: key_values, + changes: diff_attrs.keys + ) + else + create_attrs = key_values.merge(attrs) + + if persist + apply_create!(create_attrs) + else + preview_record = model_class.new(create_attrs) + raise ActiveRecord::RecordInvalid, preview_record unless preview_record.valid? + end + + build_row_result(line_number: line_number, action: :created, key_values: key_values) + end + rescue ActiveRecord::RecordInvalid => e + build_error_result( + line_number: line_number, + key_values: key_values, + error: e.record.errors.full_messages.to_sentence + ) + rescue ArgumentError, TypeError => e + build_error_result(line_number: line_number, key_values: key_values, error: e.message) + end + + def parse_rows + io = file.respond_to?(:tempfile) ? file.tempfile : file + io.rewind if io.respond_to?(:rewind) + content = io.read + CSV.parse(content, headers: true, col_sep: detect_col_sep(content)) + end + + def detect_col_sep(content) + first_line = content.to_s.each_line.first.to_s + comma_count = first_line.count(',') + semicolon_count = first_line.count(';') + semicolon_count > comma_count ? ';' : ',' + end + + def parse_key_values(row) + model_class.csv_sync_key_fields.each_with_object({}) do |field, acc| + header = field.to_s + raw_value = row[header] + if raw_value.blank? + raise ArgumentError, "Missing #{header} value" + end + + acc[field] = model_class.csv_sync_import_value(field, raw_value) + end + end + + def parse_import_attrs(row) + fields.each_with_object({}) do |field, acc| + next if model_class.csv_sync_key_fields.include?(field) + + header = field.to_s + next unless row.headers.include?(header) + + acc[field] = model_class.csv_sync_import_value(field, row[header]) + end + end + + def ensure_key_headers!(headers) + missing_headers = model_class.csv_sync_key_fields.map(&:to_s) - headers + return if missing_headers.empty? + + raise ArgumentError, "Missing required CSV headers: #{missing_headers.join(', ')}" + end + + def normalize_fields(fields) + allowed_fields = model_class.csv_sync_fields + selected_fields = Array(fields).map(&:to_sym).select { |field| allowed_fields.include?(field) } + selected_fields = model_class.csv_sync_default_import_fields if selected_fields.empty? + + (model_class.csv_sync_key_fields + selected_fields).uniq + end + + def find_record(key_values) + scope = model_class.all + + key_values.each do |field, value| + if value.is_a?(String) + column_name = model_class.connection.quote_column_name(field) + scope = scope.where("UPPER(#{column_name}) = ?", value.upcase) + else + scope = scope.where(field => value) + end + end + + scope.first + end + + def changed_attrs(record, attrs) + attrs.each_with_object({}) do |(field, value), changes| + changes[field] = value if record.public_send(field) != value + end + end + + def apply_create!(attrs) + if model_class.respond_to?(:csv_sync_create_record) + model_class.csv_sync_create_record(attrs) + return + end + + if model_class.name == 'Registrar' + create_registrar!(attrs) + return + end + + model_class.create!(attrs) + end + + def apply_update!(record, attrs) + if model_class.respond_to?(:csv_sync_update_record) + model_class.csv_sync_update_record(record, attrs) + return + end + + record.update!(attrs) + end + + def create_registrar!(attrs) + registrar = model_class.new(attrs) + registrar.reference_no ||= ::Billing::ReferenceNo.generate(owner: registrar.name) + + registrar.transaction do + registrar.save! + registrar.accounts.create!(account_type: Account::CASH, currency: 'EUR') + end + end + + def summarize(row_results) + Result.new( + created: row_results.count { |result| result[:action] == :created }, + updated: row_results.count { |result| result[:action] == :updated }, + unchanged: row_results.count { |result| result[:action] == :unchanged }, + errors: row_results.count { |result| result[:action] == :error }, + row_results: row_results + ) + end + + def build_row_result(line_number:, action:, key_values: {}, changes: []) + { + line_number: line_number, + action: action, + key_values: key_values, + changes: changes, + } + end + + def build_error_result(line_number:, key_values: {}, error:) + build_row_result( + line_number: line_number, + action: :error, + key_values: key_values, + changes: [] + ).merge(error: error) + end +end diff --git a/app/views/admin/csv_sync/_export_actions.html.erb b/app/views/admin/csv_sync/_export_actions.html.erb new file mode 100644 index 0000000000..6fb89c386d --- /dev/null +++ b/app/views/admin/csv_sync/_export_actions.html.erb @@ -0,0 +1,17 @@ +<% export_label = local_assigns.fetch(:export_label, t('admin.csv_sync.export_actions.export_btn')) %> +<% reset_label = local_assigns.fetch(:reset_label, t('admin.csv_sync.export_actions.reset_btn')) %> + +
<%= instructions %>
+ +| <%= t('admin.csv_sync.preview_table.line') %> | +<%= t('admin.csv_sync.preview_table.key') %> | +<%= t('admin.csv_sync.preview_table.action') %> | +<%= t('admin.csv_sync.preview_table.changes') %> | +<%= t('admin.csv_sync.preview_table.error') %> | +
|---|---|---|---|---|
| <%= row[:line_number] %> | +
+ <% key_fields.each do |field| %>
+ <% value = row.fetch(:key_values, {})[field.to_sym] || row.fetch(:key_values, {})[field.to_s] %>
+ <%= "#{field}: #{value}" %> + <% end %> + |
+ <%= t("admin.csv_sync.actions.#{action}", default: action.humanize) %> | ++ <% if row[:changes].present? %> + <%= row[:changes].map { |field| t("activerecord.attributes.#{model_name}.#{field}", default: field.to_s.humanize) }.join(', ') %> + <% else %> + - + <% end %> + | +<%= row[:error] %> | +