From c723598e68c0efa85d017741666eb07ce46ef016 Mon Sep 17 00:00:00 2001 From: Joao Vitor Barros da Silva Date: Wed, 4 Feb 2026 10:52:46 -0500 Subject: [PATCH 01/39] Add basic regex to map roles from ETDA API --- app/services/committee_role_normalizer.rb | 24 ++++++++++ .../committee_role_normalizer_spec.rb | 47 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 app/services/committee_role_normalizer.rb create mode 100644 spec/services/committee_role_normalizer_spec.rb diff --git a/app/services/committee_role_normalizer.rb b/app/services/committee_role_normalizer.rb new file mode 100644 index 00000000..084d08d5 --- /dev/null +++ b/app/services/committee_role_normalizer.rb @@ -0,0 +1,24 @@ +class CommitteeRoleNormalizer + PRIORITY_REGEX = [ + ["Co-Chairperson", /\b(co[-\s]?chair|co[-\s]?chairperson|committee chair\/co-chair)\b/i], + ["Chairperson", /\b(chairperson|chair of committee|committee chair|chair)\b/i], + ["Co-Advisor", /\b(co[-\s]?dissertation\s*advis(or|er)|co[-\s]?advisor)\b/i], + ["Advisor", /\b(dissertation\s*advis(or|er)|advisor)\b/i], + ["Supervisor", /\bsupervisor\b/i], + ["Mentor", /\bmentor\b/i], + ["Second Reader", /\bsecond\s+reader\b/i], + ["Reader", /\breader\b/i], + ["Member", /\b(member|rep|represent|representative|substitute)\b/i] + ].freeze + + def self.normalize(raw_name) + text = raw_name.to_s.strip + return "Other" if text.empty? + + PRIORITY_REGEX.each do |label, regex| + return label if text.match?(regex) + end + + "Other" + end +end \ No newline at end of file diff --git a/spec/services/committee_role_normalizer_spec.rb b/spec/services/committee_role_normalizer_spec.rb new file mode 100644 index 00000000..05b8d6a5 --- /dev/null +++ b/spec/services/committee_role_normalizer_spec.rb @@ -0,0 +1,47 @@ +require "rails_helper" + +RSpec.describe CommitteeRoleNormalizer do + describe ".normalize" do + it "maps co-chair roles correctly" do + expect( + described_class.normalize("Co-Chair & Dissertation Advisor") + ).to eq("Co-Chairperson") + end + + it "maps chair roles correctly" do + expect( + described_class.normalize("Chair of Committee") + ).to eq("Chairperson") + end + + it "prioritizes chair over advisor when both appear" do + expect( + described_class.normalize("Chair & Dissertation Advisor") + ).to eq("Chairperson") + end + + it "maps advisor roles" do + expect( + described_class.normalize("Dissertation Advisr") + ).to eq("Advisor") + end + + it "maps member and representative roles" do + expect( + described_class.normalize("Committee Member & Dean Grad Sch Rep") + ).to eq("Member") + end + + it "returns Other for unknown roles" do + expect( + described_class.normalize("Some Weird ETDA Thing") + ).to eq("Other") + end + + it "returns Other for blank values" do + expect( + described_class.normalize(nil) + ).to eq("Other") + end + end +end From 9ab610c574c779cadb34514acfb8bc817ee5c86d Mon Sep 17 00:00:00 2001 From: Joao Vitor Barros da Silva Date: Wed, 4 Feb 2026 12:05:19 -0500 Subject: [PATCH 02/39] Fix committee_role_normalizer to pass Rspec --- app/services/committee_role_normalizer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/committee_role_normalizer.rb b/app/services/committee_role_normalizer.rb index 084d08d5..d3d285ca 100644 --- a/app/services/committee_role_normalizer.rb +++ b/app/services/committee_role_normalizer.rb @@ -3,7 +3,7 @@ class CommitteeRoleNormalizer ["Co-Chairperson", /\b(co[-\s]?chair|co[-\s]?chairperson|committee chair\/co-chair)\b/i], ["Chairperson", /\b(chairperson|chair of committee|committee chair|chair)\b/i], ["Co-Advisor", /\b(co[-\s]?dissertation\s*advis(or|er)|co[-\s]?advisor)\b/i], - ["Advisor", /\b(dissertation\s*advis(or|er)|advisor)\b/i], + ["Advisor", /\b(dissertation\s*advis(?:o?r|er)|advis(?:o?r|er))\b/i], ["Supervisor", /\bsupervisor\b/i], ["Mentor", /\bmentor\b/i], ["Second Reader", /\bsecond\s+reader\b/i], From 4d4a22ca4b765fddb651cfc4dfca12dcf3617fe9 Mon Sep 17 00:00:00 2001 From: Joao Vitor Barros da Silva Date: Wed, 4 Feb 2026 14:13:30 -0500 Subject: [PATCH 03/39] Fix Dockerfile with updated image version and rubocop corrections --- Dockerfile | 2 +- app/services/committee_role_normalizer.rb | 22 +++++----- .../committee_role_normalizer_spec.rb | 44 +++++++++---------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/Dockerfile b/Dockerfile index 445fa475..87d2432c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM harbor.k8s.libraries.psu.edu/library/ruby-3.4.1-node-22:20260123 as base +FROM harbor.k8s.libraries.psu.edu/library/ruby-3.4.1-node-22:20260202 as base ARG UID=1001 WORKDIR /app diff --git a/app/services/committee_role_normalizer.rb b/app/services/committee_role_normalizer.rb index d3d285ca..1e49d280 100644 --- a/app/services/committee_role_normalizer.rb +++ b/app/services/committee_role_normalizer.rb @@ -1,24 +1,24 @@ class CommitteeRoleNormalizer PRIORITY_REGEX = [ - ["Co-Chairperson", /\b(co[-\s]?chair|co[-\s]?chairperson|committee chair\/co-chair)\b/i], - ["Chairperson", /\b(chairperson|chair of committee|committee chair|chair)\b/i], - ["Co-Advisor", /\b(co[-\s]?dissertation\s*advis(or|er)|co[-\s]?advisor)\b/i], - ["Advisor", /\b(dissertation\s*advis(?:o?r|er)|advis(?:o?r|er))\b/i], - ["Supervisor", /\bsupervisor\b/i], - ["Mentor", /\bmentor\b/i], - ["Second Reader", /\bsecond\s+reader\b/i], - ["Reader", /\breader\b/i], - ["Member", /\b(member|rep|represent|representative|substitute)\b/i] + ['Co-Chairperson', %r{\b(co[-\s]?chair|co[-\s]?chairperson|committee chair/co-chair)\b}i], + ['Chairperson', /\b(chairperson|chair of committee|committee chair|chair)\b/i], + ['Co-Advisor', /\b(co[-\s]?dissertation\s*advis(or|er)|co[-\s]?advisor)\b/i], + ['Advisor', /\b(dissertation\s*advis(?:o?r|er)|advis(?:o?r|er))\b/i], + ['Supervisor', /\bsupervisor\b/i], + ['Mentor', /\bmentor\b/i], + ['Second Reader', /\bsecond\s+reader\b/i], + ['Reader', /\breader\b/i], + ['Member', /\b(member|rep|represent|representative|substitute)\b/i] ].freeze def self.normalize(raw_name) text = raw_name.to_s.strip - return "Other" if text.empty? + return 'Other' if text.empty? PRIORITY_REGEX.each do |label, regex| return label if text.match?(regex) end - "Other" + 'Other' end end \ No newline at end of file diff --git a/spec/services/committee_role_normalizer_spec.rb b/spec/services/committee_role_normalizer_spec.rb index 05b8d6a5..e02833c8 100644 --- a/spec/services/committee_role_normalizer_spec.rb +++ b/spec/services/committee_role_normalizer_spec.rb @@ -1,47 +1,47 @@ -require "rails_helper" +require 'rails_helper' RSpec.describe CommitteeRoleNormalizer do - describe ".normalize" do - it "maps co-chair roles correctly" do + describe '.normalize' do + it 'maps co-chair roles correctly' do expect( - described_class.normalize("Co-Chair & Dissertation Advisor") - ).to eq("Co-Chairperson") + described_class.normalize('Co-Chair & Dissertation Advisor') + ).to eq('Co-Chairperson') end - it "maps chair roles correctly" do + it 'maps chair roles correctly' do expect( - described_class.normalize("Chair of Committee") - ).to eq("Chairperson") + described_class.normalize('Chair of Committee') + ).to eq('Chairperson') end - it "prioritizes chair over advisor when both appear" do + it 'prioritizes chair over advisor when both appear' do expect( - described_class.normalize("Chair & Dissertation Advisor") - ).to eq("Chairperson") + described_class.normalize('Chair & Dissertation Advisor') + ).to eq('Chairperson') end - it "maps advisor roles" do + it 'maps advisor roles' do expect( - described_class.normalize("Dissertation Advisr") - ).to eq("Advisor") + described_class.normalize('Dissertation Advisr') + ).to eq('Advisor') end - it "maps member and representative roles" do + it 'maps member and representative roles' do expect( - described_class.normalize("Committee Member & Dean Grad Sch Rep") - ).to eq("Member") + described_class.normalize('Committee Member & Dean Grad Sch Rep') + ).to eq('Member') end - it "returns Other for unknown roles" do + it 'returns Other for unknown roles' do expect( - described_class.normalize("Some Weird ETDA Thing") - ).to eq("Other") + described_class.normalize('Some Weird ETDA Thing') + ).to eq('Other') end - it "returns Other for blank values" do + it 'returns Other for blank values' do expect( described_class.normalize(nil) - ).to eq("Other") + ).to eq('Other') end end end From f2fbf9d59c9820be4c2877c6e169a73159ee127d Mon Sep 17 00:00:00 2001 From: Joao Vitor Barros da Silva Date: Thu, 5 Feb 2026 08:09:20 -0500 Subject: [PATCH 04/39] Rubocop --- app/services/committee_role_normalizer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/committee_role_normalizer.rb b/app/services/committee_role_normalizer.rb index 1e49d280..abdf46c0 100644 --- a/app/services/committee_role_normalizer.rb +++ b/app/services/committee_role_normalizer.rb @@ -21,4 +21,4 @@ def self.normalize(raw_name) 'Other' end -end \ No newline at end of file +end From 8eb239691be99ea1d78206621ca6a574027d598d Mon Sep 17 00:00:00 2001 From: Joao Vitor Barros da Silva Date: Wed, 11 Feb 2026 11:45:08 -0500 Subject: [PATCH 05/39] Add edta api importer --- .../etda_committee_memberships_importer.rb | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 app/importers/committee_data/etda_committee_memberships_importer.rb diff --git a/app/importers/committee_data/etda_committee_memberships_importer.rb b/app/importers/committee_data/etda_committee_memberships_importer.rb new file mode 100644 index 00000000..24ad5d02 --- /dev/null +++ b/app/importers/committee_data/etda_committee_memberships_importer.rb @@ -0,0 +1,82 @@ +require 'json' +require 'builder' + +module Etda + class CommitteeMembershipsXmlExporter + class ExportError < StandardError; end + + def initialize(payload) + @payload = payload.is_a?(String) ? JSON.parse(payload) : payload + rescue JSON::ParserError => e + raise ExportError, "Invalid JSON: #{e.message}" + end + + def to_xml + validate_payload! + + faculty_access_id = @payload.fetch('faculty_access_id').to_s + committees = @payload.fetch('committees') + + xml = Builder::XmlMarkup.new(indent: 2) + xml.instruct!(:xml, version: '1.0', encoding: 'UTF-8') + + xml.faculty_committees do + xml.faculty_access_id faculty_access_id + + xml.committees do + committees.each do |row| + build_committee(xml, row) + end + end + end + + xml.target! + end + + private + + def validate_payload! + raise ExportError, 'Payload must be a JSON object' unless @payload.is_a?(Hash) + + raise ExportError, 'faculty_access_id is required' if @payload['faculty_access_id'].to_s.strip.empty? + + return if @payload['committees'].is_a?(Array) + + raise ExportError, 'committees must be an array' + end + + def build_committee(xml, row) + raise ExportError, 'Each committee must be an object' unless row.is_a?(Hash) + + raw_role = row['role'] + normalized_role = CommitteeRoleNormalizer.normalize(raw_role) + + xml.committee do + xml.committee_member_id row['committee_member_id'] + xml.role raw_role + xml.normalized_role normalized_role + xml.role_code row['role_code'] + + xml.student do + xml.fname row['student_fname'] + xml.lname row['student_lname'] + xml.access_id row['student_access_id'] + end + + xml.submission do + xml.submission_id row['submission_id'] + xml.title row['title'] + xml.degree_name row['degree_name'] + xml.program_name row['program_name'] + xml.semester row['semester'] + xml.year row['year'] + xml.status row['submission_status'] + xml.final_submission_approved_at row['final_submission_approved_at'] + end + + xml.approval_started_at row['approval_started_at'] + xml.committee_member_status row['committee_member_status'] + end + end + end +end From 9ae71a26bd7b2687ac0d5d20ab7eb9f41be65f59 Mon Sep 17 00:00:00 2001 From: Joao Vitor Barros da Silva Date: Wed, 11 Feb 2026 13:31:37 -0500 Subject: [PATCH 06/39] Fix name to etda_committee_memberships_xml_builder.rb --- .../etda_committee_memberships_xml_builder.rb | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 app/importers/committee_data/etda_committee_memberships_xml_builder.rb diff --git a/app/importers/committee_data/etda_committee_memberships_xml_builder.rb b/app/importers/committee_data/etda_committee_memberships_xml_builder.rb new file mode 100644 index 00000000..2e7996ab --- /dev/null +++ b/app/importers/committee_data/etda_committee_memberships_xml_builder.rb @@ -0,0 +1,71 @@ +require 'json' +require 'builder' + +class CommitteeData::EtdaCommitteeMembershipsXmlBuilder + class ExportError < StandardError; end + + def initialize(payload) + @payload = payload.is_a?(String) ? JSON.parse(payload) : payload + rescue JSON::ParserError => e + raise ExportError, "Invalid JSON: #{e.message}" + end + + def to_xml + validate_payload! + faculty_access_id = @payload.fetch('faculty_access_id').to_s + committees = @payload.fetch('committees') + xml = Builder::XmlMarkup.new(indent: 2) + xml.instruct!(:xml, version: '1.0', encoding: 'UTF-8') + + xml.faculty_committees do + xml.faculty_access_id faculty_access_id + + xml.committees do + committees.each do |row| + build_committee(xml, row) + end + end + end + xml.target! + end + + private + + def validate_payload! + raise ExportError, 'Payload must be a JSON object' unless @payload.is_a?(Hash) + raise ExportError, 'faculty_access_id is required' if @payload['faculty_access_id'].to_s.strip.empty? + return if @payload['committees'].is_a?(Array) + + raise ExportError, 'committees must be an array' + end + + def build_committee(xml, row) + raise ExportError, 'Each committee must be an object' unless row.is_a?(Hash) + + raw_role = row['role'] + normalized_role = CommitteeRoleNormalizer.normalize(raw_role) + xml.committee do + xml.committee_member_id row['committee_member_id'] + xml.role raw_role + xml.normalized_role normalized_role + xml.role_code row['role_code'] + xml.student do + xml.fname row['student_fname'] + xml.lname row['student_lname'] + xml.access_id row['student_access_id'] + end + xml.submission do + xml.submission_id row['submission_id'] + xml.title row['title'] + xml.degree_name row['degree_name'] + xml.program_name row['program_name'] + xml.semester row['semester'] + xml.year row['year'] + xml.status row['submission_status'] + xml.final_submission_approved_at row['final_submission_approved_at'] + end + xml.approval_started_at row['approval_started_at'] + xml.committee_member_status row['committee_member_status'] + end + end +end From 804bc0f5b61473a39931adf8924d98d9fa01e9c7 Mon Sep 17 00:00:00 2001 From: Joao Vitor Barros da Silva Date: Wed, 11 Feb 2026 13:52:20 -0500 Subject: [PATCH 07/39] Add rspec tests for the api importer --- .../etda_committee_memberships_importer.rb | 82 ------------------- ..._committee_memberships_xml_builder_spec.rb | 49 +++++++++++ 2 files changed, 49 insertions(+), 82 deletions(-) delete mode 100644 app/importers/committee_data/etda_committee_memberships_importer.rb create mode 100644 spec/importers/committee_data/etda_committee_memberships_xml_builder_spec.rb diff --git a/app/importers/committee_data/etda_committee_memberships_importer.rb b/app/importers/committee_data/etda_committee_memberships_importer.rb deleted file mode 100644 index 24ad5d02..00000000 --- a/app/importers/committee_data/etda_committee_memberships_importer.rb +++ /dev/null @@ -1,82 +0,0 @@ -require 'json' -require 'builder' - -module Etda - class CommitteeMembershipsXmlExporter - class ExportError < StandardError; end - - def initialize(payload) - @payload = payload.is_a?(String) ? JSON.parse(payload) : payload - rescue JSON::ParserError => e - raise ExportError, "Invalid JSON: #{e.message}" - end - - def to_xml - validate_payload! - - faculty_access_id = @payload.fetch('faculty_access_id').to_s - committees = @payload.fetch('committees') - - xml = Builder::XmlMarkup.new(indent: 2) - xml.instruct!(:xml, version: '1.0', encoding: 'UTF-8') - - xml.faculty_committees do - xml.faculty_access_id faculty_access_id - - xml.committees do - committees.each do |row| - build_committee(xml, row) - end - end - end - - xml.target! - end - - private - - def validate_payload! - raise ExportError, 'Payload must be a JSON object' unless @payload.is_a?(Hash) - - raise ExportError, 'faculty_access_id is required' if @payload['faculty_access_id'].to_s.strip.empty? - - return if @payload['committees'].is_a?(Array) - - raise ExportError, 'committees must be an array' - end - - def build_committee(xml, row) - raise ExportError, 'Each committee must be an object' unless row.is_a?(Hash) - - raw_role = row['role'] - normalized_role = CommitteeRoleNormalizer.normalize(raw_role) - - xml.committee do - xml.committee_member_id row['committee_member_id'] - xml.role raw_role - xml.normalized_role normalized_role - xml.role_code row['role_code'] - - xml.student do - xml.fname row['student_fname'] - xml.lname row['student_lname'] - xml.access_id row['student_access_id'] - end - - xml.submission do - xml.submission_id row['submission_id'] - xml.title row['title'] - xml.degree_name row['degree_name'] - xml.program_name row['program_name'] - xml.semester row['semester'] - xml.year row['year'] - xml.status row['submission_status'] - xml.final_submission_approved_at row['final_submission_approved_at'] - end - - xml.approval_started_at row['approval_started_at'] - xml.committee_member_status row['committee_member_status'] - end - end - end -end diff --git a/spec/importers/committee_data/etda_committee_memberships_xml_builder_spec.rb b/spec/importers/committee_data/etda_committee_memberships_xml_builder_spec.rb new file mode 100644 index 00000000..2623993b --- /dev/null +++ b/spec/importers/committee_data/etda_committee_memberships_xml_builder_spec.rb @@ -0,0 +1,49 @@ +require 'rails_helper' + +RSpec.describe CommitteeData::EtdaCommitteeMembershipsXmlBuilder do + describe '#to_xml' do + it 'converts JSON payload to XML and normalizes role' do + payload = { + 'faculty_access_id' => 'aab27', + 'committees' => [ + { + 'committee_member_id' => 101_388, + 'role' => 'Major Field Member', + 'role_code' => 'M', + 'student_fname' => 'Esther', + 'student_lname' => 'Munoz', + 'student_access_id' => 'ecm5494', + 'submission_id' => 22_620, + 'title' => 'Test Title', + 'degree_name' => 'PHD', + 'program_name' => 'Geosciences (PHD)', + 'semester' => 'Spring', + 'year' => 2025, + 'approval_started_at' => '2024-12-09T16:10:33.000-05:00', + 'final_submission_approved_at' => '2025-01-15T15:13:39.000-05:00', + 'submission_status' => 'released for publication metadata only', + 'committee_member_status' => 'approved' + } + ] + } + + allow(CommitteeRoleNormalizer).to receive(:normalize) + .with('Major Field Member') + .and_return('Member') + + xml = described_class.new(payload).to_xml + + expect(xml).to include('aab27') + expect(xml).to include('101388') + expect(xml).to include('Major Field Member') + expect(xml).to include('Member') + end + + it 'raises when faculty_access_id missing' do + payload = { 'committees' => [] } + + expect { described_class.new(payload).to_xml } + .to raise_error(CommitteeData::EtdaCommitteeMembershipsXmlBuilder::ExportError) + end + end +end From f887c48c00d5f97d21e903d8787db86fa29c7176 Mon Sep 17 00:00:00 2001 From: Joao Vitor Barros da Silva Date: Thu, 12 Feb 2026 08:25:06 -0500 Subject: [PATCH 08/39] Refactor ETDA committee memberships XML builder to use Nokogiri --- .../etda_committee_memberships_xml_builder.rb | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/app/importers/committee_data/etda_committee_memberships_xml_builder.rb b/app/importers/committee_data/etda_committee_memberships_xml_builder.rb index 2e7996ab..ed2b4839 100644 --- a/app/importers/committee_data/etda_committee_memberships_xml_builder.rb +++ b/app/importers/committee_data/etda_committee_memberships_xml_builder.rb @@ -1,5 +1,5 @@ require 'json' -require 'builder' +require 'nokogiri' class CommitteeData::EtdaCommitteeMembershipsXmlBuilder class ExportError < StandardError; end @@ -14,19 +14,19 @@ def to_xml validate_payload! faculty_access_id = @payload.fetch('faculty_access_id').to_s committees = @payload.fetch('committees') - xml = Builder::XmlMarkup.new(indent: 2) - xml.instruct!(:xml, version: '1.0', encoding: 'UTF-8') - xml.faculty_committees do - xml.faculty_access_id faculty_access_id + builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| + xml.faculty_committees do + xml.faculty_access_id faculty_access_id - xml.committees do - committees.each do |row| - build_committee(xml, row) + xml.committees do + committees.each do |row| + build_committee(xml, row) + end end end end - xml.target! + builder.to_xml end private @@ -34,9 +34,7 @@ def to_xml def validate_payload! raise ExportError, 'Payload must be a JSON object' unless @payload.is_a?(Hash) raise ExportError, 'faculty_access_id is required' if @payload['faculty_access_id'].to_s.strip.empty? - return if @payload['committees'].is_a?(Array) - - raise ExportError, 'committees must be an array' + raise ExportError, 'committees must be an array' unless @payload['committees'].is_a?(Array) end def build_committee(xml, row) @@ -44,16 +42,19 @@ def build_committee(xml, row) raw_role = row['role'] normalized_role = CommitteeRoleNormalizer.normalize(raw_role) + xml.committee do xml.committee_member_id row['committee_member_id'] xml.role raw_role xml.normalized_role normalized_role xml.role_code row['role_code'] + xml.student do xml.fname row['student_fname'] xml.lname row['student_lname'] xml.access_id row['student_access_id'] end + xml.submission do xml.submission_id row['submission_id'] xml.title row['title'] @@ -64,6 +65,7 @@ def build_committee(xml, row) xml.status row['submission_status'] xml.final_submission_approved_at row['final_submission_approved_at'] end + xml.approval_started_at row['approval_started_at'] xml.committee_member_status row['committee_member_status'] end From e898cc7a9a8eb98af6b7b172d9d989e825280625 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Tue, 17 Feb 2026 10:32:11 -0500 Subject: [PATCH 09/39] Setup skeleton for the api client with initialize --- app/clients/etda/committee_records_client.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 app/clients/etda/committee_records_client.rb diff --git a/app/clients/etda/committee_records_client.rb b/app/clients/etda/committee_records_client.rb new file mode 100644 index 00000000..e69de29b From 15f502a73da0994c4756d744599f12660ce68b65 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Tue, 17 Feb 2026 12:47:38 -0500 Subject: [PATCH 10/39] Skeleton initialized, didnt save file initially --- app/clients/etda/committee_records_client.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/clients/etda/committee_records_client.rb b/app/clients/etda/committee_records_client.rb index e69de29b..be698189 100644 --- a/app/clients/etda/committee_records_client.rb +++ b/app/clients/etda/committee_records_client.rb @@ -0,0 +1,12 @@ +module ETDA + class ComitteeRecordsClient + def initialize + @base_url = ENV['ETDA_API_URL'] + @token = ENV['ETDA_API_TOKEN'] + end + + def faculty_committees (access_id) + + end + end +end From f488a2b97fc2dbe4d5581b9ab2a323b41ae2f344 Mon Sep 17 00:00:00 2001 From: madhurakhandkar Date: Thu, 19 Feb 2026 11:01:31 -0500 Subject: [PATCH 11/39] Finished setting up the parameters for faculty committee --- .gitignore | 2 +- README.md | 2 +- app/clients/etda/committee_records_client.rb | 12 ---- lib/etda/committee_records_client.rb | 54 +++++++++++++++++ .../activity_insight_committee_job_spec.rb | 58 +++++++++++++++++++ 5 files changed, 114 insertions(+), 14 deletions(-) delete mode 100644 app/clients/etda/committee_records_client.rb create mode 100644 lib/etda/committee_records_client.rb create mode 100644 spec/jobs/activity_insight_committee_job_spec.rb diff --git a/.gitignore b/.gitignore index 3cc11584..a6904f74 100644 --- a/.gitignore +++ b/.gitignore @@ -42,5 +42,5 @@ spec/examples.txt .cache .bash_history .envrc -.irb_history +.irb_historyx .DS_Store diff --git a/README.md b/README.md index 88e669ec..40edd08e 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ This app parses data from faculty CVs, and integrates data into Activity Insight Add the following untracked files/folders: - * `config/database.yml` This will be similar to `config/database.yml.sample` + * `'config/database.yml'` This will be similar to `config/database.yml.sample` * `config/activity_insight.yml` * `config/integration_passcode.yml` * `public/log` diff --git a/app/clients/etda/committee_records_client.rb b/app/clients/etda/committee_records_client.rb deleted file mode 100644 index be698189..00000000 --- a/app/clients/etda/committee_records_client.rb +++ /dev/null @@ -1,12 +0,0 @@ -module ETDA - class ComitteeRecordsClient - def initialize - @base_url = ENV['ETDA_API_URL'] - @token = ENV['ETDA_API_TOKEN'] - end - - def faculty_committees (access_id) - - end - end -end diff --git a/lib/etda/committee_records_client.rb b/lib/etda/committee_records_client.rb new file mode 100644 index 00000000..7829bede --- /dev/null +++ b/lib/etda/committee_records_client.rb @@ -0,0 +1,54 @@ +require 'httparty' +module Etda + class CommitteeRecordsClient + def faculty_committees(access_id) + response = HTTParty.post( + "#{@base_url}/api/v1/committee_records/faculty_committees", + headers: headers, + body: body(access_id) + ) + + handle_response(response) + rescue StandardError => e + { success: false, error: e.message } + end + + private + + def base_url + @base_url ||= ENV.fetch('ETDA_API_URL', 'http://localhost:3000') + end + + def api_token + @api_token ||= ENV.fetch('ETDA_API_TOKEN', 'abc123') + end + + def headers + { + 'Authorization' => "Bearer #{@token}", + 'Content-Type' => 'application/json' + } + end + + def body(access_id) + { + access_id: access_id + }.to_json + end + + def handle_response(response) + if response.success? + { + success: true, + data: response.parsed_response + } + else + { + success: false, + error: response.parsed_response['error'] || 'Unknown error', + status: response.code + } + end + end + end +end diff --git a/spec/jobs/activity_insight_committee_job_spec.rb b/spec/jobs/activity_insight_committee_job_spec.rb new file mode 100644 index 00000000..f51e8e13 --- /dev/null +++ b/spec/jobs/activity_insight_committee_job_spec.rb @@ -0,0 +1,58 @@ +require 'rails_helper' + +RSpec.describe ActivityInsightCommitteeJob, type: :job do + describe '#integrate' do + let(:target) { 'test_target' } + let(:xml_enumerator) { ['test'].to_enum } + + let(:builder_double) do + instance_double( + CommitteeData::CommitteeXmlBuilder, + xmls_enumerator: xml_enumerator + ) + end + + let(:integrator_double) do + instance_double( + ActivityInsight::IntegrateData, + integrate: [] + ) + end + + before do + allow(CommitteeData::CommitteeXmlBuilder) + .to receive(:new) + .and_return(builder_double) + + allow(ActivityInsight::IntegrateData) + .to receive(:new) + .with(xml_enumerator, target, :post) + .and_return(integrator_double) + end + + it 'calls the XML builder to get the xml enumerator' do + described_class.new.integrate(target) + expect(CommitteeData::CommitteeXmlBuilder).to have_received(:new) + end + + it 'initializes ActivityInsight::IntegrateData with correct arguments' do + described_class.new.integrate(target) + + expect(ActivityInsight::IntegrateData) + .to have_received(:new) + .with(xml_enumerator, target, :post) + end + + it 'calls integrate on the integrator and returns its errors' do + result = described_class.new.integrate(target) + expect(integrator_double).to have_received(:integrate) + expect(result).to eq([]) + end + end + + describe '#name' do + it 'returns the job name' do + expect(described_class.new.name).to eq('Committee Integration') + end + end +end From eb0e15ca38b9cec39c415e8be8d9f5fd8eaa89d5 Mon Sep 17 00:00:00 2001 From: madhurakhandkar Date: Thu, 19 Feb 2026 11:39:24 -0500 Subject: [PATCH 12/39] Created rspec tests for committee records client --- lib/etda/committee_records_client.rb | 16 +++++---- .../lib/etda/committee_records_client_spec.rb | 35 +++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 spec/lib/etda/committee_records_client_spec.rb diff --git a/lib/etda/committee_records_client.rb b/lib/etda/committee_records_client.rb index 7829bede..40a103da 100644 --- a/lib/etda/committee_records_client.rb +++ b/lib/etda/committee_records_client.rb @@ -1,6 +1,9 @@ require 'httparty' module Etda class CommitteeRecordsClient + + class CommitteeRecordsClientError < StandardError;end + def faculty_committees(access_id) response = HTTParty.post( "#{@base_url}/api/v1/committee_records/faculty_committees", @@ -9,8 +12,10 @@ def faculty_committees(access_id) ) handle_response(response) + + # Tells us the error with committe records client with the error itself rescue StandardError => e - { success: false, error: e.message } + raise CommitteeRecordsClientError.new(e.message) end private @@ -42,12 +47,11 @@ def handle_response(response) success: true, data: response.parsed_response } + else - { - success: false, - error: response.parsed_response['error'] || 'Unknown error', - status: response.code - } + + raise CommitteeRecordsClientError.new(response.parsed_response['error'] || 'Unknown error') + end end end diff --git a/spec/lib/etda/committee_records_client_spec.rb b/spec/lib/etda/committee_records_client_spec.rb new file mode 100644 index 00000000..ae208fc7 --- /dev/null +++ b/spec/lib/etda/committee_records_client_spec.rb @@ -0,0 +1,35 @@ +require 'rails_helper' + +Rspec.describe CommitteeRecordsClient do + + context"when valid API KEY" do + + context "when user has committees" do + it "returns committee member JSON" do + Etda::CommitteeRecordsClient.new.faculty_committees("abc123") + end + end + + context "when user has no committees" do + it "returns empty" do + + end + end + + + end + + context"when an unsuccessful response from ETDA e.g, API KEY invalid" do + it "raises a commitee records client error" do + + end + end + + context "when an time-out occurs" do + it "raises a commitee records time-out error with a specific message" do + + end + + end +end + From 21063fa2f0e86203f40f3c095a0eb056bdc7279f Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Mon, 23 Feb 2026 17:18:44 -0500 Subject: [PATCH 13/39] Add .envrc.example for environment configuration --- .envrc.example | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .envrc.example diff --git a/.envrc.example b/.envrc.example new file mode 100644 index 00000000..e69de29b From e8df657f10443180305a1c75d6a036229c01d960 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Mon, 23 Feb 2026 17:26:20 -0500 Subject: [PATCH 14/39] Add environment variables to .envrc.example --- .envrc.example | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.envrc.example b/.envrc.example index e69de29b..cfd0f879 100644 --- a/.envrc.example +++ b/.envrc.example @@ -0,0 +1,5 @@ +# Example environment variables for FAMS Tools +# Copy this file to .envrc and replace with actual values + +export ETDA_API_URL="http://localhost:3000" +export ETDA_API_TOKEN="your_token_here" \ No newline at end of file From 2bbed27a192ef1534c37aa2f48490c6b9558c34b Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Tue, 24 Feb 2026 14:16:58 -0500 Subject: [PATCH 15/39] Fix API client to use env variable methods --- lib/etda/committee_records_client.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/etda/committee_records_client.rb b/lib/etda/committee_records_client.rb index 40a103da..d7ca68ce 100644 --- a/lib/etda/committee_records_client.rb +++ b/lib/etda/committee_records_client.rb @@ -1,3 +1,4 @@ +require './lib/etda/committee_records_client' require 'httparty' module Etda class CommitteeRecordsClient @@ -6,7 +7,7 @@ class CommitteeRecordsClientError < StandardError;end def faculty_committees(access_id) response = HTTParty.post( - "#{@base_url}/api/v1/committee_records/faculty_committees", + "#{base_url}/api/v1/committee_records/faculty_committees", headers: headers, body: body(access_id) ) @@ -30,7 +31,7 @@ def api_token def headers { - 'Authorization' => "Bearer #{@token}", + 'Authorization' => "Bearer #{api_token}", 'Content-Type' => 'application/json' } end From f76696c3fb3831c7077bdd91a24e07e87852456d Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Tue, 24 Feb 2026 14:18:38 -0500 Subject: [PATCH 16/39] changed explanation in envr.example --- .envrc.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.envrc.example b/.envrc.example index cfd0f879..6f6c9dae 100644 --- a/.envrc.example +++ b/.envrc.example @@ -2,4 +2,4 @@ # Copy this file to .envrc and replace with actual values export ETDA_API_URL="http://localhost:3000" -export ETDA_API_TOKEN="your_token_here" \ No newline at end of file +export ETDA_API_TOKEN="generate_a_token_in_ETDA_console" \ No newline at end of file From a40040c641edb600e41af593dc63cf3f1fca3125 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Fri, 27 Feb 2026 16:09:50 -0500 Subject: [PATCH 17/39] Complete RSpec tests for CommitteeRecordsClient and fix API key header - Fill in all test cases using WebMock to stub HTTP responses - Fix RSpec.describe class name and capitalization - Fix auth header from Bearer token to X-API-KEY --- .gitignore | 1 + lib/etda/committee_records_client.rb | 2 +- .../lib/etda/committee_records_client_spec.rb | 73 +++++++++++++++---- 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index a6904f74..65fa3f1b 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ spec/examples.txt .envrc .irb_historyx .DS_Store +CLAUDE.md \ No newline at end of file diff --git a/lib/etda/committee_records_client.rb b/lib/etda/committee_records_client.rb index d7ca68ce..8a537e14 100644 --- a/lib/etda/committee_records_client.rb +++ b/lib/etda/committee_records_client.rb @@ -31,7 +31,7 @@ def api_token def headers { - 'Authorization' => "Bearer #{api_token}", + 'X-API-KEY' => api_token, 'Content-Type' => 'application/json' } end diff --git a/spec/lib/etda/committee_records_client_spec.rb b/spec/lib/etda/committee_records_client_spec.rb index ae208fc7..28556aad 100644 --- a/spec/lib/etda/committee_records_client_spec.rb +++ b/spec/lib/etda/committee_records_client_spec.rb @@ -1,35 +1,82 @@ require 'rails_helper' +require 'etda/committee_records_client' -Rspec.describe CommitteeRecordsClient do - - context"when valid API KEY" do +RSpec.describe Etda::CommitteeRecordsClient do + let(:client) { Etda::CommitteeRecordsClient.new } + let(:url) { 'http://localhost:3000/api/v1/committee_records/faculty_committees' } + + context "when valid API KEY" do context "when user has committees" do it "returns committee member JSON" do - Etda::CommitteeRecordsClient.new.faculty_committees("abc123") + stub_request(:post, url) + .to_return( + status: 200, + body: { + faculty_access_id: 'abc123', + committees: [ + { + role: 'Advisor', + student_fname: 'John', + student_lname: 'Doe', + title: 'Machine Learning in Healthcare', + degree_name: 'PhD' + } + ] + }.to_json, + headers: { 'Content-Type' => 'application/json' } + ) + + result = client.faculty_committees("abc123") + + expect(result[:success]).to eq(true) + expect(result[:data]['committees'].length).to eq(1) end end context "when user has no committees" do it "returns empty" do - + stub_request(:post, url) + .to_return( + status: 200, + body: { + faculty_access_id: 'abc123', + committees: [] + }.to_json, + headers: { 'Content-Type' => 'application/json' } + ) + + result = client.faculty_committees("abc123") + + expect(result[:success]).to eq(true) + expect(result[:data]['committees']).to be_empty end end - + end - context"when an unsuccessful response from ETDA e.g, API KEY invalid" do - it "raises a commitee records client error" do - + context "when an unsuccessful response from ETDA e.g, API KEY invalid" do + it "raises a commitee records client error" do + stub_request(:post, url) + .to_return( + status: 401, + body: { error: 'Unauthorized' }.to_json, + headers: { 'Content-Type' => 'application/json' } + ) + + expect { client.faculty_committees("abc123") } + .to raise_error(Etda::CommitteeRecordsClient::CommitteeRecordsClientError) end end context "when an time-out occurs" do - it "raises a commitee records time-out error with a specific message" do - + it "raises a commitee records time-out error with a specific message" do + stub_request(:post, url).to_raise(Net::OpenTimeout) + + expect { client.faculty_committees("abc123") } + .to raise_error(Etda::CommitteeRecordsClient::CommitteeRecordsClientError) end - + end end - From 740d5bce72fab5a397275e599a678d587fcb0097 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Fri, 27 Feb 2026 16:20:17 -0500 Subject: [PATCH 18/39] Add Committee.delete_all to ApplicationJob cleanup --- app/jobs/application_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index d536a4ef..84758189 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -86,5 +86,6 @@ def delete_all_data Yearly.delete_all ComEffort.delete_all ComQuality.delete_all + Committee.delete_all end end From 70bff3c271f00610d9654a0dfa9ff228a688a820 Mon Sep 17 00:00:00 2001 From: madhurakhandkar Date: Mon, 2 Mar 2026 10:43:35 -0500 Subject: [PATCH 19/39] Finished rspec tests for committee_records_client --- Dockerfile | 2 +- .../lib/etda/committee_records_client_spec.rb | 124 +++++++++--------- 2 files changed, 60 insertions(+), 66 deletions(-) diff --git a/Dockerfile b/Dockerfile index 445fa475..87d2432c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM harbor.k8s.libraries.psu.edu/library/ruby-3.4.1-node-22:20260123 as base +FROM harbor.k8s.libraries.psu.edu/library/ruby-3.4.1-node-22:20260202 as base ARG UID=1001 WORKDIR /app diff --git a/spec/lib/etda/committee_records_client_spec.rb b/spec/lib/etda/committee_records_client_spec.rb index 28556aad..b137f206 100644 --- a/spec/lib/etda/committee_records_client_spec.rb +++ b/spec/lib/etda/committee_records_client_spec.rb @@ -1,82 +1,76 @@ require 'rails_helper' -require 'etda/committee_records_client' - +require_relative '../../../lib/etda/committee_records_client' RSpec.describe Etda::CommitteeRecordsClient do let(:client) { Etda::CommitteeRecordsClient.new } - let(:url) { 'http://localhost:3000/api/v1/committee_records/faculty_committees' } - - context "when valid API KEY" do - - context "when user has committees" do - it "returns committee member JSON" do - stub_request(:post, url) - .to_return( - status: 200, - body: { - faculty_access_id: 'abc123', - committees: [ - { - role: 'Advisor', - student_fname: 'John', - student_lname: 'Doe', - title: 'Machine Learning in Healthcare', - degree_name: 'PhD' - } - ] - }.to_json, - headers: { 'Content-Type' => 'application/json' } - ) - result = client.faculty_committees("abc123") + # This is to instansiate the client + describe "#faculty_committees" do + + + context "when valid API KEY and user has committees" do + it "returns committee member JSON" do + + # Create a mock test but with a respone returned as a JSON - expect(result[:success]).to eq(true) - expect(result[:data]['committees'].length).to eq(1) + fake_response = double( + success?: true, + parsed_response: [{ id: 1, name: "Millennium Scholar" }]) + + allow(HTTParty).to receive(:post).and_return(fake_response) + + # Calling the method and checking if the test works + response = client.faculty_committees("abc123") + + expect(response).to be_present + expect(response[:success]).to be true + expect(response[:data]).to be_present + end end - end - context "when user has no committees" do - it "returns empty" do - stub_request(:post, url) - .to_return( - status: 200, - body: { - faculty_access_id: 'abc123', - committees: [] - }.to_json, - headers: { 'Content-Type' => 'application/json' } + context "when user has no committees" do + it "returns empty committee" do + + # Create a mock test but with no response + fake_response = double( + success?: true, + parsed_response: [] ) - result = client.faculty_committees("abc123") + allow(HTTParty).to receive(:post).and_return(fake_response) - expect(result[:success]).to eq(true) - expect(result[:data]['committees']).to be_empty - end - end + response = client.faculty_committees(" ") + expect(response).to be_present + expect(response[:success]).to be true + expect(response[:data]).to eq([]) + end + end - end + #Checking for when key is invalid + context "when API key is invalid" do + it "raises a committee records client error" do + fake_response = double( + success?: false, + parsed_response: { 'error' => 'Invalid API key' } + ) - context "when an unsuccessful response from ETDA e.g, API KEY invalid" do - it "raises a commitee records client error" do - stub_request(:post, url) - .to_return( - status: 401, - body: { error: 'Unauthorized' }.to_json, - headers: { 'Content-Type' => 'application/json' } - ) + allow(HTTParty).to receive(:post).and_return(fake_response) - expect { client.faculty_committees("abc123") } - .to raise_error(Etda::CommitteeRecordsClient::CommitteeRecordsClientError) - end - end + expect { + client.faculty_committees("invalid_key") + }.to raise_error(Etda::CommitteeRecordsClient::CommitteeRecordsClientError) + end + end - context "when an time-out occurs" do - it "raises a commitee records time-out error with a specific message" do - stub_request(:post, url).to_raise(Net::OpenTimeout) + # Checking when a timeout occurs + context "when a timeout occurs" do + it "raises a committee records client error" do + allow(HTTParty).to receive(:post).and_raise(Timeout::Error) - expect { client.faculty_committees("abc123") } - .to raise_error(Etda::CommitteeRecordsClient::CommitteeRecordsClientError) + expect { + client.faculty_committees("timeout_test") + }.to raise_error(Etda::CommitteeRecordsClient::CommitteeRecordsClientError) + end + end end - - end -end + end From 54279de498f9acee66020f770cd792d303da65c3 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Mon, 2 Mar 2026 16:39:32 -0500 Subject: [PATCH 20/39] WIP: Add EtdaImporter skeleton for pulling committee data from ETDA API Adds CommitteeData::EtdaImporter with import_all and import_for_faculty methods. Iterates over all faculty, calls the ETDA API client, and creates committee records. Error handling and tests still in progress. --- app/importers/committee_data/etda_importer.rb | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 app/importers/committee_data/etda_importer.rb diff --git a/app/importers/committee_data/etda_importer.rb b/app/importers/committee_data/etda_importer.rb new file mode 100644 index 00000000..99f2a6f9 --- /dev/null +++ b/app/importers/committee_data/etda_importer.rb @@ -0,0 +1,33 @@ +require 'etda/committee_records_client' + +module CommitteeData + class EtdaImporter + def import_all + Faculty.find_each do |faculty| + import_for_faculty(faculty.access_id) + rescue Etda::CommitteeRecordsClient::CommitteeRecordsClientError => e + Rails.logger.error("Failed to import committees for #{faculty.access_id}: #{e.message}") + end + end + + def import_for_faculty(access_id) + faculty = Faculty.find_by(access_id: access_id) + return unless faculty + + result = Etda::CommitteeRecordsClient.new.faculty_committees(access_id) + committees_data = result[:data]['committees'] + + committees_data.each do |committee| + faculty.committees.create!( + student_fname: committee['student_fname'], + student_lname: committee['student_lname'], + role: committee['role'], + thesis_title: committee['title'], + degree_type: committee['degree_name'] + ) + end + + Rails.logger.info("Imported #{committees_data.length} committees for #{access_id}") + end + end +end From be3fc7ac1445b6546028312513c4e9cb0c60cd92 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Thu, 5 Mar 2026 11:03:26 -0500 Subject: [PATCH 21/39] Wire EtdaImporter into ActivityInsightCommitteeJob and fix importer - Add CommitteeData::EtdaImporter.new.import_all call to integrate method - Pass faculty object directly to import_for_faculty instead of access_id - Move import_for_faculty to private and fix access_id references --- app/importers/committee_data/etda_importer.rb | 12 ++++++------ app/jobs/activity_insight_committee_job.rb | 4 +++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app/importers/committee_data/etda_importer.rb b/app/importers/committee_data/etda_importer.rb index 99f2a6f9..5e91c62d 100644 --- a/app/importers/committee_data/etda_importer.rb +++ b/app/importers/committee_data/etda_importer.rb @@ -4,17 +4,17 @@ module CommitteeData class EtdaImporter def import_all Faculty.find_each do |faculty| - import_for_faculty(faculty.access_id) + import_for_faculty(faculty) rescue Etda::CommitteeRecordsClient::CommitteeRecordsClientError => e Rails.logger.error("Failed to import committees for #{faculty.access_id}: #{e.message}") end end - def import_for_faculty(access_id) - faculty = Faculty.find_by(access_id: access_id) - return unless faculty + private - result = Etda::CommitteeRecordsClient.new.faculty_committees(access_id) + def import_for_faculty(faculty) + + result = Etda::CommitteeRecordsClient.new.faculty_committees(faculty.access_id) committees_data = result[:data]['committees'] committees_data.each do |committee| @@ -27,7 +27,7 @@ def import_for_faculty(access_id) ) end - Rails.logger.info("Imported #{committees_data.length} committees for #{access_id}") + Rails.logger.info("Imported #{committees_data.length} committees for #{faculty.access_id}") end end end diff --git a/app/jobs/activity_insight_committee_job.rb b/app/jobs/activity_insight_committee_job.rb index da972bfc..00692b01 100644 --- a/app/jobs/activity_insight_committee_job.rb +++ b/app/jobs/activity_insight_committee_job.rb @@ -1,7 +1,9 @@ class ActivityInsightCommitteeJob < ApplicationJob - def integrate(params) + def integrate(params, _user_uploaded = true) target = params[:target] + CommitteeData::EtdaImporter.new.import_all + builder = CommitteeData::CommitteeXmlBuilder.new xml_enum = builder.xmls_enumerator From 8fe4f476e1613f5a3bfb476c0f8415855c9016e1 Mon Sep 17 00:00:00 2001 From: madhurakhandkar Date: Thu, 5 Mar 2026 12:04:14 -0500 Subject: [PATCH 22/39] Modified the etda importer and created rspec tests for the etda importer --- app/importers/committee_data/etda_importer.rb | 4 +- .../committee_data/etda_importer_spec.rb | 59 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 spec/importers/committee_data/etda_importer_spec.rb diff --git a/app/importers/committee_data/etda_importer.rb b/app/importers/committee_data/etda_importer.rb index 5e91c62d..549e0ce0 100644 --- a/app/importers/committee_data/etda_importer.rb +++ b/app/importers/committee_data/etda_importer.rb @@ -3,6 +3,7 @@ module CommitteeData class EtdaImporter def import_all + Faculty.find_each do |faculty| import_for_faculty(faculty) rescue Etda::CommitteeRecordsClient::CommitteeRecordsClientError => e @@ -21,7 +22,8 @@ def import_for_faculty(faculty) faculty.committees.create!( student_fname: committee['student_fname'], student_lname: committee['student_lname'], - role: committee['role'], + #Adding in the normalizer method + role: CommitteeRoleNormalizer.normalize(committee['role']), thesis_title: committee['title'], degree_type: committee['degree_name'] ) diff --git a/spec/importers/committee_data/etda_importer_spec.rb b/spec/importers/committee_data/etda_importer_spec.rb new file mode 100644 index 00000000..64ce2dcd --- /dev/null +++ b/spec/importers/committee_data/etda_importer_spec.rb @@ -0,0 +1,59 @@ +require "importers/importers_helper" + +RSpec.describe CommitteeData::EtdaImporter do + subject(:importer) { described_class.new } + + let(:faculty) { create :faculty, access_id: "abc123" } + let(:facultyOne) { create :faculty, access_id: "mpk6156" } + let(:facultyTwo) { create :faculty, access_id: "aez1236" } + + + let(:client) { instance_double(Etda::CommitteeRecordsClient) } + let(:committees) { double("committees") } + + before do + allow(Etda::CommitteeRecordsClient).to receive(:new).and_return(client) + end + + describe "#import_all" do + context "when the import finds a committee" do + let(:api_response) do + { data: { "committees" => [ + { "student_fname" => "Spider", "student_lname" => "Man", + "role" => "advisor", "title" => "My Thesis", "degree_name" => "PhD" } + ] } } + end + + before do + allow(client).to receive(:faculty_committees).and_return(api_response) + allow(faculty).to receive(:committees).and_return(committees) + allow(facultyOne).to receive(:committees).and_return(committees) + allow(facultyTwo).to receive(:committees).and_return(committees) + allow(committees).to receive(:create!) + end + + it "creates a committee with the correct attributes" do + expect { importer.import_all }.to change { Committee.count }.by(3) + expect(Committee.last.student_fname).to eq("Spider") + expect(Committee.last.student_lname).to eq("Man") + expect(Committee.last.role).to eq("Advisor") + expect(Committee.last.thesis_title).to eq("My Thesis") + expect(Committee.last.degree_type).to eq("PhD") + + end + end +end + +context "when an error occurs" do + before { allow(Faculty).to receive(:find_each).and_yield(faculty) } + + it "rescues CommitteeRecordsClientError and logs it" do + allow(importer).to receive(:import_for_faculty) + .and_raise(Etda::CommitteeRecordsClient::CommitteeRecordsClientError, "API down") + allow(Rails.logger).to receive(:error) + + expect { importer.import_all }.not_to raise_error + expect(Rails.logger).to have_received(:error).with(/abc123.*API down/) + end + end +end \ No newline at end of file From ebc0229b9c5f6ab2e3a70d398e80ef7ee74ef04a Mon Sep 17 00:00:00 2001 From: madhurakhandkar Date: Tue, 17 Mar 2026 13:15:06 -0400 Subject: [PATCH 23/39] adding rubocop fixes for now --- app/importers/committee_data/etda_importer.rb | 10 +- lib/etda/committee_records_client.rb | 24 ++-- .../committee_data/etda_importer_spec.rb | 77 ++++++------- .../activity_insight_committee_job_spec.rb | 10 +- .../lib/etda/committee_records_client_spec.rb | 103 +++++++++--------- 5 files changed, 102 insertions(+), 122 deletions(-) diff --git a/app/importers/committee_data/etda_importer.rb b/app/importers/committee_data/etda_importer.rb index 549e0ce0..7256f5bd 100644 --- a/app/importers/committee_data/etda_importer.rb +++ b/app/importers/committee_data/etda_importer.rb @@ -3,7 +3,6 @@ module CommitteeData class EtdaImporter def import_all - Faculty.find_each do |faculty| import_for_faculty(faculty) rescue Etda::CommitteeRecordsClient::CommitteeRecordsClientError => e @@ -14,7 +13,6 @@ def import_all private def import_for_faculty(faculty) - result = Etda::CommitteeRecordsClient.new.faculty_committees(faculty.access_id) committees_data = result[:data]['committees'] @@ -22,10 +20,10 @@ def import_for_faculty(faculty) faculty.committees.create!( student_fname: committee['student_fname'], student_lname: committee['student_lname'], - #Adding in the normalizer method - role: CommitteeRoleNormalizer.normalize(committee['role']), - thesis_title: committee['title'], - degree_type: committee['degree_name'] + # Adding in the normalizer method + role: CommitteeRoleNormalizer.normalize(committee['role']), + thesis_title: committee['title'], + degree_type: committee['degree_name'] ) end diff --git a/lib/etda/committee_records_client.rb b/lib/etda/committee_records_client.rb index 8a537e14..464199f8 100644 --- a/lib/etda/committee_records_client.rb +++ b/lib/etda/committee_records_client.rb @@ -2,9 +2,8 @@ require 'httparty' module Etda class CommitteeRecordsClient + class CommitteeRecordsClientError < StandardError; end - class CommitteeRecordsClientError < StandardError;end - def faculty_committees(access_id) response = HTTParty.post( "#{base_url}/api/v1/committee_records/faculty_committees", @@ -13,10 +12,10 @@ def faculty_committees(access_id) ) handle_response(response) - + # Tells us the error with committe records client with the error itself rescue StandardError => e - raise CommitteeRecordsClientError.new(e.message) + raise CommitteeRecordsClientError, e.message end private @@ -43,17 +42,12 @@ def body(access_id) end def handle_response(response) - if response.success? - { - success: true, - data: response.parsed_response - } - - else - - raise CommitteeRecordsClientError.new(response.parsed_response['error'] || 'Unknown error') - - end + raise CommitteeRecordsClientError, response.parsed_response['error'] || 'Unknown error' unless response.success? + + { + success: true, + data: response.parsed_response + } end end end diff --git a/spec/importers/committee_data/etda_importer_spec.rb b/spec/importers/committee_data/etda_importer_spec.rb index 64ce2dcd..d9352cc8 100644 --- a/spec/importers/committee_data/etda_importer_spec.rb +++ b/spec/importers/committee_data/etda_importer_spec.rb @@ -1,59 +1,50 @@ -require "importers/importers_helper" +require 'importers/importers_helper' RSpec.describe CommitteeData::EtdaImporter do subject(:importer) { described_class.new } - let(:faculty) { create :faculty, access_id: "abc123" } - let(:facultyOne) { create :faculty, access_id: "mpk6156" } - let(:facultyTwo) { create :faculty, access_id: "aez1236" } - + let(:faculty) { create(:faculty, access_id: 'abc123') } + let(:faculty_one) { create(:faculty, access_id: 'mpk6156') } + let(:faculty_two) { create(:faculty, access_id: 'aez1236') } let(:client) { instance_double(Etda::CommitteeRecordsClient) } - let(:committees) { double("committees") } + let(:committees) { instance_double(committees) } before do allow(Etda::CommitteeRecordsClient).to receive(:new).and_return(client) end - describe "#import_all" do - context "when the import finds a committee" do - let(:api_response) do - { data: { "committees" => [ - { "student_fname" => "Spider", "student_lname" => "Man", - "role" => "advisor", "title" => "My Thesis", "degree_name" => "PhD" } - ] } } - end - - before do - allow(client).to receive(:faculty_committees).and_return(api_response) - allow(faculty).to receive(:committees).and_return(committees) - allow(facultyOne).to receive(:committees).and_return(committees) - allow(facultyTwo).to receive(:committees).and_return(committees) - allow(committees).to receive(:create!) - end - - it "creates a committee with the correct attributes" do - expect { importer.import_all }.to change { Committee.count }.by(3) - expect(Committee.last.student_fname).to eq("Spider") - expect(Committee.last.student_lname).to eq("Man") - expect(Committee.last.role).to eq("Advisor") - expect(Committee.last.thesis_title).to eq("My Thesis") - expect(Committee.last.degree_type).to eq("PhD") - + describe '#import_all' do + context 'when the import finds a committee' do + let(:api_response) do + { data: { 'committees' => [ + { 'student_fname' => 'Spider', 'student_lname' => 'Man', + 'role' => 'advisor', 'title' => 'My Thesis', 'degree_name' => 'PhD' } + ] } } + end + + before do + allow(client).to receive(:faculty_committees).and_return(api_response) + allow(faculty).to receive(:committees).and_return(committees) + allow(faculty_one).to receive(:committees).and_return(committees) + allow(faculty_two).to receive(:committees).and_return(committees) + allow(committees).to receive(:create!) + end + + it 'creates a committee with the correct attributes' do + expect { importer.import_all }.to change(Committee, :count).by(3) + expect(Committee.last.student_fname).to eq('Spider') + expect(Committee.last.student_lname).to eq('Man') + expect(Committee.last.role).to eq('Advisor') + expect(Committee.last.thesis_title).to eq('My Thesis') + expect(Committee.last.degree_type).to eq('PhD') + end end end -end - -context "when an error occurs" do - before { allow(Faculty).to receive(:find_each).and_yield(faculty) } - it "rescues CommitteeRecordsClientError and logs it" do - allow(importer).to receive(:import_for_faculty) - .and_raise(Etda::CommitteeRecordsClient::CommitteeRecordsClientError, "API down") - allow(Rails.logger).to receive(:error) + it 'rescues CommitteeRecordsClientError and logs it' do + expect(Rails.logger).to receive(:error).with(/abc123.*API down/) - expect { importer.import_all }.not_to raise_error - expect(Rails.logger).to have_received(:error).with(/abc123.*API down/) - end + expect { importer.import_all }.not_to raise_error end -end \ No newline at end of file +end diff --git a/spec/jobs/activity_insight_committee_job_spec.rb b/spec/jobs/activity_insight_committee_job_spec.rb index f51e8e13..20561bd0 100644 --- a/spec/jobs/activity_insight_committee_job_spec.rb +++ b/spec/jobs/activity_insight_committee_job_spec.rb @@ -31,21 +31,21 @@ end it 'calls the XML builder to get the xml enumerator' do + expect(CommitteeData::CommitteeXmlBuilder).to receive(:new) described_class.new.integrate(target) - expect(CommitteeData::CommitteeXmlBuilder).to have_received(:new) end it 'initializes ActivityInsight::IntegrateData with correct arguments' do - described_class.new.integrate(target) - expect(ActivityInsight::IntegrateData) - .to have_received(:new) + .to receive(:new) .with(xml_enumerator, target, :post) + + described_class.new.integrate(target) end it 'calls integrate on the integrator and returns its errors' do + expect(integrator_double).to receive(:integrate) result = described_class.new.integrate(target) - expect(integrator_double).to have_received(:integrate) expect(result).to eq([]) end end diff --git a/spec/lib/etda/committee_records_client_spec.rb b/spec/lib/etda/committee_records_client_spec.rb index b137f206..c5386ece 100644 --- a/spec/lib/etda/committee_records_client_spec.rb +++ b/spec/lib/etda/committee_records_client_spec.rb @@ -3,74 +3,71 @@ RSpec.describe Etda::CommitteeRecordsClient do let(:client) { Etda::CommitteeRecordsClient.new } - # This is to instansiate the client - describe "#faculty_committees" do + # This is to instansiate the client + describe '#faculty_committees' do + context 'when valid API KEY and user has committees' do + it 'returns committee member JSON' do + # Create a mock test but with a respone returned as a JSON - - context "when valid API KEY and user has committees" do - it "returns committee member JSON" do + fake_response = instance_double( + success?: true, + parsed_response: [{ id: 1, name: 'Millennium Scholar' }] + ) - # Create a mock test but with a respone returned as a JSON + allow(HTTParty).to receive(:post).and_return(fake_response) - fake_response = double( - success?: true, - parsed_response: [{ id: 1, name: "Millennium Scholar" }]) + # Calling the method and checking if the test works + response = client.faculty_committees('abc123') - allow(HTTParty).to receive(:post).and_return(fake_response) - - # Calling the method and checking if the test works - response = client.faculty_committees("abc123") - - expect(response).to be_present - expect(response[:success]).to be true - expect(response[:data]).to be_present - end + expect(response).to be_present + expect(response[:success]).to be true + expect(response[:data]).to be_present end + end - context "when user has no committees" do - it "returns empty committee" do - - # Create a mock test but with no response - fake_response = double( - success?: true, - parsed_response: [] - ) + context 'when user has no committees' do + it 'returns empty committee' do + # Create a mock test but with no response + fake_response = instance_double( + success?: true, + parsed_response: [] + ) - allow(HTTParty).to receive(:post).and_return(fake_response) + allow(HTTParty).to receive(:post).and_return(fake_response) - response = client.faculty_committees(" ") + response = client.faculty_committees(' ') - expect(response).to be_present - expect(response[:success]).to be true - expect(response[:data]).to eq([]) - end + expect(response).to be_present + expect(response[:success]).to be true + expect(response[:data]).to eq([]) end + end - #Checking for when key is invalid - context "when API key is invalid" do - it "raises a committee records client error" do - fake_response = double( - success?: false, - parsed_response: { 'error' => 'Invalid API key' } - ) + # Checking for when key is invalid + context 'when API key is invalid' do + it 'raises a committee records client error' do + fake_response = instance_double( + success?: false, + parsed_response: { 'error' => 'Invalid API key' } + ) - allow(HTTParty).to receive(:post).and_return(fake_response) + allow(HTTParty).to receive(:post).and_return(fake_response) - expect { - client.faculty_committees("invalid_key") - }.to raise_error(Etda::CommitteeRecordsClient::CommitteeRecordsClientError) - end + expect do + client.faculty_committees('invalid_key') + end.to raise_error(Etda::CommitteeRecordsClient::CommitteeRecordsClientError) end + end - # Checking when a timeout occurs - context "when a timeout occurs" do - it "raises a committee records client error" do - allow(HTTParty).to receive(:post).and_raise(Timeout::Error) + # Checking when a timeout occurs + context 'when a timeout occurs' do + it 'raises a committee records client error' do + allow(HTTParty).to receive(:post).and_raise(Timeout::Error) - expect { - client.faculty_committees("timeout_test") - }.to raise_error(Etda::CommitteeRecordsClient::CommitteeRecordsClientError) - end + expect do + client.faculty_committees('timeout_test') + end.to raise_error(Etda::CommitteeRecordsClient::CommitteeRecordsClientError) end end - end + end +end From 3a0ef49281c416b176e8c1ac71db8a79b118ba06 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Tue, 17 Mar 2026 13:24:40 -0400 Subject: [PATCH 24/39] Rspec tests passed - Fix integrate method to accept target directly instead of a hash - Rename job to 'Committee Integration' - Use send(:name) in spec to test private method --- app/jobs/activity_insight_committee_job.rb | 6 ++---- spec/jobs/activity_insight_committee_job_spec.rb | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/jobs/activity_insight_committee_job.rb b/app/jobs/activity_insight_committee_job.rb index 00692b01..bc99b6db 100644 --- a/app/jobs/activity_insight_committee_job.rb +++ b/app/jobs/activity_insight_committee_job.rb @@ -1,7 +1,5 @@ class ActivityInsightCommitteeJob < ApplicationJob - def integrate(params, _user_uploaded = true) - target = params[:target] - + def integrate(target) CommitteeData::EtdaImporter.new.import_all builder = CommitteeData::CommitteeXmlBuilder.new @@ -14,6 +12,6 @@ def integrate(params, _user_uploaded = true) private def name - 'Committee Membership Integration' + 'Committee Integration' end end diff --git a/spec/jobs/activity_insight_committee_job_spec.rb b/spec/jobs/activity_insight_committee_job_spec.rb index 20561bd0..09217e04 100644 --- a/spec/jobs/activity_insight_committee_job_spec.rb +++ b/spec/jobs/activity_insight_committee_job_spec.rb @@ -52,7 +52,7 @@ describe '#name' do it 'returns the job name' do - expect(described_class.new.name).to eq('Committee Integration') + expect(described_class.new.send(:name)).to eq('Committee Integration') end end end From 3011030b1ec18856a58596c984ffab1af9fa42f6 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Tue, 17 Mar 2026 13:45:37 -0400 Subject: [PATCH 25/39] Fix etda importer spec to correctly test committee creation and error handling --- .../committee_data/etda_importer_spec.rb | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/spec/importers/committee_data/etda_importer_spec.rb b/spec/importers/committee_data/etda_importer_spec.rb index d9352cc8..91b85ed6 100644 --- a/spec/importers/committee_data/etda_importer_spec.rb +++ b/spec/importers/committee_data/etda_importer_spec.rb @@ -3,12 +3,11 @@ RSpec.describe CommitteeData::EtdaImporter do subject(:importer) { described_class.new } - let(:faculty) { create(:faculty, access_id: 'abc123') } - let(:faculty_one) { create(:faculty, access_id: 'mpk6156') } - let(:faculty_two) { create(:faculty, access_id: 'aez1236') } + let(:faculty) { create(:faculty, access_id: 'abc123') } + let(:faculty_one) { create(:faculty, access_id: 'mpk6156') } + let(:faculty_two) { create(:faculty, access_id: 'aez1236') } - let(:client) { instance_double(Etda::CommitteeRecordsClient) } - let(:committees) { instance_double(committees) } + let(:client) { instance_double(Etda::CommitteeRecordsClient) } before do allow(Etda::CommitteeRecordsClient).to receive(:new).and_return(client) @@ -24,11 +23,10 @@ end before do + faculty + faculty_one + faculty_two allow(client).to receive(:faculty_committees).and_return(api_response) - allow(faculty).to receive(:committees).and_return(committees) - allow(faculty_one).to receive(:committees).and_return(committees) - allow(faculty_two).to receive(:committees).and_return(committees) - allow(committees).to receive(:create!) end it 'creates a committee with the correct attributes' do @@ -43,8 +41,11 @@ end it 'rescues CommitteeRecordsClientError and logs it' do - expect(Rails.logger).to receive(:error).with(/abc123.*API down/) + faculty + allow(client).to receive(:faculty_committees) + .and_raise(Etda::CommitteeRecordsClient::CommitteeRecordsClientError, 'API down') + expect(Rails.logger).to receive(:error).with(/abc123.*API down/) expect { importer.import_all }.not_to raise_error end end From aa0093ac0844bd2ae9e67018304800c944684a31 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Tue, 17 Mar 2026 13:57:16 -0400 Subject: [PATCH 26/39] Fix rspec tests for committee records client --- .../lib/etda/committee_records_client_spec.rb | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/spec/lib/etda/committee_records_client_spec.rb b/spec/lib/etda/committee_records_client_spec.rb index c5386ece..2905cc28 100644 --- a/spec/lib/etda/committee_records_client_spec.rb +++ b/spec/lib/etda/committee_records_client_spec.rb @@ -9,10 +9,9 @@ it 'returns committee member JSON' do # Create a mock test but with a respone returned as a JSON - fake_response = instance_double( - success?: true, - parsed_response: [{ id: 1, name: 'Millennium Scholar' }] - ) + fake_response = instance_double(HTTParty::Response, + success?: true, + parsed_response: [{ id: 1, name: 'Millennium Scholar' }]) allow(HTTParty).to receive(:post).and_return(fake_response) @@ -28,10 +27,9 @@ context 'when user has no committees' do it 'returns empty committee' do # Create a mock test but with no response - fake_response = instance_double( - success?: true, - parsed_response: [] - ) + fake_response = instance_double(HTTParty::Response, + success?: true, + parsed_response: []) allow(HTTParty).to receive(:post).and_return(fake_response) @@ -46,10 +44,9 @@ # Checking for when key is invalid context 'when API key is invalid' do it 'raises a committee records client error' do - fake_response = instance_double( - success?: false, - parsed_response: { 'error' => 'Invalid API key' } - ) + fake_response = instance_double(HTTParty::Response, + success?: false, + parsed_response: { 'error' => 'Invalid API key' }) allow(HTTParty).to receive(:post).and_return(fake_response) From a8321098e2e71e76d571ba1ed4ab0745f35d5bd9 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Thu, 19 Mar 2026 10:23:27 -0400 Subject: [PATCH 27/39] deleted 2 redundant files --- .../etda_committee_memberships_xml_builder.rb | 73 ------------------- ..._committee_memberships_xml_builder_spec.rb | 49 ------------- 2 files changed, 122 deletions(-) delete mode 100644 app/importers/committee_data/etda_committee_memberships_xml_builder.rb delete mode 100644 spec/importers/committee_data/etda_committee_memberships_xml_builder_spec.rb diff --git a/app/importers/committee_data/etda_committee_memberships_xml_builder.rb b/app/importers/committee_data/etda_committee_memberships_xml_builder.rb deleted file mode 100644 index ed2b4839..00000000 --- a/app/importers/committee_data/etda_committee_memberships_xml_builder.rb +++ /dev/null @@ -1,73 +0,0 @@ -require 'json' -require 'nokogiri' - -class CommitteeData::EtdaCommitteeMembershipsXmlBuilder - class ExportError < StandardError; end - - def initialize(payload) - @payload = payload.is_a?(String) ? JSON.parse(payload) : payload - rescue JSON::ParserError => e - raise ExportError, "Invalid JSON: #{e.message}" - end - - def to_xml - validate_payload! - faculty_access_id = @payload.fetch('faculty_access_id').to_s - committees = @payload.fetch('committees') - - builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| - xml.faculty_committees do - xml.faculty_access_id faculty_access_id - - xml.committees do - committees.each do |row| - build_committee(xml, row) - end - end - end - end - builder.to_xml - end - - private - - def validate_payload! - raise ExportError, 'Payload must be a JSON object' unless @payload.is_a?(Hash) - raise ExportError, 'faculty_access_id is required' if @payload['faculty_access_id'].to_s.strip.empty? - raise ExportError, 'committees must be an array' unless @payload['committees'].is_a?(Array) - end - - def build_committee(xml, row) - raise ExportError, 'Each committee must be an object' unless row.is_a?(Hash) - - raw_role = row['role'] - normalized_role = CommitteeRoleNormalizer.normalize(raw_role) - - xml.committee do - xml.committee_member_id row['committee_member_id'] - xml.role raw_role - xml.normalized_role normalized_role - xml.role_code row['role_code'] - - xml.student do - xml.fname row['student_fname'] - xml.lname row['student_lname'] - xml.access_id row['student_access_id'] - end - - xml.submission do - xml.submission_id row['submission_id'] - xml.title row['title'] - xml.degree_name row['degree_name'] - xml.program_name row['program_name'] - xml.semester row['semester'] - xml.year row['year'] - xml.status row['submission_status'] - xml.final_submission_approved_at row['final_submission_approved_at'] - end - - xml.approval_started_at row['approval_started_at'] - xml.committee_member_status row['committee_member_status'] - end - end -end diff --git a/spec/importers/committee_data/etda_committee_memberships_xml_builder_spec.rb b/spec/importers/committee_data/etda_committee_memberships_xml_builder_spec.rb deleted file mode 100644 index 2623993b..00000000 --- a/spec/importers/committee_data/etda_committee_memberships_xml_builder_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -require 'rails_helper' - -RSpec.describe CommitteeData::EtdaCommitteeMembershipsXmlBuilder do - describe '#to_xml' do - it 'converts JSON payload to XML and normalizes role' do - payload = { - 'faculty_access_id' => 'aab27', - 'committees' => [ - { - 'committee_member_id' => 101_388, - 'role' => 'Major Field Member', - 'role_code' => 'M', - 'student_fname' => 'Esther', - 'student_lname' => 'Munoz', - 'student_access_id' => 'ecm5494', - 'submission_id' => 22_620, - 'title' => 'Test Title', - 'degree_name' => 'PHD', - 'program_name' => 'Geosciences (PHD)', - 'semester' => 'Spring', - 'year' => 2025, - 'approval_started_at' => '2024-12-09T16:10:33.000-05:00', - 'final_submission_approved_at' => '2025-01-15T15:13:39.000-05:00', - 'submission_status' => 'released for publication metadata only', - 'committee_member_status' => 'approved' - } - ] - } - - allow(CommitteeRoleNormalizer).to receive(:normalize) - .with('Major Field Member') - .and_return('Member') - - xml = described_class.new(payload).to_xml - - expect(xml).to include('aab27') - expect(xml).to include('101388') - expect(xml).to include('Major Field Member') - expect(xml).to include('Member') - end - - it 'raises when faculty_access_id missing' do - payload = { 'committees' => [] } - - expect { described_class.new(payload).to_xml } - .to raise_error(CommitteeData::EtdaCommitteeMembershipsXmlBuilder::ExportError) - end - end -end From 74c5441881fc7fe438fd12640d21da98908bfedd Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Thu, 19 Mar 2026 11:52:49 -0400 Subject: [PATCH 28/39] fixed up regex in committe normalizer and updated envrc.example --- .envrc.example | 12 +++++++++++- app/services/committee_role_normalizer.rb | 19 ++++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.envrc.example b/.envrc.example index 6f6c9dae..b879a2ec 100644 --- a/.envrc.example +++ b/.envrc.example @@ -2,4 +2,14 @@ # Copy this file to .envrc and replace with actual values export ETDA_API_URL="http://localhost:3000" -export ETDA_API_TOKEN="generate_a_token_in_ETDA_console" \ No newline at end of file +export ETDA_API_TOKEN="generate_a_token_in_ETDA_console" +export FAMS_WEBSERVICES_USERNAME +export FAMS_WEBSERVICES_PASSWORD +export FAMS_MAIN_USERNAME +export FAMS_MAIN_PASSWORD +export FAMS_BACKUPS_SERVICE_USERNAME +export FAMS_BACKUPS_SERVICE_PASSWORD +export FAMS_METADATA_DB_KEY +export FAMS_S3_BUCKET_API_KEY +export FAMS_LP_SFTP_USERNAME +export FAMS_LP_SFTP_HOST diff --git a/app/services/committee_role_normalizer.rb b/app/services/committee_role_normalizer.rb index abdf46c0..c2c40764 100644 --- a/app/services/committee_role_normalizer.rb +++ b/app/services/committee_role_normalizer.rb @@ -1,14 +1,15 @@ class CommitteeRoleNormalizer PRIORITY_REGEX = [ - ['Co-Chairperson', %r{\b(co[-\s]?chair|co[-\s]?chairperson|committee chair/co-chair)\b}i], - ['Chairperson', /\b(chairperson|chair of committee|committee chair|chair)\b/i], - ['Co-Advisor', /\b(co[-\s]?dissertation\s*advis(or|er)|co[-\s]?advisor)\b/i], - ['Advisor', /\b(dissertation\s*advis(?:o?r|er)|advis(?:o?r|er))\b/i], - ['Supervisor', /\bsupervisor\b/i], - ['Mentor', /\bmentor\b/i], - ['Second Reader', /\bsecond\s+reader\b/i], - ['Reader', /\breader\b/i], - ['Member', /\b(member|rep|represent|representative|substitute)\b/i] + ['Co-Chairperson', %r{(co[-\s]?chair|co[-\s]?chairperson|committee chair/co-chair)}i], + ['Chairperson', /(chairperson|chair of committee|committee chair|chair.)/i], + ['Co-Advisor', /(co[-\s]?dissertation\s*advis(or|er)|co[-\s]?advisor)/i], + ['Advisor', /(dissertation\s*advis(?:o?r|er)|advis(?:o?r|er))/i], + ['Supervisor', /supervisor/i], + ['Mentor', /mentor/i], + ['Second Reader', /second\s+reader/i], + ['Reader', /reader/i], + ['Member', /(member|rep|represent|representative|substitute)/i] + ].freeze def self.normalize(raw_name) From 4c30e5a9d3f32f2d8826d927210d4613cf790afc Mon Sep 17 00:00:00 2001 From: Joao Vitor Barros da Silva Date: Thu, 19 Mar 2026 13:18:41 -0400 Subject: [PATCH 29/39] Rubocop corrections --- app/services/committee_role_normalizer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/committee_role_normalizer.rb b/app/services/committee_role_normalizer.rb index c2c40764..9df716ca 100644 --- a/app/services/committee_role_normalizer.rb +++ b/app/services/committee_role_normalizer.rb @@ -9,7 +9,7 @@ class CommitteeRoleNormalizer ['Second Reader', /second\s+reader/i], ['Reader', /reader/i], ['Member', /(member|rep|represent|representative|substitute)/i] - + ].freeze def self.normalize(raw_name) From d086479efd6a4be1c20bdb89ee4dcccca323b171 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Tue, 24 Mar 2026 15:05:15 -0400 Subject: [PATCH 30/39] Fix Activity Insight dropdown mappings for committee data - Fix stage_of_completion: 'In Progress' -> 'In Process' to match Activity Insight's valid dropdown values; add 'Withdrew' support - Add role_other_explanation and type_other_explanation DB columns for when role/type maps to 'Other' in Activity Insight - Improve map_type_of_work to use role context and handle more degree types (undergraduate, postdoc, honors) - Emit and XML tags when explanations present - Update all related specs --- .../committee_data/committee_xml_builder.rb | 6 + app/importers/committee_data/etda_importer.rb | 73 +++++++++- ...d_activity_insight_fields_to_committees.rb | 10 ++ ..._other_explanation_fields_to_committees.rb | 8 ++ db/schema.rb | 9 +- .../committee_xml_builder_spec.rb | 73 +++++++++- .../committee_data/etda_importer_spec.rb | 130 +++++++++++++++++- 7 files changed, 302 insertions(+), 7 deletions(-) create mode 100644 db/migrate/20260324000000_add_activity_insight_fields_to_committees.rb create mode 100644 db/migrate/20260325000000_add_other_explanation_fields_to_committees.rb diff --git a/app/importers/committee_data/committee_xml_builder.rb b/app/importers/committee_data/committee_xml_builder.rb index caec7c27..63f1458a 100644 --- a/app/importers/committee_data/committee_xml_builder.rb +++ b/app/importers/committee_data/committee_xml_builder.rb @@ -25,6 +25,12 @@ def build_xml(batch) committees.each do |committee| xml.DSL do xml.ROLE committee.role + xml.ROLE_OTHER committee.role_other_explanation if committee.role_other_explanation.present? + xml.TYPE committee.type_of_work + xml.TYPE_OTHER committee.type_other_explanation if committee.type_other_explanation.present? + xml.COMP committee.stage_of_completion + xml.DTY_START committee.start_year if committee.start_year + xml.DTY_END committee.completion_year if committee.completion_year xml.DSL_STUDENT do xml.FNAME committee.student_fname diff --git a/app/importers/committee_data/etda_importer.rb b/app/importers/committee_data/etda_importer.rb index 7256f5bd..e8d20f4e 100644 --- a/app/importers/committee_data/etda_importer.rb +++ b/app/importers/committee_data/etda_importer.rb @@ -17,17 +17,84 @@ def import_for_faculty(faculty) committees_data = result[:data]['committees'] committees_data.each do |committee| + normalized_role = CommitteeRoleNormalizer.normalize(committee['role']) + faculty.committees.create!( student_fname: committee['student_fname'], student_lname: committee['student_lname'], - # Adding in the normalizer method - role: CommitteeRoleNormalizer.normalize(committee['role']), + role: normalized_role, + role_other_explanation: normalized_role == 'Other' ? committee['role'] : nil, thesis_title: committee['title'], - degree_type: committee['degree_name'] + degree_type: committee['degree_name'], + type_of_work: map_type_of_work(committee['degree_name'], normalized_role), + stage_of_completion: determine_completion_stage( + committee['final_submission_approved_at'], + committee['submission_status'] + ), + start_year: extract_year(committee['approval_started_at']), + completion_year: extract_year(committee['final_submission_approved_at']) ) end Rails.logger.info("Imported #{committees_data.length} committees for #{faculty.access_id}") end + + def map_type_of_work(degree_name, role = nil) + return nil if degree_name.blank? + + case degree_name.upcase.strip + when 'PHD', 'PH.D.', 'DOCTOR OF PHILOSOPHY' + determine_phd_type(role) + when 'MS', 'M.S.', 'MASTER OF SCIENCE', 'MA', 'M.A.', 'MASTER OF ARTS' + determine_masters_type(role) + when 'UNDERGRADUATE', 'BS', 'BA', 'B.S.', 'B.A.' + 'Undergraduate Research' + when 'POSTDOC', 'POSTDOCTORAL' + 'Postdoctoral Mentorship' + when 'HONORS' + 'Honors Thesis' + else + if degree_name.downcase.include?('phd') || degree_name.downcase.include?('doctor') + 'Ph.D. Dissertation Committee' + elsif degree_name.downcase.include?('master') + "Master's Thesis Committee" + else + 'Dissertation Committee' + end + end + end + + def determine_phd_type(role) + case role&.downcase + when 'advisor', 'chairperson' + 'Ph.D. Dissertation' + else + 'Ph.D. Dissertation Committee' + end + end + + def determine_masters_type(role) + case role&.downcase + when 'advisor', 'chairperson' + "Master's Thesis" + else + "Master's Thesis Committee" + end + end + + def extract_year(date_string) + return nil if date_string.blank? + + Date.parse(date_string).year + rescue ArgumentError + nil + end + + def determine_completion_stage(final_submission_approved_at, submission_status) + return 'Completed' if final_submission_approved_at.present? + return 'Withdrew' if submission_status&.downcase&.include?('withdrawn') + + 'In Process' + end end end diff --git a/db/migrate/20260324000000_add_activity_insight_fields_to_committees.rb b/db/migrate/20260324000000_add_activity_insight_fields_to_committees.rb new file mode 100644 index 00000000..7b451a05 --- /dev/null +++ b/db/migrate/20260324000000_add_activity_insight_fields_to_committees.rb @@ -0,0 +1,10 @@ +class AddActivityInsightFieldsToCommittees < ActiveRecord::Migration[7.2] + def change + change_table :committees, bulk: true do |t| + t.string :type_of_work + t.string :stage_of_completion + t.integer :start_year + t.integer :completion_year + end + end +end diff --git a/db/migrate/20260325000000_add_other_explanation_fields_to_committees.rb b/db/migrate/20260325000000_add_other_explanation_fields_to_committees.rb new file mode 100644 index 00000000..1d4a7b4b --- /dev/null +++ b/db/migrate/20260325000000_add_other_explanation_fields_to_committees.rb @@ -0,0 +1,8 @@ +class AddOtherExplanationFieldsToCommittees < ActiveRecord::Migration[7.2] + def change + change_table :committees, bulk: true do |t| + t.text :role_other_explanation, limit: 20_000 + t.text :type_other_explanation, limit: 20_000 + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 1aef6b1e..e74eb0d0 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2025_12_05_172036) do +ActiveRecord::Schema[7.2].define(version: 2026_03_25_000000) do create_table "authors", charset: "utf8mb4", force: :cascade do |t| t.string "f_name" t.string "m_name" @@ -61,6 +61,12 @@ t.string "degree_type" t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.string "type_of_work" + t.string "stage_of_completion" + t.integer "start_year" + t.integer "completion_year" + t.text "role_other_explanation" + t.text "type_other_explanation" t.index ["faculty_id"], name: "index_committees_on_faculty_id" end @@ -306,6 +312,7 @@ t.string "hr_code" t.text "departments", size: :long, collation: "utf8mb4_bin" t.index ["faculty_id"], name: "index_yearlies_on_faculty_id", unique: true + t.check_constraint "json_valid(`departments`)", name: "departments" end add_foreign_key "authors", "works", on_delete: :cascade diff --git a/spec/importers/committee_data/committee_xml_builder_spec.rb b/spec/importers/committee_data/committee_xml_builder_spec.rb index ae6b16a3..47a14c1c 100644 --- a/spec/importers/committee_data/committee_xml_builder_spec.rb +++ b/spec/importers/committee_data/committee_xml_builder_spec.rb @@ -4,7 +4,7 @@ let(:xml_builder_obj) { described_class.new } describe '#xmls_enumerator' do - it 'returns an xml of DSL records' do + it 'returns an xml of DSL records with all Activity Insight fields' do faculty = FactoryBot.create(:faculty, access_id: 'test123') Committee.create!( @@ -13,7 +13,11 @@ student_lname: 'User', role: 'Mentor', thesis_title: 'Test Title', - degree_type: 'PhD' + degree_type: 'PhD', + type_of_work: 'Ph.D. Dissertation Committee', + stage_of_completion: 'Completed', + start_year: 2024, + completion_year: 2026 ) expect(xml_builder_obj.xmls_enumerator.first).to eq( @@ -22,6 +26,10 @@ Mentor + Ph.D. Dissertation Committee + Completed + 2024 + 2026 Test User @@ -35,6 +43,67 @@ ) end + it 'omits DTY_START and DTY_END when years are nil' do + faculty = FactoryBot.create(:faculty, access_id: 'test123') + + Committee.create!( + faculty: faculty, + student_fname: 'Test', + student_lname: 'User', + role: 'Mentor', + thesis_title: 'Test Title', + degree_type: 'PhD', + type_of_work: 'Ph.D. Dissertation Committee', + stage_of_completion: 'In Process', + start_year: nil, + completion_year: nil + ) + + xml = xml_builder_obj.xmls_enumerator.first + expect(xml).not_to include('') + expect(xml).not_to include('') + expect(xml).to include('In Process') + end + + it 'includes ROLE_OTHER and TYPE_OTHER when role or type is Other' do + faculty = FactoryBot.create(:faculty, access_id: 'test123') + + Committee.create!( + faculty: faculty, + student_fname: 'Test', + student_lname: 'User', + role: 'Other', + role_other_explanation: 'External International Reviewer', + type_of_work: 'Other', + type_other_explanation: 'Special Joint Program Committee', + thesis_title: 'Research', + degree_type: 'Special Degree' + ) + + xml = xml_builder_obj.xmls_enumerator.first + expect(xml).to include('Other') + expect(xml).to include('External International Reviewer') + expect(xml).to include('Other') + expect(xml).to include('Special Joint Program Committee') + end + + it 'omits ROLE_OTHER and TYPE_OTHER when explanations are absent' do + faculty = FactoryBot.create(:faculty, access_id: 'test123') + + Committee.create!( + faculty: faculty, + student_fname: 'Test', + student_lname: 'User', + role: 'Advisor', + thesis_title: 'Test Title', + degree_type: 'PhD' + ) + + xml = xml_builder_obj.xmls_enumerator.first + expect(xml).not_to include('') + expect(xml).not_to include('') + end + it 'handles faculty with no committees' do FactoryBot.create(:faculty, access_id: 'test123') diff --git a/spec/importers/committee_data/etda_importer_spec.rb b/spec/importers/committee_data/etda_importer_spec.rb index 91b85ed6..76bba4a4 100644 --- a/spec/importers/committee_data/etda_importer_spec.rb +++ b/spec/importers/committee_data/etda_importer_spec.rb @@ -18,7 +18,10 @@ let(:api_response) do { data: { 'committees' => [ { 'student_fname' => 'Spider', 'student_lname' => 'Man', - 'role' => 'advisor', 'title' => 'My Thesis', 'degree_name' => 'PhD' } + 'role' => 'advisor', 'title' => 'My Thesis', 'degree_name' => 'PhD', + 'approval_started_at' => '2024-08-15T10:30:00Z', + 'final_submission_approved_at' => '2026-01-15T10:30:00Z', + 'submission_status' => 'released for publication' } ] } } end @@ -34,9 +37,134 @@ expect(Committee.last.student_fname).to eq('Spider') expect(Committee.last.student_lname).to eq('Man') expect(Committee.last.role).to eq('Advisor') + expect(Committee.last.role_other_explanation).to be_nil expect(Committee.last.thesis_title).to eq('My Thesis') expect(Committee.last.degree_type).to eq('PhD') + expect(Committee.last.type_of_work).to eq('Ph.D. Dissertation') + expect(Committee.last.stage_of_completion).to eq('Completed') + expect(Committee.last.start_year).to eq(2024) + expect(Committee.last.completion_year).to eq(2026) end + + context 'when role does not match any valid Activity Insight value' do + let(:api_response) do + { data: { 'committees' => [ + { 'student_fname' => 'Spider', 'student_lname' => 'Man', + 'role' => 'External Reviewer', 'title' => 'My Thesis', 'degree_name' => 'PhD', + 'approval_started_at' => nil, + 'final_submission_approved_at' => nil, + 'submission_status' => 'waiting for publication release' } + ] } } + end + + it 'saves role as Other and stores original role in role_other_explanation' do + importer.import_all + committee = Committee.last + expect(committee.role).to eq('Other') + expect(committee.role_other_explanation).to eq('External Reviewer') + end + end + end + end + + describe '#map_type_of_work' do + it 'maps PhD with Advisor role to Ph.D. Dissertation' do + expect(importer.send(:map_type_of_work, 'PhD', 'Advisor')).to eq('Ph.D. Dissertation') + end + + it 'maps PhD with Chairperson role to Ph.D. Dissertation' do + expect(importer.send(:map_type_of_work, 'PhD', 'Chairperson')).to eq('Ph.D. Dissertation') + end + + it 'maps PhD with Member role to Ph.D. Dissertation Committee' do + expect(importer.send(:map_type_of_work, 'PhD', 'Member')).to eq('Ph.D. Dissertation Committee') + end + + it 'maps Ph.D. to Ph.D. Dissertation Committee when no role given' do + expect(importer.send(:map_type_of_work, 'Ph.D.')).to eq('Ph.D. Dissertation Committee') + end + + it 'maps MS with Advisor role to Master\'s Thesis' do + expect(importer.send(:map_type_of_work, 'MS', 'Advisor')).to eq("Master's Thesis") + end + + it 'maps MS with Member role to Master\'s Thesis Committee' do + expect(importer.send(:map_type_of_work, 'MS', 'Member')).to eq("Master's Thesis Committee") + end + + it 'maps MA to Master\'s Thesis Committee when no role given' do + expect(importer.send(:map_type_of_work, 'MA')).to eq("Master's Thesis Committee") + end + + it 'maps undergraduate degrees to Undergraduate Research' do + expect(importer.send(:map_type_of_work, 'BS')).to eq('Undergraduate Research') + end + + it 'maps postdoc to Postdoctoral Mentorship' do + expect(importer.send(:map_type_of_work, 'Postdoc')).to eq('Postdoctoral Mentorship') + end + + it 'maps honors to Honors Thesis' do + expect(importer.send(:map_type_of_work, 'HONORS')).to eq('Honors Thesis') + end + + it 'defaults to Dissertation Committee for unknown degree types' do + expect(importer.send(:map_type_of_work, 'DMA')).to eq('Dissertation Committee') + end + + it 'returns nil for blank degree name' do + expect(importer.send(:map_type_of_work, nil)).to be_nil + expect(importer.send(:map_type_of_work, '')).to be_nil + end + end + + describe '#extract_year' do + subject(:extract) { importer.send(:extract_year, date_string) } + + context 'with a valid ISO8601 date string' do + let(:date_string) { '2026-01-15T10:30:00Z' } + + it { is_expected.to eq(2026) } + end + + context 'with nil' do + let(:date_string) { nil } + + it { is_expected.to be_nil } + end + + context 'with an empty string' do + let(:date_string) { '' } + + it { is_expected.to be_nil } + end + + context 'with an invalid date string' do + let(:date_string) { 'not-a-date' } + + it { is_expected.to be_nil } + end + end + + describe '#determine_completion_stage' do + it 'returns Completed when final submission date is present' do + result = importer.send(:determine_completion_stage, '2026-01-15T10:30:00Z', 'released for publication') + expect(result).to eq('Completed') + end + + it 'returns In Process when final submission date is nil and not withdrawn' do + result = importer.send(:determine_completion_stage, nil, 'waiting for publication release') + expect(result).to eq('In Process') + end + + it 'returns Withdrew when submission status indicates withdrawal' do + result = importer.send(:determine_completion_stage, nil, 'withdrawn by student') + expect(result).to eq('Withdrew') + end + + it 'handles nil submission status gracefully' do + result = importer.send(:determine_completion_stage, nil, nil) + expect(result).to eq('In Process') end end From 092e2f4ee121d8799c6f0f0c4e6e1bc60fdc72de Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Tue, 24 Mar 2026 15:47:01 -0400 Subject: [PATCH 31/39] Fix XML tag COMP -> COMPSTAGE to match Activity Insight schema --- app/importers/committee_data/committee_xml_builder.rb | 2 +- spec/importers/committee_data/committee_xml_builder_spec.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/importers/committee_data/committee_xml_builder.rb b/app/importers/committee_data/committee_xml_builder.rb index 63f1458a..01d65d26 100644 --- a/app/importers/committee_data/committee_xml_builder.rb +++ b/app/importers/committee_data/committee_xml_builder.rb @@ -28,7 +28,7 @@ def build_xml(batch) xml.ROLE_OTHER committee.role_other_explanation if committee.role_other_explanation.present? xml.TYPE committee.type_of_work xml.TYPE_OTHER committee.type_other_explanation if committee.type_other_explanation.present? - xml.COMP committee.stage_of_completion + xml.COMPSTAGE committee.stage_of_completion xml.DTY_START committee.start_year if committee.start_year xml.DTY_END committee.completion_year if committee.completion_year diff --git a/spec/importers/committee_data/committee_xml_builder_spec.rb b/spec/importers/committee_data/committee_xml_builder_spec.rb index 47a14c1c..ed204388 100644 --- a/spec/importers/committee_data/committee_xml_builder_spec.rb +++ b/spec/importers/committee_data/committee_xml_builder_spec.rb @@ -27,7 +27,7 @@ Mentor Ph.D. Dissertation Committee - Completed + Completed 2024 2026 @@ -62,7 +62,7 @@ xml = xml_builder_obj.xmls_enumerator.first expect(xml).not_to include('') expect(xml).not_to include('') - expect(xml).to include('In Process') + expect(xml).to include('In Process') end it 'includes ROLE_OTHER and TYPE_OTHER when role or type is Other' do From 7cfbafbe4866c6fb4fc8e8844de98a1784b5a7df Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Thu, 26 Mar 2026 11:24:19 -0400 Subject: [PATCH 32/39] Remove wihtdrawn from determine_comp method --- .irb_history | 11 +++++++++++ app/importers/committee_data/etda_importer.rb | 2 -- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 .irb_history diff --git a/.irb_history b/.irb_history new file mode 100644 index 00000000..bf1bdb15 --- /dev/null +++ b/.irb_history @@ -0,0 +1,11 @@ +importer = CommitteeData::EtdaImporter.new +Committee.last.attributes +ENV['ETDA_API_URL'] +quit +importer = CommitteeData::EtdaImporter.new +Committee.last.attributes +importer.import_all +quit +importer = CommitteeData::EtdaImporter.new +importer.import_all +quit diff --git a/app/importers/committee_data/etda_importer.rb b/app/importers/committee_data/etda_importer.rb index e8d20f4e..07ff82ab 100644 --- a/app/importers/committee_data/etda_importer.rb +++ b/app/importers/committee_data/etda_importer.rb @@ -92,8 +92,6 @@ def extract_year(date_string) def determine_completion_stage(final_submission_approved_at, submission_status) return 'Completed' if final_submission_approved_at.present? - return 'Withdrew' if submission_status&.downcase&.include?('withdrawn') - 'In Process' end end From 1f78a666b7b98f16d471dd55498bf711e11fd3b3 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Thu, 26 Mar 2026 13:30:01 -0400 Subject: [PATCH 33/39] Remove degree_type and role_other_explanation from committee data pipeline --- .../committee_data/committee_xml_builder.rb | 2 - app/importers/committee_data/etda_importer.rb | 58 ++++---------- .../committee_xml_builder_spec.rb | 39 ++-------- .../committee_data/etda_importer_spec.rb | 78 ++++--------------- spec/models/committee_spec.rb | 1 - 5 files changed, 37 insertions(+), 141 deletions(-) diff --git a/app/importers/committee_data/committee_xml_builder.rb b/app/importers/committee_data/committee_xml_builder.rb index 01d65d26..fdefb8a9 100644 --- a/app/importers/committee_data/committee_xml_builder.rb +++ b/app/importers/committee_data/committee_xml_builder.rb @@ -25,7 +25,6 @@ def build_xml(batch) committees.each do |committee| xml.DSL do xml.ROLE committee.role - xml.ROLE_OTHER committee.role_other_explanation if committee.role_other_explanation.present? xml.TYPE committee.type_of_work xml.TYPE_OTHER committee.type_other_explanation if committee.type_other_explanation.present? xml.COMPSTAGE committee.stage_of_completion @@ -35,7 +34,6 @@ def build_xml(batch) xml.DSL_STUDENT do xml.FNAME committee.student_fname xml.LNAME committee.student_lname - xml.DEG committee.degree_type xml.TITLE committee.thesis_title end end diff --git a/app/importers/committee_data/etda_importer.rb b/app/importers/committee_data/etda_importer.rb index 07ff82ab..b8f722f4 100644 --- a/app/importers/committee_data/etda_importer.rb +++ b/app/importers/committee_data/etda_importer.rb @@ -2,6 +2,8 @@ module CommitteeData class EtdaImporter + class DegreeTypeError < RuntimeError; end + def import_all Faculty.find_each do |faculty| import_for_faculty(faculty) @@ -23,10 +25,8 @@ def import_for_faculty(faculty) student_fname: committee['student_fname'], student_lname: committee['student_lname'], role: normalized_role, - role_other_explanation: normalized_role == 'Other' ? committee['role'] : nil, thesis_title: committee['title'], - degree_type: committee['degree_name'], - type_of_work: map_type_of_work(committee['degree_name'], normalized_role), + type_of_work: map_type_of_work(committee['degree_type']), stage_of_completion: determine_completion_stage( committee['final_submission_approved_at'], committee['submission_status'] @@ -39,46 +39,21 @@ def import_for_faculty(faculty) Rails.logger.info("Imported #{committees_data.length} committees for #{faculty.access_id}") end - def map_type_of_work(degree_name, role = nil) - return nil if degree_name.blank? + def map_type_of_work(degree_type) + return nil if degree_type.blank? - case degree_name.upcase.strip - when 'PHD', 'PH.D.', 'DOCTOR OF PHILOSOPHY' - determine_phd_type(role) - when 'MS', 'M.S.', 'MASTER OF SCIENCE', 'MA', 'M.A.', 'MASTER OF ARTS' - determine_masters_type(role) - when 'UNDERGRADUATE', 'BS', 'BA', 'B.S.', 'B.A.' - 'Undergraduate Research' - when 'POSTDOC', 'POSTDOCTORAL' - 'Postdoctoral Mentorship' - when 'HONORS' - 'Honors Thesis' - else - if degree_name.downcase.include?('phd') || degree_name.downcase.include?('doctor') - 'Ph.D. Dissertation Committee' - elsif degree_name.downcase.include?('master') - "Master's Thesis Committee" - else - 'Dissertation Committee' - end - end - end + case degree_type.strip + when 'Master Thesis' + "Master's Committee" + when 'Dissertation' + 'Dissertation Committee' + when 'Thesis' + 'Undergraduate Honors Thesis' + when 'Final Paper' + "Master's Paper Committee" - def determine_phd_type(role) - case role&.downcase - when 'advisor', 'chairperson' - 'Ph.D. Dissertation' else - 'Ph.D. Dissertation Committee' - end - end - - def determine_masters_type(role) - case role&.downcase - when 'advisor', 'chairperson' - "Master's Thesis" - else - "Master's Thesis Committee" + raise DegreeTypeError, "Unexpected Degree Type: #{degree_type.strip}" end end @@ -90,8 +65,9 @@ def extract_year(date_string) nil end - def determine_completion_stage(final_submission_approved_at, submission_status) + def determine_completion_stage(final_submission_approved_at, _submission_status) return 'Completed' if final_submission_approved_at.present? + 'In Process' end end diff --git a/spec/importers/committee_data/committee_xml_builder_spec.rb b/spec/importers/committee_data/committee_xml_builder_spec.rb index ed204388..e40b7887 100644 --- a/spec/importers/committee_data/committee_xml_builder_spec.rb +++ b/spec/importers/committee_data/committee_xml_builder_spec.rb @@ -13,7 +13,6 @@ student_lname: 'User', role: 'Mentor', thesis_title: 'Test Title', - degree_type: 'PhD', type_of_work: 'Ph.D. Dissertation Committee', stage_of_completion: 'Completed', start_year: 2024, @@ -33,7 +32,6 @@ Test User - PhD Test Title @@ -52,7 +50,6 @@ student_lname: 'User', role: 'Mentor', thesis_title: 'Test Title', - degree_type: 'PhD', type_of_work: 'Ph.D. Dissertation Committee', stage_of_completion: 'In Process', start_year: nil, @@ -65,29 +62,7 @@ expect(xml).to include('In Process') end - it 'includes ROLE_OTHER and TYPE_OTHER when role or type is Other' do - faculty = FactoryBot.create(:faculty, access_id: 'test123') - - Committee.create!( - faculty: faculty, - student_fname: 'Test', - student_lname: 'User', - role: 'Other', - role_other_explanation: 'External International Reviewer', - type_of_work: 'Other', - type_other_explanation: 'Special Joint Program Committee', - thesis_title: 'Research', - degree_type: 'Special Degree' - ) - - xml = xml_builder_obj.xmls_enumerator.first - expect(xml).to include('Other') - expect(xml).to include('External International Reviewer') - expect(xml).to include('Other') - expect(xml).to include('Special Joint Program Committee') - end - - it 'omits ROLE_OTHER and TYPE_OTHER when explanations are absent' do + it 'omits TYPE_OTHER when type_other_explanation is absent' do faculty = FactoryBot.create(:faculty, access_id: 'test123') Committee.create!( @@ -95,12 +70,10 @@ student_fname: 'Test', student_lname: 'User', role: 'Advisor', - thesis_title: 'Test Title', - degree_type: 'PhD' + thesis_title: 'Test Title' ) xml = xml_builder_obj.xmls_enumerator.first - expect(xml).not_to include('') expect(xml).not_to include('') end @@ -115,8 +88,8 @@ faculty1 = FactoryBot.create(:faculty, access_id: 'fac1') faculty2 = FactoryBot.create(:faculty, access_id: 'fac2') - Committee.create!(faculty: faculty1, student_fname: 'John', student_lname: 'Doe', role: 'Mentor', thesis_title: 'Title 1', degree_type: 'PhD') - Committee.create!(faculty: faculty2, student_fname: 'Jane', student_lname: 'Smith', role: 'Chair', thesis_title: 'Title 2', degree_type: 'MS') + Committee.create!(faculty: faculty1, student_fname: 'John', student_lname: 'Doe', role: 'Mentor', thesis_title: 'Title 1') + Committee.create!(faculty: faculty2, student_fname: 'Jane', student_lname: 'Smith', role: 'Chair', thesis_title: 'Title 2') result = xml_builder_obj.xmls_enumerator.to_a expect(result.length).to eq(1) @@ -125,8 +98,8 @@ it 'handles faculty member with multiple committees' do faculty = FactoryBot.create(:faculty, access_id: 'test123') - Committee.create!(faculty: faculty, student_fname: 'John', student_lname: 'Doe', role: 'Mentor', thesis_title: 'Title 1', degree_type: 'PhD') - Committee.create!(faculty: faculty, student_fname: 'Jane', student_lname: 'Smith', role: 'Member', thesis_title: 'Title 2', degree_type: 'MS') + Committee.create!(faculty: faculty, student_fname: 'John', student_lname: 'Doe', role: 'Mentor', thesis_title: 'Title 1') + Committee.create!(faculty: faculty, student_fname: 'Jane', student_lname: 'Smith', role: 'Member', thesis_title: 'Title 2') xml = xml_builder_obj.xmls_enumerator.first expect(xml.scan('').count).to eq(2) diff --git a/spec/importers/committee_data/etda_importer_spec.rb b/spec/importers/committee_data/etda_importer_spec.rb index 76bba4a4..cc28d859 100644 --- a/spec/importers/committee_data/etda_importer_spec.rb +++ b/spec/importers/committee_data/etda_importer_spec.rb @@ -18,7 +18,7 @@ let(:api_response) do { data: { 'committees' => [ { 'student_fname' => 'Spider', 'student_lname' => 'Man', - 'role' => 'advisor', 'title' => 'My Thesis', 'degree_name' => 'PhD', + 'role' => 'advisor', 'title' => 'My Thesis', 'degree_type' => 'Dissertation', 'approval_started_at' => '2024-08-15T10:30:00Z', 'final_submission_approved_at' => '2026-01-15T10:30:00Z', 'submission_status' => 'released for publication' } @@ -37,82 +37,37 @@ expect(Committee.last.student_fname).to eq('Spider') expect(Committee.last.student_lname).to eq('Man') expect(Committee.last.role).to eq('Advisor') - expect(Committee.last.role_other_explanation).to be_nil expect(Committee.last.thesis_title).to eq('My Thesis') - expect(Committee.last.degree_type).to eq('PhD') - expect(Committee.last.type_of_work).to eq('Ph.D. Dissertation') + expect(Committee.last.type_of_work).to eq('Dissertation Committee') expect(Committee.last.stage_of_completion).to eq('Completed') expect(Committee.last.start_year).to eq(2024) expect(Committee.last.completion_year).to eq(2026) end - - context 'when role does not match any valid Activity Insight value' do - let(:api_response) do - { data: { 'committees' => [ - { 'student_fname' => 'Spider', 'student_lname' => 'Man', - 'role' => 'External Reviewer', 'title' => 'My Thesis', 'degree_name' => 'PhD', - 'approval_started_at' => nil, - 'final_submission_approved_at' => nil, - 'submission_status' => 'waiting for publication release' } - ] } } - end - - it 'saves role as Other and stores original role in role_other_explanation' do - importer.import_all - committee = Committee.last - expect(committee.role).to eq('Other') - expect(committee.role_other_explanation).to eq('External Reviewer') - end - end end end describe '#map_type_of_work' do - it 'maps PhD with Advisor role to Ph.D. Dissertation' do - expect(importer.send(:map_type_of_work, 'PhD', 'Advisor')).to eq('Ph.D. Dissertation') - end - - it 'maps PhD with Chairperson role to Ph.D. Dissertation' do - expect(importer.send(:map_type_of_work, 'PhD', 'Chairperson')).to eq('Ph.D. Dissertation') - end - - it 'maps PhD with Member role to Ph.D. Dissertation Committee' do - expect(importer.send(:map_type_of_work, 'PhD', 'Member')).to eq('Ph.D. Dissertation Committee') - end - - it 'maps Ph.D. to Ph.D. Dissertation Committee when no role given' do - expect(importer.send(:map_type_of_work, 'Ph.D.')).to eq('Ph.D. Dissertation Committee') + it 'maps Dissertation to Dissertation Committee' do + expect(importer.send(:map_type_of_work, 'Dissertation')).to eq('Dissertation Committee') end - it 'maps MS with Advisor role to Master\'s Thesis' do - expect(importer.send(:map_type_of_work, 'MS', 'Advisor')).to eq("Master's Thesis") + it "maps Master Thesis to Master's Committee" do + expect(importer.send(:map_type_of_work, 'Master Thesis')).to eq("Master's Committee") end - it 'maps MS with Member role to Master\'s Thesis Committee' do - expect(importer.send(:map_type_of_work, 'MS', 'Member')).to eq("Master's Thesis Committee") + it 'maps Thesis to Undergraduate Honors Thesis' do + expect(importer.send(:map_type_of_work, 'Thesis')).to eq('Undergraduate Honors Thesis') end - it 'maps MA to Master\'s Thesis Committee when no role given' do - expect(importer.send(:map_type_of_work, 'MA')).to eq("Master's Thesis Committee") + it "maps Final Paper to Master's Paper Committee" do + expect(importer.send(:map_type_of_work, 'Final Paper')).to eq("Master's Paper Committee") end - it 'maps undergraduate degrees to Undergraduate Research' do - expect(importer.send(:map_type_of_work, 'BS')).to eq('Undergraduate Research') + it 'raises DegreeTypeError for unknown degree types' do + expect { importer.send(:map_type_of_work, 'DMA') }.to raise_error(CommitteeData::EtdaImporter::DegreeTypeError) end - it 'maps postdoc to Postdoctoral Mentorship' do - expect(importer.send(:map_type_of_work, 'Postdoc')).to eq('Postdoctoral Mentorship') - end - - it 'maps honors to Honors Thesis' do - expect(importer.send(:map_type_of_work, 'HONORS')).to eq('Honors Thesis') - end - - it 'defaults to Dissertation Committee for unknown degree types' do - expect(importer.send(:map_type_of_work, 'DMA')).to eq('Dissertation Committee') - end - - it 'returns nil for blank degree name' do + it 'returns nil for blank degree type' do expect(importer.send(:map_type_of_work, nil)).to be_nil expect(importer.send(:map_type_of_work, '')).to be_nil end @@ -152,16 +107,11 @@ expect(result).to eq('Completed') end - it 'returns In Process when final submission date is nil and not withdrawn' do + it 'returns In Process when final submission date is nil' do result = importer.send(:determine_completion_stage, nil, 'waiting for publication release') expect(result).to eq('In Process') end - it 'returns Withdrew when submission status indicates withdrawal' do - result = importer.send(:determine_completion_stage, nil, 'withdrawn by student') - expect(result).to eq('Withdrew') - end - it 'handles nil submission status gracefully' do result = importer.send(:determine_completion_stage, nil, nil) expect(result).to eq('In Process') diff --git a/spec/models/committee_spec.rb b/spec/models/committee_spec.rb index 05b29bad..75d9cf03 100644 --- a/spec/models/committee_spec.rb +++ b/spec/models/committee_spec.rb @@ -11,7 +11,6 @@ it { is_expected.to have_db_column(:student_lname).of_type(:string) } it { is_expected.to have_db_column(:role).of_type(:string) } it { is_expected.to have_db_column(:thesis_title).of_type(:string) } - it { is_expected.to have_db_column(:degree_type).of_type(:string) } it { is_expected.to have_db_column(:faculty_id).of_type(:integer) } it 'has faculty_id as bigint in the database' do From 1442e913088eafb454e249df87e7c87af136a1c0 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Thu, 26 Mar 2026 13:30:06 -0400 Subject: [PATCH 34/39] Add migration to drop degree_type and role_other_explanation from committees --- ...0_remove_degree_type_and_role_other_from_committees.rb | 8 ++++++++ db/schema.rb | 5 +---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 db/migrate/20260326000000_remove_degree_type_and_role_other_from_committees.rb diff --git a/db/migrate/20260326000000_remove_degree_type_and_role_other_from_committees.rb b/db/migrate/20260326000000_remove_degree_type_and_role_other_from_committees.rb new file mode 100644 index 00000000..1cabf9e5 --- /dev/null +++ b/db/migrate/20260326000000_remove_degree_type_and_role_other_from_committees.rb @@ -0,0 +1,8 @@ +class RemoveDegreeTypeAndRoleOtherFromCommittees < ActiveRecord::Migration[7.0] + def change + change_table :committees, bulk: true do |t| + t.remove :degree_type, type: :string + t.remove :role_other_explanation, type: :text + end + end +end diff --git a/db/schema.rb b/db/schema.rb index e74eb0d0..c02b2725 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2026_03_25_000000) do +ActiveRecord::Schema[7.2].define(version: 2026_03_26_000000) do create_table "authors", charset: "utf8mb4", force: :cascade do |t| t.string "f_name" t.string "m_name" @@ -58,14 +58,12 @@ t.string "student_lname" t.string "role" t.string "thesis_title" - t.string "degree_type" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.string "type_of_work" t.string "stage_of_completion" t.integer "start_year" t.integer "completion_year" - t.text "role_other_explanation" t.text "type_other_explanation" t.index ["faculty_id"], name: "index_committees_on_faculty_id" end @@ -312,7 +310,6 @@ t.string "hr_code" t.text "departments", size: :long, collation: "utf8mb4_bin" t.index ["faculty_id"], name: "index_yearlies_on_faculty_id", unique: true - t.check_constraint "json_valid(`departments`)", name: "departments" end add_foreign_key "authors", "works", on_delete: :cascade From 1691ad83b42a2c836e78469bcf3d073f95e8e7fe Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Tue, 31 Mar 2026 16:14:14 -0400 Subject: [PATCH 35/39] Address PR feedback: remove type_other_explanation, fix migration, update specs and gitignore --- .gitignore | 2 +- README.md | 3 +-- .../committee_data/committee_xml_builder.rb | 1 - ..._add_other_explanation_fields_to_committees.rb | 8 -------- ..._degree_type_and_role_other_from_committees.rb | 5 +---- .../committee_data/committee_xml_builder_spec.rb | 15 --------------- spec/models/committee_spec.rb | 4 ++++ 7 files changed, 7 insertions(+), 31 deletions(-) delete mode 100644 db/migrate/20260325000000_add_other_explanation_fields_to_committees.rb diff --git a/.gitignore b/.gitignore index 65fa3f1b..8f89fd14 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,6 @@ spec/examples.txt .cache .bash_history .envrc -.irb_historyx +.irb_history .DS_Store CLAUDE.md \ No newline at end of file diff --git a/README.md b/README.md index 40edd08e..72c25bc8 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,7 @@ This app parses data from faculty CVs, and integrates data into Activity Insight ## Setup Add the following untracked files/folders: - - * `'config/database.yml'` This will be similar to `config/database.yml.sample` + * `config/activity_insight.yml` * `config/integration_passcode.yml` * `public/log` diff --git a/app/importers/committee_data/committee_xml_builder.rb b/app/importers/committee_data/committee_xml_builder.rb index fdefb8a9..b5d56fcd 100644 --- a/app/importers/committee_data/committee_xml_builder.rb +++ b/app/importers/committee_data/committee_xml_builder.rb @@ -26,7 +26,6 @@ def build_xml(batch) xml.DSL do xml.ROLE committee.role xml.TYPE committee.type_of_work - xml.TYPE_OTHER committee.type_other_explanation if committee.type_other_explanation.present? xml.COMPSTAGE committee.stage_of_completion xml.DTY_START committee.start_year if committee.start_year xml.DTY_END committee.completion_year if committee.completion_year diff --git a/db/migrate/20260325000000_add_other_explanation_fields_to_committees.rb b/db/migrate/20260325000000_add_other_explanation_fields_to_committees.rb deleted file mode 100644 index 1d4a7b4b..00000000 --- a/db/migrate/20260325000000_add_other_explanation_fields_to_committees.rb +++ /dev/null @@ -1,8 +0,0 @@ -class AddOtherExplanationFieldsToCommittees < ActiveRecord::Migration[7.2] - def change - change_table :committees, bulk: true do |t| - t.text :role_other_explanation, limit: 20_000 - t.text :type_other_explanation, limit: 20_000 - end - end -end diff --git a/db/migrate/20260326000000_remove_degree_type_and_role_other_from_committees.rb b/db/migrate/20260326000000_remove_degree_type_and_role_other_from_committees.rb index 1cabf9e5..68bc8cba 100644 --- a/db/migrate/20260326000000_remove_degree_type_and_role_other_from_committees.rb +++ b/db/migrate/20260326000000_remove_degree_type_and_role_other_from_committees.rb @@ -1,8 +1,5 @@ class RemoveDegreeTypeAndRoleOtherFromCommittees < ActiveRecord::Migration[7.0] def change - change_table :committees, bulk: true do |t| - t.remove :degree_type, type: :string - t.remove :role_other_explanation, type: :text - end + remove_column :committees, :degree_type, :string end end diff --git a/spec/importers/committee_data/committee_xml_builder_spec.rb b/spec/importers/committee_data/committee_xml_builder_spec.rb index e40b7887..86d26b23 100644 --- a/spec/importers/committee_data/committee_xml_builder_spec.rb +++ b/spec/importers/committee_data/committee_xml_builder_spec.rb @@ -62,21 +62,6 @@ expect(xml).to include('In Process') end - it 'omits TYPE_OTHER when type_other_explanation is absent' do - faculty = FactoryBot.create(:faculty, access_id: 'test123') - - Committee.create!( - faculty: faculty, - student_fname: 'Test', - student_lname: 'User', - role: 'Advisor', - thesis_title: 'Test Title' - ) - - xml = xml_builder_obj.xmls_enumerator.first - expect(xml).not_to include('') - end - it 'handles faculty with no committees' do FactoryBot.create(:faculty, access_id: 'test123') diff --git a/spec/models/committee_spec.rb b/spec/models/committee_spec.rb index 75d9cf03..8ef9eb2c 100644 --- a/spec/models/committee_spec.rb +++ b/spec/models/committee_spec.rb @@ -11,6 +11,10 @@ it { is_expected.to have_db_column(:student_lname).of_type(:string) } it { is_expected.to have_db_column(:role).of_type(:string) } it { is_expected.to have_db_column(:thesis_title).of_type(:string) } + it { is_expected.to have_db_column(:type_of_work).of_type(:string) } + it { is_expected.to have_db_column(:stage_of_completion).of_type(:string) } + it { is_expected.to have_db_column(:start_year).of_type(:integer) } + it { is_expected.to have_db_column(:completion_year).of_type(:integer) } it { is_expected.to have_db_column(:faculty_id).of_type(:integer) } it 'has faculty_id as bigint in the database' do From 2e902d91f358951e0e2507a04a052e2590bfded2 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Wed, 1 Apr 2026 18:01:12 -0400 Subject: [PATCH 36/39] Filter committee imports to last 6 months, remove .irb_history from tracking, fix README --- .irb_history | 11 --------- README.md | 1 + app/importers/committee_data/etda_importer.rb | 8 +++++++ .../committee_data/etda_importer_spec.rb | 24 +++++++++++++++++-- 4 files changed, 31 insertions(+), 13 deletions(-) delete mode 100644 .irb_history diff --git a/.irb_history b/.irb_history deleted file mode 100644 index bf1bdb15..00000000 --- a/.irb_history +++ /dev/null @@ -1,11 +0,0 @@ -importer = CommitteeData::EtdaImporter.new -Committee.last.attributes -ENV['ETDA_API_URL'] -quit -importer = CommitteeData::EtdaImporter.new -Committee.last.attributes -importer.import_all -quit -importer = CommitteeData::EtdaImporter.new -importer.import_all -quit diff --git a/README.md b/README.md index 72c25bc8..59b0dfb3 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ This app parses data from faculty CVs, and integrates data into Activity Insight Add the following untracked files/folders: + * `config/database.yml` This will be similar to `config/database.yml.sample` * `config/activity_insight.yml` * `config/integration_passcode.yml` * `public/log` diff --git a/app/importers/committee_data/etda_importer.rb b/app/importers/committee_data/etda_importer.rb index b8f722f4..435e17c4 100644 --- a/app/importers/committee_data/etda_importer.rb +++ b/app/importers/committee_data/etda_importer.rb @@ -19,6 +19,8 @@ def import_for_faculty(faculty) committees_data = result[:data]['committees'] committees_data.each do |committee| + next unless within_last_six_months?(committee['approval_started_at']) + normalized_role = CommitteeRoleNormalizer.normalize(committee['role']) faculty.committees.create!( @@ -65,6 +67,12 @@ def extract_year(date_string) nil end + def within_last_six_months?(date_string) + return false if date_string.blank? + + Date.parse(date_string) >= 6.months.ago + end + def determine_completion_stage(final_submission_approved_at, _submission_status) return 'Completed' if final_submission_approved_at.present? diff --git a/spec/importers/committee_data/etda_importer_spec.rb b/spec/importers/committee_data/etda_importer_spec.rb index cc28d859..2f3763a9 100644 --- a/spec/importers/committee_data/etda_importer_spec.rb +++ b/spec/importers/committee_data/etda_importer_spec.rb @@ -19,7 +19,7 @@ { data: { 'committees' => [ { 'student_fname' => 'Spider', 'student_lname' => 'Man', 'role' => 'advisor', 'title' => 'My Thesis', 'degree_type' => 'Dissertation', - 'approval_started_at' => '2024-08-15T10:30:00Z', + 'approval_started_at' => 1.month.ago.iso8601, 'final_submission_approved_at' => '2026-01-15T10:30:00Z', 'submission_status' => 'released for publication' } ] } } @@ -40,7 +40,7 @@ expect(Committee.last.thesis_title).to eq('My Thesis') expect(Committee.last.type_of_work).to eq('Dissertation Committee') expect(Committee.last.stage_of_completion).to eq('Completed') - expect(Committee.last.start_year).to eq(2024) + expect(Committee.last.start_year).to eq(1.month.ago.year) expect(Committee.last.completion_year).to eq(2026) end end @@ -101,6 +101,26 @@ end end + describe '#within_last_six_months?' do + it 'returns true for a date within the last 6 months' do + recent_date = 1.month.ago.iso8601 + expect(importer.send(:within_last_six_months?, recent_date)).to be true + end + + it 'returns false for a date older than 6 months' do + old_date = 1.year.ago.iso8601 + expect(importer.send(:within_last_six_months?, old_date)).to be false + end + + it 'returns false for a nil date' do + expect(importer.send(:within_last_six_months?, nil)).to be false + end + + it 'returns false for a blank date' do + expect(importer.send(:within_last_six_months?, '')).to be false + end + end + describe '#determine_completion_stage' do it 'returns Completed when final submission date is present' do result = importer.send(:determine_completion_stage, '2026-01-15T10:30:00Z', 'released for publication') From 0eb5b3b18a9bcd7f3f9a8c23e320e8733ac705ba Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Thu, 2 Apr 2026 09:59:37 -0400 Subject: [PATCH 37/39] Update base Docker image to ruby-3.4.9-node-22 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 87d2432c..32f4a694 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM harbor.k8s.libraries.psu.edu/library/ruby-3.4.1-node-22:20260202 as base +FROM harbor.k8s.libraries.psu.edu/library/ruby-3.4.9-node-22:20260317 AS base ARG UID=1001 WORKDIR /app From 4dca3394759c04f4acb6bf321a90cacb76deb195 Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Thu, 2 Apr 2026 10:33:46 -0400 Subject: [PATCH 38/39] Removed type_other_explantion from schema --- db/schema.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index c02b2725..acc3a256 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -64,7 +64,6 @@ t.string "stage_of_completion" t.integer "start_year" t.integer "completion_year" - t.text "type_other_explanation" t.index ["faculty_id"], name: "index_committees_on_faculty_id" end From 1e7b7537f945186caa6d0088d9c78972877b897f Mon Sep 17 00:00:00 2001 From: usmannsiddiqui Date: Thu, 2 Apr 2026 11:54:47 -0400 Subject: [PATCH 39/39] Removed submission status from etda_importer and fixed rails logger error in spec - Changed logic for start_year in committee_xml_builder --- Gemfile.lock | 6 +++--- .../committee_data/committee_xml_builder.rb | 2 +- app/importers/committee_data/etda_importer.rb | 5 ++--- .../committee_xml_builder_spec.rb | 6 +++--- .../committee_data/etda_importer_spec.rb | 21 +++++++------------ 5 files changed, 16 insertions(+), 24 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 69dc4a93..5f3ca30a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -102,8 +102,8 @@ GEM builder (3.3.0) bundle-audit (0.1.0) bundler-audit - bundler-audit (0.9.1) - bundler (>= 1.2.0, < 3) + bundler-audit (0.9.3) + bundler (>= 1.2.0) thor (~> 1.0) byebug (11.1.3) capistrano (3.18.0) @@ -526,4 +526,4 @@ DEPENDENCIES whenever BUNDLED WITH - 4.0.9 + 4.0.9 diff --git a/app/importers/committee_data/committee_xml_builder.rb b/app/importers/committee_data/committee_xml_builder.rb index b5d56fcd..5d662238 100644 --- a/app/importers/committee_data/committee_xml_builder.rb +++ b/app/importers/committee_data/committee_xml_builder.rb @@ -27,7 +27,7 @@ def build_xml(batch) xml.ROLE committee.role xml.TYPE committee.type_of_work xml.COMPSTAGE committee.stage_of_completion - xml.DTY_START committee.start_year if committee.start_year + xml.DTY_START committee.start_year xml.DTY_END committee.completion_year if committee.completion_year xml.DSL_STUDENT do diff --git a/app/importers/committee_data/etda_importer.rb b/app/importers/committee_data/etda_importer.rb index 435e17c4..4c14be8d 100644 --- a/app/importers/committee_data/etda_importer.rb +++ b/app/importers/committee_data/etda_importer.rb @@ -30,8 +30,7 @@ def import_for_faculty(faculty) thesis_title: committee['title'], type_of_work: map_type_of_work(committee['degree_type']), stage_of_completion: determine_completion_stage( - committee['final_submission_approved_at'], - committee['submission_status'] + committee['final_submission_approved_at'] ), start_year: extract_year(committee['approval_started_at']), completion_year: extract_year(committee['final_submission_approved_at']) @@ -73,7 +72,7 @@ def within_last_six_months?(date_string) Date.parse(date_string) >= 6.months.ago end - def determine_completion_stage(final_submission_approved_at, _submission_status) + def determine_completion_stage(final_submission_approved_at) return 'Completed' if final_submission_approved_at.present? 'In Process' diff --git a/spec/importers/committee_data/committee_xml_builder_spec.rb b/spec/importers/committee_data/committee_xml_builder_spec.rb index 86d26b23..0e4cfa56 100644 --- a/spec/importers/committee_data/committee_xml_builder_spec.rb +++ b/spec/importers/committee_data/committee_xml_builder_spec.rb @@ -41,7 +41,7 @@ ) end - it 'omits DTY_START and DTY_END when years are nil' do + it 'omits DTY_END when completion year is nil' do faculty = FactoryBot.create(:faculty, access_id: 'test123') Committee.create!( @@ -52,12 +52,12 @@ thesis_title: 'Test Title', type_of_work: 'Ph.D. Dissertation Committee', stage_of_completion: 'In Process', - start_year: nil, + start_year: 2024, completion_year: nil ) xml = xml_builder_obj.xmls_enumerator.first - expect(xml).not_to include('') + expect(xml).to include('') expect(xml).not_to include('') expect(xml).to include('In Process') end diff --git a/spec/importers/committee_data/etda_importer_spec.rb b/spec/importers/committee_data/etda_importer_spec.rb index 2f3763a9..86819b00 100644 --- a/spec/importers/committee_data/etda_importer_spec.rb +++ b/spec/importers/committee_data/etda_importer_spec.rb @@ -3,9 +3,9 @@ RSpec.describe CommitteeData::EtdaImporter do subject(:importer) { described_class.new } - let(:faculty) { create(:faculty, access_id: 'abc123') } - let(:faculty_one) { create(:faculty, access_id: 'mpk6156') } - let(:faculty_two) { create(:faculty, access_id: 'aez1236') } + let!(:faculty) { create(:faculty, access_id: 'abc123') } + let!(:faculty_one) { create(:faculty, access_id: 'mpk6156') } + let!(:faculty_two) { create(:faculty, access_id: 'aez1236') } let(:client) { instance_double(Etda::CommitteeRecordsClient) } @@ -26,9 +26,6 @@ end before do - faculty - faculty_one - faculty_two allow(client).to receive(:faculty_committees).and_return(api_response) end @@ -123,27 +120,23 @@ describe '#determine_completion_stage' do it 'returns Completed when final submission date is present' do - result = importer.send(:determine_completion_stage, '2026-01-15T10:30:00Z', 'released for publication') + result = importer.send(:determine_completion_stage, '2026-01-15T10:30:00Z') expect(result).to eq('Completed') end it 'returns In Process when final submission date is nil' do - result = importer.send(:determine_completion_stage, nil, 'waiting for publication release') - expect(result).to eq('In Process') - end - - it 'handles nil submission status gracefully' do - result = importer.send(:determine_completion_stage, nil, nil) + result = importer.send(:determine_completion_stage, nil) expect(result).to eq('In Process') end end it 'rescues CommitteeRecordsClientError and logs it' do - faculty allow(client).to receive(:faculty_committees) .and_raise(Etda::CommitteeRecordsClient::CommitteeRecordsClientError, 'API down') + expect(Rails.logger).to receive(:error).with(/mpk6156.*API down/) expect(Rails.logger).to receive(:error).with(/abc123.*API down/) + expect(Rails.logger).to receive(:error).with(/aez1236.*API down/) expect { importer.import_all }.not_to raise_error end end