diff --git a/SBA_NEXUS_Counseling-2-14.xsd b/SBA_NEXUS_Counseling-2-14.xsd
new file mode 100644
index 0000000..4ccd90f
--- /dev/null
+++ b/SBA_NEXUS_Counseling-2-14.xsd
@@ -0,0 +1,4739 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Race
+ --------------
+ Asian
+ Black or African American
+ Middle Eastern
+ Native American/Alaska Native
+ Native Hawaiian/Other Pacific Islander
+ North African
+ White
+ Prefer not to say
+ Prefer to Self-Describe
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Ethnicity
+ --------------
+ Hispanic or Latino
+ Non Hispanic or Latino
+ Prefer not to say
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Sex
+ --------------
+ Male
+ Female
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Disability
+ --------------
+ Yes
+ No
+ Prefer not to say
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Military
+ --------------
+ Active Duty
+ Member of National Guard
+ Member of the Reserve
+ No military service
+ Service Disabled Veteran
+ Spouse of Military Member
+ Veteran
+ Prefer not to say
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Branch Of Service
+ --------------
+ Air Force
+ Army
+ Coast Guard
+ Marine Corps
+ Navy
+ Space Force
+ Prefer not to say
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Media
+ --------------
+ Boots to Business
+ Business Owner
+ Chamber of Commerce
+ Educational Institution
+ Internet
+ Lender
+ Local Economic Development Official
+ Magazine/Newspaper
+ SBA District
+ SBA Web site
+ SBDC
+ SCORE
+ Television/Radio
+ USEAC
+ VBOC
+ VRE
+ WBC
+ Word of Mouth
+ Other Client
+ Other
+
+
+ Note: If media is "Other", please specify it.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Type of business
+ --------------------------------------------------
+ Agriculture, Forestry, Fishing and Hunting
+ Accommodation and Food Services
+ Administrative and Support
+ Arts, Entertainment, and Recreation
+ Construction
+ Educational Services
+ Finance and Insurance
+ Health Care and Social Assistance
+ Information
+ Management of Companies and Enterprises
+ Manufacturing
+ Mining
+ Professional, Scientific, and Technical Services
+ Public Administration
+ Real Estate and Rental and Leasing
+ Retail Trade
+ Transportation and Warehousing
+ Utilities
+ Waste Management and Remediation Services
+ Wholesale Trade
+ Other Services (except Public Administration)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Legal Entity
+ ---------------------------------
+ Corporation
+ LLC
+ Partnership
+ S-Corporation
+ Sole Proprietor
+ Other
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Nature of counseling Seeking
+ -------------------------------------
+ Business Accounting/Budget
+ Business Financial/Cash Flow
+ Business Financing/Capital Sources
+ Business Operations/Management
+ Business Plan
+ Business Start-up/Preplanning
+ Buy/Sell Business
+ Credit Counseling
+ Customer Relations
+ Cyber Security/Cyber Awareness
+ Disaster Planning/Recovery
+ eCommerce
+ Franchising
+ Government Contracting
+ Human Resources/Managing Employees
+ Intellectual Property Training
+ International Trade
+ Legal Issues
+ Marketing/Sales
+ Tax Planning
+ Technology
+ Other
+ //Unknown/Not Stated
+
+ Note: if other, please specify it.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Type of Funding Source
+ ----------------------
+ 2020 SBDC Portable Assistance – PA2003
+ 2023 Portable Assistance - PA2023
+ 2024 SBDC Supplemental Program - SP2024
+ Hurricane Dolly (TX) – 1780
+ Hurricane Gustav (LA) – 1786
+ Hurricane Gustav (MS) – 1794
+ Hurricane Ike (LA) – 1792
+ Hurricane Ike (TX) – 1791
+ Hurricane Sandy - Phase 1 – SANDY1
+ Hurricane Sandy - Phase 2 – SANDY2
+ Resiliency and Recovery Demonstration Grant – CARESRRD
+ Severe Storms, and Flooding (GA) – 1761
+ Severe Storms, and Flooding (IL) – 1747
+ Severe Storms, and Flooding (IL) – 1771
+ Severe Storms, and Flooding (IL) – 1800
+ Severe Storms, and Flooding (IN) – 1795
+ Severe Storms, and Flooding (IN) – 1766
+ Severe Storms, and Flooding (MO) – 1749
+ Severe Storms, and Flooding (MO) – 1773
+ Severe Storms, and Flooding (MS) – 1753
+ Severe Storms, and Flooding (PR) – 1798
+ Severe Storms, and Flooding, Tornadoes (MO) – 1809
+ Severe Storms, and Tornadoes (CO) – 1762
+ Severe Storms, and Tornadoes (GA) – 1750
+ Severe Storms, and Tornadoes (MO) – 1760
+ Severe Storms, and Tornadoes (MS) – 1764
+ Severe Storms, Tornadoes, and Flooding (AR) – 1744
+ Severe Storms, Tornadoes, and Flooding (AR) – 1751
+ Severe Storms, Tornadoes, and Flooding (AR) – 1758
+ Severe Storms, Tornadoes, and Flooding (IA) – 1763
+ Severe Storms, Tornadoes, and Flooding (NE) – 1770
+ Severe Storms, Tornadoes, and Flooding (WI) – 1768
+ Severe Storms, Tornadoes, Flooding, Mudslides, and Landslides (WV) – 1769
+ Severe Storms, Tornadoes, Straight Line Winds, and Flooding (KY) – 1746
+ Severe Storms, Tornadoes, Straight Line Winds, and Flooding (TN) – 1745
+ Severe Winter Storm and Flooding (IN) – 1740
+ Severe Winter Storm and Flooding (NV) – 1738
+ Tropical Storm Fay (FL) – 1785
+ Wildfires (CA) – 1810
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Name of Certification
+ --------------------------
+ 8(a)
+ Hubzones
+ Small Disadvantage Business
+ Economically Disadvantaged Women-Owned Small Business
+ Service-Disabled Veteran-Owned Small Business
+ Veteran Owned Small Business
+ Women-Owned Small Business
+ Other
+ Note: if other, please specify it.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Name of Certification
+ --------------------------
+ Community Advantage
+ Economic Impact Disaster Loan (EIDL)
+ Export Express
+ Export Working Capital
+ Micro Loan
+
+ SBIR
+ State/Local COVID-19 Loans or Grants
+ Other(SBIR, SBIC, 7(a) 504, etc)
+
+
+ Note: if other, please specify it.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Nature of your counseling provided Value
+ -------------------------------------- -----
+ Business Accounting/Budget
+ Business Financial/Cash Flow
+ Business Financing/Capital Sources
+ Business Operations/Management
+ Business Plan
+ Business Start-up/Preplanning
+ Buy/Sell Business
+ Credit Counseling
+ Customer Relations
+ Cyber Security/Cyber Awareness
+ Disaster Planning/Recovery
+ eCommerce
+ Franchising
+ Government Contracting
+ Human Resources/Managing Employees
+ Intellectual Property Training
+ International Trade
+ Legal Issues
+ Marketing/Sales
+ Tax Planning
+ Technology
+ Other
+ //Unknown/Not Stated
+
+ Note: if other, please specify it.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Referred Client Type
+ -----------------------
+ APEX Accelerator
+ Department of Agriculture
+ Department of Commerce/Commercial Services
+ Department of State
+ Export/Import Bank
+ Overseas Private Investment Corporation
+ SBA Capital Access (PPP)
+ SBA Disaster Assistance
+ SBA District Office
+ SBA Office of International Trade (OIT)
+ SCORE Chapter
+ Small Business Development Center
+ State Trade Agency
+ U.S. Trade And Development Agency
+ US Export Assistance Center
+ Veterans Business Outreach Center
+ Women's Business Center
+ Other
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Type of session
+ -----------------------
+ Face-to-face
+ Online
+ Prepare Only
+ Telephone
+ Training
+ Update Only
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Language
+ --------------
+ English
+ Spanish
+ Arabic
+ Armenian
+ ASL-American Sign Language
+ Assamese
+ Borana
+ Burmese
+ Catalan/Valencian
+ Chinese-Cantonese
+ Chinese-Mandarin
+ Chinese-Other
+ Chuukese
+ Croatian
+ Czech
+ Farsi
+ Finnish
+ French
+ Fukien
+ Galician
+ German
+ Hmong
+ Hungarian
+ Inupiat/Inupiaq
+ Italian
+ Japanese
+ Kamba
+ Kannada
+ Khmer
+ Kikuyu/Gikuyu
+ Korean
+ Kosraean
+ Luo
+ Marshallese
+ Palauan
+ Panjabi/Punjabi
+ Pohnpein
+ Portuguese
+ Russian
+ Slovenian
+ Somali
+ Swahili
+ Tagalog/Filipino
+ Tajik
+ Tamil
+ Thai
+ Vietnamese
+ Yapese
+ Yiddish
+ Other
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ State Name
+ --------------
+ Alabama
+ Alaska
+ American Samoa
+ Arizona
+ Arkansas
+ Armed Forces Europe
+ Armed Forces Pacific
+ Armed Forces the Americas
+ California
+ Colorado
+ Connecticut
+ Delaware
+ District of Columbia
+ Federated States of Micronesia
+ Florida
+ Georgia
+ Guam
+ Hawaii
+ Idaho
+ Illinois
+ Indiana
+ Iowa
+ Kansas
+ Kentucky
+ Louisiana
+ Maine
+ Marshall Islands
+ Maryland
+ Massachusetts
+ Michigan
+ Minnesota
+ Mississippi
+ Missouri
+ Montana
+ Nebraska
+ Nevada
+ New Hampshire
+ New Jersey
+ New Mexico
+ New York
+ North Carolina
+ North Dakota
+ Northern Mariana Islands
+ Ohio
+ Oklahoma
+ Oregon
+ Pennsylvania
+ Puerto Rico
+ Republic of Palau
+ Rhode Island
+ South Carolina
+ South Dakota
+ Tennessee
+ Texas
+ United States Minor Outlying Islands
+ U.S. Virgin Islands
+ Utah
+ Vermont
+ Virginia
+ Washington
+ West Virginia
+ Wisconsin
+ Wyoming
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Value
+ --------------
+ Yes
+ No
+
+
+
+
+
+
+
+
+
+
+ Value
+ --------------
+ Yes
+ No
+ Undetermined
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Country Name
+ --------------------------
+ United States
+ Canada
+ Mexico
+ Afghanistan
+ Aland Islands
+ Albania
+ Algeria
+ Andorra
+ Angola
+ Anguilla
+ Antarctica
+ Antigua and Barbuda
+ Argentina
+ Armenia
+ Aruba
+ Australia
+ Austria
+ Azerbaijan
+ Bahamas
+ Bahrain
+ Bangladesh
+ Barbados
+ Belarus
+ Belgium
+ Belize
+ Benin
+ Bermuda
+ Bhutan
+ Bolivia, Plurinational State of
+ Bonaire, Sint Eustatius, and Saba
+ Bosnia and Herzegovina
+ Botswana
+ Bouvet Island
+ Brazil
+ British Indian Ocean Territory
+ Brunei Darussalam
+ Bulgaria
+ Burkina Faso
+ Burundi
+ Cambodia
+ Cameroon
+ Cape Verde
+ Cayman Islands
+ Central African Republic
+ Chad
+ Chile
+ China
+ Christmas Island
+ Cocos (Keeling) Islands
+ Colombia
+ Comoros
+ Congo
+ Congo, the Democratic Republic of the
+ Cook Islands
+ Costa Rica
+ Cote d’Ivoire
+ Croatia
+ Cuba
+ Curaçao
+ Cyprus
+ Czechia
+ Denmark
+ Djibouti
+ Dominica
+ Dominican Republic
+ Ecuador
+ Egypt
+ El Salvador
+ Equatorial Guinea
+ Eritrea
+ Estonia
+ Eswatini
+ Ethiopia
+ Falkland Islands (Malvinas)
+ Faroe Islands
+ Fiji
+ Finland
+ France
+ French Guiana
+ French Polynesia
+ French Southern Territories
+ Gabon
+ Gambia
+ Georgia
+ Germany
+ Ghana
+ Gibraltar
+ Greece
+ Greenland
+ Grenada
+ Guadeloupe
+ Guatemala
+ Guernsey
+ Guinea
+ Guinea-Bissau
+ Guyana
+ Haiti
+ Heard Island and McDonald Islands
+ Holy See (Vatican City State)
+ Honduras
+ Hungary
+ Iceland
+ India
+ Indonesia
+ Iran, Islamic Republic of
+ Iraq
+ Ireland
+ Isle of Man
+ Israel
+ Italy
+ Jamaica
+ Japan
+ Jersey
+ Jordan
+ Kazakhstan
+ Kenya
+ Kiribati
+ Korea, Democratic People’s Republic of
+ Korea, Republic of
+ Kosovo
+ Kuwait
+ Kyrgyzstan
+ Lao People’s Democratic Republic
+ Latvia
+ Lebanon
+ Lesotho
+ Liberia
+ Libya
+ Liechtenstein
+ Lithuania
+ Luxembourg
+ Macao
+ Madagascar
+ Malawi
+ Malaysia
+ Maldives
+ Mali
+ Malta
+ Martinique
+ Mauritania
+ Mauritius
+ Mayotte
+ Moldova, Republic of
+ Monaco
+ Mongolia
+ Montenegro
+ Montserrat
+ Morocco
+ Mozambique
+ Myanmar
+ Namibia
+ Nauru
+ Nepal
+ Netherlands
+ New Caledonia
+ New Zealand
+ Nicaragua
+ Niger
+ Nigeria
+ Niue
+ Norfolk Island
+ North Macedonia
+ Norway
+ Oman
+ Pakistan
+ Palestine
+ Panama
+ Papua New Guinea
+ Paraguay
+ Peru
+ Philippines
+ Pitcairn
+ Poland
+ Portugal
+ Qatar
+ Reunion
+ Romania
+ Russian Federation
+ Rwanda
+ Saint Barthélemy
+ Saint Helena, Ascension and Tristan da Cunha
+ Saint Kitts and Nevis
+ Saint Lucia
+ Saint Martin (French part)
+ Saint Pierre and Miquelon
+ Saint Vincent and the Grenadines
+ Samoa
+ San Marino
+ Sao Tome and Principe
+ Saudi Arabia
+ Senegal
+ Serbia
+ Seychelles
+ Sierra Leone
+ Singapore
+ Sint Maarten (Dutch part)
+ Slovakia
+ Slovenia
+ Solomon Islands
+ Somalia
+ South Africa
+ South Georgia and the South Sandwich Islands
+ South Sudan
+ Spain
+ Sri Lanka
+ Sudan
+ Suriname
+ Svalbard and Jan Mayen
+ Sweden
+ Switzerland
+ Syrian Arab Republic
+ Taiwan
+ Tajikistan
+ Tanzania, United Republic of
+ Thailand
+ Timor-Leste
+ Togo
+ Tokelau
+ Tonga
+ Trinidad and Tobago
+ Tunisia
+ Türkiye
+ Turkmenistan
+ Turks and Caicos Islands
+ Tuvalu
+ Uganda
+ Ukraine
+ United Arab Emirates
+ United Kingdom
+ Uruguay
+ Uzbekistan
+ Vanuatu
+ Venezuela, Bolivarian Republic of
+ Vietnam
+ Virgin Islands, British
+ Wallis and Futuna
+ Western Sahara
+ Yemen
+ Zambia
+ Zimbabwe
+ Other
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Country Name
+ --------------------------
+ United States
+ Canada
+ Mexico
+ Afghanistan
+ Aland Islands
+ Albania
+ Algeria
+ Andorra
+ Angola
+ Anguilla
+ Antarctica
+ Antigua and Barbuda
+ Argentina
+ Armenia
+ Aruba
+ Australia
+ Austria
+ Azerbaijan
+ Bahamas
+ Bahrain
+ Bangladesh
+ Barbados
+ Belarus
+ Belgium
+ Belize
+ Benin
+ Bermuda
+ Bhutan
+ Bolivia, Plurinational State of
+ Bonaire, Sint Eustatius, and Saba
+ Bosnia and Herzegovina
+ Botswana
+ Bouvet Island
+ Brazil
+ British Indian Ocean Territory
+ Brunei Darussalam
+ Bulgaria
+ Burkina Faso
+ Burundi
+ Cambodia
+ Cameroon
+ Cape Verde
+ Cayman Islands
+ Central African Republic
+ Chad
+ Chile
+ China
+ Christmas Island
+ Cocos (Keeling) Islands
+ Colombia
+ Comoros
+ Congo
+ Congo, the Democratic Republic of the
+ Cook Islands
+ Costa Rica
+ Cote d’Ivoire
+ Croatia
+ Cuba
+ Curaçao
+ Cyprus
+ Czechia
+ Denmark
+ Djibouti
+ Dominica
+ Dominican Republic
+ Ecuador
+ Egypt
+ El Salvador
+ Equatorial Guinea
+ Eritrea
+ Estonia
+ Eswatini
+ Ethiopia
+ Falkland Islands (Malvinas)
+ Faroe Islands
+ Fiji
+ Finland
+ France
+ French Guiana
+ French Polynesia
+ French Southern Territories
+ Gabon
+ Gambia
+ Georgia
+ Germany
+ Ghana
+ Gibraltar
+ Greece
+ Greenland
+ Grenada
+ Guadeloupe
+ Guatemala
+ Guernsey
+ Guinea
+ Guinea-Bissau
+ Guyana
+ Haiti
+ Heard Island and McDonald Islands
+ Holy See (Vatican City State)
+ Honduras
+ Hungary
+ Iceland
+ India
+ Indonesia
+ Iran, Islamic Republic of
+ Iraq
+ Ireland
+ Isle of Man
+ Israel
+ Italy
+ Jamaica
+ Japan
+ Jersey
+ Jordan
+ Kazakhstan
+ Kenya
+ Kiribati
+ Korea, Democratic People’s Republic of
+ Korea, Republic of
+ Kosovo
+ Kuwait
+ Kyrgyzstan
+ Lao People’s Democratic Republic
+ Latvia
+ Lebanon
+ Lesotho
+ Liberia
+ Libya
+ Liechtenstein
+ Lithuania
+ Luxembourg
+ Macao
+ Madagascar
+ Malawi
+ Malaysia
+ Maldives
+ Mali
+ Malta
+ Martinique
+ Mauritania
+ Mauritius
+ Mayotte
+ Moldova, Republic of
+ Monaco
+ Mongolia
+ Montenegro
+ Montserrat
+ Morocco
+ Mozambique
+ Myanmar
+ Namibia
+ Nauru
+ Nepal
+ Netherlands
+ New Caledonia
+ New Zealand
+ Nicaragua
+ Niger
+ Nigeria
+ Niue
+ Norfolk Island
+ North Macedonia
+ Norway
+ Oman
+ Pakistan
+ Palestine
+ Panama
+ Papua New Guinea
+ Paraguay
+ Peru
+ Philippines
+ Pitcairn
+ Poland
+ Portugal
+ Qatar
+ Reunion
+ Romania
+ Russian Federation
+ Rwanda
+ Saint Barthélemy
+ Saint Helena, Ascension and Tristan da Cunha
+ Saint Kitts and Nevis
+ Saint Lucia
+ Saint Martin (French part)
+ Saint Pierre and Miquelon
+ Saint Vincent and the Grenadines
+ Samoa
+ San Marino
+ Sao Tome and Principe
+ Saudi Arabia
+ Senegal
+ Serbia
+ Seychelles
+ Sierra Leone
+ Singapore
+ Sint Maarten (Dutch part)
+ Slovakia
+ Slovenia
+ Solomon Islands
+ Somalia
+ South Africa
+ South Georgia and the South Sandwich Islands
+ South Sudan
+ Spain
+ Sri Lanka
+ Sudan
+ Suriname
+ Svalbard and Jan Mayen
+ Sweden
+ Switzerland
+ Syrian Arab Republic
+ Taiwan
+ Tajikistan
+ Tanzania, United Republic of
+ Thailand
+ Timor-Leste
+ Togo
+ Tokelau
+ Tonga
+ Trinidad and Tobago
+ Tunisia
+ Türkiye
+ Turkmenistan
+ Turks and Caicos Islands
+ Tuvalu
+ Uganda
+ Ukraine
+ United Arab Emirates
+ United Kingdom
+ Uruguay
+ Uzbekistan
+ Vanuatu
+ Venezuela, Bolivarian Republic of
+ Vietnam
+ Virgin Islands, British
+ Wallis and Futuna
+ Western Sahara
+ Yemen
+ Zambia
+ Zimbabwe
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/SBA_NEXUS_Training-2-25-2025.xsd b/SBA_NEXUS_Training-2-25-2025.xsd
new file mode 100644
index 0000000..18f5f6c
--- /dev/null
+++ b/SBA_NEXUS_Training-2-25-2025.xsd
@@ -0,0 +1,2269 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Type of Funding Source
+ ----------------------
+ 2020 SBDC Portable Assistance - PA2003
+ 2023 Portable Assistance - PA2023
+ 2024 SBDC Supplemental Program - SP2024
+ Hurricane Dolly (TX) - 1780
+ Hurricane Gustav (LA) - 1786
+ Hurricane Gustav (MS) - 1794
+ Hurricane Ike (LA) - 1792
+ Hurricane Ike (TX) - 1791
+ Hurricane Sandy - Phase 1 - SANDY1
+ Hurricane Sandy - Phase 2 - SANDY2
+ Resiliency and Recovery Demonstration Grant – CARESRRD
+ Severe Storms, and Flooding (GA) - 1761
+ Severe Storms, and Flooding (IL) - 1747
+ Severe Storms, and Flooding (IL) - 1771
+ Severe Storms, and Flooding (IL) - 1800
+ Severe Storms, and Flooding (IN) - 1795
+ Severe Storms, and Flooding (IN) - 1766
+ Severe Storms, and Flooding (MO) - 1749
+ Severe Storms, and Flooding (MO) - 1773
+ Severe Storms, and Flooding (MS) - 1753
+ Severe Storms, and Flooding (PR) - 1798
+ Severe Storms, and Flooding, Tornadoes (MO) - 1809
+ Severe Storms, and Tornadoes (CO) - 1762
+ Severe Storms, and Tornadoes (GA) - 1750
+ Severe Storms, and Tornadoes (MO) - 1760
+ Severe Storms, and Tornadoes (MS) - 1764
+ Severe Storms, Tornadoes, and Flooding (AR) - 1744
+ Severe Storms, Tornadoes, and Flooding (AR) - 1751
+ Severe Storms, Tornadoes, and Flooding (AR) - 1758
+ Severe Storms, Tornadoes, and Flooding (IA) - 1763
+ Severe Storms, Tornadoes, and Flooding (NE) - 1770
+ Severe Storms, Tornadoes, and Flooding (WI) - 1768
+ Severe Storms, Tornadoes, Flooding, Mudslides, and Landslides (WV) - 1769
+ Severe Storms, Tornadoes, Straight Line Winds, and Flooding (KY) - 1746
+ Severe Storms, Tornadoes, Straight Line Winds, and Flooding (TN) - 1745
+ Severe Winter Storm and Flooding (IN) - 1740
+ Severe Winter Storm and Flooding (NV) - 1738
+ Tropical Storm Fay (FL) - 1785
+ Wildfires (CA) - 1810
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Training Topic
+ ---------------------------------------
+ Business Accounting/Budget
+ Business Financial/Cash Flow
+ Business Financing/Capital Sources
+ Business Operations/Management
+ Business Plan
+ Business Start-up/Preplanning
+ Buy/Sell Business
+ Credit Counseling
+ Customer Relations
+ Cyber Security/Cyber Awareness
+ Disaster Planning/Recovery
+ eCommerce
+ Franchising
+ Government Contracting
+ Human Resources/Managing Employees
+ Intellectual Property Training
+ International Trade
+ Legal Issues
+ Marketing/Sales
+ Tax Planning
+ Technology
+ Other
+ //Unknown/Not Stated
+
+ Note: if other, please specify it.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Resource Partners Participating
+ --------------------------------------
+ Chamber Of Commerce
+ Educational Institution
+ For-Profit Organization
+ Native American Center
+ Online Training Resource
+ SBA
+ SBA District Office
+ SBDC
+ SCORE
+ Trade or Professional Association
+ VBOC
+ Women's Business Center
+ Other Government Agency
+ Other
+
+
+ Note: if SBA, Other Government Agency, or Other, please specify it.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Program Format Types
+ --------------
+ Hybrid
+ In-person
+ On Demand
+ Online
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Language
+ --------------
+ English
+ Spanish
+ Arabic
+ Armenian
+ ASL-American Sign Language
+ Assamese
+ Borana
+ Burmese
+ Catalan/Valencian
+ Chinese-Cantonese
+ Chinese-Mandarin
+ Chinese-Other
+ Chuukese
+ Croatian
+ Czech
+ Farsi
+ Finnish
+ French
+ Fukien
+ Galician
+ German
+ Hmong
+ Hungarian
+ Inupiat/Inupiaq
+ Italian
+ Japanese
+ Kamba
+ Kannada
+ Khmer
+ Kikuyu/Gikuyu
+ Korean
+ Kosraean
+ Luo
+ Marshallese
+ Palauan
+ Panjabi/Punjabi
+ Pohnpein
+ Portuguese
+ Russian
+ Slovenian
+ Somali
+ Swahili
+ Tagalog/Filipino
+ Tajik
+ Tamil
+ Thai
+ Vietnamese
+ Yapese
+ Yiddish
+ Other
+ Note: if Other, please specify it.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ State Name
+ --------------
+ Alabama
+ Alaska
+ American Samoa
+ Arizona
+ Arkansas
+ Armed Forces Europe
+ Armed Forces Pacific
+ Armed Forces the Americas
+ California
+ Colorado
+ Connecticut
+ Delaware
+ District of Columbia
+ Federated States of Micronesia
+ Florida
+ Georgia
+ Guam
+ Hawaii
+ Idaho
+ Illinois
+ Indiana
+ Iowa
+ Kansas
+ Kentucky
+ Louisiana
+ Maine
+ Marshall Islands
+ Maryland
+ Massachusetts
+ Michigan
+ Minnesota
+ Mississippi
+ Missouri
+ Montana
+ Nebraska
+ Nevada
+ New Hampshire
+ New Jersey
+ New Mexico
+ New York
+ North Carolina
+ North Dakota
+ Northern Mariana Islands
+ Ohio
+ Oklahoma
+ Oregon
+ Pennsylvania
+ Puerto Rico
+ Republic of Palau
+ Rhode Island
+ South Carolina
+ South Dakota
+ Tennessee
+ Texas
+ United States Minor Outlying Islands
+ U.S. Virgin Islands
+ Utah
+ Vermont
+ Virginia
+ Washington
+ West Virginia
+ Wisconsin
+ Wyoming
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Country Name
+ --------------------------
+ United States
+ Canada
+ Mexico
+ Afghanistan
+ Aland Islands
+ Albania
+ Algeria
+ Andorra
+ Angola
+ Anguilla
+ Antarctica
+ Antigua and Barbuda
+ Argentina
+ Armenia
+ Aruba
+ Australia
+ Austria
+ Azerbaijan
+ Bahamas
+ Bahrain
+ Bangladesh
+ Barbados
+ Belarus
+ Belgium
+ Belize
+ Benin
+ Bermuda
+ Bhutan
+ Bolivia, Plurinational State of
+ Bonaire, Sint Eustatius, and Saba
+ Bosnia and Herzegovina
+ Botswana
+ Bouvet Island
+ Brazil
+ British Indian Ocean Territory
+ Brunei Darussalam
+ Bulgaria
+ Burkina Faso
+ Burundi
+ Cambodia
+ Cameroon
+ Cape Verde
+ Cayman Islands
+ Central African Republic
+ Chad
+ Chile
+ China
+ Christmas Island
+ Cocos (Keeling) Islands
+ Colombia
+ Comoros
+ Congo
+ Congo, the Democratic Republic of the
+ Cook Islands
+ Costa Rica
+ Cote d’Ivoire
+ Croatia
+ Cuba
+ Curaçao
+ Cyprus
+ Czechia
+ Denmark
+ Djibouti
+ Dominica
+ Dominican Republic
+ Ecuador
+ Egypt
+ El Salvador
+ Equatorial Guinea
+ Eritrea
+ Estonia
+ Eswatini
+ Ethiopia
+ Falkland Islands (Malvinas)
+ Faroe Islands
+ Fiji
+ Finland
+ France
+ French Guiana
+ French Polynesia
+ French Southern Territories
+ Gabon
+ Gambia
+ Georgia
+ Germany
+ Ghana
+ Gibraltar
+ Greece
+ Greenland
+ Grenada
+ Guadeloupe
+ Guatemala
+ Guernsey
+ Guinea
+ Guinea-Bissau
+ Guyana
+ Haiti
+ Heard Island and McDonald Islands
+ Holy See (Vatican City State)
+ Honduras
+ Hungary
+ Iceland
+ India
+ Indonesia
+ Iran, Islamic Republic of
+ Iraq
+ Ireland
+ Isle of Man
+ Israel
+ Italy
+ Jamaica
+ Japan
+ Jersey
+ Jordan
+ Kazakhstan
+ Kenya
+ Kiribati
+ Korea, Democratic People’s Republic of
+ Korea, Republic of
+ Kosovo
+ Kuwait
+ Kyrgyzstan
+ Lao People’s Democratic Republic
+ Latvia
+ Lebanon
+ Lesotho
+ Liberia
+ Libya
+ Liechtenstein
+ Lithuania
+ Luxembourg
+ Macao
+ Madagascar
+ Malawi
+ Malaysia
+ Maldives
+ Mali
+ Malta
+ Martinique
+ Mauritania
+ Mauritius
+ Mayotte
+ Moldova, Republic of
+ Monaco
+ Mongolia
+ Montenegro
+ Montserrat
+ Morocco
+ Mozambique
+ Myanmar
+ Namibia
+ Nauru
+ Nepal
+ Netherlands
+ New Caledonia
+ New Zealand
+ Nicaragua
+ Niger
+ Nigeria
+ Niue
+ Norfolk Island
+ North Macedonia
+ Norway
+ Oman
+ Pakistan
+ Palestine
+ Panama
+ Papua New Guinea
+ Paraguay
+ Peru
+ Philippines
+ Pitcairn
+ Poland
+ Portugal
+ Qatar
+ Reunion
+ Romania
+ Russian Federation
+ Rwanda
+ Saint Barthélemy
+ Saint Helena, Ascension and Tristan da Cunha
+ Saint Kitts and Nevis
+ Saint Lucia
+ Saint Martin (French part)
+ Saint Pierre and Miquelon
+ Saint Vincent and the Grenadines
+ Samoa
+ San Marino
+ Sao Tome and Principe
+ Saudi Arabia
+ Senegal
+ Serbia
+ Seychelles
+ Sierra Leone
+ Singapore
+ Sint Maarten (Dutch part)
+ Slovakia
+ Slovenia
+ Solomon Islands
+ Somalia
+ South Africa
+ South Georgia and the South Sandwich Islands
+ South Sudan
+ Spain
+ Sri Lanka
+ Sudan
+ Suriname
+ Svalbard and Jan Mayen
+ Sweden
+ Switzerland
+ Syrian Arab Republic
+ Taiwan
+ Tajikistan
+ Tanzania, United Republic of
+ Thailand
+ Timor-Leste
+ Togo
+ Tokelau
+ Tonga
+ Trinidad and Tobago
+ Tunisia
+ Türkiye
+ Turkmenistan
+ Turks and Caicos Islands
+ Tuvalu
+ Uganda
+ Ukraine
+ United Arab Emirates
+ United Kingdom
+ Uruguay
+ Uzbekistan
+ Vanuatu
+ Venezuela, Bolivarian Republic of
+ Vietnam
+ Virgin Islands, British
+ Wallis and Futuna
+ Western Sahara
+ Yemen
+ Zambia
+ Zimbabwe
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/Sample641CouselingRecord-2-14.xml b/Sample641CouselingRecord-2-14.xml
new file mode 100644
index 0000000..4c930c5
--- /dev/null
+++ b/Sample641CouselingRecord-2-14.xml
@@ -0,0 +1,180 @@
+
+
+
+
+
+
+ 234347
+
+ 786
+
+
+
+ Hanks
+ Jerry
+
+
+ jomh@gmail.com
+ 2365894123
+
+
+
+ Austin
+ Alabama
+33189
+ 2344
+ United States
+
+
+
+
+ Yes
+
+
+ 1987-12-12
+ Yes
+
+
+
+ WhiteAsian
+ Non Hispanic or Latino
+Female
+ No
+ Active Duty
+ Prefer not to say
+
+ Magazine/Newspaper
+
+Other
+ testmedia
+
+
+ Yes
+ Yes
+ ABC company
+ Real Estate and Rental and Leasing
+
+ 100
+
+ No
+
+ No
+ Yes
+ 14
+ 0
+
+ 0.00
+ 0.00
+ 5660.00
+
+
+ Other
+ Other Counseling
+
+ Urban
+ 54346
+
+
+ Business Plan
+
+
+ Belgium
+
+
+
+ 371786_T50267
+
+ Resiliency and Recovery Demonstration Grant – CARESRRD
+
+
+ hanks
+ tom
+
+
+ tomh@gmail.com
+ 2365894123
+
+
+
+ Alpharetta
+ Alabama
+ 95928
+ 2344
+ United States
+
+
+
+ Yes
+ No
+
+ No
+ 2021-12-31
+ 18
+
+
+ 0.00
+ 0.00
+ 788.00
+ Yes
+
+
+ 10
+ 10
+
+ 10
+
+ 80
+
+ 80
+
+ 80
+
+ 80
+
+
+
+
+ Service-Disabled Veteran-Owned Small Business
+
+
+
+ Community Advantage
+ Other(SBIR, SBIC, 7(a) 504, etc)
+ Other SBA Disaster Loan for COVID-19
+
+
+ Tax Planning
+
+
+
+ SBA Office of International Trade (OIT)
+
+
+ Online
+
+ English
+
+
+ 2024-01-01
+ Paul Bozzo (FC)
+
+ 7
+ 6
+ 6
+
+ Test
+
+ United States
+
+
+
+
+ 2024-09-22
+ PartnerTrainingNum
+ 2
+ 3
+
+
+
+
+
\ No newline at end of file
diff --git a/Sample_Training_888-2-26-2025.xml b/Sample_Training_888-2-26-2025.xml
new file mode 100644
index 0000000..d8544c8
--- /dev/null
+++ b/Sample_Training_888-2-26-2025.xml
@@ -0,0 +1,61 @@
+
+
+
+93747
+250009
+Severe Winter Storm and Flooding (NV) – 1738
+2023-12-12
+6
+0.1
+ International Sales for Small Business - Webinar
+AuroraAlabama07501United States
+ 07501
+
+2
+3
+3
+5
+3
+4
+0
+0
+0
+0
+20
+
+3
+3
+6
+7
+5
+20
+30
+
+
+10
+6
+
+
+
+6
+
+
+Technology
+
+
+SBA
+Other Government Agency
+Educational Institution
+Other
+Denver SBA office
+new agency1
+other partners
+
+Seminar
+In-person
+100
+EnglishSpanish
+test
+test
+
+
\ No newline at end of file
diff --git a/patch_tests.diff b/patch_tests.diff
new file mode 100644
index 0000000..0c3c8db
--- /dev/null
+++ b/patch_tests.diff
@@ -0,0 +1,64 @@
+--- tests/test_data_cleaning.py
++++ tests/test_data_cleaning.py
+@@ -6,7 +6,7 @@
+ # Add the project root to the Python path
+ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+-from src.data_cleaning import format_date, standardize_state_name, map_value
++from src.data_cleaning import format_date, standardize_state_name, map_value, clean_percentage
+
+ class TestFormatDate(unittest.TestCase):
+
+@@ -165,34 +165,29 @@
+ with self.subTest(value=value):
+ self.assertEqual(standardize_country_code(value), expected)
+
+-if __name__ == '__main__':
+- unittest.main()
+-
+ class TestCleanPercentage(unittest.TestCase):
+ def test_clean_percentage_valid_strings(self):
+- from src.data_cleaning import clean_percentage
+ self.assertEqual(clean_percentage("50"), "50")
+ self.assertEqual(clean_percentage("50%"), "50")
+ self.assertEqual(clean_percentage("0.5"), "0.5")
+ self.assertEqual(clean_percentage(" 0.5% "), "0.5")
+ self.assertEqual(clean_percentage("100"), "100")
+ self.assertEqual(clean_percentage("100%"), "100")
+
+ def test_clean_percentage_valid_numbers(self):
+- from src.data_cleaning import clean_percentage
+ self.assertEqual(clean_percentage(50), "50")
+ self.assertEqual(clean_percentage(0.5), "0.5")
+ self.assertEqual(clean_percentage(100), "100")
+ self.assertEqual(clean_percentage(100.0), "100")
+ self.assertEqual(clean_percentage(0), "0")
+
+ def test_clean_percentage_empty_and_none(self):
+- from src.data_cleaning import clean_percentage
+ self.assertEqual(clean_percentage(""), "0")
+ self.assertEqual(clean_percentage(None), "0")
+ self.assertEqual(clean_percentage(" "), "0")
+ self.assertEqual(clean_percentage("nan"), "0")
+ self.assertEqual(clean_percentage("NaN"), "0")
+
+ def test_clean_percentage_out_of_bounds(self):
+- from src.data_cleaning import clean_percentage
+ self.assertEqual(clean_percentage("-10"), "0")
+ self.assertEqual(clean_percentage("-10%"), "0")
+ self.assertEqual(clean_percentage("-0.5"), "0")
+ self.assertEqual(clean_percentage("150"), "100")
+ self.assertEqual(clean_percentage("150%"), "100")
+ self.assertEqual(clean_percentage(150), "100")
+
+ def test_clean_percentage_invalid_strings(self):
+- from src.data_cleaning import clean_percentage
+ with self.assertRaises(ValueError):
+ clean_percentage("abc")
+ with self.assertRaises(ValueError):
+ clean_percentage("50 percent")
+ with self.assertRaises(ValueError):
+ clean_percentage("10.5.5")
++
++if __name__ == '__main__':
++ unittest.main()
diff --git a/requirements.txt b/requirements.txt
index fb6c7ed..78f4e90 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,4 @@
pandas
+pytest
+defusedxml
+lxml
diff --git a/src/converters/counseling_converter.py b/src/converters/counseling_converter.py
index b31812c..6532860 100644
--- a/src/converters/counseling_converter.py
+++ b/src/converters/counseling_converter.py
@@ -4,9 +4,7 @@
import csv
import xml.etree.ElementTree as ET
-import os
import re
-from datetime import datetime
from .base_converter import BaseConverter
from ..config import CounselingConfig, GeneralConfig, ValidationCategory
@@ -89,23 +87,8 @@ def _build_client_request_section(self, parent, row, record_id):
create_element(client_name, 'First', row.get('First Name', ''))
create_element(client_name, 'Middle', row.get('Middle Name', ''))
create_element(client_request, 'Email', row.get('Email', ''))
- phone = create_element(client_request, 'PhonePart1')
- create_element(phone, 'Primary', data_cleaning.clean_phone_number(row.get('Contact: Phone', '')))
- create_element(phone, 'Secondary', '')
- address = create_element(client_request, 'AddressPart1')
- create_element(address, 'Street1', row.get('Mailing Street', ''))
- create_element(address, 'Street2', '')
- create_element(address, 'City', row.get('Mailing City', ''))
- create_element(address, 'State', data_cleaning.standardize_state_name(row.get('Mailing State/Province', '')))
- zip_full = str(row.get('Mailing Zip/Postal Code', '')).strip()
- zip_5digit_match = re.match(r'^\d{5}', zip_full)
- zip_5digit = zip_5digit_match.group(0) if zip_5digit_match else ''
- if not zip_5digit and zip_full:
- self.validator.add_issue(record_id, "warning", ValidationCategory.INVALID_FORMAT, "Mailing Zip/Postal Code", f"Could not parse 5-digit ZIP from '{zip_full}'.")
- create_element(address, 'ZipCode', zip_5digit)
- create_element(address, 'Zip4Code', '')
- country = create_element(address, 'Country')
- create_element(country, 'Code', data_cleaning.standardize_country_code(row.get('Mailing Country', 'US')))
+ self._build_phone(client_request, 'PhonePart1', row)
+ self._build_address(client_request, 'AddressPart1', row, record_id)
create_element(client_request, 'SurveyAgreement', row.get('Agree to Impact Survey', 'No'))
signature = create_element(client_request, 'ClientSignature')
create_element(signature, 'Date', data_cleaning.format_date(row.get('Client Signature - Date', '')))
@@ -176,7 +159,9 @@ def _build_client_intake_section(self, parent, row, record_id):
create_element(client_intake, 'ConductingBusinessOnline', row.get('Conduct Business Online?', self.general_config.DEFAULT_BUSINESS_STATUS))
create_element(client_intake, 'ClientIntake_Certified8a', row.get('8(a) Certified?(old)', self.general_config.DEFAULT_BUSINESS_STATUS))
create_element(client_intake, 'TotalNumberOfEmployees', data_cleaning.clean_numeric(row.get('Total Number of Employees', '0')))
- create_element(client_intake, 'NumberOfEmployeesInExportingBusiness', '0')
+ exporting_employees1 = data_cleaning.clean_numeric(row.get('Number of Employees in Exporting Business', ''))
+ if exporting_employees1 and float(exporting_employees1) > 0:
+ create_element(client_intake, 'NumberOfEmployeesInExportingBusiness', str(int(float(exporting_employees1))))
income_part2 = create_element(client_intake, 'ClientAnnualIncomePart2')
create_element(income_part2, 'GrossRevenues', data_cleaning.clean_numeric(row.get('Gross Revenues/Sales', '0')))
@@ -229,26 +214,15 @@ def _build_counselor_record_section(self, parent, row, record_id):
create_element(counselor_record, 'Email', row.get('Email', ''))
- phone_part3 = create_element(counselor_record, 'PhonePart3')
- create_element(phone_part3, 'Primary', data_cleaning.clean_phone_number(row.get('Contact: Phone', '')))
- create_element(phone_part3, 'Secondary', '')
-
- address_part3 = create_element(counselor_record, 'AddressPart3')
- create_element(address_part3, 'Street1', row.get('Mailing Street', ''))
- create_element(address_part3, 'Street2', '')
- create_element(address_part3, 'City', row.get('Mailing City', ''))
- create_element(address_part3, 'State', data_cleaning.standardize_state_name(row.get('Mailing State/Province', '')))
- zip_full_p3 = str(row.get('Mailing Zip/Postal Code', '')).strip()
- zip_5digit_match_p3 = re.match(r'^\d{5}', zip_full_p3)
- zip_5digit_p3 = zip_5digit_match_p3.group(0) if zip_5digit_match_p3 else ''
- create_element(address_part3, 'ZipCode', zip_5digit_p3)
- create_element(address_part3, 'Zip4Code', '')
- country_p3 = create_element(address_part3, 'Country')
- create_element(country_p3, 'Code', data_cleaning.standardize_country_code(row.get('Mailing Country', 'US')))
+ self._build_phone(counselor_record, 'PhonePart3', row)
+
+ self._build_address(counselor_record, 'AddressPart3', row, record_id)
create_element(counselor_record, 'VerifiedToBeInBusiness', 'Undetermined')
create_element(counselor_record, 'ReportableImpact', row.get('Reportable Impact', self.general_config.DEFAULT_BUSINESS_STATUS))
- create_element(counselor_record, 'DateOfReportableImpact', data_cleaning.format_date(row.get('Reportable Impact Date', '')))
+ impact_date = data_cleaning.format_date(row.get('Reportable Impact Date', ''))
+ if impact_date:
+ create_element(counselor_record, 'DateOfReportableImpact', impact_date)
create_element(counselor_record, 'CurrentlyExporting', self.general_config.DEFAULT_BUSINESS_STATUS)
business_start_date = data_cleaning.format_date(row.get('Business Start Date', '')) or data_cleaning.format_date(row.get('Date Started (Meeting)', ''))
@@ -256,7 +230,9 @@ def _build_counselor_record_section(self, parent, row, record_id):
create_element(counselor_record, 'BusinessStartDatePart3', business_start_date)
create_element(counselor_record, 'TotalNumberOfEmployees', data_cleaning.clean_numeric(row.get('Total No. of Employees (Meeting)', row.get('Total Number of Employees', '0'))))
- create_element(counselor_record, 'NumberOfEmployeesInExportingBusiness', '0')
+ exporting_employees2 = data_cleaning.clean_numeric(row.get('Number of Employees in Exporting Business', ''))
+ if exporting_employees2 and float(exporting_employees2) > 0:
+ create_element(counselor_record, 'NumberOfEmployeesInExportingBusiness', str(int(float(exporting_employees2))))
income_part3 = create_element(counselor_record, 'ClientAnnualIncomePart3')
create_element(income_part3, 'GrossRevenues', data_cleaning.clean_numeric(row.get('Gross Revenues/Sales (Meeting)', row.get('Gross Revenues/Sales', '0'))))
@@ -297,3 +273,27 @@ def _build_counselor_record_section(self, parent, row, record_id):
create_element(counselor_record, 'SBALoanAmount', data_cleaning.clean_numeric(row.get('SBA Loan Amount', '0')))
create_element(counselor_record, 'NonSBALoanAmount', data_cleaning.clean_numeric(row.get('Non-SBA Loan Amount', '0')))
create_element(counselor_record, 'EquityCapitalReceived', data_cleaning.clean_numeric(row.get('Amount of Equity Capital Received', '0')))
+
+
+ def _build_address(self, parent, element_name, row, record_id):
+ address = create_element(parent, element_name)
+ create_element(address, 'Street1', row.get('Mailing Street', ''))
+ create_element(address, 'Street2', '')
+ create_element(address, 'City', row.get('Mailing City', ''))
+ create_element(address, 'State', data_cleaning.standardize_state_name(row.get('Mailing State/Province', '')))
+ zip_full = str(row.get('Mailing Zip/Postal Code', '')).strip()
+ zip_5digit_match = re.match(r'^\d{5}', zip_full)
+ zip_5digit = zip_5digit_match.group(0) if zip_5digit_match else ''
+ if not zip_5digit and zip_full:
+ self.validator.add_issue(record_id, "warning", ValidationCategory.INVALID_FORMAT, "Mailing Zip/Postal Code", f"Could not parse 5-digit ZIP from '{zip_full}'.")
+ create_element(address, 'ZipCode', zip_5digit)
+ create_element(address, 'Zip4Code', '')
+ country = create_element(address, 'Country')
+ create_element(country, 'Code', data_cleaning.standardize_country_code(row.get('Mailing Country', 'US')))
+
+ def _build_phone(self, parent, element_name, row):
+ phone = create_element(parent, element_name)
+ create_element(phone, 'Primary', data_cleaning.clean_phone_number(row.get('Contact: Phone', '')))
+ secondary_phone = data_cleaning.clean_phone_number(row.get('Contact: Secondary Phone', ''))
+ if secondary_phone:
+ create_element(phone, 'Secondary', secondary_phone)
diff --git a/src/converters/training_converter.py b/src/converters/training_converter.py
index dc8de5e..6482a66 100644
--- a/src/converters/training_converter.py
+++ b/src/converters/training_converter.py
@@ -1,3 +1,4 @@
+from .. import data_validation
"""
Handles the conversion of SBA Management Training Reports from CSV to XML.
"""
@@ -10,7 +11,7 @@
from .base_converter import BaseConverter
from ..config import TrainingConfig, GeneralConfig, ValidationCategory
from .. import data_cleaning
-from ..xml_utils import create_element, escape_xml
+from ..xml_utils import create_element
class TrainingConverter(BaseConverter):
"""
@@ -79,6 +80,12 @@ def convert(self, input_path: str, output_path: str):
record = create_element(root, 'ManagementTrainingRecord')
create_element(record, 'PartnerTrainingNumber', str(event_id))
+
+ # FundingSource - optional element based on XSD, but placing it correctly.
+ funding_source = self._get_column_value(first_record, 'funding_source')
+ if funding_source:
+ create_element(record, 'FundingSource', funding_source)
+
location = create_element(record, 'Location')
create_element(location, 'LocationCode', self.general_config.DEFAULT_LOCATION_CODE)
@@ -92,7 +99,7 @@ def convert(self, input_path: str, output_path: str):
title_val = self._get_column_value(first_record, "event_name")
if not title_val:
title_val = f"{self.config.DEFAULT_TRAINING_EVENT_TITLE_PREFIX}{event_id}"
- create_element(record, 'TrainingTitle', escape_xml(title_val))
+ create_element(record, 'TrainingTitle', title_val)
self._build_location_section(record, first_record)
demographics = self._calculate_demographics(group_df)
@@ -116,7 +123,7 @@ def convert(self, input_path: str, output_path: str):
cosponsor_name = self._get_column_value(first_record, "cosponsor")
if cosponsor_name and cosponsor_name.lower() != 'n/a':
- create_element(record, 'CosponsorsName', escape_xml(cosponsor_name))
+ create_element(record, 'CosponsorsName', cosponsor_name)
self.validator.record_processed(success=True)
@@ -149,9 +156,9 @@ def _build_location_section(self, parent, record):
state = self.config.DEFAULT_LOCATION['state']
zip_code = self.config.DEFAULT_LOCATION['zip']
- create_element(training_location, 'City', escape_xml(city))
+ create_element(training_location, 'City', city)
create_element(training_location, 'State', data_cleaning.standardize_state_name(state))
- create_element(training_location, 'ZipCode', escape_xml(zip_code))
+ create_element(training_location, 'ZipCode', zip_code)
country_element = create_element(training_location, 'Country')
create_element(country_element, 'Code', self.config.DEFAULT_LOCATION['country'])
@@ -160,18 +167,25 @@ def _calculate_demographics(self, df):
total = len(df)
demographics['total'] = max(total, 2) # XSD minimum
+ # Helper to resolve a config column key to the actual DataFrame column name
+ def resolve_column(column_key):
+ possible = self.config.COLUMN_MAPPING.get(column_key, [])
+ if isinstance(possible, str):
+ possible = [possible]
+ return next((c for c in possible if c in df.columns), None)
+
# Helper to count matches for a given set of keywords in a specified column
def count_matches(column_key, keywords_map):
- column_name = self._get_column_value(df.iloc[0], column_key)
- if not column_name or column_name not in df.columns:
+ column_name = resolve_column(column_key)
+ if not column_name:
return 0
pattern = '|'.join(keywords_map)
return sum(df[column_name].fillna('').astype(str).str.lower().str.contains(pattern))
# Business Status
- business_status_col = self._get_column_value(df.iloc[0], 'business_status')
- if business_status_col and business_status_col in df.columns:
+ business_status_col = resolve_column('business_status')
+ if business_status_col:
currently_in_business = sum(df[business_status_col].fillna('').astype(str).str.lower().str.contains('yes|true|1|y'))
demographics['currently_in_business'] = currently_in_business
demographics['not_in_business'] = total - currently_in_business
@@ -191,9 +205,9 @@ def count_matches(column_key, keywords_map):
# Ethnicity
hispanic_count = count_matches('ethnicity', self.config.DEMOGRAPHIC_KEYWORDS['ethnicity']['hispanic'])
- ethnicity_col_name = self._get_column_value(df.iloc[0], 'ethnicity')
+ ethnicity_col_name = resolve_column('ethnicity')
non_hispanic_count = 0
- if ethnicity_col_name and ethnicity_col_name in df.columns:
+ if ethnicity_col_name:
non_hispanic_mask = (~df[ethnicity_col_name].fillna('').astype(str).str.lower().str.contains('hispanic|latino')) & (df[ethnicity_col_name] != '')
non_hispanic_count = sum(non_hispanic_mask)
demographics['ethnicity'] = {'hispanic': hispanic_count, 'non_hispanic': non_hispanic_count}
diff --git a/src/data_cleaning.py b/src/data_cleaning.py
index 280fa9f..84b7bd7 100644
--- a/src/data_cleaning.py
+++ b/src/data_cleaning.py
@@ -331,38 +331,45 @@ def split_multi_value(value, delimiter=";"):
def clean_numeric(value):
"""
- Cleans numeric values to ensure they're valid.
- Returns empty string if invalid or None.
+ Cleans a numeric string by removing commas, currency symbols, and whitespace.
+ Extracts digits and optional decimal point.
"""
- if not value or str(value).strip() == "" or str(value).lower() == "nan":
+ if value is None or str(value).strip() == "" or str(value).strip().lower() == "nan":
return ""
+ cleaned_str = str(value).replace(" ", "").replace("$", "").replace(",", "")
+
try:
- # Try to convert to float and then string (removes redundant .0)
- float_val = float(value)
- # If it's a whole number, return it as an integer
+ float_val = float(cleaned_str)
if float_val.is_integer():
return str(int(float_val))
- # Otherwise return as float
return str(float_val)
except (ValueError, TypeError):
return ""
def clean_percentage(value):
"""
- Cleans percentage values ensuring they're valid.
+ Cleans a percentage string, removing the % symbol and converting to a decimal.
Returns a number between 0 and 100.
"""
if not value or str(value).strip() == "" or str(value).lower() == "nan":
return "0"
+ value_str = str(value).strip()
+ if value_str.endswith('%'):
+ value_str = value_str[:-1].strip()
+
try:
- float_val = float(value)
+ float_val = float(value_str)
# Ensure it's between 0 and 100
- float_val = max(0, min(100, float_val))
+ float_val = float(max(0, min(100, float_val)))
+
+ if float_val.is_integer():
+ return str(int(float_val))
+
return str(float_val)
except (ValueError, TypeError):
- raise ValueError(f"Invalid percentage value: {value}")
+ return "0"
def truncate_counselor_notes(notes, max_length=CounselingConfig.MAX_FIELD_LENGTHS["CounselorNotes"]):
"""
diff --git a/src/data_validation.py b/src/data_validation.py
index ec96661..976db0f 100644
--- a/src/data_validation.py
+++ b/src/data_validation.py
@@ -5,7 +5,7 @@
from .data_cleaning import (
clean_phone_number, format_date, validate_counseling_date,
- clean_percentage, standardize_country_code, standardize_state_name
+ standardize_country_code, standardize_state_name
)
from .config import ValidationCategory as VC, CounselingConfig, TrainingConfig
diff --git a/src/fix-sba-xml.py b/src/fix-sba-xml.py
index b84d201..ca945e5 100644
--- a/src/fix-sba-xml.py
+++ b/src/fix-sba-xml.py
@@ -8,32 +8,34 @@
import os
import sys
-import xml.etree.ElementTree as ET
+
import argparse
import logging # Keep standard logging import for levels like logging.INFO
from datetime import datetime
-# Local setup_logging will be removed.
-# from .logging_util import logger # Default instance not used here.
-from .logging_util import ConversionLogger # Import ConversionLogger
-# Import necessary functions from xml_validator
-# Note: If xml_validator.py is in the same directory or PYTHONPATH, this should work.
-# Otherwise, sys.path manipulations might be needed, or a proper package structure.
+import sys
+import os
+
+# Ensure the script can be run from anywhere by adding its directory to sys.path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from logging_util import ConversionLogger # Import ConversionLogger
+
+# Import necessary functions from xml-validator
+# Note: Since the file name is xml-validator.py (with a hyphen), we use importlib.
+import importlib
try:
- from .xml_validator import fix_client_intake_element_order as validator_fix_order
- from .xml_validator import process_directory as validator_process_directory
- from .xml_validator import validate_against_xsd
+ xml_validator = importlib.import_module("xml-validator")
+ validator_fix_order = xml_validator.fix_client_intake_element_order
+ validator_process_directory = xml_validator.process_directory
except ImportError:
- # Fallback or error handling if xml_validator is not found directly
- # This might happen if they are not in the same directory and PYTHONPATH isn't set up.
- # For this tool's context, we assume they are accessible.
- print("Error: Could not import from xml_validator. Ensure it's in the Python path.")
+ print("Error: Could not import from xml-validator. Ensure it's in the same directory.")
sys.exit(1)
-def main():
- """Command-line entry point."""
+def parse_arguments():
+ """Parse command-line arguments."""
parser = argparse.ArgumentParser(description='Fix SBA counseling XML files')
# File/directory selection arguments
@@ -54,9 +56,11 @@ def main():
default='INFO', help='Logging level')
parser.add_argument('--log-file', action='store_true', help='Save log to file')
- args = parser.parse_args()
-
- # Setup logger using ConversionLogger
+ return parser.parse_args()
+
+
+def setup_logger(args):
+ """Set up the logger based on command-line arguments."""
log_level_val = getattr(logging, args.log_level.upper(), logging.INFO)
# Determine log file path for ConversionLogger
# fix-sba-xml.py had a --log-file flag which meant "create a timestamped file in current dir"
@@ -69,12 +73,73 @@ def main():
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
log_file_path_for_fixer = f"sba_xml_fixer_wrapper_{timestamp}.log"
- logger = ConversionLogger(
+ return ConversionLogger(
logger_name="SBAXMLFixerWrapper",
log_level=log_level_val,
log_to_file=args.log_file, # True if --log-file is present
log_file_path=log_file_path_for_fixer # Specific path if needed, else None
).logger # Get the actual logger instance
+
+
+def process_single_file(args, logger, mimic_original_add_missing):
+ """Process a single XML file."""
+ logger.info(f"[fix-sba-xml wrapper] Processing single file: {args.file}")
+
+ output_file_path = args.output if args.output else args.file
+
+ # Backup logic (simplified, xml-validator doesn't handle backups directly in its fix function)
+ if not args.no_backup and output_file_path == args.file:
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+ backup_file = f"{args.file}.{timestamp}.bak.fromwrapper"
+ try:
+ import shutil
+ shutil.copy2(args.file, backup_file)
+ logger.info(f"[fix-sba-xml wrapper] Created backup at {backup_file}")
+ except Exception as e:
+ logger.warning(f"[fix-sba-xml wrapper] Could not create backup: {str(e)}")
+
+ fix_success = validator_fix_order(
+ xml_file=args.file,
+ output_file=output_file_path,
+ add_missing_elements_flag=mimic_original_add_missing # Match original behavior
+ )
+
+ if fix_success:
+ logger.info(f"[fix-sba-xml wrapper] Successfully fixed XML file: {output_file_path} (via xml_validator)")
+ return 0
+ else:
+ logger.error("[fix-sba-xml wrapper] Failed to fix XML file (via xml_validator)")
+ return 1
+
+
+def process_directory(args, logger, always_fix, mimic_original_add_missing):
+ """Process a directory of XML files."""
+ logger.info(f"[fix-sba-xml wrapper] Processing directory: {args.directory} (via xml_validator)")
+ # Note: The new process_directory in xml-validator does not handle backups internally.
+ # Backups were handled per-file in the old fix-sba-xml.py if output_dir was None.
+ # This wrapper will not replicate the backup functionality for directory mode to keep it thin.
+ # Users should rely on xml-validator's output directory behavior.
+ if args.output and not os.path.exists(args.output):
+ os.makedirs(args.output)
+ logger.info(f"[fix-sba-xml wrapper] Created output directory: {args.output}")
+
+ count = validator_process_directory(
+ input_dir=args.directory,
+ output_dir=args.output, # Pass output dir. If None, xml-validator will modify in-place.
+ recursive=args.recursive,
+ pattern=args.pattern,
+ xsd_file=None, # fix-sba-xml didn't use XSD for its directory processing.
+ fix=always_fix,
+ add_missing_elements_flag=mimic_original_add_missing # Match original behavior
+ )
+ logger.info(f"[fix-sba-xml wrapper] Successfully processed {count} XML files (via xml_validator)")
+ return 0
+
+
+def main():
+ """Command-line entry point."""
+ args = parse_arguments()
+ logger = setup_logger(args)
# Note: fix-sba-xml.py implicitly always fixes and adds missing elements.
# We map its behavior to the new flags in xml-validator.
@@ -92,63 +157,15 @@ def main():
# this should be False.
mimic_original_add_missing = False
-
try:
if args.file:
- logger.info(f"[fix-sba-xml wrapper] Processing single file: {args.file}")
-
- output_file_path = args.output if args.output else args.file
-
- # Backup logic (simplified, xml-validator doesn't handle backups directly in its fix function)
- if not args.no_backup and output_file_path == args.file:
- timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
- backup_file = f"{args.file}.{timestamp}.bak.fromwrapper"
- try:
- import shutil
- shutil.copy2(args.file, backup_file)
- logger.info(f"[fix-sba-xml wrapper] Created backup at {backup_file}")
- except Exception as e:
- logger.warning(f"[fix-sba-xml wrapper] Could not create backup: {str(e)}")
-
- fix_success = validator_fix_order(
- xml_file=args.file,
- output_file=output_file_path,
- add_missing_elements_flag=mimic_original_add_missing # Match original behavior
- )
-
- if fix_success:
- logger.info(f"[fix-sba-xml wrapper] Successfully fixed XML file: {output_file_path} (via xml_validator)")
- return 0
- else:
- logger.error("[fix-sba-xml wrapper] Failed to fix XML file (via xml_validator)")
- return 1
-
+ return process_single_file(args, logger, mimic_original_add_missing)
elif args.directory:
- logger.info(f"[fix-sba-xml wrapper] Processing directory: {args.directory} (via xml_validator)")
- # Note: The new process_directory in xml-validator does not handle backups internally.
- # Backups were handled per-file in the old fix-sba-xml.py if output_dir was None.
- # This wrapper will not replicate the backup functionality for directory mode to keep it thin.
- # Users should rely on xml-validator's output directory behavior.
- if args.output and not os.path.exists(args.output):
- os.makedirs(args.output)
- logger.info(f"[fix-sba-xml wrapper] Created output directory: {args.output}")
-
-
- count = validator_process_directory(
- input_dir=args.directory,
- output_dir=args.output, # Pass output dir. If None, xml-validator will modify in-place.
- recursive=args.recursive,
- pattern=args.pattern,
- xsd_file=None, # fix-sba-xml didn't use XSD for its directory processing.
- fix=always_fix,
- add_missing_elements_flag=mimic_original_add_missing # Match original behavior
- )
- logger.info(f"[fix-sba-xml wrapper] Successfully processed {count} XML files (via xml_validator)")
- return 0
+ return process_directory(args, logger, always_fix, mimic_original_add_missing)
except Exception as e:
logger.error(f"[fix-sba-xml wrapper] Error: {str(e)}")
return 1
if __name__ == "__main__":
- sys.exit(main())
\ No newline at end of file
+ sys.exit(main())
diff --git a/src/training_client_xml.py b/src/training_client_xml.py
index 1539645..6b2e680 100644
--- a/src/training_client_xml.py
+++ b/src/training_client_xml.py
@@ -12,16 +12,16 @@
import logging # Keep standard logging import for levels like logging.INFO
# Import data cleaning functions from existing module
-from data_cleaning import (
- clean_phone_number, format_date, clean_whitespace,
- map_gender_to_sex, split_multi_value, clean_numeric, clean_percentage,
+from .data_cleaning import (
+ clean_phone_number, format_date, clean_whitespace,
+ map_gender_to_sex, split_multi_value, clean_numeric,
truncate_counselor_notes, standardize_country_code, standardize_state_name
)
-# Import constants from config (if needed)
-from config import (
- DEFAULT_LOCATION_CODE, DEFAULT_LANGUAGE, ValidationCategory
-)
+# Import constants from config
+from .config import GeneralConfig, ValidationCategory
+
+DEFAULT_LOCATION_CODE = GeneralConfig.DEFAULT_LOCATION_CODE
# ================ DEFAULT VALUES ================
# Iowa-specific defaults
@@ -69,7 +69,7 @@ def get_value_with_default(row, field_name, default_value):
return value
# ================ XML GENERATION FUNCTIONS ================
-from xml_utils import create_element
+from .xml_utils import create_element
def build_client_request_section(counseling_record, row, record_id, logger):
"""
@@ -127,12 +127,8 @@ def build_client_request_section(counseling_record, row, record_id, logger):
signature_onfile = 'No'
create_element(signature, 'OnFile', signature_onfile)
-def build_client_intake_section(counseling_record, row, record_id, logger):
- """
- Builds the ClientIntake section of the XML with comprehensive defaults.
- """
- client_intake = create_element(counseling_record, 'ClientIntake')
-
+
+def _build_race_info(client_intake, row):
# Race information (multi-value field)
race = create_element(client_intake, 'Race')
race_codes = split_multi_value(row.get('Race', ''))
@@ -143,7 +139,8 @@ def build_client_intake_section(counseling_record, row, record_id, logger):
for code in race_codes:
create_element(race, 'Code', code)
create_element(race, 'SelfDescribedRace', '')
-
+
+def _build_demographics_info(client_intake, row):
# Demographics - required fields with defaults
ethnicity = get_value_with_default(row, 'Ethnicity', DEFAULT_ETHNICITY)
create_element(client_intake, 'Ethnicity', ethnicity)
@@ -155,7 +152,8 @@ def build_client_intake_section(counseling_record, row, record_id, logger):
disability = get_value_with_default(row, 'Disability', DEFAULT_DISABILITY)
create_element(client_intake, 'Disability', disability)
-
+
+def _build_military_info(client_intake, row):
# Military information - required with default
military_status = get_value_with_default(row, 'Veteran Status', DEFAULT_MILITARY_STATUS)
create_element(client_intake, 'MilitaryStatus', military_status)
@@ -164,7 +162,8 @@ def build_client_intake_section(counseling_record, row, record_id, logger):
if military_status not in ['Prefer not to say', 'No military service']:
branch = get_value_with_default(row, 'Branch Of Service', 'Prefer not to say')
create_element(client_intake, 'BranchOfService', branch)
-
+
+def _build_media_info(client_intake, row):
# Referral information (Media)
media_codes = split_multi_value(row.get('What Prompted you to contact us?', ''))
if media_codes or row.get('Internet (specify)'):
@@ -174,7 +173,8 @@ def build_client_intake_section(counseling_record, row, record_id, logger):
media_other = row.get('Internet (specify)', '')
if media_other:
create_element(media, 'Other', media_other)
-
+
+def _build_business_info(client_intake, row):
# Business information - required fields with defaults
currently_in_business = get_value_with_default(row, 'Currently in Business?', DEFAULT_BUSINESS_STATUS)
create_element(client_intake, 'CurrentlyInBusiness', currently_in_business)
@@ -200,7 +200,8 @@ def build_client_intake_section(counseling_record, row, record_id, logger):
certified_8a = get_value_with_default(row, '8(a) Certified?', DEFAULT_BUSINESS_STATUS)
create_element(client_intake, 'ClientIntake_Certified8a', certified_8a)
-
+
+def _build_financial_info(client_intake, row):
# Employee and financial information
employees = row.get('Total Number of Employees', '0')
create_element(client_intake, 'TotalNumberOfEmployees', clean_numeric(employees))
@@ -214,7 +215,8 @@ def build_client_intake_section(counseling_record, row, record_id, logger):
create_element(client_annual_income, 'ProfitLoss', clean_numeric(profit_loss))
create_element(client_annual_income, 'ExportGrossRevenuesOrSales', '0')
-
+
+def _build_legal_entity_info(client_intake, row):
# Legal entity information
legal_entity_codes = split_multi_value(row.get('Legal Entity of Business', ''))
if legal_entity_codes or row.get('Other legal entity (specify)'):
@@ -224,10 +226,8 @@ def build_client_intake_section(counseling_record, row, record_id, logger):
legal_entity_other = row.get('Other legal entity (specify)', '')
if legal_entity_other:
create_element(legal_entity, 'Other', legal_entity_other)
-
- # Rural/Urban status - required with default
- create_element(client_intake, 'Rural_vs_Urban', 'Undetermined')
-
+
+def _build_counseling_seeking_info(client_intake, row):
# Counseling seeking information
counseling_seeking_codes = split_multi_value(row.get('Nature of the Counseling Seeking?', ''))
if counseling_seeking_codes:
@@ -236,27 +236,29 @@ def build_client_intake_section(counseling_record, row, record_id, logger):
create_element(counseling_seeking, 'Code', code)
create_element(counseling_seeking, 'Other', '')
-def build_training_counselor_record_section(counseling_record, row, record_id, logger, training_hours=DEFAULT_TRAINING_HOURS):
+
+def build_client_intake_section(counseling_record, row, record_id, logger):
"""
- Builds the CounselorRecord section with training-specific elements and defaults.
-
- Args:
- counseling_record: The parent XML element
- row: Dictionary of field values
- record_id: ID of the record
- logger: Logger instance
- training_hours: Default training hours to use if not specified in CSV
+ Builds the ClientIntake section of the XML with comprehensive defaults.
"""
- counselor_record = create_element(counseling_record, 'CounselorRecord')
-
- # CHANGE 3: Use Class Member ID as the PartnerSessionNumber, generate if missing
- session_number = get_value_with_default(row, 'Class Member ID', f"TRN{record_id}")
- create_element(counselor_record, 'PartnerSessionNumber', session_number)
-
- # Still need Class/Event ID for the training section
- class_id = get_value_with_default(row, 'Class/Event ID', f"CLS{record_id}")
-
- # Contact information - repeat from ClientRequest with defaults
+ client_intake = create_element(counseling_record, 'ClientIntake')
+
+ _build_race_info(client_intake, row)
+ _build_demographics_info(client_intake, row)
+ _build_military_info(client_intake, row)
+ _build_media_info(client_intake, row)
+ _build_business_info(client_intake, row)
+ _build_financial_info(client_intake, row)
+ _build_legal_entity_info(client_intake, row)
+
+ # Rural/Urban status - required with default
+ create_element(client_intake, 'Rural_vs_Urban', 'Undetermined')
+
+ _build_counseling_seeking_info(client_intake, row)
+
+
+def _add_training_contact_info(counselor_record, row):
+ """Helper to add contact information to a counselor record."""
counselor_name = create_element(counselor_record, 'ClientNamePart3')
create_element(counselor_name, 'Last', get_value_with_default(row, 'Last Name', DEFAULT_LAST_NAME))
create_element(counselor_name, 'First', get_value_with_default(row, 'First Name', DEFAULT_FIRST_NAME))
@@ -268,8 +270,10 @@ def build_training_counselor_record_section(counseling_record, row, record_id, l
phone = create_element(counselor_record, 'PhonePart3')
create_element(phone, 'Primary', clean_phone_number(row.get('Phone', '')))
create_element(phone, 'Secondary', '')
-
- # Address information (optional but adding for completeness)
+
+
+def _add_training_address_info(counselor_record, row):
+ """Helper to add address information to a counselor record."""
address = create_element(counselor_record, 'AddressPart3')
create_element(address, 'Street1', row.get('Mailing Street', ''))
create_element(address, 'Street2', '')
@@ -290,6 +294,52 @@ def build_training_counselor_record_section(counseling_record, row, record_id, l
country_value = get_value_with_default(row, 'Mailing Country', DEFAULT_COUNTRY)
standardized_country = standardize_country_code(country_value)
create_element(country, 'Code', standardized_country)
+
+
+def _add_training_session_info(counselor_record, row, class_id, training_hours):
+ """Helper to add training session specific information."""
+ training_session = create_element(counselor_record, 'TrainingSession')
+
+ # DateTrainingStarted - use the Start Date or current date if missing
+ training_date = format_date(row.get('Start Date', ''))
+ if not training_date:
+ training_date = datetime.now().strftime("%Y-%m-%d")
+ create_element(training_session, 'DateTrainingStarted', training_date)
+
+ # Partner Training Number - use Class/Event ID or generate one
+ create_element(training_session, 'PartnerTrainingNumber', class_id)
+
+ # Employees Trained - default to 1 (the attendee)
+ create_element(training_session, 'EmployeesTrained', str(DEFAULT_EMPLOYEES_TRAINED))
+
+ # Hours Trained - use default value
+ create_element(training_session, 'HoursTrained', str(training_hours))
+
+def build_training_counselor_record_section(counseling_record, row, record_id, logger, training_hours=DEFAULT_TRAINING_HOURS):
+ """
+ Builds the CounselorRecord section with training-specific elements and defaults.
+
+ Args:
+ counseling_record: The parent XML element
+ row: Dictionary of field values
+ record_id: ID of the record
+ logger: Logger instance
+ training_hours: Default training hours to use if not specified in CSV
+ """
+ counselor_record = create_element(counseling_record, 'CounselorRecord')
+
+ # CHANGE 3: Use Class Member ID as the PartnerSessionNumber, generate if missing
+ session_number = get_value_with_default(row, 'Class Member ID', f"TRN{record_id}")
+ create_element(counselor_record, 'PartnerSessionNumber', session_number)
+
+ # Still need Class/Event ID for the training section
+ class_id = get_value_with_default(row, 'Class/Event ID', f"CLS{record_id}")
+
+ # Contact information - repeat from ClientRequest with defaults
+ _add_training_contact_info(counselor_record, row)
+
+ # Address information (optional but adding for completeness)
+ _add_training_address_info(counselor_record, row)
# Status fields (optional but recommended)
create_element(counselor_record, 'VerifiedToBeInBusiness', 'Undetermined')
@@ -324,22 +374,7 @@ def build_training_counselor_record_section(counseling_record, row, record_id, l
create_element(counseling_provided, 'Code', counseling_type)
# Training-specific section (required for training clients)
- training_session = create_element(counselor_record, 'TrainingSession')
-
- # DateTrainingStarted - use the Start Date or current date if missing
- training_date = format_date(row.get('Start Date', ''))
- if not training_date:
- training_date = datetime.now().strftime("%Y-%m-%d")
- create_element(training_session, 'DateTrainingStarted', training_date)
-
- # Partner Training Number - use Class/Event ID or generate one
- create_element(training_session, 'PartnerTrainingNumber', class_id)
-
- # Employees Trained - default to 1 (the attendee)
- create_element(training_session, 'EmployeesTrained', str(DEFAULT_EMPLOYEES_TRAINED))
-
- # Hours Trained - use default value
- create_element(training_session, 'HoursTrained', str(training_hours))
+ _add_training_session_info(counselor_record, row, class_id, training_hours)
def create_training_xml_from_csv(csv_file_path, xml_file_path, training_hours=DEFAULT_TRAINING_HOURS, logger=None):
"""
@@ -427,7 +462,7 @@ def main():
args = parser.parse_args()
# Setup logger using ConversionLogger
- from logging_util import ConversionLogger
+ from .logging_util import ConversionLogger
log_level_val = getattr(logging, args.log_level.upper(), logging.INFO)
# For this script, file logging is not explicitly configured via CLI.
# Defaulting log_to_file=False for now, or add a --log-file arg if needed.
diff --git a/src/validation_report.py b/src/validation_report.py
index 6f8fee7..84bd51c 100644
--- a/src/validation_report.py
+++ b/src/validation_report.py
@@ -11,17 +11,18 @@
class ValidationTracker:
"""Tracks validation issues during the conversion process."""
+
def __init__(self):
"""Initialize the validation tracker."""
- # Store validation issues as a list of dictionaries
self.issues = []
-
- # Track counts by category and type
self.issue_counts = defaultdict(Counter)
-
- # Track processed records
self.total_records = 0
self.successful_records = 0
+ self.failed_records = 0
+ self.current_record_id = None
+
+ def set_current_record_id(self, record_id):
+ self.current_record_id = record_id
def add_issue(self, record_id, severity, category, field_name, message):
"""
@@ -134,26 +135,10 @@ def save_issues_to_csv(self, output_dir="."):
return csv_file
- def generate_html_report(self, output_dir="."):
- """
- Generate an HTML report of validation issues.
-
- Args:
- output_dir: Directory to save the HTML report
-
- Returns:
- Path to the created HTML file
- """
- if not os.path.exists(output_dir):
- os.makedirs(output_dir)
-
- timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
- html_file = os.path.join(output_dir, f"validation_report_{timestamp}.html")
-
- summary = self.get_summary()
-
- # Generate HTML content
- html_content = f"""
+
+ def _generate_html_header(self):
+ """Generate the HTML header and styles."""
+ return f"""
CSV to XML Conversion Validation Report
@@ -173,7 +158,11 @@ def generate_html_report(self, output_dir="."):
CSV to XML Conversion Validation Report
Generated on: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
-
+"""
+
+ def _generate_summary_section(self, summary):
+ """Generate the summary section HTML."""
+ return f"""
Summary
Total records processed: {summary['total_records']}
@@ -183,52 +172,35 @@ def generate_html_report(self, output_dir="."):
Total warnings: {summary['warning_count']}
"""
-
- # Add error categories table if there are errors
- if summary['errors_by_category']:
- html_content += """
- Errors by Category
-
-
- | Category |
- Count |
-
-"""
- for category, count in sorted(summary['errors_by_category'].items(), key=lambda x: x[1], reverse=True):
- html_content += f"""
-
- | {category} |
- {count} |
-
-"""
- html_content += """
-
-"""
-
- # Add warning categories table if there are warnings
- if summary['warnings_by_category']:
- html_content += """
- Warnings by Category
+
+ def _generate_category_table(self, title, categories):
+ """Generate a table for issue categories."""
+ if not categories:
+ return ""
+
+ html_content = f"""
+ {title}
| Category |
Count |
"""
- for category, count in sorted(summary['warnings_by_category'].items(), key=lambda x: x[1], reverse=True):
- html_content += f"""
-
+ for category, count in sorted(categories.items(), key=lambda x: x[1], reverse=True):
+ html_content += f"""
| {category} |
{count} |
"""
- html_content += """
-
-"""
-
- # Add detailed issues table if there are issues
- if self.issues:
- html_content += """
+ html_content += " \n"
+ return html_content
+
+ def _generate_issues_table(self):
+ """Generate the detailed issues table."""
+ if not self.issues:
+ return ""
+
+ html_content = """
Detailed Issues
@@ -239,14 +211,13 @@ def generate_html_report(self, output_dir="."):
| Message |
"""
-
- # Sort issues by severity (errors first) and then by record ID
- sorted_issues = sorted(self.issues, key=lambda x: (0 if x['severity'] == 'error' else 1, x['record_id']))
-
- for issue in sorted_issues:
- severity_class = "error" if issue['severity'] == 'error' else "warning"
- html_content += f"""
-
+
+ # Sort issues by severity (errors first) and then by record ID
+ sorted_issues = sorted(self.issues, key=lambda x: (0 if x['severity'] == 'error' else 1, x['record_id']))
+
+ for issue in sorted_issues:
+ severity_class = "error" if issue['severity'] == 'error' else "warning"
+ html_content += f"""
| {issue['record_id']} |
{issue['severity'].upper()} |
{issue['category']} |
@@ -254,13 +225,36 @@ def generate_html_report(self, output_dir="."):
{issue['message']} |
"""
+
+ html_content += "
\n"
+ return html_content
+
+ def generate_html_report(self, output_dir="."):
+ """
+ Generate an HTML report of validation issues.
+
+ Args:
+ output_dir: Directory to save the HTML report
- html_content += """
-
-"""
+ Returns:
+ Path to the created HTML file
+ """
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
+
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+ html_file = os.path.join(output_dir, f"validation_report_{timestamp}.html")
+
+ summary = self.get_summary()
+
+ # Assemble HTML content
+ html_content = self._generate_html_header()
+ html_content += self._generate_summary_section(summary)
+ html_content += self._generate_category_table("Errors by Category", summary['errors_by_category'])
+ html_content += self._generate_category_table("Warnings by Category", summary['warnings_by_category'])
+ html_content += self._generate_issues_table()
- html_content += """
-
+ html_content += """
"""
diff --git a/src/xml-validator.py b/src/xml-validator.py
index 058669d..56bd394 100644
--- a/src/xml-validator.py
+++ b/src/xml-validator.py
@@ -4,45 +4,52 @@
"""
import os
-import sys
-import xml.etree.ElementTree as ET
+import defusedxml.ElementTree as ET
from lxml import etree
+
import logging # Keep standard logging import for levels like logging.INFO
import re
-# Logger will be instantiated in main() using ConversionLogger
-# logger = logging.getLogger(__name__) # To be replaced
+# Logger will be instantiated in main() using ConversionLogger,
+# but for standalone functions we provide a fallback
+logger = logging.getLogger(__name__)
+
+import sys
+import os
+# Ensure the script can be run from anywhere by adding its directory to sys.path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from logging_util import ConversionLogger # Import ConversionLogger
def validate_against_xsd(xml_file, xsd_file):
"""
Validate XML against an XSD schema.
-
+
Args:
xml_file: Path to the XML file
xsd_file: Path to the XSD schema file
-
+
Returns:
Tuple (is_valid, errors)
"""
try:
# Parse the XSD schema
- xmlschema_doc = etree.parse(xsd_file)
+ parser = etree.XMLParser(resolve_entities=False)
+ xmlschema_doc = etree.parse(xsd_file, parser=parser)
xmlschema = etree.XMLSchema(xmlschema_doc)
-
+
# Parse the XML file
- xml_doc = etree.parse(xml_file)
-
+ xml_doc = etree.parse(xml_file, parser=parser)
+
# Validate
is_valid = xmlschema.validate(xml_doc)
-
+
# Get validation errors
errors = []
if not is_valid:
for error in xmlschema.error_log:
errors.append(f"Line {error.line}: {error.message}")
-
+
return is_valid, errors
except Exception as e:
return False, [f"Validation error: {str(e)}"]
@@ -50,71 +57,26 @@ def validate_against_xsd(xml_file, xsd_file):
def extract_validation_details(error_message):
"""
Extract element names and expected elements from a validation error message.
-
+
Args:
error_message: The validation error message
-
+
Returns:
Tuple (invalid_element, expected_elements)
"""
invalid_match = re.search(r"Invalid content was found starting with element '([^']+)'", error_message)
expected_match = re.search(r"One of '{([^}]+)}' is expected", error_message)
-
+
invalid_element = invalid_match.group(1) if invalid_match else None
expected_elements = expected_match.group(1).split(', ') if expected_match else []
-
- return invalid_element, expected_elements
-def fix_client_intake_element_order(xml_file, output_file=None):
- """
- Fix the order of elements in the ClientIntake section according to the XSD schema.
-
- Args:
- xml_file: Path to the XML file
- output_file: Path to save the fixed XML file (if None, will modify the original)
-
- Returns:
- Boolean indicating success
- """
- if output_file is None:
- output_file = xml_file
-
- try:
- # Parse the XML file
- tree = ET.parse(xml_file)
- root = tree.getroot()
-
- # Define the correct order of elements in ClientIntake
- client_intake_order = [
- 'Race', 'Ethnicity', 'Sex', 'Disability', 'MilitaryStatus',
- 'BranchOfService', 'Media', 'Internet', 'CurrentlyInBusiness',
- 'CurrentlyExporting', 'CompanyName', 'BusinessType',
- 'BusinessOwnership', 'ConductingBusinessOnline',
- 'ClientIntake_Certified8a', 'Employee_Owned', 'TotalNumberOfEmployees',
- 'NumberOfEmployeesInExportingBusiness', 'ClientAnnualIncomePart2',
- 'LegalEntity', 'Rural_vs_Urban', 'FIPS_Code', 'CounselingSeeking',
- 'ExportCountries'
- ]
-
- # Process each CounselingRecord
- for counseling_record in root.findall('CounselingRecord'):
- client_intake = counseling_record.find('ClientIntake')
- if client_intake is not None:
- # Reorder elements in ClientIntake
- reorder_elements(client_intake, client_intake_order)
-
- # Save the fixed XML
- tree.write(output_file, encoding='utf-8', xml_declaration=True)
- return True
- except Exception as e:
- logger.error(f"Error fixing XML file: {str(e)}")
- return False
+ return invalid_element, expected_elements
def add_missing_required_elements(client_intake, record_id):
"""
Add any missing required elements to ClientIntake.
(Function moved from fix-sba-xml.py)
-
+
Args:
client_intake: ClientIntake element
record_id: ID of the counseling record (for logging)
@@ -122,10 +84,10 @@ def add_missing_required_elements(client_intake, record_id):
# Define required elements and their default values
# This list might need to be configurable or expanded later.
required_elements = {
- 'CurrentlyInBusiness': 'No',
+ 'CurrentlyInBusiness': 'No',
# Add other known required elements for ClientIntake here if they have simple defaults
}
-
+
elements_added = False
for tag, default_value in required_elements.items():
if client_intake.find(tag) is None:
@@ -138,47 +100,47 @@ def fix_client_intake_element_order(xml_file, output_file=None, add_missing_elem
"""
Fix the order of elements in the ClientIntake section according to the XSD schema.
Optionally adds missing required elements.
-
+
Args:
xml_file: Path to the XML file
output_file: Path to save the fixed XML file (if None, will modify the original)
add_missing_elements_flag: If True, add missing required elements.
-
+
Returns:
Boolean indicating success
"""
if output_file is None:
output_file = xml_file
-
+
try:
# Parse the XML file
tree = ET.parse(xml_file)
root = tree.getroot()
-
+
# Define the correct order of elements in ClientIntake
client_intake_order = [
- 'Race', 'Ethnicity', 'Sex', 'Disability', 'MilitaryStatus',
- 'BranchOfService', 'Media', 'Internet', 'CurrentlyInBusiness',
- 'CurrentlyExporting', 'CompanyName', 'BusinessType',
- 'BusinessOwnership', 'ConductingBusinessOnline',
+ 'Race', 'Ethnicity', 'Sex', 'Disability', 'MilitaryStatus',
+ 'BranchOfService', 'Media', 'Internet', 'CurrentlyInBusiness',
+ 'CurrentlyExporting', 'CompanyName', 'BusinessType',
+ 'BusinessOwnership', 'ConductingBusinessOnline',
'ClientIntake_Certified8a', 'Employee_Owned', 'TotalNumberOfEmployees',
'NumberOfEmployeesInExportingBusiness', 'ClientAnnualIncomePart2',
'LegalEntity', 'Rural_vs_Urban', 'FIPS_Code', 'CounselingSeeking',
'ExportCountries'
]
-
+
# Process each CounselingRecord
for counseling_record in root.findall('CounselingRecord'):
record_id_element = counseling_record.find('PartnerClientNumber')
record_id = record_id_element.text if record_id_element is not None else "UNKNOWN_RECORD"
-
+
client_intake = counseling_record.find('ClientIntake')
if client_intake is not None:
if add_missing_elements_flag:
add_missing_required_elements(client_intake, record_id)
# Reorder elements in ClientIntake
reorder_elements(client_intake, client_intake_order)
-
+
# Save the fixed XML
tree.write(output_file, encoding='utf-8', xml_declaration=True)
return True
@@ -189,7 +151,7 @@ def fix_client_intake_element_order(xml_file, output_file=None, add_missing_elem
def reorder_elements(parent, element_order):
"""
Reorder child elements according to the specified order.
-
+
Args:
parent: Parent element
element_order: List of element names in the correct order
@@ -206,10 +168,10 @@ def reorder_elements(parent, element_order):
elements[tag] = [elements[tag], child]
else:
elements[tag] = child
-
+
# Remove the child from the parent
parent.remove(child)
-
+
# Add elements back in the correct order
for tag in element_order:
if tag in elements:
@@ -220,7 +182,7 @@ def reorder_elements(parent, element_order):
else:
# Add the single element
parent.append(elements[tag])
-
+
# Add any remaining elements that weren't in the order list
for tag, element in elements.items():
if tag not in element_order:
@@ -233,20 +195,20 @@ def reorder_elements(parent, element_order):
def check_element_order(parent, element_order):
"""
Check if elements are in the correct order.
-
+
Args:
parent: Parent element
element_order: List of element names in the correct order
-
+
Returns:
Boolean indicating if there are ordering issues
"""
# Get tags of child elements
child_tags = [child.tag for child in parent]
-
+
# Find elements from order list that exist in the XML
expected_order = [tag for tag in element_order if tag in child_tags]
-
+
# Check if the actual order matches the expected order
# This simple check assumes all expected_order elements are present and in sequence.
# A more robust check might be needed if elements can be optional and still affect order.
@@ -256,13 +218,13 @@ def check_element_order(parent, element_order):
# Find the current tag's first occurrence in the actual child_tags list
# starting from where the last tag was found.
idx = child_tags.index(tag_in_expected_order, current_pos_in_xml)
- current_pos_in_xml = idx + 1
+ current_pos_in_xml = idx + 1
except ValueError:
# Tag in expected_order is not in child_tags (or not after the previous one)
# This might indicate an issue or an optional element not present.
# For strict ordering of present elements, this is an issue.
return True # Order issue or missing element that breaks sequence
-
+
# Check if all elements from child_tags that are in element_order are in the correct sequence
# This is a more complex check. The current logic in fix-sba-xml.py is simpler:
last_index_in_parent = -1
@@ -272,21 +234,21 @@ def check_element_order(parent, element_order):
indices_in_parent = [i for i, child in enumerate(parent) if child.tag == tag_in_schema_order]
if not indices_in_parent:
continue # This element is not in the parent, skip
-
+
current_element_first_index = indices_in_parent[0]
-
+
if current_element_first_index < last_index_in_parent:
return True # Element appeared sooner than a preceding element in schema order
last_index_in_parent = current_element_first_index
-
+
# Additionally, ensure all instances of this tag are contiguous if that's a requirement
# (The current reorder logic groups them, so this check might be for pre-existing state)
# For now, just checking first occurrence order.
except ValueError:
# Element from element_order not found in parent, which is fine if it's optional.
- pass
-
+ pass
+
return False # No order issues based on first occurrence
def process_directory(input_dir, output_dir=None, recursive=False, pattern="*.xml", xsd_file=None, fix=False, add_missing_elements_flag=False):
@@ -302,13 +264,13 @@ def process_directory(input_dir, output_dir=None, recursive=False, pattern="*.xm
xsd_file: Path to XSD schema for validation (optional)
fix: Boolean, if True, fix the XML files.
add_missing_elements_flag: Boolean, if True and fix is True, add missing elements.
-
+
Returns:
Number of files processed successfully.
"""
import glob
import os
-
+
logger.info(f"Processing XML files in directory: {input_dir}")
if recursive:
logger.info(f"Recursive mode enabled, pattern: {pattern}")
@@ -317,17 +279,17 @@ def process_directory(input_dir, output_dir=None, recursive=False, pattern="*.xm
if not os.path.exists(output_dir):
os.makedirs(output_dir)
logger.info(f"Created output directory: {output_dir}")
-
+
# Find XML files
search_pattern = os.path.join(input_dir, "**", pattern) if recursive else os.path.join(input_dir, pattern)
files = glob.glob(search_pattern, recursive=recursive)
-
+
logger.info(f"Found {len(files)} XML files to process.")
-
+
processed_count = 0
for file_path in files:
logger.info(f"--- Processing file: {file_path} ---")
-
+
current_output_path = file_path
if output_dir:
rel_path = os.path.relpath(file_path, input_dir)
@@ -363,16 +325,17 @@ def process_directory(input_dir, output_dir=None, recursive=False, pattern="*.xm
elif not xsd_file: # If not fixing and no XSD, then we are just listing files.
logger.info(f"File {file_path} found (no fix requested, no XSD for validation).")
processed_count +=1 # Count as processed for listing purposes
-
+
logger.info(f"Finished processing directory. {processed_count} files processed successfully (or listed).")
return processed_count
-def main():
- """Main entry point for the script."""
+
+def parse_arguments():
+ """Parse command line arguments."""
import argparse
-
+
parser = argparse.ArgumentParser(description='XML Validator and Fixer for SBA Counseling Information.')
-
+
# Input: single file or directory
input_group = parser.add_mutually_exclusive_group(required=True)
input_group.add_argument('--xmlfile', help='Path to a single XML file to process.')
@@ -380,10 +343,10 @@ def main():
# XSD for validation
parser.add_argument('--xsd', help='Path to the XSD schema file for validation.')
-
+
# Output options
parser.add_argument('--output', help='Path to save the fixed XML file (for single file mode) or output directory (for directory mode).')
-
+
# Directory processing options
parser.add_argument('--recursive', '-r', action='store_true', help='Recursively process subdirectories (used with --directory).')
parser.add_argument('--pattern', default="*.xml", help='File pattern for XML files (default: *.xml, used with --directory).')
@@ -391,27 +354,80 @@ def main():
# Fixing options
parser.add_argument('--fix', action='store_true', help='Enable fixing of XML files (currently fixes ClientIntake element order).')
parser.add_argument('--add-missing', action='store_true', help='When fixing, also add missing required elements in ClientIntake (e.g., CurrentlyInBusiness).')
-
+
# Logging options
- parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
+ parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
default='INFO', help='Logging level.')
-
- args = parser.parse_args()
-
+
+ return parser.parse_args()
+
+def process_single_file(args, logger):
+ """Process a single XML file for validation and/or fixing."""
+ logger.info(f"Mode: Processing single file '{args.xmlfile}'")
+
+ # Validate original file if XSD is provided
+ if args.xsd:
+ logger.info(f"Validating {args.xmlfile} against {args.xsd}...")
+ is_valid, errors = validate_against_xsd(args.xmlfile, args.xsd)
+ if is_valid:
+ logger.info("XML is valid!")
+ else:
+ logger.error(f"XML is not valid. Found {len(errors)} errors:")
+ for i, error_msg in enumerate(errors, 1):
+ logger.error(f"Error {i}: {error_msg}")
+ invalid_element, expected_elements = extract_validation_details(error_msg)
+ if invalid_element: # expected_elements can be empty
+ logger.info(f" Invalid element: '{invalid_element}'")
+ if expected_elements:
+ logger.info(f" Expected elements: {', '.join(expected_elements)}")
+
+ # Fix the XML file if requested
+ if args.fix:
+ # Determine output path for single file mode
+ # If --output is not provided, fix in-place (output_file = args.xmlfile)
+ # If --output is provided, save to new file.
+ output_file_path = args.output if args.output else args.xmlfile
+
+ logger.info(f"Fixing XML file '{args.xmlfile}' and saving to '{output_file_path}'...")
+ fix_success = fix_client_intake_element_order(
+ args.xmlfile,
+ output_file_path,
+ add_missing_elements_flag=args.add_missing
+ )
+
+ if fix_success:
+ logger.info("XML file fixed successfully!")
+ # Re-validate if XSD provided and file was fixed
+ if args.xsd:
+ logger.info(f"Re-validating fixed file {output_file_path} against {args.xsd}...")
+ is_valid_after_fix, errors_after_fix = validate_against_xsd(output_file_path, args.xsd)
+ if is_valid_after_fix:
+ logger.info(f"Fixed file {output_file_path} is valid.")
+ else:
+ logger.error(f"Fixed file {output_file_path} is NOT valid after fixing. Errors: {errors_after_fix}")
+ else:
+ logger.error(f"Failed to fix XML file '{args.xmlfile}'.")
+ elif not args.xsd: # No fix, no xsd
+ logger.info(f"XML file '{args.xmlfile}' processed (no fix requested, no XSD for validation).")
+
+def main():
+ """Main entry point for the script."""
+ args = parse_arguments()
+
# Setup logger using ConversionLogger
log_level_val = getattr(logging, args.log_level.upper(), logging.INFO)
# For xml-validator, default to console-only logging unless a --log-file arg is added later
logger = ConversionLogger(
logger_name="XMLValidator",
log_level=log_level_val,
- log_to_file=False
+ log_to_file=False
).logger # Get the actual logger instance
-
+
if args.directory:
# Process directory
logger.info(f"Mode: Processing directory '{args.directory}'")
output_dir_for_process = args.output # If None, process_directory handles it (in-place if fix is True)
-
+
process_directory(
input_dir=args.directory,
output_dir=output_dir_for_process,
@@ -422,58 +438,10 @@ def main():
add_missing_elements_flag=args.add_missing
)
elif args.xmlfile:
- # Process single file
- logger.info(f"Mode: Processing single file '{args.xmlfile}'")
-
- # Validate original file if XSD is provided
- if args.xsd:
- logger.info(f"Validating {args.xmlfile} against {args.xsd}...")
- is_valid, errors = validate_against_xsd(args.xmlfile, args.xsd)
- if is_valid:
- logger.info("XML is valid!")
- else:
- logger.error(f"XML is not valid. Found {len(errors)} errors:")
- for i, error_msg in enumerate(errors, 1):
- logger.error(f"Error {i}: {error_msg}")
- invalid_element, expected_elements = extract_validation_details(error_msg)
- if invalid_element: # expected_elements can be empty
- logger.info(f" Invalid element: '{invalid_element}'")
- if expected_elements:
- logger.info(f" Expected elements: {', '.join(expected_elements)}")
-
- # Fix the XML file if requested
- if args.fix:
- # Determine output path for single file mode
- # If --output is not provided, fix in-place (output_file = args.xmlfile)
- # If --output is provided, save to new file.
- output_file_path = args.output if args.output else args.xmlfile
-
- logger.info(f"Fixing XML file '{args.xmlfile}' and saving to '{output_file_path}'...")
- fix_success = fix_client_intake_element_order(
- args.xmlfile,
- output_file_path,
- add_missing_elements_flag=args.add_missing
- )
-
- if fix_success:
- logger.info("XML file fixed successfully!")
- # Re-validate if XSD provided and file was fixed
- if args.xsd:
- logger.info(f"Re-validating fixed file {output_file_path} against {args.xsd}...")
- is_valid_after_fix, errors_after_fix = validate_against_xsd(output_file_path, args.xsd)
- if is_valid_after_fix:
- logger.info(f"Fixed file {output_file_path} is valid.")
- else:
- logger.error(f"Fixed file {output_file_path} is NOT valid after fixing. Errors: {errors_after_fix}")
- else:
- logger.error(f"Failed to fix XML file '{args.xmlfile}'.")
- elif not args.xsd: # No fix, no xsd
- logger.info(f"XML file '{args.xmlfile}' processed (no fix requested, no XSD for validation).")
-
+ process_single_file(args, logger)
else:
# Should not happen due to mutually_exclusive_group
logger.error("No input specified. Use --xmlfile or --directory.")
- parser.print_help()
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/tests/test_base_converter.py b/tests/test_base_converter.py
new file mode 100644
index 0000000..5ffbe9e
--- /dev/null
+++ b/tests/test_base_converter.py
@@ -0,0 +1,50 @@
+import unittest
+import os
+import sys
+
+# Add the project root to the Python path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from src.converters.base_converter import BaseConverter
+from src.logging_util import ConversionLogger
+from src.validation_report import ValidationTracker
+
+class TestBaseConverter(unittest.TestCase):
+
+ def setUp(self):
+ self.logger = ConversionLogger("test_base", log_level="DEBUG", log_to_file=False).logger
+ self.validator = ValidationTracker()
+
+ def test_cannot_instantiate_abc(self):
+ """
+ Tests that BaseConverter cannot be instantiated directly because it's an ABC.
+ """
+ with self.assertRaisesRegex(TypeError, "Can't instantiate abstract class BaseConverter"):
+ BaseConverter(self.logger, self.validator)
+
+ def test_subclass_must_implement_convert(self):
+ """
+ Tests that a subclass must implement the 'convert' method.
+ """
+ class IncompleteConverter(BaseConverter):
+ pass
+
+ with self.assertRaisesRegex(TypeError, "Can't instantiate abstract class IncompleteConverter"):
+ IncompleteConverter(self.logger, self.validator)
+
+ def test_subclass_with_convert_can_be_instantiated(self):
+ """
+ Tests that a subclass that implements 'convert' can be instantiated.
+ """
+ class CompleteConverter(BaseConverter):
+ def convert(self, input_path: str, output_path: str):
+ pass
+
+ converter = CompleteConverter(self.logger, self.validator)
+ self.assertIsInstance(converter, CompleteConverter)
+ self.assertIsInstance(converter, BaseConverter)
+ self.assertEqual(converter.logger, self.logger)
+ self.assertEqual(converter.validator, self.validator)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/test_data_cleaning.py b/tests/test_data_cleaning.py
index 7e5244f..7bce9c8 100644
--- a/tests/test_data_cleaning.py
+++ b/tests/test_data_cleaning.py
@@ -6,7 +6,7 @@
# Add the project root to the Python path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-from src.data_cleaning import format_date, standardize_state_name, map_value
+from src.data_cleaning import format_date, standardize_state_name, map_value, clean_percentage
class TestFormatDate(unittest.TestCase):
@@ -42,6 +42,27 @@ def test_format_date_output_format_and_default(self):
self.assertEqual(format_date("2023-1-1"), "2023-01-01") # Check zero padding
self.assertEqual(format_date("bad", default_return="---"), "---")
+ def test_format_date_value_error_path(self):
+ # Specifically malformed date string that causes ValueError inside the date parsing loop
+ # and tests that it continues to try the next format
+ self.assertEqual(format_date("10/26/2023", input_formats=["%Y-%m-%d", "%m/%d/%Y"]), "2023-10-26")
+
+ # Test a date that raises ValueError for logical reasons (e.g., Feb 29 on non-leap year)
+ self.assertEqual(format_date("2023-02-29", input_formats=["%Y-%m-%d"]), "")
+
+ # Test a date that raises ValueError for the first format but succeeds on the second
+ # (leap year case)
+ self.assertEqual(format_date("2024-02-29", input_formats=["%m/%d/%Y", "%Y-%m-%d"]), "2024-02-29")
+
+ # Test complete exhaustion of formats due to ValueError
+ self.assertEqual(format_date("2023-13-01", input_formats=["%Y-%m-%d", "%m/%d/%Y"]), "")
+
+ def test_format_date_regex_fallback(self):
+ # Test the regex fallback logic for missing zero-padding
+ self.assertEqual(format_date("2023-1-1", input_formats=["%Y/%m/%d"]), "2023-01-01")
+ # Test the regex fallback failing due to invalid date elements
+ self.assertEqual(format_date("2023-30-30", input_formats=["%Y/%m/%d"]), "")
+
class TestStandardizeStateName(unittest.TestCase):
# Using DEFAULT_VALID_STATES from data_cleaning for some tests
# These are the states the function itself knows about if no list is passed
@@ -168,5 +189,80 @@ def test_standardize_country_code(self):
with self.subTest(value=value):
self.assertEqual(standardize_country_code(value), expected)
+
+class TestCleanPercentage(unittest.TestCase):
+ def test_clean_percentage_valid_strings(self):
+ self.assertEqual(clean_percentage("50"), "50")
+ self.assertEqual(clean_percentage("50%"), "50")
+ self.assertEqual(clean_percentage("0.5"), "0.5")
+ self.assertEqual(clean_percentage(" 0.5% "), "0.5")
+ self.assertEqual(clean_percentage("100"), "100")
+ self.assertEqual(clean_percentage("100%"), "100")
+
+ def test_clean_percentage_valid_numbers(self):
+ self.assertEqual(clean_percentage(50), "50")
+ self.assertEqual(clean_percentage(0.5), "0.5")
+ self.assertEqual(clean_percentage(100), "100")
+ self.assertEqual(clean_percentage(100.0), "100")
+ self.assertEqual(clean_percentage(0), "0")
+
+ def test_clean_percentage_empty_and_none(self):
+ self.assertEqual(clean_percentage(""), "0")
+ self.assertEqual(clean_percentage(None), "0")
+ self.assertEqual(clean_percentage(" "), "0")
+ self.assertEqual(clean_percentage("nan"), "0")
+ self.assertEqual(clean_percentage("NaN"), "0")
+
+ def test_clean_percentage_out_of_bounds(self):
+ self.assertEqual(clean_percentage("-10"), "0")
+ self.assertEqual(clean_percentage("-10%"), "0")
+ self.assertEqual(clean_percentage("-0.5"), "0")
+ self.assertEqual(clean_percentage("150"), "100")
+ self.assertEqual(clean_percentage("150%"), "100")
+ self.assertEqual(clean_percentage(150), "100")
+
+ def test_clean_percentage_invalid_strings(self):
+ self.assertEqual(clean_percentage("abc"), "0")
+ self.assertEqual(clean_percentage("50 percent"), "0")
+ self.assertEqual(clean_percentage("10.5.5"), "0")
+
if __name__ == '__main__':
unittest.main()
+
+class TestCleanNumeric(unittest.TestCase):
+
+ def test_clean_numeric_valid(self):
+ from src.data_cleaning import clean_numeric
+
+ self.assertEqual(clean_numeric("1000"), "1000")
+ self.assertEqual(clean_numeric("10.5"), "10.5")
+ self.assertEqual(clean_numeric("10.0"), "10") # Removes redundant .0
+ self.assertEqual(clean_numeric("0"), "0")
+ self.assertEqual(clean_numeric(100), "100")
+ self.assertEqual(clean_numeric(10.5), "10.5")
+
+ def test_clean_numeric_with_symbols(self):
+ from src.data_cleaning import clean_numeric
+
+ self.assertEqual(clean_numeric("1,000"), "1000")
+ self.assertEqual(clean_numeric("1,234,567.89"), "1234567.89")
+ self.assertEqual(clean_numeric("$10.5"), "10.5")
+ self.assertEqual(clean_numeric("$1,000.00"), "1000")
+ self.assertEqual(clean_numeric(" $ 1,000.50 "), "1000.5")
+ self.assertEqual(clean_numeric("-$500"), "-500")
+
+ def test_clean_numeric_empty_none_nan(self):
+ from src.data_cleaning import clean_numeric
+
+ self.assertEqual(clean_numeric(""), "")
+ self.assertEqual(clean_numeric(None), "")
+ self.assertEqual(clean_numeric(" "), "")
+ self.assertEqual(clean_numeric("NaN"), "")
+ self.assertEqual(clean_numeric("nan"), "")
+
+ def test_clean_numeric_invalid(self):
+ from src.data_cleaning import clean_numeric
+
+ self.assertEqual(clean_numeric("invalid_string"), "")
+ self.assertEqual(clean_numeric("1000a"), "")
+ self.assertEqual(clean_numeric("abc"), "")
diff --git a/tests/test_data_validation.py b/tests/test_data_validation.py
new file mode 100644
index 0000000..f9b9f03
--- /dev/null
+++ b/tests/test_data_validation.py
@@ -0,0 +1,152 @@
+import unittest
+from unittest.mock import MagicMock
+import sys
+import os
+
+# Add the project root to the Python path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from src.data_validation import (
+ validate_counseling_record,
+ validate_training_record,
+ analyze_counseling_csv,
+ analyze_training_csv
+)
+from src.config import ValidationCategory as VC, CounselingConfig, TrainingConfig
+
+class TestDataValidation(unittest.TestCase):
+
+ def setUp(self):
+ self.validator = MagicMock()
+
+ def test_validate_counseling_record_success(self):
+ row = {
+ CounselingConfig.REQUIRED_FIELDS[0]: "C-123",
+ 'Last Name': 'Doe',
+ 'First Name': 'John',
+ 'Date': '2023-10-15'
+ }
+
+ result = validate_counseling_record(row, 1, self.validator)
+
+ self.assertTrue(result)
+ self.validator.set_current_record_id.assert_called_once_with("C-123")
+ self.validator.add_issue.assert_not_called()
+
+ def test_validate_counseling_record_missing_id(self):
+ row = {
+ 'Last Name': 'Doe',
+ 'First Name': 'John',
+ 'Date': '2023-10-15'
+ }
+
+ result = validate_counseling_record(row, 2, self.validator)
+
+ self.assertFalse(result)
+ self.validator.set_current_record_id.assert_not_called()
+ self.validator.add_issue.assert_called_once_with(
+ "Row_2", "error", VC.MISSING_REQUIRED, CounselingConfig.REQUIRED_FIELDS[0], "Missing required Contact ID."
+ )
+
+ def test_validate_counseling_record_missing_last_name(self):
+ row = {
+ CounselingConfig.REQUIRED_FIELDS[0]: "C-124",
+ 'First Name': 'John',
+ 'Date': '2023-10-15'
+ }
+
+ result = validate_counseling_record(row, 3, self.validator)
+
+ self.assertTrue(result)
+ self.validator.set_current_record_id.assert_called_once_with("C-124")
+ self.validator.add_issue.assert_called_once_with(
+ "C-124", "warning", VC.MISSING_FIELD, "Last Name", "Missing Last Name."
+ )
+
+ def test_validate_counseling_record_invalid_date_format(self):
+ row = {
+ CounselingConfig.REQUIRED_FIELDS[0]: "C-125",
+ 'Last Name': 'Doe',
+ 'Date': 'invalid-date'
+ }
+
+ result = validate_counseling_record(row, 4, self.validator)
+
+ self.assertTrue(result)
+ self.validator.set_current_record_id.assert_called_once_with("C-125")
+ self.validator.add_issue.assert_called_once_with(
+ "C-125", "warning", VC.INVALID_FORMAT, "Date Counseled", "Invalid date format: invalid-date"
+ )
+
+ def test_validate_counseling_record_early_date(self):
+ row = {
+ CounselingConfig.REQUIRED_FIELDS[0]: "C-126",
+ 'Last Name': 'Doe',
+ 'Date': '2020-01-01'
+ }
+
+ result = validate_counseling_record(row, 5, self.validator)
+
+ self.assertTrue(result)
+ self.validator.set_current_record_id.assert_called_once_with("C-126")
+ self.validator.add_issue.assert_called_once_with(
+ "C-126", "warning", VC.INVALID_DATE, "Date Counseled", f"Date 2020-01-01 is before minimum of {CounselingConfig.MIN_COUNSELING_DATE}"
+ )
+
+ def test_validate_training_record_success(self):
+ event_id_col = TrainingConfig.COLUMN_MAPPING['event_id']
+ row = {
+ event_id_col: "T-999",
+ 'Other': 'Data'
+ }
+
+ result = validate_training_record(row, 1, self.validator)
+
+ self.assertTrue(result)
+ self.validator.set_current_record_id.assert_called_once_with("T-999")
+ self.validator.add_issue.assert_not_called()
+
+ def test_validate_training_record_missing_id(self):
+ event_id_col = TrainingConfig.COLUMN_MAPPING['event_id']
+ row = {
+ 'Other': 'Data'
+ }
+
+ result = validate_training_record(row, 2, self.validator)
+
+ self.assertFalse(result)
+ self.validator.set_current_record_id.assert_not_called()
+ self.validator.add_issue.assert_called_once_with(
+ "Row_2", "error", VC.MISSING_REQUIRED, event_id_col, "Missing required Class/Event ID."
+ )
+
+ def test_analyze_counseling_csv(self):
+ rows = [
+ {CounselingConfig.REQUIRED_FIELDS[0]: "C-1", 'Last Name': 'Doe', 'First Name': 'John', 'Date': '2023-10-15'},
+ {'Last Name': 'Smith', 'First Name': 'Alice'}, # missing id
+ {CounselingConfig.REQUIRED_FIELDS[0]: "C-3", 'First Name': 'Bob'}, # missing last name
+ {CounselingConfig.REQUIRED_FIELDS[0]: "C-4", 'Last Name': 'Brown', 'Date': 'invalid'}, # invalid date, missing first name
+ ]
+
+ analysis = analyze_counseling_csv(rows)
+
+ self.assertEqual(analysis['row_count'], 4)
+ self.assertEqual(analysis['missing_contact_id'], 1)
+ self.assertEqual(analysis['missing_names'], 2)
+ self.assertEqual(analysis['invalid_dates'], 1)
+
+ def test_analyze_training_csv(self):
+ event_id_col = TrainingConfig.COLUMN_MAPPING['event_id']
+ rows = [
+ {event_id_col: "T-1"},
+ {}, # missing event id
+ {event_id_col: "T-3"}
+ ]
+
+ analysis = analyze_training_csv(rows)
+
+ self.assertEqual(analysis['row_count'], 3)
+ self.assertEqual(analysis['missing_event_id'], 1)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/test_training_converter.py b/tests/test_training_converter.py
index 3ca9adc..ac064ef 100644
--- a/tests/test_training_converter.py
+++ b/tests/test_training_converter.py
@@ -1,6 +1,7 @@
import unittest
import os
import sys
+import pandas as pd
# Add the project root to the Python path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
@@ -25,5 +26,45 @@ def test_converter_instantiation(self):
except Exception as e:
self.fail(f"TrainingConverter instantiation failed with an exception: {e}")
+ def test_calculate_demographics(self):
+ """
+ Tests the _calculate_demographics method.
+ To test the current logic, we construct a dataframe where the first row contains
+ the column names as their values for demographic fields.
+ """
+ converter = TrainingConverter(self.logger, self.validator)
+
+ # Build a DataFrame that matches the exact behavior expected by the current implementation
+ data = {
+ 'Currently in Business?': ['Currently in Business?', 'Yes', 'No', 'Yes', 'y', 'unknown'],
+ 'Gender': ['Gender', 'Female', 'Male', 'Female', 'M', 'O'],
+ 'Disabilities': ['Disabilities', 'Yes', 'No', '1', 'False', ''],
+ 'Military Status': ['Military Status', 'Active Duty', 'Veteran', 'Spouse', 'None', ''],
+ 'Race': ['Race', 'Asian', 'Black', 'White', 'Black', 'Hawaiian'],
+ 'Ethnicity': ['Ethnicity', 'Hispanic', 'Non-Hispanic', 'Latino', '', 'Non-Hispanic']
+ }
+ df = pd.DataFrame(data)
+
+ demographics = converter._calculate_demographics(df)
+
+ self.assertIsNotNone(demographics)
+ self.assertEqual(demographics.get('total'), 6)
+ # 'currently in business?' contains 'y' so it matches
+ self.assertEqual(demographics.get('currently_in_business'), 4)
+ self.assertEqual(demographics.get('not_in_business'), 2)
+
+ # Test existence of all keys to ensure it calculated completely
+ self.assertIn('female', demographics)
+ self.assertIn('male', demographics)
+ self.assertIn('disabilities', demographics)
+ self.assertIn('active_duty', demographics)
+ self.assertIn('veterans', demographics)
+ self.assertIn('service_disabled_veterans', demographics)
+ self.assertIn('reserve_guard', demographics)
+ self.assertIn('military_spouse', demographics)
+ self.assertIn('race', demographics)
+ self.assertIn('ethnicity', demographics)
+ self.assertIn('minorities', demographics)
+
if __name__ == '__main__':
unittest.main()
diff --git a/tests/test_xml_validator.py b/tests/test_xml_validator.py
new file mode 100644
index 0000000..c59a1e4
--- /dev/null
+++ b/tests/test_xml_validator.py
@@ -0,0 +1,29 @@
+import unittest
+from unittest.mock import patch
+import sys
+import os
+import importlib
+
+# Add the project root to the Python path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src')))
+
+# Import using importlib because of the dash in the filename
+xml_validator = importlib.import_module("xml-validator")
+
+class TestValidateAgainstXsd(unittest.TestCase):
+
+ def test_validate_against_xsd_exception(self):
+ # We can patch using getattr since the module has a dash
+ with patch.object(xml_validator.etree, 'parse') as mock_parse:
+ # Setup mock to raise an exception
+ mock_parse.side_effect = Exception("Test exception")
+
+ # Call the function
+ is_valid, errors = xml_validator.validate_against_xsd("dummy.xml", "dummy.xsd")
+
+ # Verify the exception was caught and returned correctly
+ self.assertFalse(is_valid)
+ self.assertEqual(errors, ["Validation error: Test exception"])
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/update_validation.py b/update_validation.py
new file mode 100644
index 0000000..97ac5c6
--- /dev/null
+++ b/update_validation.py
@@ -0,0 +1,150 @@
+import os
+
+with open("src/validation_report.py", "r") as f:
+ content = f.read()
+
+import re
+
+# We will rewrite the generate_html_report function
+# and add helper functions.
+
+new_methods = """
+ def _generate_html_header(self):
+ \"\"\"Generate the HTML header and styles.\"\"\"
+ return f\"\"\"
+
+
+ CSV to XML Conversion Validation Report
+
+
+
+ CSV to XML Conversion Validation Report
+ Generated on: {{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}}
+\"\"\"
+
+ def _generate_summary_section(self, summary):
+ \"\"\"Generate the summary section HTML.\"\"\"
+ return f\"\"\"
+
+
Summary
+
Total records processed: {{summary['total_records']}}
+
Successfully processed: {{summary['successful_records']}} ({{summary['success_rate']:.1f}}%)
+
Failed records: {{summary['failed_records']}}
+
Total errors: {{summary['error_count']}}
+
Total warnings: {{summary['warning_count']}}
+
{{title}}
+
+
+ | Category |
+ Count |
+
+\"\"\"
+ for category, count in sorted(categories.items(), key=lambda x: x[1], reverse=True):
+ html_content += f\"\"\"
+ | {{category}} |
+ {{count}} |
+
+\"\"\"
+ html_content += "
\\n"
+ return html_content
+
+ def _generate_issues_table(self):
+ \"\"\"Generate the detailed issues table.\"\"\"
+ if not self.issues:
+ return ""
+
+ html_content = \"\"\"
+ Detailed Issues
+
+
+ | Record ID |
+ Severity |
+ Category |
+ Field |
+ Message |
+
+\"\"\"
+
+ # Sort issues by severity (errors first) and then by record ID
+ sorted_issues = sorted(self.issues, key=lambda x: (0 if x['severity'] == 'error' else 1, x['record_id']))
+
+ for issue in sorted_issues:
+ severity_class = "error" if issue['severity'] == 'error' else "warning"
+ html_content += f\"\"\"
+ | {{issue['record_id']}} |
+ {{issue['severity'].upper()}} |
+ {{issue['category']}} |
+ {{issue['field_name']}} |
+ {{issue['message']}} |
+
+\"\"\"
+
+ html_content += "
\\n"
+ return html_content
+
+ def generate_html_report(self, output_dir="."):
+ \"\"\"
+ Generate an HTML report of validation issues.
+
+ Args:
+ output_dir: Directory to save the HTML report
+
+ Returns:
+ Path to the created HTML file
+ \"\"\"
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
+
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+ html_file = os.path.join(output_dir, f"validation_report_{timestamp}.html")
+
+ summary = self.get_summary()
+
+ # Assemble HTML content
+ html_content = self._generate_html_header()
+ html_content += self._generate_summary_section(summary)
+ html_content += self._generate_category_table("Errors by Category", summary['errors_by_category'])
+ html_content += self._generate_category_table("Warnings by Category", summary['warnings_by_category'])
+ html_content += self._generate_issues_table()
+
+ html_content += \"\"\"
+
+\"\"\"
+
+ # Write HTML content to file
+ with open(html_file, 'w') as f:
+ f.write(html_content)
+
+ return html_file
+"""
+
+start_str = ' def generate_html_report(self, output_dir="."):'
+end_str = '# Create a default validator instance'
+
+start_idx = content.find(start_str)
+end_idx = content.find(end_str)
+
+new_content = content[:start_idx] + new_methods + '\n' + content[end_idx:]
+
+with open("src/validation_report.py", "w") as f:
+ f.write(new_content)