diff --git a/code/unit_tests.sh b/code/unit_tests.sh index 19cd1abe..0b3b2bf0 100644 --- a/code/unit_tests.sh +++ b/code/unit_tests.sh @@ -1,4 +1,4 @@ pip install -U pip -pip install . +pip install -r requirements.txt python3 -m pytest tests/ \ No newline at end of file diff --git a/code/uqid_bug_correction.py b/code/uqid_bug_correction.py index fdc61f8b..52631caa 100644 --- a/code/uqid_bug_correction.py +++ b/code/uqid_bug_correction.py @@ -155,13 +155,11 @@ def post_correction_checks( def add_unique_constraint(): - constraint_query = text( - """ + constraint_query = text(""" ALTER TABLE question ADD CONSTRAINT uq_question_uqid UNIQUE (uqid, year) - """ - ) + """) with SQL_UTILS.engine.begin() as conn: conn.execute(constraint_query) @@ -175,15 +173,13 @@ def revert_or_drop_temp_table(revert: bool = False): for column in QUESTION.c: if not column.primary_key: update_string.append(f"{column.name} = EXCLUDED.{column.name}") - query = text( - f""" + query = text(f""" INSERT INTO question SELECT * FROM question_temp ON CONFLICT ON CONSTRAINT pk_question DO UPDATE SET {', '.join(update_string)} - """ - ) + """) queries.insert(0, query) # Remove unique constraint. diff --git a/documentation/PIR ERD.pptx b/documentation/PIR ERD.pptx index bd43a4f6..764bfd4d 100644 Binary files a/documentation/PIR ERD.pptx and b/documentation/PIR ERD.pptx differ diff --git a/documentation/csv/PIR Data Dictionary.csv b/documentation/csv/PIR Data Dictionary.csv index 3da26cec..117c83ed 100644 --- a/documentation/csv/PIR Data Dictionary.csv +++ b/documentation/csv/PIR Data Dictionary.csv @@ -1,11 +1,11 @@ table, column, column_proper, description response, uid, Unique Program ID,"Uniquely identifies each program within a year. Hash of grant_number, program_number, and program_type." response, question_id, Question ID,"Uniquely identifies questions within a year. Hash of question_number, and question_name." -response, year, Year, Year from which question is drawn. -response, answer, Answer, Response to present question. +response, year, Year,Year from which question is drawn. +response, answer, Answer,Response to present question. question, question_id, Question ID,"Uniquely identifies questions within a year. Hash of question_number, and question_name." -question, year, Year, Year from which question is drawn. -question, uqid, Unique Question ID, Uniquely identifies a question across years. +question, year, Year,Year from which question is drawn. +question, uqid, Unique Question ID,Uniquely identifies a question across years. question, category, Category, Category of question. question, question_name, Question Name, Name of question. question, question_number, Question Number, Number of question. @@ -33,8 +33,19 @@ program, program_zip1, Zip 5, Five-digit zip code. program, program_zip2, Zip 4, Additional four digits for nine-digit zip code. program, region, Region, Region in which the program is located. uqid_changelog, id, ID, Auto-incremented integer ID -uqid_changelog, question_id, Question ID, "Uniquely identifies questions within a year. Hash of question_number, and question_name." -uqid_changelog, original_uqid, Original Unique Question ID, "Original UQID, if any, of the target question" -uqid_changelog, new_uqid, New Unique Question ID, "New UQID, if any, of the target question" +uqid_changelog, question_id, Question ID, Uniquely identifies questions within a year. Hash of question_number and question_name. +uqid_changelog, original_uqid, Original Unique Question ID, Original UQID if any of the target question +uqid_changelog, new_uqid, New Unique Question ID, New UQID if any of the target question uqid_changelog, timestamp, Timestamp, Auto-generated timestamp uqid_changelog, complete_series_flag, Complete Series Flag, Indicator for whether this question should be marked as complete/confirmed +budget,region,Region,Region in which the program is located +budget,prg,Program, +budget,type,Type, +budget,program_state,Program State,State in which the program is located +budget,afm,Annual Funding Month, +budget,grant_number,Grant Number,Grant number. +budget,program_name,Program Name,Program name. +budget,year,Year,Fiscal year. +budget,quarter,Quarter,Fiscal quarter. +budget,metric,Metric,Reported metric. +budget,value,Value,Value of reported metric. diff --git a/documentation/csv/budget.csv b/documentation/csv/budget.csv new file mode 100644 index 00000000..f5ec99bd --- /dev/null +++ b/documentation/csv/budget.csv @@ -0,0 +1,12 @@ +field," type" +region,int +prg,varchar(2) +type,varchar(10) +program_state,varchar(10) +afm,varchar(3) +grant_number,varchar(20) +program_name,varchar(255) +year,int +quarter,int +metric,varchar(50) +value,numeric diff --git a/documentation/csv/program.csv b/documentation/csv/program.csv index d3e1cd0a..1f25d54b 100644 --- a/documentation/csv/program.csv +++ b/documentation/csv/program.csv @@ -1,19 +1,19 @@ -field, type -uid, varchar(255) -year, year -grantee_name, varchar(255) -grant_number, varchar(255) -program_address_line_1, varchar(255) -program_address_line_2, varchar(255) -program_agency_description, varchar(255) -program_agency_type, varchar(255) -program_city, varchar(255) -program_email, varchar(255) -program_name, varchar(255) -program_number, varchar(255) -program_phone, varchar(255) -program_type, varchar(255) -program_state, varchar(255) -program_zip1, varchar(255) -program_zip2, varchar(255) -region, int +field," type" +uid," varchar(255)" +year,int +grantee_name," varchar(255)" +grant_number," varchar(255)" +program_address_line_1," varchar(255)" +program_address_line_2," varchar(255)" +program_agency_description," varchar(255)" +program_agency_type," varchar(255)" +program_city," varchar(255)" +program_email," varchar(255)" +program_name," varchar(255)" +program_number," varchar(255)" +program_phone," varchar(255)" +program_type," varchar(255)" +program_state," varchar(255)" +program_zip1," varchar(255)" +program_zip2," varchar(255)" +region," int" diff --git a/documentation/csv/question.csv b/documentation/csv/question.csv index 2678536e..e162f831 100644 --- a/documentation/csv/question.csv +++ b/documentation/csv/question.csv @@ -1,12 +1,12 @@ -field, type -question_id, varchar(255) -year, year -uqid, varchar(255) -category, varchar(255) -question_name, text -question_number, varchar(255) -question_order, float -question_text, text -question_type, varchar(255) -section, varchar(255) -subsection, varchar(255) +field," type" +question_id," varchar(255)" +year,int +uqid," varchar(255)" +category," varchar(255)" +question_name," text" +question_number," varchar(255)" +question_order,numeric +question_text," text" +question_type," varchar(255)" +section," varchar(255)" +subsection," varchar(255)" diff --git a/documentation/csv/response.csv b/documentation/csv/response.csv index aec1b66c..dfd3a010 100644 --- a/documentation/csv/response.csv +++ b/documentation/csv/response.csv @@ -1,5 +1,5 @@ -field, type -uid, varchar(255) -question_id, varchar(255) -year, year -answer, text +field," type" +uid," varchar(255)" +question_id," varchar(255)" +year,int +answer," text" diff --git a/documentation/source/database.rst b/documentation/source/database.rst index 6277f8fe..2c1e6731 100644 --- a/documentation/source/database.rst +++ b/documentation/source/database.rst @@ -10,6 +10,13 @@ found in `workflow `__. Tables ------ +Budget +~~~~~~ + +.. csv-table:: + :file: ../csv/budget.csv + :header-rows: 1 + Question ~~~~~~~~ diff --git a/requirements.txt b/requirements.txt index c5bac055..21602cf7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ openpyxl==3.1.5 outcome==1.3.0.post0 packaging==25.0 pandas==2.2.3 --e git+https://github.com/HHS/ACF-pir-data.git@101d4d438bdd301fd96fd88c4c4580ff902b9705#egg=pir_pipeline +-e git+https://github.com/HHS/ACF-pir-data.git@0104f01060b03cdcbef74e0f8008782c8059cd32#egg=pir_pipeline pluggy==1.5.0 psycopg==3.2.7 psycopg-binary==3.2.7 @@ -66,6 +66,7 @@ typing-inspection==0.4.0 typing_extensions==4.13.2 tzdata==2025.2 urllib3==2.4.0 +waitress==3.0.2 websocket-client==1.8.0 Werkzeug==3.1.3 wsproto==1.2.0 diff --git a/src/pir_pipeline/utils/SQLAlchemyUtils.py b/src/pir_pipeline/utils/SQLAlchemyUtils.py index 301cebe0..448473f2 100644 --- a/src/pir_pipeline/utils/SQLAlchemyUtils.py +++ b/src/pir_pipeline/utils/SQLAlchemyUtils.py @@ -165,13 +165,11 @@ def get_columns(self, table: str, where: str = "") -> list[str]: elif self._dialect == "postgresql": table_schema = "table_catalog" - query = text( - f""" + query = text(f""" SELECT column_name FROM information_schema.columns WHERE table_name = :table AND {table_schema} = :schema {where} - """ - ) + """) with self._engine.connect() as conn: result = conn.execute( query, {"table": table, "schema": self._database, "where": where} diff --git a/tests/dashboard/test_review_ui.py b/tests/dashboard/test_review_ui.py index 5634b049..7c37e681 100644 --- a/tests/dashboard/test_review_ui.py +++ b/tests/dashboard/test_review_ui.py @@ -50,7 +50,7 @@ def count_modal_rows(table: str): By.CSS_SELECTOR, "tr#flashcard-matches-table-tr-10000 td[name='question_id']", ).get_attribute("textContent") - == "0e93c25d3a95604f40d3a64e2298093b4faed6f2" + == "d27e8217ba30000a78e5d92ea54f4d9a2e69cb54" ) # Click the storeLink button in the first row diff --git a/tests/dashboard/test_search.py b/tests/dashboard/test_search.py index a89cd5cc..371d48ef 100644 --- a/tests/dashboard/test_search.py +++ b/tests/dashboard/test_search.py @@ -75,4 +75,4 @@ def test_post_data(self, client): if __name__ == "__main__": - pytest.main([__file__, "-sk", "test_post_data"]) + pytest.main([__file__, "-s", "-vv", "-k", "test_post_data"]) diff --git a/tests/dashboard/test_search_ui.py b/tests/dashboard/test_search_ui.py index dbe7474a..acbb13cf 100644 --- a/tests/dashboard/test_search_ui.py +++ b/tests/dashboard/test_search_ui.py @@ -1,4 +1,5 @@ import os +import time import pytest from selenium.common.exceptions import TimeoutException @@ -65,8 +66,8 @@ def count_modal_rows(table: str): edit_button = wait.until( EC.element_to_be_clickable( ( - By.XPATH, - '//table[@id="search-results-table"]//button[@onclick="getFlashcardData(event)"]', + By.CSS_SELECTOR, + "tr[id='search-results-table-tr-10002'] button[onclick='getFlashcardData(event)']", ) ) ) @@ -164,4 +165,3 @@ def count_modal_rows(table: str): if __name__ == "__main__": pytest.main([__file__, "-sk", "test_search_ui"]) - pytest.main([__file__, "-sk", "test_search_ui"])