diff --git a/django/parse_m2/initiate_post_parsing.py b/django/parse_m2/initiate_post_parsing.py index 9753215c..7613b130 100644 --- a/django/parse_m2/initiate_post_parsing.py +++ b/django/parse_m2/initiate_post_parsing.py @@ -20,8 +20,8 @@ def calculate_date_range(event: Metro2Event): def associate_previous_records(event: Metro2Event): logger = logging.getLogger('parse_m2.associate_previous_records') - logger.info("First, make sure all previous_values pointers are empty") - event.get_all_account_activity().update(previous_values_id=None) + # logger.info("First, make sure all previous_values pointers are empty") + # event.get_all_account_activity().update(previous_values_id=None) logger.info(f"Beginning to update all records for event: {event.id}") query_sql = """ diff --git a/django/parse_m2/m2_parser.py b/django/parse_m2/m2_parser.py index 0abf347c..bc363df4 100644 --- a/django/parse_m2/m2_parser.py +++ b/django/parse_m2/m2_parser.py @@ -22,9 +22,9 @@ class M2FileParser: # Parser version is saved on each file record. # Increment this version for all updates to parser functionality. - parser_version = "3.1" + parser_version = "3.1.2" - chunk_size = 2000 # TODO: determine a good number for this + chunk_size = 500 # Try a smaller chunk size any_non_whitespace = r'\S' activity_date = None diff --git a/django/parse_m2/parse_utils.py b/django/parse_m2/parse_utils.py index a58be80a..ae1151dc 100644 --- a/django/parse_m2/parse_utils.py +++ b/django/parse_m2/parse_utils.py @@ -117,7 +117,7 @@ def decode_if_needed(input: any) -> str: return input elif isinstance(input, bytes): try: - return input.decode('utf-8', errors='replace').replace('\x00', '\uFFFD') + return input.decode('utf-8', errors='replace') except (UnicodeDecodeError, AttributeError) as e: raise UnreadableLineException(f"Decode failed: {e}") from e else: