From 4fc2ef499302bdee583b5f2224e77d60b6bbbab3 Mon Sep 17 00:00:00 2001 From: Betsy Lorton Date: Mon, 8 Jun 2026 10:45:54 -0400 Subject: [PATCH 1/3] Try a smaller chunk size - for testing --- django/parse_m2/m2_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/parse_m2/m2_parser.py b/django/parse_m2/m2_parser.py index 0abf347c..96480649 100644 --- a/django/parse_m2/m2_parser.py +++ b/django/parse_m2/m2_parser.py @@ -22,9 +22,9 @@ class M2FileParser: # Parser version is saved on each file record. # Increment this version for all updates to parser functionality. - parser_version = "3.1" + parser_version = "3.1.1" - chunk_size = 2000 # TODO: determine a good number for this + chunk_size = 500 # Try a smaller chunk size any_non_whitespace = r'\S' activity_date = None From 97aa633cfb84529d3fe070e2e125910ef31106e1 Mon Sep 17 00:00:00 2001 From: Betsy Lorton Date: Wed, 17 Jun 2026 16:12:35 -0400 Subject: [PATCH 2/3] Skip 'replace' command, for testing --- django/parse_m2/m2_parser.py | 2 +- django/parse_m2/parse_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/django/parse_m2/m2_parser.py b/django/parse_m2/m2_parser.py index 96480649..bc363df4 100644 --- a/django/parse_m2/m2_parser.py +++ b/django/parse_m2/m2_parser.py @@ -22,7 +22,7 @@ class M2FileParser: # Parser version is saved on each file record. # Increment this version for all updates to parser functionality. - parser_version = "3.1.1" + parser_version = "3.1.2" chunk_size = 500 # Try a smaller chunk size any_non_whitespace = r'\S' diff --git a/django/parse_m2/parse_utils.py b/django/parse_m2/parse_utils.py index a58be80a..ae1151dc 100644 --- a/django/parse_m2/parse_utils.py +++ b/django/parse_m2/parse_utils.py @@ -117,7 +117,7 @@ def decode_if_needed(input: any) -> str: return input elif isinstance(input, bytes): try: - return input.decode('utf-8', errors='replace').replace('\x00', '\uFFFD') + return input.decode('utf-8', errors='replace') except (UnicodeDecodeError, AttributeError) as e: raise UnreadableLineException(f"Decode failed: {e}") from e else: From 7d56be0cf224ab5962eff45c0f1df423e02641b8 Mon Sep 17 00:00:00 2001 From: Betsy Lorton Date: Fri, 26 Jun 2026 10:10:59 -0400 Subject: [PATCH 3/3] Temp: skip steps that are already done on test event already calculated date range and made sure pointers are empty so skip them for the next attempt --- django/parse_m2/initiate_post_parsing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/parse_m2/initiate_post_parsing.py b/django/parse_m2/initiate_post_parsing.py index 9753215c..7613b130 100644 --- a/django/parse_m2/initiate_post_parsing.py +++ b/django/parse_m2/initiate_post_parsing.py @@ -20,8 +20,8 @@ def calculate_date_range(event: Metro2Event): def associate_previous_records(event: Metro2Event): logger = logging.getLogger('parse_m2.associate_previous_records') - logger.info("First, make sure all previous_values pointers are empty") - event.get_all_account_activity().update(previous_values_id=None) + # logger.info("First, make sure all previous_values pointers are empty") + # event.get_all_account_activity().update(previous_values_id=None) logger.info(f"Beginning to update all records for event: {event.id}") query_sql = """