From d277df6dd2235ddc921712d05a2890d2cbc6f46e Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Tue, 10 Mar 2026 12:58:53 -0400 Subject: [PATCH 01/10] Ensure no escaping is done on writing/reading by default --- src/AmiUtilityService.php | 80 ++++++++++++------- .../Action/AmiStrawberryfieldCSVexport.php | 2 +- src/Plugin/QueueWorker/LoDQueueWorker.php | 4 +- 3 files changed, 52 insertions(+), 34 deletions(-) diff --git a/src/AmiUtilityService.php b/src/AmiUtilityService.php index b22345d..9ee803f 100644 --- a/src/AmiUtilityService.php +++ b/src/AmiUtilityService.php @@ -1088,7 +1088,6 @@ public function csv_touch(string $filename = NULL, ?string $subpath = NULL, bool return $file->id(); } - /** * Creates an CSV from array and returns file. * @@ -1102,11 +1101,14 @@ public function csv_touch(string $filename = NULL, ?string $subpath = NULL, bool * @param boolean $auto_uuid * Defines if we are going to generate UUIDs when not valid/not present * Or leave the $uuid_key field as it is and let this fail/if later. + * @param bool $permanent + * @param bool $escape_character + * If used, CSV might not end being RFC 4180 compliant. + * @param string $logger_channel * * @return int|string|null - * @throws \Drupal\Core\Entity\EntityStorageException */ - public function csv_save(array $data, $uuid_key = 'node_uuid', $auto_uuid = TRUE, $permanent = TRUE, $logger_channel = 'ami') { + public function csv_save(array $data, $uuid_key = 'node_uuid', $auto_uuid = TRUE, $permanent = TRUE, bool $escape_character = FALSE, $logger_channel = 'ami') { //$temporary_directory = $this->fileSystem->getTempDirectory(); // We should be allowing downloads for this from temp @@ -1160,9 +1162,12 @@ public function csv_save(array $data, $uuid_key = 'node_uuid', $auto_uuid = TRUE if ($haskey === FALSE) { array_unshift($data['headers'], $uuid_key); } - - $fh->fputcsv($data['headers']); - + if ($escape_character) { + $fh->fputcsv($data['headers'], separator: ',', enclosure: '"', escape: "\\"); + } + else { + $fh->fputcsv($data['headers'], separator: ',', enclosure: '"', escape: ""); + } foreach ($data['data'] as $row) { if ($haskey === FALSE) { array_unshift($row, $uuid_key); @@ -1185,8 +1190,13 @@ public function csv_save(array $data, $uuid_key = 'node_uuid', $auto_uuid = TRUE } } } + if ($escape_character) { + $fh->fputcsv($row, separator: ',', enclosure: '"', escape: "\\"); + } + else { + $fh->fputcsv($row, separator: ',', enclosure: '"', escape: ""); + } - $fh->fputcsv($row); } // PHP Bug! This should happen automatically clearstatcache(TRUE, $url); @@ -1213,7 +1223,6 @@ public function csv_save(array $data, $uuid_key = 'node_uuid', $auto_uuid = TRUE return $file->id(); } - /** * Appends CSV from array and returns file. * @@ -1224,20 +1233,19 @@ public function csv_save(array $data, $uuid_key = 'node_uuid', $auto_uuid = TRUE * * @param \Drupal\file\Entity\File $file * - * @param string|null $uuid_key + * @param string $uuid_key * IF NULL then no attempt of using UUIDS will be made. * Needed for LoD Reconciling CSVs * @param bool $append_header * - * @param bool $escape_characters - * Defaults to internal PHP mechanism for escaping characters (a "/") - * Set to FALSE if you are passing JSON encoded strings into cells. - * NOTE: Make sure you also disable it IF reading back from files generated through this + * @param bool $escape_character + * If used, CSV might not end being RFC 4180 compliant. + * @param bool $auto_uuid * * @return int|string|null * @throws \Drupal\Core\Entity\EntityStorageException */ - public function csv_append(array $data, File $file, $uuid_key = 'node_uuid', bool $append_header = TRUE, $escape_characters = TRUE, $auto_uuid = TRUE) { + public function csv_append(array $data, File $file, $uuid_key = 'node_uuid', bool $append_header = TRUE, bool $escape_character = TRUE, $auto_uuid = TRUE) { $wrapper = $this->streamWrapperManager->getViaUri($file->getFileUri()); if (!$wrapper) { @@ -1259,7 +1267,12 @@ public function csv_append(array $data, File $file, $uuid_key = 'node_uuid', boo } } if ($append_header) { - $fh->fputcsv($data['headers']); + if ($escape_character) { + $fh->fputcsv($data['headers'], separator: ',', enclosure: '"', escape: "\\"); + } + else { + $fh->fputcsv($data['headers'], separator: ',', enclosure: '"', escape: ""); + } } foreach ($data['data'] as $row) { @@ -1282,11 +1295,11 @@ public function csv_append(array $data, File $file, $uuid_key = 'node_uuid', boo } } } - if ($escape_characters) { - $fh->fputcsv($row); + if ($escape_character) { + $fh->fputcsv($row, separator: ',', enclosure: '"', escape: "\\"); } else { - $fh->fputcsv($row, ',', '"', ""); + $fh->fputcsv($row, separator: ',', enclosure: '"', escape: ""); } } // PHP Bug! This should happen automatically @@ -1310,32 +1323,32 @@ public function csv_append(array $data, File $file, $uuid_key = 'node_uuid', boo * @param bool $always_include_header * Always return header even with an offset. * - * @param bool $escape_characters - * + * @param bool $escape_character + * If used, CSV might not end being RFC 4180 compliant. * @return array|null * Returning array will be in this form: * 'headers' => $rowHeaders_utf8 or [] if $always_include_header == FALSE * 'data' => $table, * 'totalrows' => $maxRow, */ - public function csv_read(File $file, int $offset = 0, int $count = 0, bool $always_include_header = TRUE, bool $escape_characters = TRUE) { + public function csv_read(File $file, int $offset = 0, int $count = 0, bool $always_include_header = TRUE, bool $escape_character = FALSE) { // 1.6.0: wrapper around this function now that we moved it to strawberryfield so webform module does not depend on AMI for CSV reading // This was needed bc if not we would have an undeclared circular dependency/or would have to change all the code (many parts) // there this service was used to call csv_read(). Only thing new is that the logging/caller_module is used for logging now. - return $this->strawberryfieldUtility->csv_read($file, $offset, $count, $always_include_header, $escape_characters, 'ami'); + return $this->strawberryfieldUtility->csv_read($file, $offset, $count, $always_include_header, $escape_character, 'ami'); } - /** * Removes columns from an existing CSV. * * @param \Drupal\file\Entity\File $file * @param array $headerwithdata - * + * @param bool $escape_character + * If used, CSV might not end being RFC 4180 compliant. * @return int|mixed|string|null * @throws \Drupal\Core\Entity\EntityStorageException */ - public function csv_clean(File $file, array $headerwithdata = []) { + public function csv_clean(File $file, array $headerwithdata = [], bool $escape_character = FALSE) { $wrapper = $this->streamWrapperManager->getViaUri($file->getFileUri()); if (!$wrapper) { return NULL; @@ -1382,7 +1395,12 @@ public function csv_clean(File $file, array $headerwithdata = []) { unset($data[$key]); } $data = array_values($data); - $spltmp->fputcsv($data); + if ($escape_character) { + $spltmp->fputcsv($data, separator: ',', enclosure: '"', escape: "\\"); + } + else { + $spltmp->fputcsv($data, separator: ',', enclosure: '"', escape: ""); + } $i++; } $size = $spltmp->getSize(); @@ -1399,11 +1417,11 @@ public function csv_clean(File $file, array $headerwithdata = []) { /** * @param \Drupal\file\Entity\File $file * - * @param bool $escape_characters - * + * @param bool $escape_character + * If used, CSV might not end being RFC 4180 compliant. * @return int */ - public function csv_count(File $file, $escape_characters = TRUE) { + public function csv_count(File $file, $escape_character = FALSE) { $wrapper = $this->streamWrapperManager->getViaUri($file->getFileUri()); if (!$wrapper) { return NULL; @@ -1419,11 +1437,11 @@ public function csv_count(File $file, $escape_characters = TRUE) { SplFileObject::DROP_NEW_LINE ); while (!$spl->eof()) { - if (!$escape_characters) { + if (!$escape_character) { $spl->fgetcsv( ',', '"', ""); } else { - $spl->fgetcsv(); + $spl->fgetcsv( ',', '"', "\\"); } $key = $spl->key(); } diff --git a/src/Plugin/Action/AmiStrawberryfieldCSVexport.php b/src/Plugin/Action/AmiStrawberryfieldCSVexport.php index 0d565eb..55b8392 100644 --- a/src/Plugin/Action/AmiStrawberryfieldCSVexport.php +++ b/src/Plugin/Action/AmiStrawberryfieldCSVexport.php @@ -373,7 +373,7 @@ protected function sendToFile($output) { $ami_set = TRUE; } $logger_channel = (string) $this->context['sandbox']['logger_channel'] ?? 'ami'; - $file_id = $this->AmiUtilityService->csv_save($data, 'node_uuid', TRUE, $ami_set, $logger_channel); + $file_id = $this->AmiUtilityService->csv_save($data, 'node_uuid', TRUE, $ami_set, FALSE, $logger_channel); if ($file_id && $this->configuration['create_ami_set'] && $this->context['sandbox']['ado_type_exists']) { $amisetdata = new \stdClass(); $amisetdata->plugin = 'spreadsheet'; diff --git a/src/Plugin/QueueWorker/LoDQueueWorker.php b/src/Plugin/QueueWorker/LoDQueueWorker.php index aa7a9ef..4f92e8a 100644 --- a/src/Plugin/QueueWorker/LoDQueueWorker.php +++ b/src/Plugin/QueueWorker/LoDQueueWorker.php @@ -202,9 +202,9 @@ public function processItem($data) { } } - $newdata['data'][0][$lod_route_column_name] = json_encode($lod, JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES|JSON_UNESCAPED_UNICODE) ?? ''; + $newdata['data'][0][$lod_route_column_name] = json_encode($lod, JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES|JSON_UNESCAPED_UNICODE|JSON_HEX_QUOT) ?? ''; $newdata['data'][0]['original'] = (string) $data->info['label']; - $newdata['data'][0]['csv_columns'] = json_encode((array)$data->info['csv_columns']) ?? ''; + $newdata['data'][0]['csv_columns'] = json_encode((array)$data->info['csv_columns'], JSON_HEX_QUOT) ?? ''; // Adds a "Checked" column used to mark manually reconciliated elements. $newdata['data'][0]['checked'] = FALSE; // Context data is simpler From f136afb46bf9980eca3b8fd2f91d7efbdcc12327 Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Tue, 10 Mar 2026 14:25:12 -0400 Subject: [PATCH 02/10] Fixes missing argument for Error message when AMI set preview is called with header/row that does not exist --- src/Controller/AmiRowAutocompleteHandler.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Controller/AmiRowAutocompleteHandler.php b/src/Controller/AmiRowAutocompleteHandler.php index c3066da..df328a2 100644 --- a/src/Controller/AmiRowAutocompleteHandler.php +++ b/src/Controller/AmiRowAutocompleteHandler.php @@ -463,7 +463,7 @@ public static function ajaxPreviewAmiSet($form, FormStateInterface $form_state) else { $message = !$file ? 'The AMI set has no CSV File. The AMI set is empty.': 'The AMI set has no data for chosen row. The AMI set is empty.'; if (!empty($message)) { - $preview_error = MetadataDisplayForm::buildAjaxPreviewError($message); + $preview_error = MetadataDisplayForm::buildAjaxPreviewError($message, TRUE); $output['preview_error'] = $preview_error; } $response->addCommand(new OpenOffCanvasDialogCommand(t('Preview'), $output, ['width' => '50%'])); From beeaddc1d344736c0e033941e2e18d99ae546dbb Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Tue, 17 Mar 2026 12:06:55 -0400 Subject: [PATCH 03/10] Ensures that existing moderation status also permeates into update ops --- src/Plugin/QueueWorker/IngestADOQueueWorker.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Plugin/QueueWorker/IngestADOQueueWorker.php b/src/Plugin/QueueWorker/IngestADOQueueWorker.php index bf34e23..88d6729 100644 --- a/src/Plugin/QueueWorker/IngestADOQueueWorker.php +++ b/src/Plugin/QueueWorker/IngestADOQueueWorker.php @@ -952,6 +952,7 @@ private function persistEntity(\stdClass $data, array $processed_metadata) { // Ignore status for updates if status_keep == TRUE. if ($status && is_string($status) && $status_keep == FALSE) { $node->set('moderation_state', $status); + $nodeValues['moderation_state'] = $status; $status = 0; } /** @var \Drupal\strawberryfield\Field\StrawberryFieldItemList $field */ From aa65d9675231317b1592394aaae929cd89ebc2a5 Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Thu, 19 Mar 2026 15:55:48 -0400 Subject: [PATCH 04/10] For LoD Reconciliation. Ensure we do not attempt to extract either delimited strings OR JSON , but both. Processing might be slower, but more precise @alliomeria as explained in slack, PHP decodes some primitives as JSON. With this new approach, when we check for Strings we discard ANY JSON that would end in an ARRAY (when decoded) and instead treat them as strings. Then we re-parse the same data (more time processing but once does not do LoD of a complete AMI set in realtime over and over, so that is Ok) as JSON. This also allows Mixed case CSVs, where some rows are simple strings to be processed by One Metadata Display, but other rows proper JSON, and still have LoD reconciliation --- src/AmiUtilityService.php | 31 ++++++++++++++------ src/Controller/AmiRowAutocompleteHandler.php | 8 +++-- src/Form/amiSetEntityReconcileForm.php | 2 +- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/AmiUtilityService.php b/src/AmiUtilityService.php index 9ee803f..e779eb6 100644 --- a/src/AmiUtilityService.php +++ b/src/AmiUtilityService.php @@ -1554,15 +1554,17 @@ public function provideDifferentColumnValuesFromCSV(File $file, array $columns): $data = $this->csv_read($file); $column_keys = $data['headers'] ?? []; $alldifferent = []; + $alldifferent_json = []; foreach ($columns as $column) { $column_index = array_search($column, $column_keys); if ($column_index !== FALSE) { + // New for 1.7.0/2.1.0 We process both. Strings and JSON. More expensive + // but also more precise. $alldifferent[$column] = $this->getDifferentValuesfromColumnSplit($data, $column_index); - if (empty($alldifferent[$column])) { - $alldifferent[$column] = $this->getDifferentValuesfromColumnJSON($data, + $alldifferent_json[$column] = $this->getDifferentValuesfromColumnJSON($data, $column_index); - } + $alldifferent[$column] = array_unique(array_merge($alldifferent[$column], $alldifferent_json[$column])); } } return $alldifferent; @@ -2619,10 +2621,12 @@ public function processMetadataDisplay(\stdClass $data, array $additional_contex $data_to_clean['data'][0] = [$context['data'][$source_column]]; $labels = $this->getDifferentValuesfromColumnSplit($data_to_clean, 0); - if (empty($labels)) { - $labels = $this->getDifferentValuesfromColumnJSON($data_to_clean, + // New for 1.7.0/2.1.0 We process both. Strings and JSON. More expensive + // but also more precise. The Preview does the same now + $labels_json= $this->getDifferentValuesfromColumnJSON($data_to_clean, 0); - } + // WE merge both results and make them unique + $labels = array_unique(array_merge($labels, $labels_json)); foreach ($labels as $label) { $lod_for_label = $this->AmiLoDService->getKeyValuePerAmiSet($label, $set_id); if (is_array($lod_for_label) && count($lod_for_label) > 0) { @@ -2886,15 +2890,24 @@ public function getDifferentValuesfromColumnJSON(array $data, int $key, array $v /** - * Checks if a string is valid JSON + * Checks if a string is valid RFC JSON (object or array) + * Skips if its a valid JSON-y-fable native, like a pure string + * or a number * * @param $string * * @return bool */ public function isJson($string) { - json_decode($string); - return json_last_error() === JSON_ERROR_NONE; + try { + $decoded = json_decode($string, TRUE, 512,JSON_THROW_ON_ERROR); + if (is_array($decoded) ) { + return TRUE; + } + } + catch (\Throwable $e) { + return FALSE; + } } /** diff --git a/src/Controller/AmiRowAutocompleteHandler.php b/src/Controller/AmiRowAutocompleteHandler.php index df328a2..fb08a08 100644 --- a/src/Controller/AmiRowAutocompleteHandler.php +++ b/src/Controller/AmiRowAutocompleteHandler.php @@ -213,10 +213,12 @@ public static function ajaxPreviewAmiSet($form, FormStateInterface $form_state) $labels = \Drupal::service('ami.utility') ->getDifferentValuesfromColumnSplit($data_to_clean, 0); - if (empty($labels)) { - $labels = \Drupal::service('ami.utility')->getDifferentValuesfromColumnJSON($data_to_clean, + // New for 1.7.0/2.1.0 We process both. Strings and JSON. More expensive + // but also more precise. + $labels_from_json = \Drupal::service('ami.utility')->getDifferentValuesfromColumnJSON($data_to_clean, 0); - } + $labels = array_unique(array_merge($labels, $labels_from_json)); + foreach ($labels as $label) { $lod_for_label = \Drupal::service('ami.lod') ->getKeyValuePerAmiSet($label, $id); diff --git a/src/Form/amiSetEntityReconcileForm.php b/src/Form/amiSetEntityReconcileForm.php index 074e6f0..f917246 100644 --- a/src/Form/amiSetEntityReconcileForm.php +++ b/src/Form/amiSetEntityReconcileForm.php @@ -258,7 +258,7 @@ public function buildForm(array $form, FormStateInterface $form_state) { '#title' => $this->t('Choose a Column to Preview'), '#options' => array_combine($source_options, $source_options), '#default_value' => $form_state->getValue(['lod_options','select_preview']), - '#description' => $this->t('We will attempt to fetch first cells holding a string of delimited values (by "|@|" or ";"). If no luck, and the selected column cell\'s holds a valid JSON, any simple lists of values (e.g ["pup","dog","canine"], and/or any property where the JSON key name contains one of the following strings: "label, value, name". Any URL/URN or URI will be not taken in account'), + '#description' => $this->t('We will attempt to fetch first cells holding a string of delimited values (by "|@|" or ";"). Additionally, if any selected column cell\'s holds a valid JSON, e.g. any simple lists of values (e.g ["pup","dog","canine"], and/or an object with any property where the JSON key name contains one of the following strings: "label, value, type, name". URL/URN or URI will be not taken in account.'), ]; $form['lod_options']['preview'] = [ '#type' => 'button', From 01e211e7ed83d6da8d0b3c52e43c248b7a3c4a3a Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Thu, 19 Mar 2026 16:10:54 -0400 Subject: [PATCH 05/10] Remove unused methods (duplicates, live already in AMIUtilityService) and ensure RETURN FALSE if not ARRAY --- src/AmiLoDService.php | 23 ----------------------- src/AmiUtilityService.php | 5 ++++- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/src/AmiLoDService.php b/src/AmiLoDService.php index 43509cd..3602eee 100644 --- a/src/AmiLoDService.php +++ b/src/AmiLoDService.php @@ -571,29 +571,6 @@ public function invokeCustomLoD(string $query, string $lod_custom_lod_id):array return $results_processed; } - - /** - * Checks if a string is valid JSON - * - * @param $string - * - * @return bool - */ - public function isJson($string) { - json_decode($string); - return json_last_error() === JSON_ERROR_NONE; - } - - /** - * Helper function that negates ::isJson. - * @param $string - * - * @return bool - */ - public function isNotJson($string) { - return !$this->isJson($string); - } - public function getCustomLoDEndpoints($as_arguments = FALSE) { $active_plugins = []; /* @var $plugin_config_entities \Drupal\webform_strawberryfield\Entity\LoDendpointEntity[] */ diff --git a/src/AmiUtilityService.php b/src/AmiUtilityService.php index e779eb6..99e984f 100644 --- a/src/AmiUtilityService.php +++ b/src/AmiUtilityService.php @@ -2900,10 +2900,13 @@ public function getDifferentValuesfromColumnJSON(array $data, int $key, array $v */ public function isJson($string) { try { - $decoded = json_decode($string, TRUE, 512,JSON_THROW_ON_ERROR); + $decoded = json_decode($string, TRUE, 512, JSON_THROW_ON_ERROR); if (is_array($decoded) ) { return TRUE; } + else { + return FALSE; + } } catch (\Throwable $e) { return FALSE; From 4e78ac12fc66c1d164845b4b1c12f79b6d93f22a Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Fri, 27 Mar 2026 15:19:55 -0400 Subject: [PATCH 06/10] CSV escaping (non RFC) checking helper function Will be used to pre-check on an Hook Update existing AMIs that have changed date prior to the request time (run time) of the hook Won't commit yet the hook bc the post/update logic is still missing (new tab at an AMI set + a warning letting the user know the attached CSV needs to be fixed (automatically or manually). This function could also be used on "replace/insert/add CSV" to any AMI set --- src/AmiUtilityService.php | 89 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 4 deletions(-) diff --git a/src/AmiUtilityService.php b/src/AmiUtilityService.php index 99e984f..898c9f5 100644 --- a/src/AmiUtilityService.php +++ b/src/AmiUtilityService.php @@ -661,9 +661,9 @@ public function retrieve_remote_file( * @param \Drupal\file\Entity\File $zip_file * A Zip file with that may contain the $uri * - * @return mixed + * @return false|string * One of these possibilities: - * - If it succeeds an managed file object + * - If it succeeds a Path to a managed file object * - If it fails or NULL, FALSE. */ public function retrieve_fromzip_file($uri, $destination = NULL, $replace = FileExists::Rename, File $zip_file = NULL) { @@ -1563,7 +1563,7 @@ public function provideDifferentColumnValuesFromCSV(File $file, array $columns): $alldifferent[$column] = $this->getDifferentValuesfromColumnSplit($data, $column_index); $alldifferent_json[$column] = $this->getDifferentValuesfromColumnJSON($data, - $column_index); + $column_index); $alldifferent[$column] = array_unique(array_merge($alldifferent[$column], $alldifferent_json[$column])); } } @@ -2624,7 +2624,7 @@ public function processMetadataDisplay(\stdClass $data, array $additional_contex // New for 1.7.0/2.1.0 We process both. Strings and JSON. More expensive // but also more precise. The Preview does the same now $labels_json= $this->getDifferentValuesfromColumnJSON($data_to_clean, - 0); + 0); // WE merge both results and make them unique $labels = array_unique(array_merge($labels, $labels_json)); foreach ($labels as $label) { @@ -2998,4 +2998,85 @@ public static function invalidateAmiSetDeleteAdosAccessCache(EntityInterface $en \Drupal::cache()->invalidate($cache_id); } + /** + * Checks if a CSV has escaped special characters + * + * Old PHP defaults used "\" as escaping mechanis + * Which can break amongst other JSON encoded ROWS + * Starting with AMI 1.1.0 and 2.1.0 we read/write CSV unescaped + * following RFC 4180 to ensure a safe round trip and also + * Excel and Google Sheets editing and export compatibility + * + * @param \Drupal\file\Entity\File $file + * + * @return bool + * TRUE means it has escaping or some other sanity issue + * FALSE means all is good + */ + public function csv_check_escaped(File $file): bool { + $needs_review = FALSE; + $wrapper = $this->streamWrapperManager->getViaUri($file->getFileUri()); + if (!$wrapper) { + return FALSE; + } + $url = $wrapper->getUri(); + $fh = new \SplFileObject($url, 'r'); + if (!$fh) { + $this->messenger()->addError( + $this->t('Error reading the CSV file!.') + ); + return FALSE; + } + // Instead or using PHP's CSV read line, we will get the complete lines first + // Then decode and apply a temporary fix to output unescaped. + // We compare the original read with the unescaped generation using md5 + // ,and we also compare that each ROW has exactly the same columns + // as the header. + // In this case, since we are only checking, we bail out on the first encountered + // abnormality. Of course if all is OK we have to still iterate over all of them + $i = 0; + $header_count = FALSE; + while (!$fh->eof()) { + if ($i == 0) { + // Read the header unescaped. We do not support headers with double quotes + $header = $fh->fgetcsv(',', '"', ""); + $header_count = is_array($header) ? count($header) : FALSE; + $needs_review = !($header_count); + $i++; + } + elseif ($header_count) { + $row_string = $fh->fgets(); + if (!empty($row_string)) { + $row_unescaped = str_getcsv($row_string, ',', '"', ""); + // Try replacing \" with \"" + // but only if: + // - not before a comma. + // - 2 x double quote. + // - or a double quote followed by a space. + $pattern = '}\\\\"(?!(,|""|"\s))}'; + $replacement = '\""'; + $row_string_replaced = preg_replace($pattern, $replacement, $row_string); + $row_unescaped_after_replace = str_getcsv($row_string_replaced, ',', '"', ""); + if (count($row_unescaped_after_replace) != $header_count) { + $needs_review = TRUE; + break; + } + if (md5(implode(";", $row_unescaped)) != md5(implode(";", $row_unescaped_after_replace))) { + $needs_review = TRUE; + break; + } + else { + $needs_review = FALSE; + } + } + } + else { + // $header_count is FALSE, means the CSV is malformed. + $needs_review = TRUE; + } + } + // Closes the SPL File Object. + $fh = NULL; + return $needs_review; + } } From 816d5e0d999a0f01255ee0102168c3ec98c4d9b3 Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Fri, 27 Mar 2026 15:20:31 -0400 Subject: [PATCH 07/10] Adds new status amiSetEntity::STATUS_NEEDS_REVIEW to mark CSV data that was detected as "escaped" and needs to be fixed --- src/Entity/amiSetEntity.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Entity/amiSetEntity.php b/src/Entity/amiSetEntity.php index f2bfff8..15c852c 100644 --- a/src/Entity/amiSetEntity.php +++ b/src/Entity/amiSetEntity.php @@ -147,6 +147,7 @@ class amiSetEntity extends ContentEntityBase implements amiSetEntityInterface { public const STATUS_READY = 'READY'; + public const STATUS_NEEDS_REVIEW = 'NEEDS_REVIEW'; public const STATUS_NOT_READY = 'NOT_READY'; public const STATUS_PROCESSING = 'PROCESSING'; public const STATUS_PROCESSED = 'PROCESSED'; @@ -166,6 +167,7 @@ class amiSetEntity extends ContentEntityBase implements amiSetEntityInterface { amiSetEntity::STATUS_PROCESSED_WITH_ERRORS => 'Processed with errors', amiSetEntity::STATUS_FAILED => 'Failed', amiSetEntity::STATUS_ENTITIES_DELETED => 'ADOs Deleted', + amiSetEntity::STATUS_NEEDS_REVIEW => 'CSV associated data needs review' ]; /** From 650161e20af04578b75f83d73a85bca198ed6c29 Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Fri, 27 Mar 2026 15:51:25 -0400 Subject: [PATCH 08/10] Cleanup. Remove unused method, fix broken HTML (double escaped) when setting the status on CSV export --- .../Action/AmiStrawberryfieldCSVexport.php | 26 +++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/src/Plugin/Action/AmiStrawberryfieldCSVexport.php b/src/Plugin/Action/AmiStrawberryfieldCSVexport.php index 55b8392..d3507f5 100644 --- a/src/Plugin/Action/AmiStrawberryfieldCSVexport.php +++ b/src/Plugin/Action/AmiStrawberryfieldCSVexport.php @@ -372,7 +372,10 @@ protected function sendToFile($output) { if ($this->configuration['create_ami_set'] && $this->context['sandbox']['ado_type_exists']) { $ami_set = TRUE; } - $logger_channel = (string) $this->context['sandbox']['logger_channel'] ?? 'ami'; + $logger_channel = 'ami'; + if (isset($this->context['sandbox']['logger_channel']) && !empty($this->context['sandbox']['logger_channel'])) { + $logger_channel = (string) $this->context['sandbox']['logger_channel']; + } $file_id = $this->AmiUtilityService->csv_save($data, 'node_uuid', TRUE, $ami_set, FALSE, $logger_channel); if ($file_id && $this->configuration['create_ami_set'] && $this->context['sandbox']['ado_type_exists']) { $amisetdata = new \stdClass(); @@ -408,15 +411,15 @@ protected function sendToFile($output) { if ($amiset_id) { $url = Url::fromRoute('entity.ami_set_entity.canonical', ['ami_set_entity' => $amiset_id]); - $message = $this->t('Well Done! New AMI Set was created and you can see it here', - ['@url' => $url->toString()]); + $message = $this->t('Well Done! New AMI Set was created and you can see it here', + [':url' => $url->toString()]); $this->messenger() ->addStatus($message); } return $message; } else if ($this->configuration['create_ami_set'] && !$this->context['sandbox']['ado_type_exists']) { - $message = $this->t('AMI Set could not be created because object(s) are missing the type key.'); + $message = $this->t('AMI Set could not be created because object(s) are missing the "type" key.'); $this->messenger() ->addStatus($message); return $message; @@ -578,21 +581,6 @@ protected function getCid() { return $this->context['sandbox']['cid_prefix'] . $this->context['sandbox']['current_batch']; } - /** - * Prepares sandbox data (header and cache ID). - * - * @return array - * Table header. - */ - protected function getHeader() { - // Build output header array. - $header = &$this->context['sandbox']['header']; - if (!empty($header)) { - return $header; - } - return $this->setHeader(); - } - public function getConfiguration() { return parent::getConfiguration(); // TODO: Change the autogenerated stub } From 6ebb4bee421f160fcb5c334d819b0c3c3433892e Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Mon, 30 Mar 2026 10:16:14 -0400 Subject: [PATCH 09/10] Cleanup unused "Use" statements on CSV exporter --- src/Plugin/Action/AmiStrawberryfieldCSVexport.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Plugin/Action/AmiStrawberryfieldCSVexport.php b/src/Plugin/Action/AmiStrawberryfieldCSVexport.php index d3507f5..4288a21 100644 --- a/src/Plugin/Action/AmiStrawberryfieldCSVexport.php +++ b/src/Plugin/Action/AmiStrawberryfieldCSVexport.php @@ -5,7 +5,6 @@ use Drupal\Core\Render\RendererInterface; use Drupal\Core\StreamWrapper\StreamWrapperManagerInterface; use Drupal\Core\Url; -use Drupal\Core\Link; use Drupal\Core\Access\AccessResult; use Drupal\Core\Action\ConfigurableActionBase; use Drupal\Core\Entity\EntityTypeManagerInterface; @@ -24,7 +23,6 @@ use Drupal\views_bulk_operations\Action\ViewsBulkOperationsPreconfigurationInterface; use Psr\Log\LoggerInterface; use Symfony\Component\DependencyInjection\ContainerInterface; -use Symfony\Component\HttpFoundation\RedirectResponse; /** * Provides an action that export SBFs to CSV. From 313a296fc64b1e7048dbcdd1eb67807e64d93db2 Mon Sep 17 00:00:00 2001 From: Diego Pino Navarro Date: Mon, 30 Mar 2026 10:18:19 -0400 Subject: [PATCH 10/10] When type column is missing send a warning instead of a status --- src/Plugin/Action/AmiStrawberryfieldCSVexport.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugin/Action/AmiStrawberryfieldCSVexport.php b/src/Plugin/Action/AmiStrawberryfieldCSVexport.php index 4288a21..94a772d 100644 --- a/src/Plugin/Action/AmiStrawberryfieldCSVexport.php +++ b/src/Plugin/Action/AmiStrawberryfieldCSVexport.php @@ -419,7 +419,7 @@ protected function sendToFile($output) { else if ($this->configuration['create_ami_set'] && !$this->context['sandbox']['ado_type_exists']) { $message = $this->t('AMI Set could not be created because object(s) are missing the "type" key.'); $this->messenger() - ->addStatus($message); + ->addWarning($message); return $message; } }