From 80202975b46425cb9432f4c22e9aa67d3a8c36dc Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Tue, 20 Jan 2026 12:18:58 +0100 Subject: [PATCH 01/21] First test xlsx handler --- geonode/upload/handlers/xlsx/__init__.py | 0 geonode/upload/handlers/xlsx/handler.py | 85 ++++++++++++++++++++++++ geonode/upload/settings.py | 1 + 3 files changed, 86 insertions(+) create mode 100644 geonode/upload/handlers/xlsx/__init__.py create mode 100644 geonode/upload/handlers/xlsx/handler.py diff --git a/geonode/upload/handlers/xlsx/__init__.py b/geonode/upload/handlers/xlsx/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py new file mode 100644 index 00000000000..58c37e16512 --- /dev/null +++ b/geonode/upload/handlers/xlsx/handler.py @@ -0,0 +1,85 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging +from pathlib import Path + +from geonode.upload.handlers.common.vector import BaseVectorFileHandler +from geonode.upload.handlers.csv.handler import CSVFileHandler + +logger = logging.getLogger("importer") + + +class XLSXFileHandler(CSVFileHandler): + + @property + def supported_file_extension_config(self): + return { + "id": "xlsx", + "formats": [ + { + "label": "XLSX", + "required_ext": ["xlsx"], + "optional_ext": ["sld", "xml"], + } + ], + "actions": list(self.TASKS.keys()), + "type": "vector", + } + + @staticmethod + def can_handle(_data) -> bool: + """ + This endpoint will return True or False if with the info provided + the handler is able to handle the file or not + """ + base = _data.get("base_file") + if not base: + return False + return ( + base.lower().endswith(".xlsx") or base.lower().endswith(".xls") + if isinstance(base, str) + else base.name.lower().endswith(".xlsx") or base.name.lower().endswith(".xlsx") + ) and BaseVectorFileHandler.can_handle(_data) + + + def pre_processing(self, files, execution_id, **kwargs): + from geonode.upload.orchestrator import orchestrator + import pandas as pd + # calling the super function + _data, execution_id = super().pre_processing(files, execution_id, **kwargs) + # convert the XLSX file into a CSV + xlsx_file = _data.get("files", {}).get("base_file", "") + if not xlsx_file: + raise Exception("File not found") + output_file = str(Path(xlsx_file).with_suffix('.csv')) + + wb = pd.read_excel(xlsx_file, parse_dates=False, engine='calamine', dtype={ + "latitude": float, + "longitude": float + }) + wb.to_csv(output_file, index=False) + + # update the file path in the payload + _data['files']['base_file'] = output_file + _data['temporary_files']['base_file'] = output_file + + # updating the execution id params + orchestrator.update_execution_request_obj(orchestrator.get_execution_object(execution_id), {"input_params": _data}) + return _data, execution_id + \ No newline at end of file diff --git a/geonode/upload/settings.py b/geonode/upload/settings.py index aee5c730dfc..e23ed43b127 100644 --- a/geonode/upload/settings.py +++ b/geonode/upload/settings.py @@ -39,4 +39,5 @@ "geonode.upload.handlers.remote.tiles3d.RemoteTiles3DResourceHandler", "geonode.upload.handlers.remote.wms.RemoteWMSResourceHandler", "geonode.upload.handlers.empty_dataset.handler.EmptyDatasetHandler", + "geonode.upload.handlers.xlsx.handler.XLSXFileHandler", ] From 3dd170cdac13caa9693ba3cc804cbf611e7b1b98 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Mon, 2 Feb 2026 13:28:20 +0200 Subject: [PATCH 02/21] extending xlsx_hander to cover the error handling by using calamine --- geonode/upload/handlers/xlsx/handler.py | 151 ++++++++++++++++++++++-- 1 file changed, 142 insertions(+), 9 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index 58c37e16512..b5cf64c18a6 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -18,9 +18,17 @@ ######################################################################### import logging from pathlib import Path +import csv +from python_calamine import CalamineWorkbook from geonode.upload.handlers.common.vector import BaseVectorFileHandler from geonode.upload.handlers.csv.handler import CSVFileHandler +from geonode.upload.utils import UploadLimitValidator +from geonode.upload.api.exceptions import ( + UploadParallelismLimitException, + InvalidInputFileException + ) +from osgeo import ogr logger = logging.getLogger("importer") @@ -57,29 +65,154 @@ def can_handle(_data) -> bool: else base.name.lower().endswith(".xlsx") or base.name.lower().endswith(".xlsx") ) and BaseVectorFileHandler.can_handle(_data) + + @staticmethod + def is_valid(files, user, **kwargs): + BaseVectorFileHandler.is_valid(files, user) + + upload_validator = UploadLimitValidator(user) + upload_validator.validate_parallelism_limit_per_user() + actual_upload = upload_validator._get_parallel_uploads_count() + max_upload = upload_validator._get_max_parallel_uploads() + + datasource = ogr.GetDriverByName("CSV").Open(files.get("base_file")) + if not datasource: + raise InvalidInputFileException("The converted XLSX data is invalid; no layers found.") + + layers = [datasource.GetLayer(i) for i in range(datasource.GetLayerCount())] + layers_count = len(layers) + + if layers_count >= max_upload: + raise UploadParallelismLimitException( + detail=f"The number of layers ({layers_count}) exceeds the limit of {max_upload}." + ) + + schema_keys = [x.name.lower() for layer in layers for x in layer.schema] + + has_lat = any(x in CSVFileHandler.possible_lat_column for x in schema_keys) + has_long = any(x in CSVFileHandler.possible_long_column for x in schema_keys) + + if has_lat and not has_long: + raise InvalidInputFileException( + f"Longitude is missing. Supported names: {', '.join(CSVFileHandler.possible_long_column)}" + ) + + if not has_lat and has_long: + raise InvalidInputFileException( + f"Latitude is missing. Supported names: {', '.join(CSVFileHandler.possible_lat_column)}" + ) + if not (has_lat and has_long): + raise InvalidInputFileException( + "XLSX uploads require both a Latitude and a Longitude column. " + f"Accepted Lat: {', '.join(CSVFileHandler.possible_lat_column)}. " + f"Accepted Lon: {', '.join(CSVFileHandler.possible_long_column)}." + ) + + return True + def pre_processing(self, files, execution_id, **kwargs): from geonode.upload.orchestrator import orchestrator - import pandas as pd - # calling the super function + + # calling the super function (CSVFileHandler logic) _data, execution_id = super().pre_processing(files, execution_id, **kwargs) + # convert the XLSX file into a CSV xlsx_file = _data.get("files", {}).get("base_file", "") if not xlsx_file: raise Exception("File not found") + output_file = str(Path(xlsx_file).with_suffix('.csv')) - wb = pd.read_excel(xlsx_file, parse_dates=False, engine='calamine', dtype={ - "latitude": float, - "longitude": float - }) - wb.to_csv(output_file, index=False) + try: + workbook = CalamineWorkbook.from_path(xlsx_file) + + # Sheet Validation (Uses the validated sheet name) + sheet_name = self._validate_sheets(workbook) + sheet = workbook.get_sheet_by_name(sheet_name) + + # We iterate until we find the first non-empty row + rows_gen = iter(sheet.to_python()) + try: + # We strictly take the first row. No skipping allowed. + headers = next(rows_gen) + except StopIteration: + raise InvalidInputFileException(detail="The file is empty.") + + # Restrictive File Structure Validation + self._validate_headers(headers) + + # Conversion with row cleanup + # Note: rows_gen continues from the row after the headers + self._convert_to_csv(headers, rows_gen, output_file) + + except Exception as e: + logger.exception("XLSX Pre-processing failed") + raise InvalidInputFileException(detail=f"Failed to securely parse XLSX: {str(e)}") # update the file path in the payload _data['files']['base_file'] = output_file _data['temporary_files']['base_file'] = output_file # updating the execution id params - orchestrator.update_execution_request_obj(orchestrator.get_execution_object(execution_id), {"input_params": _data}) + orchestrator.update_execution_request_obj( + orchestrator.get_execution_object(execution_id), + {"input_params": _data} + ) return _data, execution_id - \ No newline at end of file + + def _validate_sheets(self, workbook): + """Returns the first sheet name and logs warnings if others exist.""" + sheets = workbook.sheet_names + if not sheets: + raise Exception("No sheets found in workbook.") + if len(sheets) > 1: + logger.warning(f"Multiple sheets found. Ignoring: {sheets[1:]}") + return sheets[0] + + def _validate_headers(self, headers): + """ + Ensures the candidate row is a valid header row: + -- Not empty. + -- All column names are unique (required for DB import). + -- No empty column names (required for schema creation). + """ + # Basic Content Check + if self._detect_empty_rows(headers): + raise Exception("The first row found is empty. Column headers are required.") + + # Normalization and Empty Name Check + # We strip whitespace and convert to string to check for valid names + clean_headers = [str(h).strip().lower() if h is not None else "" for h in headers] + + if any(h == "" for h in clean_headers): + raise Exception( + "One or more columns are missing a header name. Every column must have a title." + ) + + # Uniqueness Check + if len(clean_headers) != len(set(clean_headers)): + duplicates = set([h for h in clean_headers if clean_headers.count(h) > 1]) + raise Exception(f"Duplicate column headers found: {', '.join(duplicates)}") + + return True + + def _detect_empty_rows(self, row): + return not row or all(cell is None or str(cell).strip() == "" for cell in row) + + def _convert_to_csv(self, headers, rows_gen, output_path): + """Streams valid data to CSV, skipping empty rows.""" + with open(output_path, 'w', newline='', encoding='utf-8') as f: + writer = csv.writer(f) + writer.writerow(headers) + for row in rows_gen: + # Skip row if it contains no data + if self._detect_empty_rows(row): + continue + + # Cleanup: handle Excel's float-based integers (1.0 -> 1) + cleaned_row = [ + int(cell) if isinstance(cell, float) and cell.is_integer() else cell + for cell in row + ] + writer.writerow(cleaned_row) \ No newline at end of file From 790c46976e280d1eee89bf06a2bf3c9a84a3264d Mon Sep 17 00:00:00 2001 From: gpetrak Date: Mon, 2 Feb 2026 17:03:58 +0200 Subject: [PATCH 03/21] improving error handling and adding the dep of XLSX handler --- geonode/upload/handlers/xlsx/handler.py | 77 ++++++++++++++----------- pyproject.toml | 2 + 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index b5cf64c18a6..808bd4c7fd0 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -23,18 +23,15 @@ from geonode.upload.handlers.common.vector import BaseVectorFileHandler from geonode.upload.handlers.csv.handler import CSVFileHandler -from geonode.upload.utils import UploadLimitValidator -from geonode.upload.api.exceptions import ( - UploadParallelismLimitException, - InvalidInputFileException - ) -from osgeo import ogr logger = logging.getLogger("importer") class XLSXFileHandler(CSVFileHandler): + lat_names = CSVFileHandler.possible_lat_column + lon_names = CSVFileHandler.possible_long_column + @property def supported_file_extension_config(self): return { @@ -68,6 +65,10 @@ def can_handle(_data) -> bool: @staticmethod def is_valid(files, user, **kwargs): + from osgeo import ogr + from geonode.upload.utils import UploadLimitValidator + from geonode.upload.api.exceptions import UploadParallelismLimitException, InvalidInputFileException + BaseVectorFileHandler.is_valid(files, user) upload_validator = UploadLimitValidator(user) @@ -79,40 +80,40 @@ def is_valid(files, user, **kwargs): if not datasource: raise InvalidInputFileException("The converted XLSX data is invalid; no layers found.") - layers = [datasource.GetLayer(i) for i in range(datasource.GetLayerCount())] - layers_count = len(layers) + # In XLSX handler, we always expect 1 layer (the first sheet) + layer = datasource.GetLayer(0) + if not layer: + raise InvalidInputFileException("No data found in the converted CSV.") - if layers_count >= max_upload: + if 1 + actual_upload > max_upload: raise UploadParallelismLimitException( - detail=f"The number of layers ({layers_count}) exceeds the limit of {max_upload}." + detail=f"Upload limit exceeded. Max allowed parallel uploads: {max_upload}" ) - schema_keys = [x.name.lower() for layer in layers for x in layer.schema] + schema_keys = [x.name.lower() for x in layer.schema] - has_lat = any(x in CSVFileHandler.possible_lat_column for x in schema_keys) - has_long = any(x in CSVFileHandler.possible_long_column for x in schema_keys) + # Accessing class-level constants explicitly + has_lat = any(x in XLSXFileHandler.lat_names for x in schema_keys) + has_long = any(x in XLSXFileHandler.lon_names for x in schema_keys) if has_lat and not has_long: - raise InvalidInputFileException( - f"Longitude is missing. Supported names: {', '.join(CSVFileHandler.possible_long_column)}" - ) + raise InvalidInputFileException(f"Longitude is missing. Supported names: {', '.join(XLSXFileHandler.lon_names)}") if not has_lat and has_long: - raise InvalidInputFileException( - f"Latitude is missing. Supported names: {', '.join(CSVFileHandler.possible_lat_column)}" - ) + raise InvalidInputFileException(f"Latitude is missing. Supported names: {', '.join(XLSXFileHandler.lat_names)}") if not (has_lat and has_long): raise InvalidInputFileException( - "XLSX uploads require both a Latitude and a Longitude column. " - f"Accepted Lat: {', '.join(CSVFileHandler.possible_lat_column)}. " - f"Accepted Lon: {', '.join(CSVFileHandler.possible_long_column)}." + "XLSX uploads require both a Latitude and a Longitude column in the first row. " + f"Accepted Lat: {', '.join(XLSXFileHandler.lat_names)}. " + f"Accepted Lon: {', '.join(XLSXFileHandler.lon_names)}." ) return True def pre_processing(self, files, execution_id, **kwargs): from geonode.upload.orchestrator import orchestrator + from geonode.upload.api.exceptions import InvalidInputFileException # calling the super function (CSVFileHandler logic) _data, execution_id = super().pre_processing(files, execution_id, **kwargs) @@ -172,28 +173,36 @@ def _validate_sheets(self, workbook): def _validate_headers(self, headers): """ - Ensures the candidate row is a valid header row: - -- Not empty. - -- All column names are unique (required for DB import). - -- No empty column names (required for schema creation). + Strictly validates Row 1 for headers: + - Must not be empty. + - Must contain geometry 'fingerprints' (Lat/Lon). + - Must have unique and non-empty column names. """ - # Basic Content Check - if self._detect_empty_rows(headers): - raise Exception("The first row found is empty. Column headers are required.") + # Existence Check + if not headers or self._detect_empty_rows(headers): + raise Exception("No data or headers found in the selected sheet.") - # Normalization and Empty Name Check - # We strip whitespace and convert to string to check for valid names + # Normalization clean_headers = [str(h).strip().lower() if h is not None else "" for h in headers] - if any(h == "" for h in clean_headers): + # Geometry Fingerprint Check + has_lat = any(h in self.lat_names for h in clean_headers) + has_lon = any(h in self.lon_names for h in clean_headers) + + if not (has_lat and has_lon): raise Exception( - "One or more columns are missing a header name. Every column must have a title." + "The headers does not contain valid geometry headers. " + "GeoNode requires Latitude and Longitude labels in the first row." ) + # Integrity Check (No Empty Names) + if any(h == "" for h in clean_headers): + raise Exception("One or more columns in the first row are missing a header name.") + # Uniqueness Check if len(clean_headers) != len(set(clean_headers)): duplicates = set([h for h in clean_headers if clean_headers.count(h) > 1]) - raise Exception(f"Duplicate column headers found: {', '.join(duplicates)}") + raise Exception(f"Duplicate headers found in Row 1: {', '.join(duplicates)}") return True diff --git a/pyproject.toml b/pyproject.toml index 8f19f409594..650e183b0c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -159,6 +159,8 @@ dependencies = [ # Security and audit "cryptography==46.0.3", "jwcrypto>=1.5.6", + # dependency for XLSX handler + "python-calamine==0.6.1", ] [project.optional-dependencies] From d39e05b61b162929df19a9330e673ad234fe47ab Mon Sep 17 00:00:00 2001 From: gpetrak Date: Tue, 3 Feb 2026 11:24:53 +0200 Subject: [PATCH 04/21] simpifying the ogr-based method in order to not handle the WKT geometries --- geonode/upload/handlers/xlsx/handler.py | 71 ++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index 808bd4c7fd0..d05eac8e672 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -19,10 +19,16 @@ import logging from pathlib import Path import csv +from celery import group from python_calamine import CalamineWorkbook +from osgeo import ogr + +from dynamic_models.models import ModelSchema from geonode.upload.handlers.common.vector import BaseVectorFileHandler from geonode.upload.handlers.csv.handler import CSVFileHandler +from geonode.upload.celery_tasks import create_dynamic_structure +from geonode.upload.handlers.utils import GEOM_TYPE_MAPPING logger = logging.getLogger("importer") @@ -65,7 +71,6 @@ def can_handle(_data) -> bool: @staticmethod def is_valid(files, user, **kwargs): - from osgeo import ogr from geonode.upload.utils import UploadLimitValidator from geonode.upload.api.exceptions import UploadParallelismLimitException, InvalidInputFileException @@ -111,6 +116,70 @@ def is_valid(files, user, **kwargs): return True + @staticmethod + def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate, **kwargs): + """ + Customized for XLSX: Only looks for X/Y (Point) data. + Ignores WKT/Geom columns as per requirements. + """ + + base_command = BaseVectorFileHandler.create_ogr2ogr_command( + files, original_name, ovverwrite_layer, alternate + ) + + # We only define X and Y possible names instead of WKT columns + lat_mapping = ",".join(XLSXFileHandler.lat_names) + lon_mapping = ",".join(XLSXFileHandler.lon_names) + + additional_option = ( + f' -oo "X_POSSIBLE_NAMES={lon_mapping}" ' + f' -oo "Y_POSSIBLE_NAMES={lat_mapping}"' + ) + + return ( + f"{base_command} -oo KEEP_GEOM_COLUMNS=NO " + f"-lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} " + + additional_option + ) + + def create_dynamic_model_fields( + self, + layer: str, + dynamic_model_schema: ModelSchema = None, + overwrite: bool = None, + execution_id: str = None, + layer_name: str = None, + return_celery_group: bool = True, + ): + # retrieving the field schema from ogr2ogr and converting the type to Django Types + layer_schema = [ + {"name": x.name.lower(), "class_name": self._get_type(x), "null": True} + for x in layer.schema + ] + + class_name = GEOM_TYPE_MAPPING.get(self.promote_to_multi("Point")) + # Get the geometry type name from OGR (e.g., 'Point' or 'Point 25D') + geom_type_name = ogr.GeometryTypeToName(layer.GetGeomType()) + + layer_schema += [ + { + "name": layer.GetGeometryColumn() or self.default_geometry_column_name, + "class_name": class_name, + "dim": (3 if geom_type_name.lower().startswith("3d") or "z" in geom_type_name.lower() else 2), + } + ] + + if not return_celery_group: + return layer_schema + + list_chunked = [layer_schema[i : i + 30] for i in range(0, len(layer_schema), 30)] + celery_group = group( + create_dynamic_structure.s(execution_id, schema, dynamic_model_schema.id, overwrite, layer_name) + for schema in list_chunked + ) + + return dynamic_model_schema, celery_group + def pre_processing(self, files, execution_id, **kwargs): from geonode.upload.orchestrator import orchestrator from geonode.upload.api.exceptions import InvalidInputFileException From ecc8f30728ebc44582087d78896333ec64952330 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Tue, 3 Feb 2026 12:29:29 +0200 Subject: [PATCH 05/21] implementing a data validation check --- geonode/upload/handlers/xlsx/handler.py | 63 +++++++++++++++++++++---- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index d05eac8e672..0b4c30e6d6a 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -19,6 +19,8 @@ import logging from pathlib import Path import csv +from datetime import datetime +import math from celery import group from python_calamine import CalamineWorkbook from osgeo import ogr @@ -29,6 +31,10 @@ from geonode.upload.handlers.csv.handler import CSVFileHandler from geonode.upload.celery_tasks import create_dynamic_structure from geonode.upload.handlers.utils import GEOM_TYPE_MAPPING +from geonode.upload.api.exceptions import ( + UploadParallelismLimitException, + InvalidInputFileException, +) logger = logging.getLogger("importer") @@ -72,7 +78,6 @@ def can_handle(_data) -> bool: @staticmethod def is_valid(files, user, **kwargs): from geonode.upload.utils import UploadLimitValidator - from geonode.upload.api.exceptions import UploadParallelismLimitException, InvalidInputFileException BaseVectorFileHandler.is_valid(files, user) @@ -180,9 +185,9 @@ def create_dynamic_model_fields( return dynamic_model_schema, celery_group + def pre_processing(self, files, execution_id, **kwargs): from geonode.upload.orchestrator import orchestrator - from geonode.upload.api.exceptions import InvalidInputFileException # calling the super function (CSVFileHandler logic) _data, execution_id = super().pre_processing(files, execution_id, **kwargs) @@ -274,23 +279,63 @@ def _validate_headers(self, headers): raise Exception(f"Duplicate headers found in Row 1: {', '.join(duplicates)}") return True + + def _data_sense_check(self, x, y): + """ + High-speed coordinate validation for large datasets + """ + try: + # Catch Excel Date objects immediately (Calamine returns these as datetime) + if isinstance(x, datetime) or isinstance(y, datetime): + return False + + f_x = float(x) + f_y = float(y) + + # Finiteness check (Catches NaN, Inf, and None) + # This is extremely fast in Python + if not (math.isfinite(f_x) and math.isfinite(f_y)): + return False + + # Magnitude check + # Limits to +/- 40 million (covers all CRS including Web Mercator) + # but blocks 'serial date numbers' or corrupted scientific notation + if not (-40000000 < f_x < 40000000 and -40000000 < f_y < 40000000): + return False + + return True + except (ValueError, TypeError): + return False def _detect_empty_rows(self, row): return not row or all(cell is None or str(cell).strip() == "" for cell in row) def _convert_to_csv(self, headers, rows_gen, output_path): """Streams valid data to CSV, skipping empty rows.""" + + # Define clean_headers once here to find the indices + clean_headers = [str(h).strip().lower() for h in headers] + + # Get the indices for the Lat and Lon columns + lat_idx = next(i for i, h in enumerate(clean_headers) if h in self.lat_names) + lon_idx = next(i for i, h in enumerate(clean_headers) if h in self.lon_names) + + # Local binding of the check function for loop speed + check_func = self._data_sense_check + with open(output_path, 'w', newline='', encoding='utf-8') as f: writer = csv.writer(f) writer.writerow(headers) - for row in rows_gen: + + for row_num, row in enumerate(rows_gen, start=2): # Skip row if it contains no data if self._detect_empty_rows(row): continue - # Cleanup: handle Excel's float-based integers (1.0 -> 1) - cleaned_row = [ - int(cell) if isinstance(cell, float) and cell.is_integer() else cell - for cell in row - ] - writer.writerow(cleaned_row) \ No newline at end of file + if not check_func(row[lon_idx], row[lat_idx]): + raise InvalidInputFileException( + detail=f"Coordinate error at row {row_num}. " + "Check for dates or non-numeric values in Lat/Lon." + ) + + writer.writerow(row) \ No newline at end of file From 5557898e522da876600bd069da5883f20a43b232 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Tue, 3 Feb 2026 13:12:04 +0200 Subject: [PATCH 06/21] make the XLSX handler configurable --- .env.sample | 5 ++++- .env_dev | 5 ++++- .env_local | 3 +++ .env_test | 3 +++ geonode/upload/handlers/xlsx/handler.py | 13 +++++++++++++ 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/.env.sample b/.env.sample index fee9293c82d..0d22ec7574f 100644 --- a/.env.sample +++ b/.env.sample @@ -245,4 +245,7 @@ RESTART_POLICY_WINDOW=120s DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 -# FORCE_READ_ONLY_MODE=False Override the read-only value saved in the configuration \ No newline at end of file +# FORCE_READ_ONLY_MODE=False Override the read-only value saved in the configuration + +# Enable or not the XLSX / XLS upload +XLSX_UPLOAD_ENABLED=True \ No newline at end of file diff --git a/.env_dev b/.env_dev index 0cfa9dad6c7..702c481b960 100644 --- a/.env_dev +++ b/.env_dev @@ -207,4 +207,7 @@ RESTART_POLICY_WINDOW=120s DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 UPSERT_CHUNK_SIZE= 100 -UPSERT_LIMIT_ERROR_LOG=100 \ No newline at end of file +UPSERT_LIMIT_ERROR_LOG=100 + +# Enable or not the XLSX / XLS upload +XLSX_UPLOAD_ENABLED=True \ No newline at end of file diff --git a/.env_local b/.env_local index 583a9fc32d6..63009d33dc4 100644 --- a/.env_local +++ b/.env_local @@ -209,3 +209,6 @@ RESTART_POLICY_MAX_ATTEMPTS="3" RESTART_POLICY_WINDOW=120s DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 + +# Enable or not the XLSX / XLS upload +XLSX_UPLOAD_ENABLED=True diff --git a/.env_test b/.env_test index a770063d228..37a4eaeeed6 100644 --- a/.env_test +++ b/.env_test @@ -224,3 +224,6 @@ MICROSOFT_TENANT_ID= AZURE_CLIENT_ID= AZURE_SECRET_KEY= AZURE_KEY= + +# Enable or not the XLSX / XLS upload +XLSX_UPLOAD_ENABLED=True diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index 0b4c30e6d6a..9c96dc5116a 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -17,6 +17,8 @@ # ######################################################################### import logging +import os +from distutils.util import strtobool from pathlib import Path import csv from datetime import datetime @@ -41,11 +43,18 @@ class XLSXFileHandler(CSVFileHandler): + XLSX_UPLOAD_ENABLED = strtobool(os.getenv("XLSX_UPLOAD_ENABLED", "False")) + lat_names = CSVFileHandler.possible_lat_column lon_names = CSVFileHandler.possible_long_column @property def supported_file_extension_config(self): + + # If disabled, return an empty list or None so the UI doesn't show XLSX options + if not XLSXFileHandler.XLSX_UPLOAD_ENABLED: + return None + return { "id": "xlsx", "formats": [ @@ -65,6 +74,10 @@ def can_handle(_data) -> bool: This endpoint will return True or False if with the info provided the handler is able to handle the file or not """ + # Availability Check for the back-end + if not XLSXFileHandler.XLSX_UPLOAD_ENABLED: + return False + base = _data.get("base_file") if not base: return False From 345951610b7b1812f4500a4df769fbcd1ebbd784 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Tue, 3 Feb 2026 13:28:07 +0200 Subject: [PATCH 07/21] adding xls extension --- geonode/upload/handlers/xlsx/handler.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index 9c96dc5116a..f0b5e8d4191 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -59,8 +59,8 @@ def supported_file_extension_config(self): "id": "xlsx", "formats": [ { - "label": "XLSX", - "required_ext": ["xlsx"], + "label": "Excel (XLSX/XLS)", + "required_ext": ["xlsx", "xls"], "optional_ext": ["sld", "xml"], } ], @@ -81,11 +81,17 @@ def can_handle(_data) -> bool: base = _data.get("base_file") if not base: return False - return ( - base.lower().endswith(".xlsx") or base.lower().endswith(".xls") + + # Support both XLSX and XLS + valid_extensions = (".xlsx", ".xls") + + is_excel = ( + base.lower().endswith(valid_extensions) if isinstance(base, str) - else base.name.lower().endswith(".xlsx") or base.name.lower().endswith(".xlsx") - ) and BaseVectorFileHandler.can_handle(_data) + else base.name.lower().endswith(valid_extensions) + ) + + return is_excel and BaseVectorFileHandler.can_handle(_data) @staticmethod From 4084871bdc9a210b786599f1a7d5475a031d01ef Mon Sep 17 00:00:00 2001 From: gpetrak Date: Tue, 3 Feb 2026 17:24:07 +0200 Subject: [PATCH 08/21] extend the handler for xls files --- geonode/upload/handlers/xlsx/handler.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index f0b5e8d4191..285ade6f12a 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -56,11 +56,16 @@ def supported_file_extension_config(self): return None return { - "id": "xlsx", + "id": "excel", # Use a generic ID that doesn't imply a specific extension "formats": [ { - "label": "Excel (XLSX/XLS)", - "required_ext": ["xlsx", "xls"], + "label": "Excel (OpenXML)", + "required_ext": ["xlsx"], + "optional_ext": ["sld", "xml"], + }, + { + "label": "Excel (Binary/Legacy)", + "required_ext": ["xls"], "optional_ext": ["sld", "xml"], } ], From e105ca7bafafb8b71e1ccf8abbede79d5f1654a0 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Wed, 4 Feb 2026 17:54:18 +0200 Subject: [PATCH 09/21] adding tests --- geonode/upload/handlers/xlsx/handler.py | 4 + geonode/upload/handlers/xlsx/tests.py | 106 ++++++++++++++++++ geonode/upload/tests/fixture/missing_lat.xlsx | Bin 0 -> 7961 bytes geonode/upload/tests/fixture/valid_excel.xls | Bin 0 -> 15872 bytes geonode/upload/tests/fixture/valid_excel.xlsx | Bin 0 -> 7982 bytes .../fixture/valid_leading_empty_rows.xlsx | Bin 0 -> 7960 bytes .../tests/fixture/valid_with_empty_rows.xlsx | Bin 0 -> 7958 bytes geonode/upload/tests/fixture/wrong_data.csv | 1 + geonode/upload/tests/fixture/wrong_data.xlsx | Bin 0 -> 7997 bytes 9 files changed, 111 insertions(+) create mode 100644 geonode/upload/handlers/xlsx/tests.py create mode 100644 geonode/upload/tests/fixture/missing_lat.xlsx create mode 100644 geonode/upload/tests/fixture/valid_excel.xls create mode 100644 geonode/upload/tests/fixture/valid_excel.xlsx create mode 100644 geonode/upload/tests/fixture/valid_leading_empty_rows.xlsx create mode 100644 geonode/upload/tests/fixture/valid_with_empty_rows.xlsx create mode 100644 geonode/upload/tests/fixture/wrong_data.csv create mode 100644 geonode/upload/tests/fixture/wrong_data.xlsx diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index 285ade6f12a..269c0b168a2 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -251,6 +251,10 @@ def pre_processing(self, files, execution_id, **kwargs): # update the file path in the payload _data['files']['base_file'] = output_file + + if 'temporary_files' not in _data or not isinstance(_data['temporary_files'], dict): + _data['temporary_files'] = {} + _data['temporary_files']['base_file'] = output_file # updating the execution id params diff --git a/geonode/upload/handlers/xlsx/tests.py b/geonode/upload/handlers/xlsx/tests.py new file mode 100644 index 00000000000..ba403e7e15a --- /dev/null +++ b/geonode/upload/handlers/xlsx/tests.py @@ -0,0 +1,106 @@ +import os +import uuid +import math +from unittest.mock import patch, MagicMock +from django.test import TestCase +from django.contrib.auth import get_user_model + +from geonode.upload import project_dir +from geonode.upload.api.exceptions import InvalidInputFileException +from geonode.upload.handlers.xlsx.handler import XLSXFileHandler + +class TestXLSXHandler(TestCase): + databases = ("default", "datastore") + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.handler = XLSXFileHandler() + + # Consistent with CSV handler's fixture path + cls.valid_xlsx = f"{project_dir}/tests/fixture/valid_excel.xlsx" + cls.valid_xls = f"{project_dir}/tests/fixture/valid_excel.xls" + cls.empty_rows_xlsx = f"{project_dir}/tests/fixture/valid_with_empty_rows.xlsx" + cls.leading_empty_xlsx = f"{project_dir}/tests/fixture/valid_leading_empty_rows.xlsx" + cls.missing_lat_xlsx = f"{project_dir}/tests/fixture/missing_lat.xlsx" + cls.wrong_data_xlsx = f"{project_dir}/tests/fixture/wrong_data.xlsx" + + cls.user, _ = get_user_model().objects.get_or_create(username="admin") + + def setUp(self): + # Force the handler to be enabled for testing + XLSXFileHandler.XLSX_UPLOAD_ENABLED = True + + @patch("geonode.upload.handlers.common.vector.BaseVectorFileHandler.can_handle") + def test_can_handle_xlsx_and_xls(self, mock_base_can_handle): + """Check if the handler identifies both extensions.""" + mock_base_can_handle.return_value = True + + self.assertTrue(self.handler.can_handle({"base_file": self.valid_xlsx})) + self.assertTrue(self.handler.can_handle({"base_file": self.valid_xls})) + + # Also verify it returns False when the file is wrong + self.assertFalse(self.handler.can_handle({"base_file": "random.txt"})) + + @patch("geonode.upload.orchestrator.orchestrator.get_execution_object") + @patch("geonode.upload.orchestrator.orchestrator.update_execution_request_obj") + def test_pre_processing_success_with_valid_files(self, mock_update, mock_get_exec): + test_files = [self.valid_xlsx, self.valid_xls, self.empty_rows_xlsx, self.leading_empty_xlsx] + + for file_path in test_files: + exec_id = str(uuid.uuid4()) + files = {"base_file": file_path} + + with patch('geonode.upload.handlers.csv.handler.CSVFileHandler.pre_processing', + return_value=({"files": files, "temporary_files": {}}, exec_id)): + + data, _ = self.handler.pre_processing(files, exec_id) + + output_csv = data['files']['base_file'] + self.assertTrue(output_csv.endswith(".csv")) + self.assertTrue(os.path.exists(output_csv)) + + # Cleanup + if os.path.exists(output_csv): + os.remove(output_csv) + + @patch("geonode.upload.orchestrator.orchestrator.get_execution_object") + def test_pre_processing_fails_on_missing_lat(self, mock_get_exec): + """Should fail when header fingerprinting doesn't find Latitude.""" + exec_id = str(uuid.uuid4()) + files = {"base_file": self.missing_lat_xlsx} + + with patch('geonode.upload.handlers.csv.handler.CSVFileHandler.pre_processing', + return_value=({"files": files}, exec_id)): + with self.assertRaises(InvalidInputFileException) as context: + self.handler.pre_processing(files, exec_id) + + self.assertIn("geometry headers", str(context.exception)) + + @patch("geonode.upload.orchestrator.orchestrator.get_execution_object") + def test_pre_processing_fails_on_wrong_data(self, mock_get_exec): + """Should fail on row 1 of the data due to 'nan' and extreme magnitude.""" + exec_id = str(uuid.uuid4()) + files = {"base_file": self.wrong_data_xlsx} + + with patch('geonode.upload.handlers.csv.handler.CSVFileHandler.pre_processing', + return_value=({"files": files}, exec_id)): + with self.assertRaises(InvalidInputFileException) as context: + self.handler.pre_processing(files, exec_id) + + # The error should specifically mention the coordinate error and the row + self.assertIn("Coordinate error at row 2", str(context.exception)) + + def test_data_sense_check_logic(self): + """Directly test the coordinate validation math.""" + # Valid + self.assertTrue(self.handler._data_sense_check(37.8, -122.4)) + # NaN + self.assertFalse(self.handler._data_sense_check("nan", 40.0)) + # Infinite + self.assertFalse(self.handler._data_sense_check(float('inf'), 40.0)) + # Extreme Magnitude + self.assertFalse(self.handler._data_sense_check(40000001, 10.0)) + # Excel Date (as datetime object) + from datetime import datetime + self.assertFalse(self.handler._data_sense_check(datetime.now(), 40.0)) \ No newline at end of file diff --git a/geonode/upload/tests/fixture/missing_lat.xlsx b/geonode/upload/tests/fixture/missing_lat.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..70d5aea451e7e8dcb4e21ace28f1ef2b9db95366 GIT binary patch literal 7961 zcma)h1yqz<_ctXygmg(a0z;>Ogmfb0HG|i2CXL%un-1?G!QL3ZYKHjfzrGIYlM&#Cu6D%~RQ84MJE`NZ~P#3_m zq`nqefdi>-CJgFRO*_a)@E7+#%Nb!2eiBZgIzhn=Z<>ny_F>IvW0rB*CL&WFnA#~H z_vv$q^yhEiF`!B<>y6kv&@&l!fI}Aq2!(j5`y%B%ahTn_4c_pt^V)W4r~ud#XHPo! zGmM&jgbZ<@Xp*6x^|)AE)wyz*do}6K2X!zVl)p4Ft=xM_Jk!^YLis9PQe{o4xic5X zkyr`?1WOd1NtJ!U)fYjCY2KYII;+cy9M2R*B5Wg{5|d<%xsmp$iX$(lKwep#hFXtg zc4~@OUw=yWCFi^v*az zar$yy?6|@E4{pW#z0+%)(gS(m6ZQGAG8h(u*2>U$ETvbT+_YFuPlov?FMAJn#m6T^ z1lY!r@m1)ktX059bp@$LDQkcC-@$^xl& zi>y8Ta0!V=4E=lElUDa_VT!J^;k_M4CzwcWn#XfI`;%61U+o5HnmDvAnqSFOU|&1^ zsPpjJt4L!S-<5?ip7$Z%>tcHAm@m3Orfoy{{^2b~Q}0(`f?Dk32PUOGZuAKa>UFUgrjwrXOd zj?4KC=beQK5;x*;2aKfp02}enI(dg^NW;w_7hlP`wCpdRJ=!7{Y_ohi2}We5>xsNd z^QzuZJ*2FQl9NDrtW4U&EHM>W>7Uqi;c|u1;ihATRV?*=)75lWyIL;zuv;L$=)%)` z^rA?|O{K}#0gpbxJlmYOg7fx1_<;N$$lgyPL#XsQh;QRM{0}qAI z4g53H{5t{0F9KIPM{6ewQ&Y(OtQ0Jp7e)e=0ziU+A^bnIKUsi3vy+B`(=0nyi)-Pn zSiZPow}EIR3SGP?$CiA$QCy8MnkTFR_+ug{V{O`%Ho$};MBY1wwq?HFPSe`m@iNXF zgr@4C@?pYNNH#}WVj#j<-dA}3Oe%rlQ+R@Do?l9WG@WX^d3aZ(~)59E#j z!U7{LCn|WF)jjKyPt?3hK-?RR)8nyxb}>oAwoOtb6Tq4%7%O*sH8uwsLw>t=)xrM;7FX{q)b(3&8d(mQ*j4{ z<+WhuYtmASS6R|p-6sR&izkeGx)F<{R+FhXpOT_u#PTS+pXYw4X;%$3H@P6I7wWPB zpi|3nEq0<;u@qLrG;LNyy;MVAs>5zP_O8*L<3f{B)q2M?&KY~XT(V%OzDRT?h&i7w z6#501iS^da?7DBdHFRkKbL^a zeTbxn;4E{~EAW93q;>0IcKyuDp_>qaE1qo?87RXfQ5_^|s3)IXfLbKne-kT{*J2nG z&K&EN|MI&L*5l$+n?RoVFA%Z*se|}_ZZ_P0!yqepBYE+GLg90{$FgATUJ^v3t0mJW zT;jHg=or9@EWef-dM%V(L2EDw=L#F55f_(8L4HZHFoRv!vgW2eE67n;5TnEKmazo) zi(|}Yoz}kop+}JXmmm>Ol?f>odyo(FJ7@it4#?s?(TBnZ?d1nKSORZQ6U-{a>4tF z-^r;Kn`ETL{E(RkLOirekm_16!*I~+a6T{0AMFde!0`pXNw%8YSc@cYJ6?V#?D>Py z?LaosEKy$O2Ue`#K-oYli`vn(`>~}woQ>#4JF|?MV_urjkcdx8$HY!a4(!`x4^+m_ zBx*`s%H|e`*UFRXD|#x7u!z1IL>s6J%9;oESd#h5F!3}o6v}71DwUCOm)b6$RCI}L z07RO)%(A*?VY|4e_-nm0#yW{jw3;2v2MH2;)XQG@oiz{MZ#j{i*Xp6I^>$>RkhUvi zH;p>MfQSmm&dH()WD;sxg^P*KYV$!{+7Ndok*)4#>A2IV)&Sz`VPT)hle;IMnbqX@ zevWW~L&4(ZXRa4wq!ohA0cRQgCed{y@1Kklye__&Zo051Wa}}|Ib;rM^0g%YmRJ0G ziz|Nm85~C9EQO&MMsR<7hLw(JFrZNx{e5Bx~0GIMTGeM zrNeE6YXh{U0m{ZnilIKf#Wb8ErP;)%ni=aPT(dH+&0Fgklb)n?8vBm>HGPE;9JXYi z6Fo#bMj~4A=Fc3(YN|_9E6Nk@vNbF+FwGxI**FJG4VZ17=bV2MUU9VIdCo4b;goA{ zDR+3wX!5SHJ6BSSb2a&6^$cmc0XUHEu&LVi(jW+^IIK1o+%$HULvCHa`~~MwiF?E5 zL!IU@(vYzH-h4)|S$3tRv0w&7}m+$e{3jh!dBti6>@X!8?5i-`(7fNIbnS1rLOL+?wrJ?9|c8@s>VU9y$2r! zWT@FfDzbSw`P!08Jy%#ToOFgJ| zgN9joeSF&BxQUhReWU4_$d5Db>n+}XC3MfuCk$s-N{hw$s?;nC z+f+B|)?+X}HX19tb7tHpjl?qt)2MZ)wMPiYgJio&_=kmZ)K=vY9?{{US=r_B?@`Ng z#XR%Lfw6OxlBN)ulE>*DTE$OxSvB=!qEPz2WBf2LoX%P6EH8D|EmO+jtHzji;%kylC zNS>D0hXSFMh^57u4!l*hx~F<#!xRP`sB^GcN{xZrmsRr9kfN)E+W8LAAvXo5dEr^Q z7<(z=z0^Qcm&TMAd$r6T-TF{Ci1sy-mLw8*Wap%h=FLa^&En+skLsVr)xOc^RsrcG7gr@d9<%Vk3?BINh4o@VexSb{5kZEV1Ln$Z!HF*|Zhn)p_hUet=ik z8?t*?y;Qzfe=XI9{^gh?&o5%g?@3y{p zrL3T&B^(6VeWR%_KYEzipZHA=`xt{Q;x%C}e`T3JCyquxyrWuPrlQSU)>iyyu$gY* zLcBr8*jC52&J1cjUqLzDi~w1*xyd{a%luyN^s4L);o98BU>4z$W7L`d6Ss9f7o|EX zgLCU|CNIbtr$Q9)f>POifw;2g=}+8JJRPd_8!n~Ij@ZcyconW>Vi+t*yFW1uUef`5 zTpemFJ&7;204uP)8F>MZJl-=3xdQ2YuPY%sv;nJ?AMBSaTy>-4+(s^3a4#9Ap7l!? zgLaawqrXMu&qQ+^ajq)aaV*!J(aC65QoP2s2J(DSsZ#rDp4&E*n;{)ACwP=fPok(+YlMYyq4AYJ z4&V|pU8ATw$>da#zA9Mf;qwIV7U)&)3z8bEG~NMfTJo_vl`Y3h8DMY`h#U-Aw@ z1Aa|NN3Bv144$*0_?D1``w+|Rz0uNMHH^PLm7b({D39BFv0iw2crEtX+F^V4f-Jo` zF8N3bCP1D2%f1AOhgNWXJpeIBqt@aIS>i+K&FGj!5-uT_DCunGjeg|ZwPTQiK-m0# zzr5$v4}ykNW)1wNRp51~)N?-44Zlj}sgVMb3;Le#m0+5BNmFppC`>El`8#ugye|yJ z=%*UyiI__o5l@sSd5DJ8y{dW}0eFq;~8kB`&CCpDy)x7R}=?i0YE>3$H$M=6-IEea;H=qn5G2f%N= z4HAga?lp}~wWo>L<1H7rM}E)Zm>uG`gJM)d&mTC{MHR^f-uJ)2^ex~UQMXKDJwi|| z{y?cE6Y40&9?MY3o7fFfl<3&Neq?t%=r8% z42LjtWFL+j*4ysvrQ1$sDt$vqJ^v3L&!uD8DidhMIE49w_tZ-+;MBYy4tn~g7ilo% z#kV0-?=JJLByt^K8guQ4#!BnDm5vcWOdiD1MDd(G*hi{CERIn|m7~$8=dY4oL-OpY ze6*fje`%KPbSh!&XF&5b!zQhBPW%y{G$^vNy+029BD0;KVVVtO_W zWIDHCT9C()HtdKeR(%3)!d2MeX~g(7&(lXDT7+cC6RdBudI^sZq@oMR8Xze20&5(R z{#R(JgEZkn>zcc+dswl=s#t}H$VU9|@BO-?3LKg+9A_TsP+5J6G3$zJ)Ov>YhHN6J zK=^3!1u;&rp-3FjgTR18sTFKql&-ZlcCKrfXv88)bhU!kwHNpu1(t6j;gmX)`UqzH zC}Ou_80>*mny))t_I|9BcP?3mk-jXWjSaw|`dUbPgpk23=5wH?I_5wonIuhpKI2Oz zZb4qm^nyXw36lqNTp_NRDM-5FM{Pae*!pHwul$v7p67VST(<_f|FDc=dP?c!psP6O8Nu9>B)-Z3VvC)zJXAqS>smi*f4B#fL|1tV%se)$f}jHlIc^+S=`p zI0|^r8&5ZbWK}BV<5{%77Rk-nW|vKOjQReMrS$|QvuXHCEr2NX67hTR2YkzhK53i; zl_+&`z}0@fod{LwF*t+V@cy$uQfQerwFW)3MMC={cl|@?-N4Dh$k7y}2642sHUD{r zGgVfQ`vARZV ztJ9e64Z4gIQk?NiJQ6tmUe(sfO%YCNKOmoZNe2{kbsr69m|&4Jmzum>&^e^0O;E@R zD|H;C&3&$g$m~Rk&hgMPZO3{1%S>P_;G((GWn-wl^Ux&s2D9mTp7mt+h>ml)aIQfx za5QavH?71bq*#ux8u6jQO4;dxb0PhdKaP-*yMeW*ygPPEDMIY>VYz`IhCAv9dr{%J zOJ!ri%>+-WU~jQA6|b9Y!*9IvnEynDR@_dQ4~hsZ^wxy=3zfUBhOwQU_1&R1O68+s z7duun^&#feg5_9Et}=H?Aj~M*8@GgN5oFux$bE$e`59qFoSZq|ZK?bc825y27~h~hjIRC;K@ zUEDza86Tiv^^FTty-|NfH$?9O5onPJvh>KbUrPj7dg_vKrLp_*sSCw; zE%r@`Hxw6ieyYS?wYeJP-g?4h7=Cp?0TNYd50sr-Zit0<3OU=Oz*DfJw?ve=tX`YF zW(2o0aj0_YRL0I+9 zp=;H}j^48$&=N4}?JyPvl*)Y3lQUiJ31h^S{NoQrkt_)N|(N2<^sDWp6Kmp|# zJ$g|~H*ILg7y*yCr4Bqzk}TB0oFcP{@`(RBEO345=eS&QwjertMXwy<5VLmMVxm!t zv6y$Y2RukbR)|vA6yfi8qv_$9>_fy~mk$*)2yxfL3+`2X;Z&G%QhxC%?T|&Vlixee zLm)yIy*mxPDCN}z#nGSxV=P!sfPk>&!eRa1eYW^e$H^QT{D23)+9kT;cD4{xTZng<(Y|QN^mYLbm->ZBg(vHn?Xt{dhqEhxvgn8?Nnjdr^5| zI$}V4U<;}o&-)`sQg2bk;5Tg=uVzlhOOiy1(d4kzN9xK%6JOF$(g8%WZb5S7%Z?YU z`E}ooSK#97lQ}Vole9PFF9z(v5(uYRRGHE{^1WDM3}Bcz?#ZmPy<*acmebfN#G83h zd|@#~NQvK9yEZ2n?G1#3kG+>g=f^{pT{VoS1E{-0OJ5&9o%MdRwQ$-RTi3?YeB5zt z!t(=BSq>Hs_h(u9HP*WCzEBT_frX)f{uP1tlmELj-iyZ;9q+HL0-?HpZ`I`Y ir0stx|7RH8iTW)-S?5ZUAC@35UnIq z#%D}Kt7s)!(#BMqv`t!HX_MAO=!do+*0yOhwJjg8U;2vCQcbm0roZ1gbMD-^bMAE) zwXKyixv&2@=Xd_E^FNP!=U2Z>oO$AzxvwQ-5n(#1&T{!0DU)-#N~zG#>g?b=&pgi% znTvbU&gF7Bk{95>?S=FS(m-VlhO8);A6LIL zjH^T-GIhQtG&f8lk-kuE%m_r@^C2bb3q+XZ6Bc~y{8o_^lP_@s=)d&U|`Fse#fK0)dP||KE+74@7 z=W8A|z@?kQA;LNFf{jY6bc?gYmAF-sCHg4NrFIG$BTuWKH={AuZ;}oiT9?d8tNVuN zbtt<`Hx|?k=|K&IK_7w%FAkQ&tKVW?iyJGb#xBzUT(wlL5otp}R7xzC%inCNbp8el zGTNJ34zHah&UScb!3G7dEv9RKSt+v-%B@w3{lr#z6~>o&{>omlsK4Th1(6z}>(O#D zta=y+1)wuhizp%Tb}p%|>0}x=CzGT*ijEQcEtQ2+IGuyR+qu&6L3E z9r(wqj)ShtPcwAH4X>XgGHAQI(j+~SrC=dbBtV-ro0co%FtE$ldc{F zL*vBc)pBuOZ{m`df5C`u!@)^M#Kwu?g4X~}YCRz18*Q9SKpb{c!%1fNJVEGd z1|-!(4>M^JI-#Q(5H9ZuX%Zleb_Qg8qm7dZ=mdS;1;GNiz=%wO6Ydz5L5y1A3?!Fz z)V>k12twLF4~t6+Ffq5F3gWQKnPfPbfH?C}$AF}I*bhxIAm*9Z08Y5!Wk9ft;;AtS zkOmLP_(mHi6A-7-1Ckj&PmrVsL=Y_sX%gZr^ni?Sj219lKbe3yO&*ZU@OgqP@_-1! zWFbvLoW(AP^4PzgaJ`ctq@5W0K#yJGf-q2@CnwD=h~ngDzbNHIDqLEC$1Zh2$|N&v z>%|u9T*5*XdCLW;tPNF7C(jJqI=Mx%!#>H{0aD>g_UyCIx||rvyfh=3mu4jM z(u`!e+_;OEm%^{w4lqzq?U5r#+zN&PFU>IEr5OgiG?#(#$6UOEYHx9Ad8RsiTImE+ z5jQT-aZC!3Vmb0ZJ}GPtGJ)}^3ruMTp_l#ijna;Zy<)#A{ByaK0@3{^H((197WMe` z*$oh_PjcclRDf^ycv>y)$)?v@TJ^Kur$F>HG&5|Qw^=%fpY?v1b+%|})n9=d`~_K| zUcZL@6;>8i$V%2v`p~LO~9ok?!0UKQ2a9bCq!(NLH+e38NOH`%9dcO|IpogJn_ z8&_>1IxKKC$Ilg^{9G;bab>E{!`^b0I2-Xsa(d5__n^Jx-%cui^~|HmQbjmA;czB z5<&%+*RWO6jM3FuhrulaZsy})hE9$%FIOfyj}#7kOT$!a3plMIsx=WKj(1h~XyY0T zv*+u3QTFnwtc6W-G+mtQ5Lmol8dW$SjQD-V(ZMR9dsq90h6%6UY`^ znnMIK-MAWi^j73dsz)v zjo}yaUxDiTlw4!zwnDjKUzi)(CY0SFZpZ}+ zyK5Z%U-hd|2t1}7(dScTan-L(zOELPi?UO|)dE=|)5N(B`a1wE8s92tu+yi(s@yaQ zDs7-wW-9?*kBUWI z(2ja&Ecrm}dK~#mxIBm>f!{SS)^26%t+4qab$1n%!q9Lt7BStj1$Vcoz*H!~`PT?k zauAzZV%|-#Eh8t!CiyHp0niLZW2$kIuRt*d zW_*XVPPLYSa)ywhYi&g~i(Rf7+zXCI6<_=IL(@r{F3oc@E82T^6@AuQ>8Y5R33t zCSN(^_Yj=8qB#~Ct>-QGzVZoyb(!>&gD2M2PZ4E#^Hhc>dWg=8;)$01as;6cSE9T* zz&z?8j+VeD_;nyjT$ViZ(a5LbDa|~3VH|T+ zjTTX8JImkM`}aSsQ)m(J6qnm8Pygs^U;lzai-0d^0X2Su`VC;N1-)aFtclE&9}ry9 zI5NCJx)0&(K95Z@Xkh{BmuW??aojnNfqZY(NUUB)JcYp=H;u|J)BQ^$UO0a$G{9i$ zR7m{s$Sdk?5Z&Tm@ruaiJD(g(V^8l09PeblSN^B3G{6}gZ1*F&o@@MQ@ji$z)809% zkaFzI>oYHpq9e=A$34rr&!(K=+vJ;PA0D%d+A+`htKrA+;cv&#SsXb3h%=a1%Bi&? zcj0Xfhu`8vfC@FYDW*eAIdk_n&c3^O&y4Tl1E{v8&;6D!O12<_j>h~n4*J`bI5?Zx zj)U#(!ND((4&s=JBZ~tmdnGC{@;!}+P5BvW3x7(`H0E<9kj5di-}ufWUpeil``4pu z|MJz-4(si}!PyzV-5ce|tq_$)hw68(7Y z!>2B$L8Jbm zQe~4CT=~@&ZN))3|38Xs8u!Gc?wQYIu7;NvK;wYP#QL-`ZOD4J;oz`)FAk*a^EfFU zab?>ZFqhhfE6Wh?GqM$y*`hR=!Q6?OT!-tZnp19rh7O=qfhXFP?J@D5ev<+hJ@|Ft zE(L4y=u+2|(mU4?hIib4Did`^J5cYm1MSCt#Xbu6QvdB@ucxiqH`y;aedB+==pP9j z)HU_UDLCmD(kDj)T&HkDEr}(KUhf@(6(^qRI2q3;`jP*Cgf=98)RxW%itEt&0cK8^~UX~gK`<-Eq6*Mm+)x`-e~InLmnntyPL{F zDo32-9s<{j+)Ud5D-0{0kYX0(x~%Qol9!)-ccSfctj&^Xq;kl;57Cu8=oSCu_*sGI zVRS#y_PH$LSbib>_ccH>gvEdQiY@-rhPL=W3v}+haU9@xr~GV`_dMq3@874RTuA>D G8u&MOmzt*l literal 0 HcmV?d00001 diff --git a/geonode/upload/tests/fixture/valid_excel.xlsx b/geonode/upload/tests/fixture/valid_excel.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..38c4918b526f959f3a422cbd6df241f949f9f9bb GIT binary patch literal 7982 zcma)B1yqz<*QP^2x|Ie=>F$(nq(K@a2Zru$r9-+wLCFE>knZjj5TqMqfPc{Mdhg}C z_gnv2Yvw&`X2*G-9p{|AmE~aJ9zorYP{lQo`{y4688R`oH&J%9cW`1>hLGVven9$3 z7QyWJ_XP|u<@%#ZwFG!Q^)_J6i4)z`n8(7sy5Zt5s?l3(eKs?|)dxTL?C<0$~f{YW^EEVY^R`qCorg7;O0)4IxrBfb` z*h`7Dmme!=A4#q1jo8BLnG6G9Q2EgWLcP>|5p#tAb6a;In*kuN?=GJ|p*av`O*;3| zj+*xahuDxcpFTS8ak-xv!!YH|e zatUpHD*krC#y?A|!R zaig*wEI1(t@XI;t%?fk%bUA|}W}1)T7$RKBd3OSAiQ?+)?Co_L_*4$rz;{=dj&a7$ zmT8vgQzeSYvi&8=o{d)E01csRcAo&;MUQ` zROhuPSbl5DA>gkFOlqLw03y|9*nUQTp=;97$^Qx3B8U-yk}0>(OX63N8g3~j%St&I zYAfZ3LqKFH>KQst)w!&X4s!N=9M!8`rEzQ7o~T@d2o#J_G<`r$Q!bW7*6 zC?F868J$StLJd`6B#h?jgZ=We2ATcaQmz$Se_urL>2?D&`_{1vR#Fr?DO-9*2XwYa z6sL06361Nn7n4L4QKtPV`2ZLF0$zPaI&yn>p_)Q>F4+0rT|;BM_z(2@BaB7vE>Z%} zGnU{J_Hw1aqaE}yT;VYU?S04x`)_DFI|1!&@9{2(RZ0oYKKou$rUvO;r`?ZRmB3DrVr#G!$9hEp8p_@ zkafXd9nxAaRWq2K*t0^xxUfF0xt%wf$MO*$&n>8uP=Jq{hCtT`r?SGIltFmAIyU;G zjQ7=|vmk!rW;{-xkyH=bX1udb?hy*&aHFY^*${oNr(u@VVLi z%ryT@@c0*jtG%POlckv%@L^Uu9@7sXgh)Y0gn}aYFWR3htUt4phJxBW3r3S`;hkt6 zY*Cltl>;h(Ag*;JHK`ZMb~vI*hGQm)p_lb=ry(h zT@Q!#?CD!9TCCV(WBhT$3(Y_}YS}6Zvs7BLLZ$4jEAURTHq(0G9IRykYH-B1!Oro7 zvqoiTI^Bx|WwJ_KHVY!sjUkN!xTJUa0#2~QX?46`;cHNfj@h3WH*6r6N~mrqTgp+8 zn-HCS5M-W=-DADMF{IayZX!7YQhj6`JHE73vrn43Pe=^C}IW4c528Xd0!hT} zB)!qRYNrXXc~(=fRJNn=4`Cn2kQ0q(w}j-gKIJsViZwD5h)@4&!IDd5@per*EVZORs3a)Pxx$FFVb?eh&q>9E0q5J1|46H}7o~}RMHv_%XQ87z7Zv+0XuJZysaH=M zi19eXlC2Z^Jf6AXOFZ7r_7k6CCre;6VK0n{ZibPZh4WmL(U$xT?W0_w`pT_VbQlX9 z#lR!o9Q{qJH0xj;Ze9EY8${vMg(I<8R8Vmtp}dK-)t!0?-q^SG}rRxZ6MNn)>i>k9K#9Xbof ztmPX@jupg1nrBTet6DE@G>%gq3xXL|gvCzss{$rVGa2%Xg|MwWY~hBKvww$b12zr9)+zw?E z%@gHiz!?k86AFt#F_tqd}1j=ApyhJ<}m+9q~OvY|gFdpx3>C00}F zP`0o{xKW-|U(-{egGTVxAl!Vkq^x=9fFYTu3>8lmL#BMLt5O~ncctwzsG>_`i$jlJItCqjcO^baNol&9II>IP(I=ny8w1bNdrjVgi0g#L@$-u>ryDLE2$;JJb&eQ<8+@%ucXEsKw>jdc zU%)(0oFjWB`Z%PwHT|`YNC;ZJGHP97C>{K8b1vfL5tfbUcXO`smaI-oK1I0rgO#K2 za5si1D}ChklVn3ZJj?H}iKXRf7Q&BG((#pnxlp)yDP(BR>fhp`H+UO^K4S<+SLW@BPGsF+bUrAF_h8<&h_t}A=|wiI*&G!hIwd(J*_=O?9)|*q!nlU|CDEoKNq(26uhBhia6RwU z_Mm1BlOAQoh{dt^c9u4~I;4tkPcj{T#M>9iBUe=DZI>#Q@VwG2eMxE1cLBiydz`k9 zML$F)NlNX6j(&hU*%#a)sUk!@8X8JVk48YjRJbXwZD)Xv(78sg6WZwx3Rk@}Pu$ok z4;lyiDBB#<>fr@hW}rUlCqkPtN`IG(zwPq0w7*@;biO+MU1pUn{{}EzHaFrR z%Kama`?~w$`&agyW+PHP+pi?{rABx!}c_-EyR;vGYxgDN2 zLP4gIsy`ZhYxf$#AO0>NyLUw$CPMdl2cKdUm{9oXC#=@TS~mIO;#{s;Otq?!Rk@mD z=9@q0Gm0hASzi~cMLB!yzXyoi>bT&B!ozl9y3^effp(dL3E507R$|&XE6YrB4e<5S zlZorv+<+2qx7`4+Trf?8aignSa~n+1oga>zHKk6!#XNp%B_Uahd^8kbzWg$#n8ow) zwUmK%%6s67Wxrv=gjlQV+f+RCffQ`Np7j(_Z;=+iTgnmMF^;Nk*IO?R0oBq`ihMsl z8bvJa%KGOeHYDOZRUgf10>Lxhe(#k5ajc2FcQh~SCL?W??xY<=tsO+SA4<%R*sH}h-(H$^V zw8M&P3SV00vscFlCu6|Q@!2W?0?3Y|aHC~qW}BAWj;8Pc_7nd1sF+?YV+E(=*d*2m zm0rkb}_X4o0}ZO{<=c^ZeuSo}NuvoW$fX>(fO z%JNG-zezfBL0d?0raI^7bF1zWGbej1knL_VOyntUA%UW(9K7Y4Qah68mCeLd2>(K& zrd5zEX@$l{T69pMY%vB3OtCN)vqr!-^yK&jmbThcOiA{qn4;p4r`l6dXgG&5Xm>uN znF;C20g$WA1bu=vqv@tUr$9ZC2hL1fqALh$cA_9GH&qVPzbmF*OAJRrpFX{qn}|pm zI~~y~bH_bKB5zytop=2)5T(}>le!l%PU)^Dj&OmJ4h#s^SsK}R#-u)`@hr4lxffl# z-uxK`VndiTLn`*Ha3OoURfV0>$$Ah+ArK+ zN=3%)OO=c4Dldp}(Ip%O0Hg3lFe}b?zrL=G$=@3a24){_6~*+#J%4gxAdOf$r$k=S zU^k_W0qsOonQ*@2$ zyfV;C>AOcNBcc^Nt@_4O>{5Hi#}WG(fv?iY_hk-LpY6nyc z_(7*bW*+(%l7_ zXaQ>MTD3}OHOk8`^maf-jxRar6;~wdV9$eiHeRZ%!pM~o$ue36y@i5 z8?U>TT`Wn?vwc#<)NdE%m_zC&`Bdlcjgf2Q6e`uOU55j5P!DRY{`i@6UaSjxF;1ly zDY!i3n2RCyCB4uLe~2^lowu$Xr~pR^9$`kZ!?bzO>GCY*nM^)~II*P`HL~|yTB*8Y zjkzx(e=yZET=ZRT>hDB1=@h}178Shg?b3dyA$vu5|d?A+wc(dYxLS zGE6V6FbLli>s-CDRje`OTe_QQrAkGv^EsGY$4iw)NvRXblnISA9fO)n5ckwyH~g%thwTVD5VEr=^1j z9;CiQvV31AO5ZHUc^@%Ck(Dmtl@AjOMca}n4c;vcblsjQtfVWH&aMwQ<*lX)r|V~- zHC^rg;5%gMes_nC^Gvw#^vCFV=1uk(pTW9zY+2t4yiM)!%MWgtf_F01zAKTW8UdmP z_{a;1PG+AEN#9$mVP~)$h3g`aab0K8v(m1=71X~a{d>uTy0MyM6LOM^gOp6}ONx*S zgOjC^qnW81(9z1y;^)E6HbztqjukcdAmA?Nbl>Haj6UNzPq`#gI-<6bs1QZ+SCzRtO9 zUOD`bv9VQJM^K7sF{pCG14lS*025vrq}DQW$&A#~+@(D?x$_ZPhMwvS>CAS}C8odq zwVkx2D*C4m4`V}B5mnEac(Z+LR7w&jC6U*f{$8erw?y*SCD#ZKa1pl`|nuYq&z~OTCy6e&Qazp34$6$dGD4`MQl&>!AAll*SB1y zsQ!ps@w(E&+ycKp@8vFbM)DL63rRuii26(EDuxwEx$!$f3i<-l$I>>K=8=xd#zrmR z)L!k>J|Q%Lp$N7ZzY>-qXYps!Sv)j_*E<~O>h=0-x}kcP2&|TgrdA#qme0{pk(@3_ zQFW&)+dXxka=c^l<53rg@mlVg68~IW&^}m>`PKG%fOA`j{#C^FA(^R&N^6kp{Oade zSf|kQeKK4H`)5`NGFMgWb2oH0t@LcF>^kMKGgl{HwJH^fj2h7C#MDH%(l$(YM?A$o zTf?|#Y!JpGG<%01qx6h(IMTF`c~&9Sza2PDYm|cR1|||{QYATZCnwY;5?mk^+2d+X z&Q%cn2pzY|k9tzHleqlvms$h}$cBz)HV^Aao;)s%caZ;`_6rL} z*L`7ovsu9v7jqWKOtGtA>g|^ze>Xyd9jCq5+$5^PI8l%IHuY*VZQ0ATfq0S3fVJ>V zV}P?tfQWsMOS8(Mmz&mNfOKioB&@r-Lpxl<_haMhpukOmgt_NbvHD)BBW$7kbn#ZB z+v>_9S^S)mMZCf$l!07LcFJ~r$5LHcB+3c`olEp#o(%{9p5VqSd;9l0Lo0%*dE&$D z4D0H#8O}B_xKV*#c{42-XOajAyu*&pLj4D>vZ1AA=1gdkdci4b5epl7Eda*UZaqcP z2Z|!Zdqq^1o(tJVGxjT1MY~5ehliYYz_OwT1T476=x7UrTi)*q60dGS-Ow=&cT@N>; zThYL&Fy*xDa`4>|BY!)ucbo@bq%LabJJh0-Hxp#X0|2^M8#z9Ff~HHr#)E5a@sW;` z1tj><;D1$7bj9uMfM#|;Lv?otGbjD~1W=aHuL#466LLY*+>x)#uI6AAW%O#*tb}R} zcH+F1<1{HV;P5ZqiorU9T4G{pvefWQ3tG{mV6oJkG(Pz_ z6E`JDt=$cgQx~>!^zg@P7H!1VfcHxn>{I-@*&rsk>VSE>NfmjZTppwf5Y7*ROVNfIGKk;7CU z0hNj*`csk9pb2N*naYu_I$l1{1N|^wgNd(AW=AJV(%zK6>~pY@fIG{i$dKNZ@5T_N zwSkJ`oXkAmFMbl)bQU{>aJwjiCn%~2Eb;qh-{J(Ny$N^N-+fhdaWYiiQB8N&_h@fu z+C8uuCAeaLEqK;c=x^WuQATq zdz>?4?QhoFb1F!KL%@MNtsuD#fv4+V3l?}|WMimcZ)5B5S^-Fg4*UV;Cs`=Fn}jte z2uLs(2nfpG$@Fb)>0PZX(_=fOx&=@Ik9?0%Fe^qV7-a#mc`zh|aG0eVWtMCJXYvKa z=B4plIvpOEwhxqRVx4`qfcFI(153j{_#Zu)EqRvSy?UMJ39j!Iln-a_p%dd>;g$f(ujXPj}iMAbZgxxDqM=5>;aSrMr5RO{< zs%-<}$nVfUXJtw8FxbnlOZ5Qu@3x_IyY}P*+C~Ft8`Qts#?Z#z_{lf{u@h3=4CsMJ zkZakSZL*72Z-kaU43^`=$e|wz+0R0zvXI%WtghDX?}>B{9Y22iIwTpoCCEBKSPCek zS4=@c|FB&a-9$ez^bTV!r{Apr(n1rFoXb~uXz#XUJ@9n43t)w){<>52Lmt6afe7~z zV$b{u0o8Ww_&D)$y(;6&$ok&E6J%BJQJ8@$urY`;gsYBu84(r|$js>NDK>BcTLeU^ z@zpOZk|!>6es2!PJ`gvW((Fc2ESpWY8o_(R6<&4@6wf=oog9IRSkK&OxcGWzJEOkm zt%k2Kb5y85TG36@@Z$4IRSBZ2*7};@T3u!01D?w6cv57*x*yvXW)d_iB&z!)A7w?& zUC;>J8=QgDq&z$48c8h27-gYVcu(tyvl$4L+=cefbQEV99F&e3x+r7s%^7~&;FLTI z1po37StKCXhrkcozk%)K;Ams@1b2Rnyu3dHO3x96cTmCABtL`;afg^5J0l^B-Ikox z0(tf;GkB@+wH51GR`=tahUDSRtJPFV_6mED8ix8g8b1~y6qp{TrH{f;nOEGkfgMd^ zbt75v{p%!j%bRm*ySWp&4CPpuuKv|HJY4J)*jnD`)m0x}((>=t#zdTzaq6u&@nXeq z1JDQc#rhGq0Zy7ZC-Bf?Ek@2hqV*}6%M0%9;j7je-d$MZl5-6>o*#3nOGsW2*GEW; zz`jzz@1+%)^{@1eZ@zZEL+W(Ze1}pbcDv(Ze4tSy9eC2i1t`4s@S3>+Bdff`fYRzx@FVt40-~4RbN`uWDt(*6ydb|;`lAYpf(;dh- zf%Gunxh2R4J0B;OToE%>tWtDVx*aqs_XMwdh9iGZ1#S@8=xkN^rl}^*6R-HAEQ3Kb z#OYv5|I(!1{N>w{m<~8Rm3VJ-h(7NgS6PDYkb5kos)b{GRT3N8qB_zT1*VAGnMn4% zj!X4>G+knipgO{H1hR7c$+Lhd_FI|^>Fm#99*sqwlm}Mv#hHK->lt?sHXG#3jy+(Lf=>p}jn>exs>ETYxjNeF?uT!JmqS-)zdq z8&jz*EHh!v>utX?y`FAb-{$wi9uYDUj{F_OtPl@&-)mDiga#*uaJo59WNE4) z$X1w%w-5D_!`ubOclg~`!fmwNZ0xbLl1GUoeL)@ zPh*o_Ozg$Z9b;T7kThjYRUt}7AogtLd>(eC#OSkn?~UP6J9fxvu{p9qmL_(Xu?VLf zSJv&wmMT4gn$`q6yU0*O*#*G%h{gNLsi~OKb-@R}{ey65G_xv<7&EeJ3ni>~-aJ^g zcYX^c^)e|6*CSi~8@EuQ>IN+i*3QyUln0mA;-shy`0L~z!CM;pt?^u=jr=}l%XN-x zdkN2FL(=r?S8Eeyn$2T6b(>&b9_f(uSHAKzv>&n%m734Oox*00o{adWUo7fJx&dhycaBBaV3BQC(H8Xm_!=TD zAG}ixhFAS7QR=cw)>0r#DX3fD<9c%X#=LZ7Up_B6RxPTMbT2D;nJ+DUd|PMbkyRa= z(I;h>ogd;^K8@xaqYm$VAn(OZGmz<$Ms^#kX{sBkLM><+?mm}nJDMG3dN&AVu72gr z?RVMfFDp`ixWTNq8ps1L)5TusyIJ^+?YfWhf#YHVfvECn8~Tu0IaYfUjQ?4DBKZ?d1nKO=4?UN&lx0eBpUzkNxv)+{L@ z_#TFKFvwlK7`na{IRp)%9&KTD<@AW#8H~ffB+-0kdo%oH``MQ&UXOcf*JG*pckwck z_b4&%2TKQ&>6A|&+|JBoz^nweJ7^`9?Q>FiM)|#yI;X!EXMvU{y2DW~;3>;@E0~x< zJt)kmY-lS|gF^Yd!P$meRZu&&MG?(a00EFi6DeG3DOQHZ-e@?FC~DzaA>ycM(MoAu zh8&<@U~cx!8)(KildE;o9><9sl78{T?5c(QwCjL(Ri_QV+1HtQj^81h**xI@VuVvL zc|{O~B^g)S##@AQS(j_XqT%Q!&%fK#A`yEL(dLKyFvja0e*SdlJ^wWkFu)Ybb;4cr zMcn0D5Wj-E#qTn0z%Z&F?-Sn?R$kHdT=TUp_Um3<%@f*yW*@Vc`#D8xq`-lWG;>XXK*S~mgirB7)R1FsIndW9FDwQ3-my)!XZDzK$w2^))=%~z9(3W? z2Z@_zh(`N4)>6<4 zZdqU|QdU}@-B6fzld7eY1ZnyF@ip_Hu`aEZYSz^V?}oiOn<|6w8;5LLGwG8@YQw6g zo@`M;=C6rEHS_qXx|aSFC(Sk1H@X3ga%0L%fz6YbSuZUbzAU4i$g^%+eXds%#IO8gfbPdA1pN-7JkAo>KXYwuepb zJjP?YQaiZ8qPffT=azdtwo%LP{)bXiyApts_@ulI?{8^$zF}8Uc6nZi)jTqQM(Yf`CQP!Ajsz4}4i6O#s|zx7r(N>1-f5J`OQh@`}ed#4vD(DH5lRKNQfHTyhoL++hs(`Q;k~!PMByFz+DmhY!VX z*x>J-CR+4=0AgWK&ck z>tv4UI33(#9>wqW#NTuJr7T5(U*6HcKLn>~@?L8o9UHhTAaSxA)P8?Nk_%Rqk%|>F zD=@xtaacJ+xISxPi%^E8vTj7MOUhdyY;Oj?hXdhRpRn#8G;+r`na#phE<v1*jQmUK|3hs+TvxFU#E1DkH_jO(R`w--2`2s=WKbopZHm>luG@+97 z7L#^t4T4UBof82kVL3WV2n*3?r2P%LlP2Oc=Lj_C-^rO1;O@H;I=&m{q&N%N@AUaA zk~Njq%eu4KZl4Iv0#@cWg}N9^n7 z%i(nO!(o;pbAr>?`#D>kg@$iAE4DU!wN}#t6TiJmJPh#+W?+HD8XX?6WOE=`%G2Zg zfSjE`msnV}Ka=FiwcYb^HZL?{YBB!oXwl0zLPD@m7Br1tO0YG_ZLub`kW$>?wRK=} zqz)}X=(Y~ctKpf8Tptay*3^I{r-W_d2pXj|^?maE@p8+c0JWqWGA|MTWu2Cg7mjQ9 zccb{NS;?bN&QaUt9z!I^^o8kBV3*BoYr*(j{r>s~Z&S+j1tq>(Fw%!pEff!hQO%=A zBUc~)_~41Pn_v`FOErm&G@I>{N#P9GU{8DsO`r|dNHmX<=ZjfQh;7S>~n?$GbH`Gm3* zj$uRrkvuCxr#g^|=gFH>?KUD%E)sA)%UN}n>RXi3MMiJZ@w-IG`Hgl{9zZe)NJyjy z)A|G7xXI+-ELokO+hMIiCX3Jgphk9g7NnKFxTAqP0&rwG0Juo+lj%d6j+T(XTW4~$=}x<$MR^g((_Kqu z-vG(T;zrH?+dw z2PMw-eF?86x`HIi?;Es@ghQpBA2Fvov7&9NUj!ansuB(Rac=f9og7z7X92#YtCn6> zwe6Ch6UQmLr75=Q3zGDj$+#C(g11l#8P&)}Io2sg;v8#42GCO=NFkBA%=j`zCP7x+ zpJBveH}i;%kpMgc9fdY|id00Uow9`UXDY2`O88Q!`Kv6!t3WOZ64-EX*b;Nk0zq9I>4w!X@ciAK8Ti`yML^bq5@{$A=|;s|W0GFs zjl$Z@6eM`GEe(0foQ!&kz@zx#c#Z<^66TE8|Mfn!GRR+b`CEqwEOp5zRP1O+< zbqXdkNS*kv5a}?YXT2;qogDefc8b<6QA2sa=8Dc2$n&ANZ%}~V4K7QVzFUQ%U+qNZ z>-^jg{d?sdN?~D1|7=hh_0dXW)9`rjW<>b;VJ)TU%Ix7la>uIh$TWsH)EjnwX&9kO zf&ou3IIkjhxG+~U^!&%b$#XOwoHS8|0x7ov1y2bOD3(U0 zROzjrFskiRJyS7^U>ee$(3Bz{WW{@7vDE}`sQV=n<@%{|MRps#?gn7885J&Evz0Zk zD`!#hd#_gg5;*k0eWOT|SCHE^+E{`MS z(C#-0+q(rEx}DZ{jy14qoj>s6mt(Zrlwon*62)!RdgHgce9A{hoX-71Pisup}o zK2u>Gxi{VO|9dNnyrq_K8+c}m1$IZCnuoww0|!%mdt)PIM|(4Clb=U8t7t)K2u6f} zBj3mD^F!xzBARr1j!F+SL0(q`E9Zjw0z&}m!<92C>O0=Ht1aHx`2!QVJ#gASq1Lu# zM@{0J7bFp_-fpXJmrZ-X(1|d*bbN^S%eeV#hI2F%KFtC8JxvhqV405#h9x5KrACNk zXD5!eQNiScV*!nOO%?#+Sk*QRllR9C(fcX5axG` ztEfuU(5?d;i+fN>$jdP0rJZ1H3evuYVl<66GILKirAI`7akzSkpfy+B<)KBulEUzwLxm^WbFF_?xUnd| zYorqOtJU2w>n8YU%X z&B~bho3pR#)pEG{&B)Y3$^vYuTSniU%?dGV5rS3R$<=;O_}9|;}`D7ul@Jl5)=?*U3+88 z=Pe@-4(FCf;D2jCL%{#0p?{(*iyf2t%z)DS)WY6wlXbyBU2uF&@?F-*>wS{U<2VIc ztj0lGtDqv?bQ5$`^38VYnx|1S-czb6h-~q7QuGzD>342ZJoa=|U-6YFZJhykQ?c!W zXx=+Dxbyeh=80RRCyROOsWi?M#7zw_o7h!R5Xj_a6)oiCHzf6AYqeIe9y}H6%_LNi z<>^_a3HE4)vf~J7xv{Y+*&ki!P0ke_W2W6yiAi^|jK&D}^UR%ZN4XG%g5n&rcj6m5 za*+xuEqnJ0QB*r1NjY?ROS|2UF1b%z?qw)>*2s;()YP}G&X@_U+93%pfAXCDM>3|q z&zlKue5|?>_NT8ty8L%6|J4fkCpfo`bFdG9Fg5|<#06e~@&1eK@810vy5`*s2)##s zt$q_;c9Ri|V(B`)S#xC`AcnL)rUuZ`TB!}0{n;8>j)~d`nf~oZltW7^jYreS6y}Bu z3Pzk%2=upX(@u6(3k$pT+HULF3vJPfrh^GE_z@w0^-HvbZLA%QtsQk$+-!{<-aaLO zvbZ5RFh=yiD~h)6JS}ErTm5i-y)VYaWE5|Mxk9sQfmcr)-#*@?<)&|yURO%iL}bI~=~SVUp~+A?_xMWY zE2zq2jagrs5`q$*KoX$r={p%{J@&&U!%D5mO-YXFM87zx-lv6#!b-Jmbo6ZwG@H4$RJ6V_%73jB1mNk63aT1ad}vTAJ%#iGYj=&MF5jmP|mUV z{WqI-2M~>Ih~uHYo5HKJ(aP>x>We|RgVFVGuY?x8N_JN-+G6V4=~~V@&kWh_fk_z@ z4E<+e`Zd*h?!G_|1_1>j0)F!W`^o>^nV-vfZs9+23~@j}{=weA1pN2T+mnD_gEp+c zZ{)e&=h^Iu>DT^`F#pPbKcnwC(rEMRvHust)BNVwdSZzF51gKF_4#b=iTBr58GyQfZ`JVkr0stxf2vx) d)|2w5^;>{~G~`e8P=Vu8XyC5yo=O+U{{gQOgJJ*x literal 0 HcmV?d00001 diff --git a/geonode/upload/tests/fixture/valid_with_empty_rows.xlsx b/geonode/upload/tests/fixture/valid_with_empty_rows.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..a6979d5e7eedc28daec72659a992a5ac8ebeb9d1 GIT binary patch literal 7958 zcma)B1yodP*QPtAyO9zB=?>`*=?>`}8U*Q%Vd(BqI;BfsNC{~WL_k8i1>_&}yWV^G z?)}#Pt~GPcn%VK5XV-i7R*{E=LxH+qVM=SF_t!rbGUUeG(M-j~(FyQW1ww`k`2p!C zStO5_i~}?jR5%P26xLtKOq`rpJ?-tX61wGjMX^E;0uQk8t48VB6+sDwNYvyg_~mOA zcHAI$x_R{0#qsYfhJr{P0gqRtx(A#&k6Cc{nb_ zQW)mYL~&WvIhUN^C_+rjz7){~Jys+TQw)iay+UeyvMJ_fI(aQeer}P1iUci}Kc3)w|B`*#&tn|VSlkMvlDJ&}27%wxIWSiM%?!S=h;=0h{~h1cg) z?m(*#4T~Mqi7?JQkYusDwllZ2i;5dH23#|PqJ89!!4n9 zWBmOP&f))#c5r2cA_fHQeaHvrZ)m##TpjK2@h$=?D+jY-^&Qarg%w{+iokhLbx9fX zu#>{bQ{@6x$JRK@6vRo1h1BP^_*McT z=R%-9w5vs`VKg@hyh_cov_7k~T`*C=R!M;G8C**w$j?JhsON`UTN6OREV5k>j5(>` zGhTKRB1qZ<;SQNdfzdZXZo2tLsE8lh%-#JZ8`E=^=Dj2TvtFDW@Cfn#opqXTy~BS9^ZU` zhJGUZ8Cl#T26f<(3(k}grq%$CT;fZMeiv17g&eVoy7k=@FPim&1sJP9w4%su&0U+# z7N%wK?nD|U25cQv9kv+xX6F}l0T%S7;N`9prKP~51l`cI8?-R(DlZgy;0%1?LV^r9 zP3qQWnAy$PXGpj&MRvxx0#QgPLO#mX_s3;cM_`E>L(Pm56UwZdkI}}*EQ}qNNss3g zoNtYfYbs-b?|RlW1;2iuPxQfp`<_%{OOqI#<`KZ&H9ilzsgz&jLW zj691quoDpiR>y8Pi?pIKF2oEtV#MsUQ&7vBnwEwimUkI!Rhs+$=QlX z=gk!ce*9PV_S7}pTQ0ssJe+q1p+Byv7L$n5^<+7;>-c9;fp?$NKACE+AWOGLk?Ze2 z2cY}bO>g1ws>(!^x{b+SLJNSgW41l5BqnDpl?kg4m7+ z0@@c*>=Mfg_G~#)@~LwryF`;u76eJ$0QN#jy-C^ReG)FPfsHc1DVPsj1_9is_**J>gj^F8BtS7!$tt%+g z&vls`TI5_7_9E~f>G_S1H#W4cE6THuH%$?_xoVcg5K4}{&ru56#xl@7ox_B&BvlMN ztQZ$QDU50m-{V|lES*E}<7oa|M9o*Os<@cyPFY0up;s*mMf&|w59JQ4pq-ZZ7o}UI z`L)l;yz+>F4OlHr{bCM)m7q#ywHlOlA^&b1lBVoWmic26J8>Jy(D1t(>RK{M-TRb=k94=qRvO+3N86@WJ^WSf>pTs}IZ8}LYZh;MQ7)bg~LA}x{ zD(B{HvOon2L>o`3a;~RZ9i4Ec<36gYM{JKyq@~9! zr*|H)hkJ&%ERH}nvA#p7l<2&nz?@UZ)k|4qyRS_q;Vh;ji1_w{kYDua{hi<3mlV(-M1k!Bd#1=An3);U}mg|I7qP+tsfojlK_ux(mjYrq5_UEjVo)`z4{oJbu?!_j35e$U5AG;hRt^aZ_fUB3C1>io9PwV?Q z7R=$@b*gp3nl9LQe0<+_W6V8f_bqr|ZfaWw^d_lgI(&{-Y`7O*P*L}+^!zmba*eS)}T7VUcZlgGCF1YKQMd znE&47TG&cWRntO-NQal1@70g6cGON^>8H2(@7`N|m?J2uv1{I_gmSDPXCpq4zJu-a&*L64NFUv>W(S0bQu*I`R*0Ps(Us;-Zc9Y!%>Rq z8nF!uhb(*mFW{Z@^^WvwzM%BQIN0-RCe+*TOkJT%9gIU$k9iC}9@F}7cNj|HnhJ6W zQr5*57aoA+6ST{dHtra8WCn+3H0RX9dGgLy*c%im!PNGT`jvt*Q;p5(4?7WwJ!;x1bQ z!K$<^<&BEo+)kUvrn>}^@pR-j3Uyhp8Vvxgby;RngMrEAPT!p<^j=Uqh zY&{>z(%v88EVUs!e!82#(OqI@z*n`g-mmv5Gc@Jvla&1k-*7fgc!IH!K|5{$*q>fv^_U%48c8av(jgn8k4moD-Pzom3t+elph>yvS*N9VIn5;#g?5lnqj_dIErBo zHy*S0@JE0z!FIBFSUtleA;wIee-6DLTv$Ou{*6O!@EM)9T>7qaHHHVf6Np|`1@?=4N!9`#UDOcvPNNZ6ciY3qd@lreGHf{*$rkFr0^eVcODDO_BP%rhS;sGDC+D80tL-V_$k zRXIeG1jPz2iys@ptDdH8Ufp3VfRM><>mvZ+9_f`Lk;#lmifNE&#+&bCXykSDZfj#fv;(^Eb@2%QuunIlD>2q{uoqWvgW>P*FcWOjA8JJ~+d&vnkQ?tYkDV2v)g$Vxy2->x3}W9ARI+O^&GQ}7&82x)C`}QkpfDn03!afGlv;+_ z_r6DxN?0$XvcLcd4tA9|7OFB(mUk$9N@RX|IS@NP-*t}}nuZ9USrWnO z+*%`SJw4mD&ls^KjGydv^Ng^7*qGS|1aiFzV`>q-%uw$*rfu5@aSa^9C~#P<^GZaJ zXJ^=Zv3>>%>ZHvMs%~yLi>5QA!RMjS9D%49;362NqGWO zpAZ#Hn;r-6=eZfF6v7!AlnlgZa~1DNL|Jj^8b;yYa2;!`>)*-DM9>&8FOKiKo4^U^ zNJhB@MjZLH+3TUQK@s?BfvC44xnZ!BtLXqYHsvG?L1lWM!bTDti8v|_q zZ@?L16Ga&$90hTnLM~)RPhcg-nxu#8Eb^1WEK0a*WAPTpdZq9NSmZQ!b2Vh=LXu}Y z!3qN1QE&Y{`>BXUY?L!tv~^X&!P9}%$K%Unf(hie+vB<`!EZF+ zs+i-g60u{244q-GDIKcirKmpl7^O5;n+JFGw66Ktp<*LK8IK@)_n-_d){E*6UK+;{ z&Ip~S7}PA|i3<`D8WX^SHdu#OH>i{YWu!tm#;fIEzH6cr%GLP?$A|}lq7l#KB3h(G z9?MX*0O2j5=>q)%o07VsR5(p+`9lm;YaTP~`X>9;FGvc{_i7AD^T@t%O`eqYmz!#! zEbSIcW0N}yUZyl;$IYcEKAs%?!hMX>DO2}&kJ}ShI8-p8Y+y)~)e9w8g0)wJ4Xkyf z@MUiHhslll)?*0?+2A~A1?{nF3#;fPzgBeAxe-0JkJWi2p>(b_(Xp9qiP%>>BJxP$ z)ntRdFetA|c~GJ}t#OO)LMKmg1c@>wRf^@j237EZ%u^cM@W5a)qNmk?MXSxUhFEsy zXE%Kk~g=Dd5yXqu3hVpwYqB9xanU~1<5 z-@IoaIPh+{5&3&-gs!cgd=qkJOMtXS?pueDR|9~RiHn7~x~q$|gXPa7oPE5QJRCb( z$U)#;-s!&kDJ4@DD{r+Aj+l@qn!S7RT(KDl`}V>e8~deD$Hj(F!rY#v(he;1fOvby zlB+J&)g$Vdc0aFA21{0bFu0U>J%;{NyA=W=btCz@$?s-CU|&nLYvgB##UnB?q;jK_ ziZc_3`q(ha;R>ofcaVRf1**3vGMHv`yY+$LitRtc|VmcZB8GmZPe-yl_ObhAvP`sRNN4t=?(xGtHV!h9YM52M-lj%sqG~?zAd7ulG#YY% zvZ#$#fUmjH4`PMuvTK9~xJWn(@j@_xhP*am{=((HrD5vmXnTLEjZqy|>Se=fqdCHy z`D8s=pQplA77R6k`o=S{j>M7pt>5!T`Jt6MB@GlFrR!SDmzD&>1)AS*vQlPn*~p8! z#x*o$>X=s{ZN(jgRLmu$%JOcQj`7~wwl;0onZ2f&eIjT=BT*c20p&+VT&2wv^Z4kB zHoKgd8Z8EEdSUvP2<%o#=GNX>R;=h~NPr6pG`-o{9v?k2&UCidyc&Y>zANAviRRLx zp3!RTFZS0XT-(A-#*x>Dl;)zUUBPmTpPPZOfUxs@N<2l!$JPk4S9R+Pw~TgOOdM*@ zbgO}LS0`VzYn6yiS}_^L)kV28Hq5_``-nH&!gytE5CIW7UPTX2IE zMou%@q+kbJlL)nFlU;aH65k~eULch?;%QAU)DV6To3bv9CN0@bTKV_aR|E*iMlKe1 z5Bo(vyzXsxkpEU7hCuyI!+K9yk?=w8JsVd4eFJ;5L(u~dd*1aa^*2THSFckQ?#AhH z5_I-D+Qn2^KDHpnrCn`itoWL@lH8}7;@Gx8H`PEY=Zv_Hv+1dHXc1mxfah zT2+|>reNX%D=pB#S8bdlOn?z&J+ZB!BAO$>C0W8JVn!3h-R_{`Fmx={pF^&q zDA@OjDcq+O!I?Lt?aI;d&F|^d)ep%%sGDjuHEcK1I9}J^*;My z5|Uz!;+6>CpeJoV_cRy*e?uW$%qYxDA1`!3$q-PSdRlQgntsG0(8KpC!J9uy53Mg9 ztt2(}BjxdkGb7MWo}YlQ{nB~k!3%BakuJaz68z}!zxpM55{?e877ngP8eUEo0E7Dk zP?0#S1jCLSdO_dOTd4O;-N_`{#Q3vC8SNVE$MY`E)8y>H!}DxLV;Hf3=nPr@y{pc< zn|3rQX4^=*=N~^!mm8gFLo0a~EtkKf0HqvfA}`c9DcW9(?RSCd_y)< zF3TwB6k7|;Tzw2pI;!xho_d09%oEFn^<|f%PrCqXyZowJnl3swsbIGTivm-&)~U}I z+CVW)K?nTC^7II-=wxc3dWY-IYc&UCyce5Xb6c-@%!7^!g55Vgx*GRm@*{M`*zwuh zQRKPb9lMae5>*O))A=HH?sTdwS(F%69$RC)v0OAMfR>6LT_pR?T%O{y%Oz_;<9E|F z7*JEnGfd)SolS+yAt$@%aA(=nSu)=g2C&2!?4T03rnArYOG%^J&ww)sKbA%Dg~XIx z%U*wV>;ypRY{DH54_uX8oQzfX)-#?Bq3n&Vetjao@aoO>r?U=VV<$`7N%x5v_YEW| zL&MaqWvks9Pg)uzh;2{ z|Kp+{{bc^5?f7?dJ?sqqsWTn|<^8UI_lX{E_hHZSPgeH7YW%Z<`B1>aLBpQ{+Tngb z?|()Ve~uj=50*w`%r#%=W*O d|2>TEMg10_A`kx)J#5IZ6cMtktNYRg^?yBKX}|yg literal 0 HcmV?d00001 diff --git a/geonode/upload/tests/fixture/wrong_data.csv b/geonode/upload/tests/fixture/wrong_data.csv new file mode 100644 index 00000000000..967a1f66be8 --- /dev/null +++ b/geonode/upload/tests/fixture/wrong_data.csv @@ -0,0 +1 @@ +id,name,latitude,longitude,description diff --git a/geonode/upload/tests/fixture/wrong_data.xlsx b/geonode/upload/tests/fixture/wrong_data.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..36b35f808b9ac425afa2e61e8c62bdde9ee38578 GIT binary patch literal 7997 zcma)h1yqz<_xI2p(%m54A|fCy-Q8Ud4BcHLNH<7HgMbJ)h%^Y&p@1|<2_i7`Q1T7> zUcHz9ec$z+wT5{Xv+McoIOpuGu7re)3AkBc)fI`G@4prX{J|PzrS1lDapzHolM%uH zf&P;$=Dx4IGXenc6cGTx|2vt5iwn1p6EHWqQ?XkDKWZ;*51-`KFbkhbN^&te0}Uog z)k-x`D8-9;2B&#$?3~k76um9?=Z#`WCXtKFw@sPHLER*h#?{r}MUf9rPIC$Izq3Jn8kazkqo2+XXZmEkt&*L#sorY-)EyDW?1SG3eu;%WABtwRF88 zbfb{NwMHOM$Yscf3HCiB!?WwjkeD^%Mo(dnqZD&e&P+_V#M{WGspBs!C{b3IW@0p^ zoSmAwZ(=f~IC!*re0H_7f4IfmE4R1w&dr{BYDjHi#5_jYigL`I>`S6czX!&*N`3r_ zYk(KHgIYRaZq*nB@Ob3j_t^=^ay8KV$c*(0@t?lg-OL9|zM8BlhWk0k9jKEX}}^e#);Wj2bj5{8_b#`Qp)QoQ)>J$?|i zSJUQ$U-k|hH4}uy1Un5-eor30z2 zoY3O&j{AW~Q!uMuxx9i%0!G{?Sp`y{kX=NJ9c*4?n3PdHf9MBuDPLB)M{j9kY%%G3 zfdaDF;Gn1#tbf8CQ4_6#3x|6b{vrGa+@9_pAg3F^OOn*oB6#t8_E>_zrKcYsA$!wz z$eG{gqY-pnS5=&0e!=B{r5L-k;5>ED4_ermHMDlR_*~)sD>pzrFXS_OxF8)qdXMMa zOKH^n(}xXF9q;7c3>T#JEi-V=uYK0rDjF~1tsy1xiKrtN6}itsW)wtJ_d1l8w>G19b+6ruqOxjvkY6i>s#eJr%m7Hh9ySx9#N{ET0F?4omCP#5W#!skZv zfR}}Jl?(**kT@k73@&x5UJjY5h}y8UW|-FnZl{l-Eq=M&`KGtcjzPUr)P9dhN*T;Q za2!@<=%djb><6Wvjc&I528NJqfQ6YYp0eUxen3>zzX-fRZcp6pZEQSlm!-uuWDa7u6g)Hlfb9R!{$%0%S)O!M+@JH}w|bXe zOBPA1;*bG9(DGRmJk0Va9rj6X5F4zW&IS}F^C|=d>UEg|zb3Mw(Qj2RwwY8H&rCi^ zPv(;Ek)=@|-O@nxWwOkBg>=4swi}=v{gE@lfZYv!lakesIOvN1Wki4mB|s&SUJqfx z-bC3%PnNPT(@h590Yh{m`6uDjCwjyL`an{fitSTBQs-VD)hAIEm)Kx=0-8=K85`85 z8U?0e<)WZBUm_i+O;^?Sdy;j?o)ncax@s_T<=+<`&)Ap8WG2{=VQ_!qP^O_e-!GYC z;30J+S;x;}Hn2h!jfL&pqgAs6d>cnM=ATztnNz8*n!2a$L9iBQJa(w*ki?Z~E0{Aw zt*>}Lr-!QE9+D?7kLb6f0=gjjTHOP}P5X8fZTs@BJpD&}s|s~XMRVr&&-gCmwZyue zaIhJl2`+U}eH1Kkr+vBk>NwQIV;Ms8?y7e#mn*x3iI1sD`PjHP*r)3L=A?Vn)yQMY zl*bwiNM<0xS=flZd^D@)fIAGqNd68EtK;Sfy@LdroHCQvSllqiRS2-26gcqY)vzmKqi>SRm z^|VL>7ke(Hf#W^r;AtmP%7)Bm6h<#>)l?(Tbyt#NDI1*D@r*<$vuEZ=nuXt!^nz6s zp74n@yxpmxX`Bun`vI1^(B0HfbsmFw-gDS;d~qD}qmSG_B|12Q+wo}rFwl;d;-b5V zci;SbR_hhXl;XBu&31wL6IlOR_Scx=#DbhW0gC6zsh*)i(p;EBai1v9i%zjkW1npNH55 zE4|>!-_O%nLtsB(h36=MxH1k_+1@(s*Ibv>vHzHE^?AcZNBPW{A4qc%4A%)o0{!-&EOhg{Ey zL(%9YdWYzkWbMPWrD0k#vtB0$G*xDo@k;F19T z*?(C6<*_$RlIqpNmhh9DeyaosavTjMqHx}+W4}!Uv!RT(6{5lR34oH{Y=u6x=l9r)s3NEAE$@Fr zzB0pF9-x0WNjK6byp&B?rZ$@efz^z``e~}?X21xR{pNb-GqXrO)lsdvsOFC+1+-5ZzswWvtKHjhdJoYX zMH>-Q-bFdif9L4kai#wy_pmF@*>Shv>{fWSrUn+&!Nc#FA^?E%E4$52_%5B4J z-|;T2)xP}AgrLgyw#yX|sly4``5tY+qfg5u8Y1Jgb-=62&q`nMehes$uD+q|QXoa2hJo^C#wauMCGAbM+cVVk4vmn zkVK!^ar1n> z`~Is18Se%0OZNq7=Mg6Jx5GC2C>lB6q0L2?RmopSLD6jxu@m_uDZN9;~+o)=#lUwT_- ztV0@%2dQxqhB&}z_1^_v)OOmGryu9A%zxNJudGPCCK5tb+Z%S!=wSs%3WVqQ0QJ{e zR7p&m?&Ny|EBF{_h4T3zO!&CaUUwn+d_z7vAu3TVDQj7IOiyL}s6y*EoK?W(q4l2G zLHIiUCTTCVF~d=EO`Ah(B9mIoT!W^D@zVO zCOFVN!4_k0A$L_|EX2V(9d1DE538@Oj8MCm_APuFgm{Cw3qb%!k*R6&>;k82Yi`-H zJZ$lzBCxr*O9!xpx_RRgjrm%4*&hg7a}?ov`GEZ_Uns!sYDO7G)GuGql*X4Wqs3v* z;_NINOfbyVsSIm(x@hdmCqi%~MOm`0ufB)ANrUCOK{H+sh^4aVtx8~) z@_&L!Al9l_WFxWVRjx8D&PTS@>ebjZ#gu=nOS07)l_Q8OjT2Ox)L9?(Q=w$_ItJ3I z+s{b2mdE3BGZmzo_iM2yphPke`zvM8vPUJm;#O8f2Mb4Qhbr2jNx5cLV1!C_qLP5W zu|B7W7q)oe*Ar_w7GX=djU=Jlt^GMCiURh1(8IE$&Qe+JWfz|K7 z3;yCiRCa9`gY9gExx}r&L;3DUY@4Iq93(* z+d|du<$Igf;NI}u+`6O^(KgKR3;Xr@Hww9gZAvRc`F6njQnc?+Vw7A`q)3KdTZy7! z<~Q0k4qi!7OtX#Z)JqIID=H{Ebe-;EVQ9T^##Fe%d5$TL#HKRsKCWV9>CJ zTHBB%aKxvRPkoSCU(lVs%R)#dib|2f z$90TImWKAbToj}50W9cW6D2uiC+WX#o=Hb%YofsRt9_a9OpYI-NZ>^pt6xrJpDGi@ zG*PPIRFzBlxwq-58txL(V=Ut>RzlZSynN2?yCwMihVrT}GxpaM5+1QnPb>?E5h}#& z)CEdF5LYkQ`{R96<&`m&`g@)3;x<@gMTf6xusMYY-}60s+kGc1F|1J}bzn@N3x@uI z$TQ{5$m25k=VG(G8#`7XAM}=ao*swrF`RBTmL|KNsXbT$X zXt&%4O`KoEMU8$}If5S=aXqUO<(ub_>AzEw9_V01U9~^4o1LWc9}2NlMcZZcK$=Q; zMKH@Qg&f@P(&14~pc${;>l%;W5;3+hx=K=gW!xB(s4t$;Qt%b8M=p(&SQFaS1faK0 z*w)5SDa@sN?;(vo6V=F0KZ(n30sOGm0EcR7;KUaHHhT~DFhD~z-Td`Iep z&Ll1IkbtSmSJ^(**`H7*7>4D9s>(KDdH6LMH$8#E7+^Yrz=G7HDUy&)P&3MAQ>?Fe z*X2slnO~c;%uA(A2=cTLK8(KvQ^1ILolW;M5Rri;DE}^K&VaWpq zngc$efsB}n&T5j~>C@3mX^Z~bpt@pZSttW?jFP^QF7yQrPg7_hdZER2T$tN8E=;3Z z6;v6KlEw-y8LHaj(PV-cN0hQTVMfD#*fJiEZtLYI79$WpeI7P;?X$3cUj5iLU@{+y$B9l(W^=$vLT9PyR@cto{9 zLVfYdsIBhVP>V|&hS)r{PTvQJS(?XKgtp6t9Tk?a1d#@;Baab9l0L7N@;ygM*bWJ1 z<;<`bwO>+3Pk)iuHYhTIUQPX5D1RbW9^z)=ASOm6DDE)*qrxdUy{yrPR7#5lAYdr>h^m?TgU04hlbNr^}Gg_VQQKtAn>nhelZQ#*-_MZE=w9Q z>W%hSWq8`J`-ED<-yFVNM4?@cU9c?7SXx?+@}~jd>w+|=TX6GdSuj}6gF&T?U(b8* zjrLw`!G<80omm2zrujJEXO7NiJOGlvw$bETnFfVf#%{ahA3Ge;JwE(kUh0n8zb+dnVP8MGg z8}Gmmb;Iz^$xUk!es^%Uw{Wwu*79(3aJKt-#&gmbR(#Kk--gvD-M65kCnGb zqmD-dwua!aq-1Bsw6z)_NA8a7Tu?_jc$=|Rs&P{Bd-^pyG)TG4piw-$I_EcDaOgo? zqqpr+4QAZVZEdm7lAYvqExVg^wn*kh$g-@GmsIK2#~iAyFW2oZ2^FPRjM;eS;H(lR zVc67?IMHfUO@c{_brKxm^k7;S2(7*|m(6%?gHBM>gic_+UlvfhU$vZMf>A%w{Mke- znds>^jDZV7M&Z{>f?sW^sp!@tCygKK&>h33UQu6Ai6zg5=NRv+htck5`lD(}Be;c{ z`~dMDQ=g#y6CP%1keDzW8U*&EzlCuhZ)(xuW5Wye zdleCYajZ(8)Otz~#mgWWh|<7Py{b0mebtLPJ6k)_!6M!7gt-}0M7%U59b?+M^7S0c z@b2R`)J(j2^qQ(p#I~{Sx|SAwkExx;sa$?1ns0FimLxi?&c7c0UBb}O^ArMk>oFF>WP_cyR!I%c ziFarTS!Wy1mjbMtDHrHW`AXwk!aOy?BtU&$Z5q%3AN_?e`Ks1QBwuZpF68EK2bLER z;TxiAvFk z%;en;xL$hycP#(YGWa(**P(@EH);P44krct8=m&R$o?MPf1&Hx&5PZ;7v35^9_acp zj!!Pvw71}MwLid$BiP;&P08qaV}9QYg948XW96+R=n!g4_L9e z&xIyX+i>^rcnY36K_Sl&aAM_aIrjD-w9^LZN)ru8Y_1sA1r{`>Z&*H?gV>%Sp>ApO z9jD8d8sbe+JH`2>d>s|JIu3FBQgN~hJ9zTrp#18q7)gViJ#3sk%(Q)7Y}`$53P5%0peiCC zQPe3*TX(UMfR>9ztcCd(n+m2Cq=}Ob!K3uNFz87hn>nInXzX(Zk)5;l>&sSbIgTeW z%rX;;lT~KN`Ut9iB@0!yv?&<}dBm9!T3tS9_l%J0#-Znqyas3;VU^TVRSImPF7b5; z9JL1sRKv=F4GiPFBR=?U+_oJc|JH}xtxC(9S%x_9i&m#Lr!sq<-jV+*Q<$W#s51$q zDmxlKHk~0!tIgwEh^8|JvChW5>CKS)*5Ug_5ndbK9gy9)!e~QDJ`%oGOeLYW2X0h> z5~@*^?K<((N1rOvB`C0z?r4ueswC1vndn(?9_3wIE75*&gK-x@&Mj9EQyMb_@F>y^ zHk4rlE?M{2tp*p3G}p53PhQQPq_Y5Q%>7PrpWA zx5F16!2kpR9sI8qyyg7ggZZ|c+aCTS=P5Y=@Gt)UCE(v1<2UK{Yu!xaME`EcZN0av z*$vw-`-w>Zs((MT?>6J@?fMTR5Wai=+va^+!tEXHkA!E$KPCLV0{s8KE(ZEf=D*sL z|8&>w4$>c;32-QHtp43kx^3?5-sT@x-oI=7)9Jh|;PyD;kAODh-|zdMLB^lq#BC|J z=Xrmmbi)h8?*Z*!)4l(N+szL5Yu$til;kge-&T2hynDm*D@N|ZpMI3l2{xX#xuKTyCR= Date: Thu, 5 Feb 2026 10:43:00 +0200 Subject: [PATCH 10/21] adding a security-check test --- geonode/upload/handlers/xlsx/handler.py | 130 +++++++++++------------- geonode/upload/handlers/xlsx/tests.py | 102 ++++++++++++++----- 2 files changed, 140 insertions(+), 92 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index 269c0b168a2..2d1f9ff2c92 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -34,7 +34,7 @@ from geonode.upload.celery_tasks import create_dynamic_structure from geonode.upload.handlers.utils import GEOM_TYPE_MAPPING from geonode.upload.api.exceptions import ( - UploadParallelismLimitException, + UploadParallelismLimitException, InvalidInputFileException, ) @@ -47,14 +47,14 @@ class XLSXFileHandler(CSVFileHandler): lat_names = CSVFileHandler.possible_lat_column lon_names = CSVFileHandler.possible_long_column - + @property def supported_file_extension_config(self): - + # If disabled, return an empty list or None so the UI doesn't show XLSX options if not XLSXFileHandler.XLSX_UPLOAD_ENABLED: return None - + return { "id": "excel", # Use a generic ID that doesn't imply a specific extension "formats": [ @@ -67,7 +67,7 @@ def supported_file_extension_config(self): "label": "Excel (Binary/Legacy)", "required_ext": ["xls"], "optional_ext": ["sld", "xml"], - } + }, ], "actions": list(self.TASKS.keys()), "type": "vector", @@ -82,11 +82,11 @@ def can_handle(_data) -> bool: # Availability Check for the back-end if not XLSXFileHandler.XLSX_UPLOAD_ENABLED: return False - + base = _data.get("base_file") if not base: return False - + # Support both XLSX and XLS valid_extensions = (".xlsx", ".xls") @@ -95,16 +95,15 @@ def can_handle(_data) -> bool: if isinstance(base, str) else base.name.lower().endswith(valid_extensions) ) - + return is_excel and BaseVectorFileHandler.can_handle(_data) - @staticmethod def is_valid(files, user, **kwargs): from geonode.upload.utils import UploadLimitValidator - + BaseVectorFileHandler.is_valid(files, user) - + upload_validator = UploadLimitValidator(user) upload_validator.validate_parallelism_limit_per_user() actual_upload = upload_validator._get_parallel_uploads_count() @@ -117,7 +116,7 @@ def is_valid(files, user, **kwargs): # In XLSX handler, we always expect 1 layer (the first sheet) layer = datasource.GetLayer(0) if not layer: - raise InvalidInputFileException("No data found in the converted CSV.") + raise InvalidInputFileException("No data found in the converted CSV.") if 1 + actual_upload > max_upload: raise UploadParallelismLimitException( @@ -125,16 +124,20 @@ def is_valid(files, user, **kwargs): ) schema_keys = [x.name.lower() for x in layer.schema] - + # Accessing class-level constants explicitly has_lat = any(x in XLSXFileHandler.lat_names for x in schema_keys) has_long = any(x in XLSXFileHandler.lon_names for x in schema_keys) if has_lat and not has_long: - raise InvalidInputFileException(f"Longitude is missing. Supported names: {', '.join(XLSXFileHandler.lon_names)}") + raise InvalidInputFileException( + f"Longitude is missing. Supported names: {', '.join(XLSXFileHandler.lon_names)}" + ) if not has_lat and has_long: - raise InvalidInputFileException(f"Latitude is missing. Supported names: {', '.join(XLSXFileHandler.lat_names)}") + raise InvalidInputFileException( + f"Latitude is missing. Supported names: {', '.join(XLSXFileHandler.lat_names)}" + ) if not (has_lat and has_long): raise InvalidInputFileException( @@ -144,33 +147,27 @@ def is_valid(files, user, **kwargs): ) return True - + @staticmethod def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate, **kwargs): """ Customized for XLSX: Only looks for X/Y (Point) data. Ignores WKT/Geom columns as per requirements. """ - - base_command = BaseVectorFileHandler.create_ogr2ogr_command( - files, original_name, ovverwrite_layer, alternate - ) - + + base_command = BaseVectorFileHandler.create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate) + # We only define X and Y possible names instead of WKT columns lat_mapping = ",".join(XLSXFileHandler.lat_names) lon_mapping = ",".join(XLSXFileHandler.lon_names) - - additional_option = ( - f' -oo "X_POSSIBLE_NAMES={lon_mapping}" ' - f' -oo "Y_POSSIBLE_NAMES={lat_mapping}"' - ) - + + additional_option = f' -oo "X_POSSIBLE_NAMES={lon_mapping}" ' f' -oo "Y_POSSIBLE_NAMES={lat_mapping}"' + return ( f"{base_command} -oo KEEP_GEOM_COLUMNS=NO " - f"-lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} " - + additional_option + f"-lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} " + additional_option ) - + def create_dynamic_model_fields( self, layer: str, @@ -181,11 +178,8 @@ def create_dynamic_model_fields( return_celery_group: bool = True, ): # retrieving the field schema from ogr2ogr and converting the type to Django Types - layer_schema = [ - {"name": x.name.lower(), "class_name": self._get_type(x), "null": True} - for x in layer.schema - ] - + layer_schema = [{"name": x.name.lower(), "class_name": self._get_type(x), "null": True} for x in layer.schema] + class_name = GEOM_TYPE_MAPPING.get(self.promote_to_multi("Point")) # Get the geometry type name from OGR (e.g., 'Point' or 'Point 25D') geom_type_name = ogr.GeometryTypeToName(layer.GetGeomType()) @@ -208,28 +202,27 @@ def create_dynamic_model_fields( ) return dynamic_model_schema, celery_group - - + def pre_processing(self, files, execution_id, **kwargs): from geonode.upload.orchestrator import orchestrator - + # calling the super function (CSVFileHandler logic) _data, execution_id = super().pre_processing(files, execution_id, **kwargs) - + # convert the XLSX file into a CSV xlsx_file = _data.get("files", {}).get("base_file", "") if not xlsx_file: raise Exception("File not found") - - output_file = str(Path(xlsx_file).with_suffix('.csv')) - + + output_file = str(Path(xlsx_file).with_suffix(".csv")) + try: workbook = CalamineWorkbook.from_path(xlsx_file) # Sheet Validation (Uses the validated sheet name) sheet_name = self._validate_sheets(workbook) sheet = workbook.get_sheet_by_name(sheet_name) - + # We iterate until we find the first non-empty row rows_gen = iter(sheet.to_python()) try: @@ -240,30 +233,29 @@ def pre_processing(self, files, execution_id, **kwargs): # Restrictive File Structure Validation self._validate_headers(headers) - + # Conversion with row cleanup # Note: rows_gen continues from the row after the headers self._convert_to_csv(headers, rows_gen, output_file) except Exception as e: logger.exception("XLSX Pre-processing failed") - raise InvalidInputFileException(detail=f"Failed to securely parse XLSX: {str(e)}") - + raise InvalidInputFileException(detail=f"Failed to securely parse Excel: {str(e)}") + # update the file path in the payload - _data['files']['base_file'] = output_file + _data["files"]["base_file"] = output_file - if 'temporary_files' not in _data or not isinstance(_data['temporary_files'], dict): - _data['temporary_files'] = {} + if "temporary_files" not in _data or not isinstance(_data["temporary_files"], dict): + _data["temporary_files"] = {} + + _data["temporary_files"]["base_file"] = output_file - _data['temporary_files']['base_file'] = output_file - # updating the execution id params orchestrator.update_execution_request_obj( - orchestrator.get_execution_object(execution_id), - {"input_params": _data} + orchestrator.get_execution_object(execution_id), {"input_params": _data} ) return _data, execution_id - + def _validate_sheets(self, workbook): """Returns the first sheet name and logs warnings if others exist.""" sheets = workbook.sheet_names @@ -272,7 +264,7 @@ def _validate_sheets(self, workbook): if len(sheets) > 1: logger.warning(f"Multiple sheets found. Ignoring: {sheets[1:]}") return sheets[0] - + def _validate_headers(self, headers): """ Strictly validates Row 1 for headers: @@ -286,7 +278,7 @@ def _validate_headers(self, headers): # Normalization clean_headers = [str(h).strip().lower() if h is not None else "" for h in headers] - + # Geometry Fingerprint Check has_lat = any(h in self.lat_names for h in clean_headers) has_lon = any(h in self.lon_names for h in clean_headers) @@ -307,7 +299,7 @@ def _validate_headers(self, headers): raise Exception(f"Duplicate headers found in Row 1: {', '.join(duplicates)}") return True - + def _data_sense_check(self, x, y): """ High-speed coordinate validation for large datasets @@ -316,7 +308,7 @@ def _data_sense_check(self, x, y): # Catch Excel Date objects immediately (Calamine returns these as datetime) if isinstance(x, datetime) or isinstance(y, datetime): return False - + f_x = float(x) f_y = float(y) @@ -325,7 +317,7 @@ def _data_sense_check(self, x, y): if not (math.isfinite(f_x) and math.isfinite(f_y)): return False - # Magnitude check + # Magnitude check # Limits to +/- 40 million (covers all CRS including Web Mercator) # but blocks 'serial date numbers' or corrupted scientific notation if not (-40000000 < f_x < 40000000 and -40000000 < f_y < 40000000): @@ -334,36 +326,36 @@ def _data_sense_check(self, x, y): return True except (ValueError, TypeError): return False - + def _detect_empty_rows(self, row): return not row or all(cell is None or str(cell).strip() == "" for cell in row) - + def _convert_to_csv(self, headers, rows_gen, output_path): """Streams valid data to CSV, skipping empty rows.""" # Define clean_headers once here to find the indices clean_headers = [str(h).strip().lower() for h in headers] - + # Get the indices for the Lat and Lon columns lat_idx = next(i for i, h in enumerate(clean_headers) if h in self.lat_names) lon_idx = next(i for i, h in enumerate(clean_headers) if h in self.lon_names) - + # Local binding of the check function for loop speed check_func = self._data_sense_check - - with open(output_path, 'w', newline='', encoding='utf-8') as f: + + with open(output_path, "w", newline="", encoding="utf-8") as f: writer = csv.writer(f) writer.writerow(headers) - + for row_num, row in enumerate(rows_gen, start=2): # Skip row if it contains no data if self._detect_empty_rows(row): continue - + if not check_func(row[lon_idx], row[lat_idx]): raise InvalidInputFileException( detail=f"Coordinate error at row {row_num}. " - "Check for dates or non-numeric values in Lat/Lon." + "Check for dates or non-numeric values in Lat/Lon." ) - writer.writerow(row) \ No newline at end of file + writer.writerow(row) diff --git a/geonode/upload/handlers/xlsx/tests.py b/geonode/upload/handlers/xlsx/tests.py index ba403e7e15a..7c62c86df85 100644 --- a/geonode/upload/handlers/xlsx/tests.py +++ b/geonode/upload/handlers/xlsx/tests.py @@ -1,7 +1,8 @@ import os +import tempfile +import zipfile import uuid -import math -from unittest.mock import patch, MagicMock +from unittest.mock import patch from django.test import TestCase from django.contrib.auth import get_user_model @@ -9,6 +10,7 @@ from geonode.upload.api.exceptions import InvalidInputFileException from geonode.upload.handlers.xlsx.handler import XLSXFileHandler + class TestXLSXHandler(TestCase): databases = ("default", "datastore") @@ -16,7 +18,7 @@ class TestXLSXHandler(TestCase): def setUpClass(cls): super().setUpClass() cls.handler = XLSXFileHandler() - + # Consistent with CSV handler's fixture path cls.valid_xlsx = f"{project_dir}/tests/fixture/valid_excel.xlsx" cls.valid_xls = f"{project_dir}/tests/fixture/valid_excel.xls" @@ -24,7 +26,7 @@ def setUpClass(cls): cls.leading_empty_xlsx = f"{project_dir}/tests/fixture/valid_leading_empty_rows.xlsx" cls.missing_lat_xlsx = f"{project_dir}/tests/fixture/missing_lat.xlsx" cls.wrong_data_xlsx = f"{project_dir}/tests/fixture/wrong_data.xlsx" - + cls.user, _ = get_user_model().objects.get_or_create(username="admin") def setUp(self): @@ -38,7 +40,7 @@ def test_can_handle_xlsx_and_xls(self, mock_base_can_handle): self.assertTrue(self.handler.can_handle({"base_file": self.valid_xlsx})) self.assertTrue(self.handler.can_handle({"base_file": self.valid_xls})) - + # Also verify it returns False when the file is wrong self.assertFalse(self.handler.can_handle({"base_file": "random.txt"})) @@ -46,20 +48,22 @@ def test_can_handle_xlsx_and_xls(self, mock_base_can_handle): @patch("geonode.upload.orchestrator.orchestrator.update_execution_request_obj") def test_pre_processing_success_with_valid_files(self, mock_update, mock_get_exec): test_files = [self.valid_xlsx, self.valid_xls, self.empty_rows_xlsx, self.leading_empty_xlsx] - + for file_path in test_files: exec_id = str(uuid.uuid4()) files = {"base_file": file_path} - - with patch('geonode.upload.handlers.csv.handler.CSVFileHandler.pre_processing', - return_value=({"files": files, "temporary_files": {}}, exec_id)): - + + with patch( + "geonode.upload.handlers.csv.handler.CSVFileHandler.pre_processing", + return_value=({"files": files, "temporary_files": {}}, exec_id), + ): + data, _ = self.handler.pre_processing(files, exec_id) - - output_csv = data['files']['base_file'] + + output_csv = data["files"]["base_file"] self.assertTrue(output_csv.endswith(".csv")) self.assertTrue(os.path.exists(output_csv)) - + # Cleanup if os.path.exists(output_csv): os.remove(output_csv) @@ -69,12 +73,14 @@ def test_pre_processing_fails_on_missing_lat(self, mock_get_exec): """Should fail when header fingerprinting doesn't find Latitude.""" exec_id = str(uuid.uuid4()) files = {"base_file": self.missing_lat_xlsx} - - with patch('geonode.upload.handlers.csv.handler.CSVFileHandler.pre_processing', - return_value=({"files": files}, exec_id)): + + with patch( + "geonode.upload.handlers.csv.handler.CSVFileHandler.pre_processing", + return_value=({"files": files}, exec_id), + ): with self.assertRaises(InvalidInputFileException) as context: self.handler.pre_processing(files, exec_id) - + self.assertIn("geometry headers", str(context.exception)) @patch("geonode.upload.orchestrator.orchestrator.get_execution_object") @@ -82,12 +88,14 @@ def test_pre_processing_fails_on_wrong_data(self, mock_get_exec): """Should fail on row 1 of the data due to 'nan' and extreme magnitude.""" exec_id = str(uuid.uuid4()) files = {"base_file": self.wrong_data_xlsx} - - with patch('geonode.upload.handlers.csv.handler.CSVFileHandler.pre_processing', - return_value=({"files": files}, exec_id)): + + with patch( + "geonode.upload.handlers.csv.handler.CSVFileHandler.pre_processing", + return_value=({"files": files}, exec_id), + ): with self.assertRaises(InvalidInputFileException) as context: self.handler.pre_processing(files, exec_id) - + # The error should specifically mention the coordinate error and the row self.assertIn("Coordinate error at row 2", str(context.exception)) @@ -98,9 +106,57 @@ def test_data_sense_check_logic(self): # NaN self.assertFalse(self.handler._data_sense_check("nan", 40.0)) # Infinite - self.assertFalse(self.handler._data_sense_check(float('inf'), 40.0)) + self.assertFalse(self.handler._data_sense_check(float("inf"), 40.0)) # Extreme Magnitude self.assertFalse(self.handler._data_sense_check(40000001, 10.0)) # Excel Date (as datetime object) from datetime import datetime - self.assertFalse(self.handler._data_sense_check(datetime.now(), 40.0)) \ No newline at end of file + + self.assertFalse(self.handler._data_sense_check(datetime.now(), 40.0)) + + def test_security_billion_laughs_protection(self): + """ + Security Test: Verifies protection against XML Entity Expansion (Billion Laughs). + Ensures the parser handles malicious DTD entities without crashing. + """ + # Create the malicious payload in memory + xml_payload = """ + + + + + ]> + + + """ + + # Use a temporary file so we don't pollute the project's fixture folder + with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as tf: + with zipfile.ZipFile(tf, "w") as zf: + zf.writestr("xl/workbook.xml", xml_payload) + zf.writestr( + "[Content_Types].xml", + '', + ) + malicious_path = tf.name + + exec_id = str(uuid.uuid4()) + files = {"base_file": malicious_path} + + try: + # Patch the super().pre_processing to return our temp file + with patch( + "geonode.upload.handlers.csv.handler.CSVFileHandler.pre_processing", + return_value=({"files": files, "temporary_files": {}}, exec_id), + ): + + # The test passes if it raises the exception OR if it handles it safely + # without timing out (hanging). + with self.assertRaises(InvalidInputFileException) as context: + self.handler.pre_processing(files, exec_id) + + self.assertIn("Failed to securely parse Excel", str(context.exception)) + finally: + if os.path.exists(malicious_path): + os.remove(malicious_path) From a63ffea524778bca66675e8a42b458e520ad9c1b Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 5 Feb 2026 11:56:49 +0200 Subject: [PATCH 11/21] fixing flake issues --- geonode/upload/handlers/xlsx/tests.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/geonode/upload/handlers/xlsx/tests.py b/geonode/upload/handlers/xlsx/tests.py index 7c62c86df85..75500263b54 100644 --- a/geonode/upload/handlers/xlsx/tests.py +++ b/geonode/upload/handlers/xlsx/tests.py @@ -136,9 +136,11 @@ def test_security_billion_laughs_protection(self): with zipfile.ZipFile(tf, "w") as zf: zf.writestr("xl/workbook.xml", xml_payload) zf.writestr( - "[Content_Types].xml", - '', - ) + "[Content_Types].xml", + '' + '' + '' + ) malicious_path = tf.name exec_id = str(uuid.uuid4()) From e2988667ecd982c28e5c3516fea6c07fee17b520 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 5 Feb 2026 12:04:15 +0200 Subject: [PATCH 12/21] black re-format --- geonode/upload/handlers/xlsx/tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/geonode/upload/handlers/xlsx/tests.py b/geonode/upload/handlers/xlsx/tests.py index 75500263b54..fb432f799b0 100644 --- a/geonode/upload/handlers/xlsx/tests.py +++ b/geonode/upload/handlers/xlsx/tests.py @@ -136,11 +136,11 @@ def test_security_billion_laughs_protection(self): with zipfile.ZipFile(tf, "w") as zf: zf.writestr("xl/workbook.xml", xml_payload) zf.writestr( - "[Content_Types].xml", - '' - '' - '' - ) + "[Content_Types].xml", + '' + '' + '', + ) malicious_path = tf.name exec_id = str(uuid.uuid4()) From 4fa8944483bee7c6e68bf47855e8deb6893612f6 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 5 Feb 2026 12:10:42 +0200 Subject: [PATCH 13/21] use InvalidInputFileException in more simple exceptions --- geonode/upload/handlers/xlsx/handler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index 2d1f9ff2c92..f69419745a8 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -260,7 +260,7 @@ def _validate_sheets(self, workbook): """Returns the first sheet name and logs warnings if others exist.""" sheets = workbook.sheet_names if not sheets: - raise Exception("No sheets found in workbook.") + raise InvalidInputFileException("No sheets found in workbook.") if len(sheets) > 1: logger.warning(f"Multiple sheets found. Ignoring: {sheets[1:]}") return sheets[0] @@ -274,7 +274,7 @@ def _validate_headers(self, headers): """ # Existence Check if not headers or self._detect_empty_rows(headers): - raise Exception("No data or headers found in the selected sheet.") + raise InvalidInputFileException("No data or headers found in the selected sheet.") # Normalization clean_headers = [str(h).strip().lower() if h is not None else "" for h in headers] @@ -284,19 +284,19 @@ def _validate_headers(self, headers): has_lon = any(h in self.lon_names for h in clean_headers) if not (has_lat and has_lon): - raise Exception( + raise InvalidInputFileException( "The headers does not contain valid geometry headers. " "GeoNode requires Latitude and Longitude labels in the first row." ) # Integrity Check (No Empty Names) if any(h == "" for h in clean_headers): - raise Exception("One or more columns in the first row are missing a header name.") + raise InvalidInputFileException("One or more columns in the first row are missing a header name.") # Uniqueness Check if len(clean_headers) != len(set(clean_headers)): duplicates = set([h for h in clean_headers if clean_headers.count(h) > 1]) - raise Exception(f"Duplicate headers found in Row 1: {', '.join(duplicates)}") + raise InvalidInputFileException(f"Duplicate headers found in Row 1: {', '.join(duplicates)}") return True From af1d9ee3a8f6300af94317c12d55d0bc417b3af8 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 5 Feb 2026 12:44:59 +0200 Subject: [PATCH 14/21] avoid exposing a row exception to the user --- geonode/upload/handlers/xlsx/handler.py | 26 +++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index f69419745a8..2612127ce0e 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -111,12 +111,12 @@ def is_valid(files, user, **kwargs): datasource = ogr.GetDriverByName("CSV").Open(files.get("base_file")) if not datasource: - raise InvalidInputFileException("The converted XLSX data is invalid; no layers found.") + raise InvalidInputFileException(detail="The converted XLSX data is invalid; no layers found.") # In XLSX handler, we always expect 1 layer (the first sheet) layer = datasource.GetLayer(0) if not layer: - raise InvalidInputFileException("No data found in the converted CSV.") + raise InvalidInputFileException(detail="No data found in the converted CSV.") if 1 + actual_upload > max_upload: raise UploadParallelismLimitException( @@ -131,17 +131,17 @@ def is_valid(files, user, **kwargs): if has_lat and not has_long: raise InvalidInputFileException( - f"Longitude is missing. Supported names: {', '.join(XLSXFileHandler.lon_names)}" + detail=f"Longitude is missing. Supported names: {', '.join(XLSXFileHandler.lon_names)}" ) if not has_lat and has_long: raise InvalidInputFileException( - f"Latitude is missing. Supported names: {', '.join(XLSXFileHandler.lat_names)}" + detail=f"Latitude is missing. Supported names: {', '.join(XLSXFileHandler.lat_names)}" ) if not (has_lat and has_long): raise InvalidInputFileException( - "XLSX uploads require both a Latitude and a Longitude column in the first row. " + detail="XLSX uploads require both a Latitude and a Longitude column in the first row. " f"Accepted Lat: {', '.join(XLSXFileHandler.lat_names)}. " f"Accepted Lon: {', '.join(XLSXFileHandler.lon_names)}." ) @@ -212,7 +212,7 @@ def pre_processing(self, files, execution_id, **kwargs): # convert the XLSX file into a CSV xlsx_file = _data.get("files", {}).get("base_file", "") if not xlsx_file: - raise Exception("File not found") + raise InvalidInputFileException(detail="The base file was not found in the upload payload.") output_file = str(Path(xlsx_file).with_suffix(".csv")) @@ -240,7 +240,9 @@ def pre_processing(self, files, execution_id, **kwargs): except Exception as e: logger.exception("XLSX Pre-processing failed") - raise InvalidInputFileException(detail=f"Failed to securely parse Excel: {str(e)}") + raise InvalidInputFileException( + detail="Failed to securely parse Excel file." + ) # update the file path in the payload _data["files"]["base_file"] = output_file @@ -260,7 +262,7 @@ def _validate_sheets(self, workbook): """Returns the first sheet name and logs warnings if others exist.""" sheets = workbook.sheet_names if not sheets: - raise InvalidInputFileException("No sheets found in workbook.") + raise InvalidInputFileException(detail="No sheets found in workbook.") if len(sheets) > 1: logger.warning(f"Multiple sheets found. Ignoring: {sheets[1:]}") return sheets[0] @@ -274,7 +276,7 @@ def _validate_headers(self, headers): """ # Existence Check if not headers or self._detect_empty_rows(headers): - raise InvalidInputFileException("No data or headers found in the selected sheet.") + raise InvalidInputFileException(detail="No data or headers found in the selected sheet.") # Normalization clean_headers = [str(h).strip().lower() if h is not None else "" for h in headers] @@ -285,18 +287,18 @@ def _validate_headers(self, headers): if not (has_lat and has_lon): raise InvalidInputFileException( - "The headers does not contain valid geometry headers. " + detail="The headers does not contain valid geometry headers. " "GeoNode requires Latitude and Longitude labels in the first row." ) # Integrity Check (No Empty Names) if any(h == "" for h in clean_headers): - raise InvalidInputFileException("One or more columns in the first row are missing a header name.") + raise InvalidInputFileException(detail="One or more columns in the first row are missing a header name.") # Uniqueness Check if len(clean_headers) != len(set(clean_headers)): duplicates = set([h for h in clean_headers if clean_headers.count(h) > 1]) - raise InvalidInputFileException(f"Duplicate headers found in Row 1: {', '.join(duplicates)}") + raise InvalidInputFileException(detail=f"Duplicate headers found in Row 1: {', '.join(duplicates)}") return True From 083addf7c79a2a424deef1aa541822a6db20a563 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 5 Feb 2026 13:08:05 +0200 Subject: [PATCH 15/21] simplifying is_valid method --- geonode/upload/handlers/xlsx/handler.py | 46 +++---------------------- 1 file changed, 4 insertions(+), 42 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index 2612127ce0e..23d59cf47b5 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -102,50 +102,14 @@ def can_handle(_data) -> bool: def is_valid(files, user, **kwargs): from geonode.upload.utils import UploadLimitValidator + # Basic GeoNode validation BaseVectorFileHandler.is_valid(files, user) + # Parallelism check (This is fast and doesn't need to open the file) upload_validator = UploadLimitValidator(user) upload_validator.validate_parallelism_limit_per_user() - actual_upload = upload_validator._get_parallel_uploads_count() - max_upload = upload_validator._get_max_parallel_uploads() - - datasource = ogr.GetDriverByName("CSV").Open(files.get("base_file")) - if not datasource: - raise InvalidInputFileException(detail="The converted XLSX data is invalid; no layers found.") - - # In XLSX handler, we always expect 1 layer (the first sheet) - layer = datasource.GetLayer(0) - if not layer: - raise InvalidInputFileException(detail="No data found in the converted CSV.") - - if 1 + actual_upload > max_upload: - raise UploadParallelismLimitException( - detail=f"Upload limit exceeded. Max allowed parallel uploads: {max_upload}" - ) - - schema_keys = [x.name.lower() for x in layer.schema] - - # Accessing class-level constants explicitly - has_lat = any(x in XLSXFileHandler.lat_names for x in schema_keys) - has_long = any(x in XLSXFileHandler.lon_names for x in schema_keys) - - if has_lat and not has_long: - raise InvalidInputFileException( - detail=f"Longitude is missing. Supported names: {', '.join(XLSXFileHandler.lon_names)}" - ) - - if not has_lat and has_long: - raise InvalidInputFileException( - detail=f"Latitude is missing. Supported names: {', '.join(XLSXFileHandler.lat_names)}" - ) - - if not (has_lat and has_long): - raise InvalidInputFileException( - detail="XLSX uploads require both a Latitude and a Longitude column in the first row. " - f"Accepted Lat: {', '.join(XLSXFileHandler.lat_names)}. " - f"Accepted Lon: {', '.join(XLSXFileHandler.lon_names)}." - ) + # We handle the deep inspection (lat/lon) later. return True @staticmethod @@ -240,9 +204,7 @@ def pre_processing(self, files, execution_id, **kwargs): except Exception as e: logger.exception("XLSX Pre-processing failed") - raise InvalidInputFileException( - detail="Failed to securely parse Excel file." - ) + raise InvalidInputFileException(detail=f"Failed to securely parse Excel: {str(e)}") # update the file path in the payload _data["files"]["base_file"] = output_file From cbede0b98d2484a24c94ebf3b911f1235d8c01f4 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 5 Feb 2026 13:16:32 +0200 Subject: [PATCH 16/21] fixing security in the create_ogr2ogr_command method as gemini reviewer proposed --- geonode/upload/handlers/xlsx/handler.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index 23d59cf47b5..dab4e1e5239 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -18,6 +18,7 @@ ######################################################################### import logging import os +import shlex from distutils.util import strtobool from pathlib import Path import csv @@ -116,20 +117,28 @@ def is_valid(files, user, **kwargs): def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate, **kwargs): """ Customized for XLSX: Only looks for X/Y (Point) data. - Ignores WKT/Geom columns as per requirements. + Sanitized with shlex.quote to prevent Command Injection. """ + # Sanitize user-controlled strings immediately + safe_original_name = shlex.quote(original_name) + safe_alternate = shlex.quote(alternate) - base_command = BaseVectorFileHandler.create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate) + # Pass the safe versions to the base handler + base_command = BaseVectorFileHandler.create_ogr2ogr_command( + files, safe_original_name, ovverwrite_layer, safe_alternate + ) - # We only define X and Y possible names instead of WKT columns + # Define mapping (these are safe as they are class-level constants) lat_mapping = ",".join(XLSXFileHandler.lat_names) lon_mapping = ",".join(XLSXFileHandler.lon_names) - additional_option = f' -oo "X_POSSIBLE_NAMES={lon_mapping}" ' f' -oo "Y_POSSIBLE_NAMES={lat_mapping}"' + additional_option = f' -oo "X_POSSIBLE_NAMES={lon_mapping}" ' f'-oo "Y_POSSIBLE_NAMES={lat_mapping}"' + # Return the combined, safe command string return ( f"{base_command} -oo KEEP_GEOM_COLUMNS=NO " - f"-lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} " + additional_option + f"-lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} " + f"{additional_option}" ) def create_dynamic_model_fields( From bdbd20054ffac03af55a1811523b8c985c238993 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 5 Feb 2026 13:18:48 +0200 Subject: [PATCH 17/21] fixing flake issues --- geonode/upload/handlers/xlsx/handler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index dab4e1e5239..b53de9b4fba 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -34,10 +34,7 @@ from geonode.upload.handlers.csv.handler import CSVFileHandler from geonode.upload.celery_tasks import create_dynamic_structure from geonode.upload.handlers.utils import GEOM_TYPE_MAPPING -from geonode.upload.api.exceptions import ( - UploadParallelismLimitException, - InvalidInputFileException, -) +from geonode.upload.api.exceptions import InvalidInputFileException logger = logging.getLogger("importer") From 1937fd9e57cd290b5ec74185b6951a456ca7ef0d Mon Sep 17 00:00:00 2001 From: gpetrak Date: Tue, 10 Feb 2026 12:22:42 +0200 Subject: [PATCH 18/21] wrapping the xlsx enable check in a separate method --- geonode/upload/handlers/xlsx/handler.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index b53de9b4fba..629c28b8357 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -46,11 +46,21 @@ class XLSXFileHandler(CSVFileHandler): lat_names = CSVFileHandler.possible_lat_column lon_names = CSVFileHandler.possible_long_column + @classmethod + def is_xlsx_enabled(cls): + """ + Unified check for the feature toggle. + Returns True if enabled, None if disabled. + """ + if not cls.XLSX_UPLOAD_ENABLED: + return None + return True + @property def supported_file_extension_config(self): # If disabled, return an empty list or None so the UI doesn't show XLSX options - if not XLSXFileHandler.XLSX_UPLOAD_ENABLED: + if not self.is_xlsx_enabled(): return None return { @@ -78,7 +88,7 @@ def can_handle(_data) -> bool: the handler is able to handle the file or not """ # Availability Check for the back-end - if not XLSXFileHandler.XLSX_UPLOAD_ENABLED: + if not XLSXFileHandler.is_xlsx_enabled(): return False base = _data.get("base_file") @@ -255,7 +265,7 @@ def _validate_headers(self, headers): if not (has_lat and has_lon): raise InvalidInputFileException( - detail="The headers does not contain valid geometry headers. " + detail="The headers do not contain valid geometry headers. " "GeoNode requires Latitude and Longitude labels in the first row." ) From 38a4102dec61a124ab2521a77e5aa0ed67a30f11 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Wed, 11 Feb 2026 14:27:38 +0200 Subject: [PATCH 19/21] moving the XLSX conf variable in the settings and define it as False --- .env.sample | 2 +- .env_dev | 2 +- .env_local | 2 +- .env_test | 2 +- geonode/settings.py | 3 +++ geonode/upload/handlers/xlsx/handler.py | 7 ++++--- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.env.sample b/.env.sample index 0d22ec7574f..3727b7078f2 100644 --- a/.env.sample +++ b/.env.sample @@ -248,4 +248,4 @@ DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 # FORCE_READ_ONLY_MODE=False Override the read-only value saved in the configuration # Enable or not the XLSX / XLS upload -XLSX_UPLOAD_ENABLED=True \ No newline at end of file +XLSX_UPLOAD_ENABLED=False \ No newline at end of file diff --git a/.env_dev b/.env_dev index 702c481b960..f4d32a94fa3 100644 --- a/.env_dev +++ b/.env_dev @@ -210,4 +210,4 @@ UPSERT_CHUNK_SIZE= 100 UPSERT_LIMIT_ERROR_LOG=100 # Enable or not the XLSX / XLS upload -XLSX_UPLOAD_ENABLED=True \ No newline at end of file +XLSX_UPLOAD_ENABLED=False \ No newline at end of file diff --git a/.env_local b/.env_local index 63009d33dc4..bc9a975fe1a 100644 --- a/.env_local +++ b/.env_local @@ -211,4 +211,4 @@ RESTART_POLICY_WINDOW=120s DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 # Enable or not the XLSX / XLS upload -XLSX_UPLOAD_ENABLED=True +XLSX_UPLOAD_ENABLED=False diff --git a/.env_test b/.env_test index 37a4eaeeed6..04e8407217a 100644 --- a/.env_test +++ b/.env_test @@ -226,4 +226,4 @@ AZURE_SECRET_KEY= AZURE_KEY= # Enable or not the XLSX / XLS upload -XLSX_UPLOAD_ENABLED=True +XLSX_UPLOAD_ENABLED=False diff --git a/geonode/settings.py b/geonode/settings.py index d9335455510..761f4e17675 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -2221,3 +2221,6 @@ def get_geonode_catalogue_service(): FILE_UPLOAD_DIRECTORY_PERMISSIONS = 0o777 FILE_UPLOAD_PERMISSIONS = 0o777 + +# Enable or not the XLSX / XLS upload +XLSX_UPLOAD_ENABLED = ast.literal_eval(os.getenv("XLSX_UPLOAD_ENABLED", "False")) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index 629c28b8357..9340ed3173d 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -29,6 +29,7 @@ from osgeo import ogr from dynamic_models.models import ModelSchema +from django.conf import settings from geonode.upload.handlers.common.vector import BaseVectorFileHandler from geonode.upload.handlers.csv.handler import CSVFileHandler @@ -41,7 +42,7 @@ class XLSXFileHandler(CSVFileHandler): - XLSX_UPLOAD_ENABLED = strtobool(os.getenv("XLSX_UPLOAD_ENABLED", "False")) + XLSX_UPLOAD_ENABLED = getattr(settings, "XLSX_UPLOAD_ENABLED", False) lat_names = CSVFileHandler.possible_lat_column lon_names = CSVFileHandler.possible_long_column @@ -67,12 +68,12 @@ def supported_file_extension_config(self): "id": "excel", # Use a generic ID that doesn't imply a specific extension "formats": [ { - "label": "Excel (OpenXML)", + "label": "Excel (xlsx)", "required_ext": ["xlsx"], "optional_ext": ["sld", "xml"], }, { - "label": "Excel (Binary/Legacy)", + "label": "Excel (xls)", "required_ext": ["xls"], "optional_ext": ["sld", "xml"], }, From db358f454258a940b3a3a904753f2d6b5f4fa405 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 12 Feb 2026 12:53:46 +0200 Subject: [PATCH 20/21] fixing flake issues --- geonode/upload/handlers/xlsx/handler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index 9340ed3173d..e53c0d5c3f2 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -17,9 +17,7 @@ # ######################################################################### import logging -import os import shlex -from distutils.util import strtobool from pathlib import Path import csv from datetime import datetime From 7f6c03007e462e79dfeb87715b195a756aa5c8b5 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Tue, 17 Feb 2026 16:48:37 +0200 Subject: [PATCH 21/21] remove the shlex-based modification --- geonode/upload/handlers/xlsx/handler.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/geonode/upload/handlers/xlsx/handler.py b/geonode/upload/handlers/xlsx/handler.py index e53c0d5c3f2..d44e99cb586 100644 --- a/geonode/upload/handlers/xlsx/handler.py +++ b/geonode/upload/handlers/xlsx/handler.py @@ -17,7 +17,6 @@ # ######################################################################### import logging -import shlex from pathlib import Path import csv from datetime import datetime @@ -125,20 +124,17 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate, ** Customized for XLSX: Only looks for X/Y (Point) data. Sanitized with shlex.quote to prevent Command Injection. """ - # Sanitize user-controlled strings immediately - safe_original_name = shlex.quote(original_name) - safe_alternate = shlex.quote(alternate) # Pass the safe versions to the base handler - base_command = BaseVectorFileHandler.create_ogr2ogr_command( - files, safe_original_name, ovverwrite_layer, safe_alternate - ) + base_command = BaseVectorFileHandler.create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate) # Define mapping (these are safe as they are class-level constants) lat_mapping = ",".join(XLSXFileHandler.lat_names) lon_mapping = ",".join(XLSXFileHandler.lon_names) - additional_option = f' -oo "X_POSSIBLE_NAMES={lon_mapping}" ' f'-oo "Y_POSSIBLE_NAMES={lat_mapping}"' + additional_option = ( + f' -oo "X_POSSIBLE_NAMES={lon_mapping}" ' f'-oo "Y_POSSIBLE_NAMES={lat_mapping}" ' f'-nln "{alternate}"' + ) # Return the combined, safe command string return (