diff --git a/README.md b/README.md
index 8a7e784..c90a06b 100644
--- a/README.md
+++ b/README.md
@@ -28,6 +28,10 @@ This plugin is available from the official plugin repository and can be installe
+## Requirements
+
+- **GDAL 3.7 or higher.**
+
## Usage and installation
Please read [Instructions.md](documentation/instructions.md) for more information.
diff --git a/nlsgpkgloader/nls_geopackage_loader_tasks.py b/nlsgpkgloader/nls_geopackage_loader_tasks.py
index 6a7498f..f93c77b 100644
--- a/nlsgpkgloader/nls_geopackage_loader_tasks.py
+++ b/nlsgpkgloader/nls_geopackage_loader_tasks.py
@@ -17,10 +17,10 @@
# You should have received a copy of the GNU General Public License
# along with NLSgpkgloadert. If not, see .
-import os
import sqlite3
+from pathlib import Path
-from osgeo import ogr
+from osgeo import gdal, ogr
from qgis import processing
from qgis.core import (
QgsFeature,
@@ -28,7 +28,6 @@
QgsMessageLog,
QgsProcessingContext,
QgsTask,
- QgsVectorFileWriter,
QgsVectorLayer,
)
@@ -43,109 +42,45 @@ def __init__(self, description, urls, dlcount, products, dlpath, path):
self.all_urls = urls
self.total_download_count = dlcount
self.products = products
- self.data_download_dir = dlpath
- self.gpkg_path = path
+ self.data_download_dir = Path(dlpath)
+ self.gpkg_path = Path(path)
def run(self):
- for dl_index in range(0, self.total_download_count):
- url = self.all_urls[dl_index][0]
- url_parts = url.split("/")
- file_name = url_parts[-1].split("?")[0]
- data_dir_name = self.all_urls[dl_index][1]
- data_dir_name = data_dir_name.replace(":", "_suhde_")
- dir_path = os.path.join(self.data_download_dir, data_dir_name)
- dir_path = os.path.join(dir_path, file_name.split(".")[0])
- data_type = self.all_urls[dl_index][3]
-
- percentage = dl_index / float(self.total_download_count) * 100.0
- self.setProgress(percentage)
-
- if not os.path.exists(dir_path):
- QgsMessageLog.logMessage(
- "Skipping directory: " + dir_path, "NLSgpkgloader", 1
+ gdal.UseExceptions()
+ all_gml_files = []
+ try:
+ for dl_index in range(0, self.total_download_count):
+ url = self.all_urls[dl_index][0]
+ url_parts = url.split("/")
+ file_name = url_parts[-1].split("?")[0]
+ data_dir_name = self.all_urls[dl_index][1]
+ data_dir_name = data_dir_name.replace(":", "_suhde_")
+ dir_path = (
+ self.data_download_dir / data_dir_name / file_name.split(".")[0]
)
- continue
+ data_type = self.all_urls[dl_index][3]
- for listed_file_name in os.listdir(dir_path):
- if data_type == "gml" and listed_file_name.endswith(".xml"):
- driver = ogr.GetDriverByName("GML")
- data_source = driver.Open(
- os.path.join(dir_path, listed_file_name), 0
- )
- layer_count = data_source.GetLayerCount()
-
- mtk_layer_count = 0 # Used for breaking from the for loop
- # when all MTK layers chosen by the user have been added
- for i in range(layer_count):
- if self.isCanceled():
- return False
- layer = data_source.GetLayerByIndex(i)
- layer_name = layer.GetName()
- if layer_name in self.products:
- new_layer = QgsVectorLayer(
- os.path.join(dir_path, listed_file_name)
- + "|layerid="
- + str(i),
- layer_name,
- "ogr",
- )
- if new_layer.isValid():
- options = QgsVectorFileWriter.SaveVectorOptions()
- options.layerName = layer_name
- options.driverName = "GPKG"
- options.fileEncoding = "UTF-8"
- if os.path.isfile(self.gpkg_path):
- if QgsVectorLayer(
- self.gpkg_path + "|layername=" + layer_name
- ).isValid():
- options.actionOnExistingFile = (
- QgsVectorFileWriter.AppendToLayerNoNewFields
- )
- else:
- options.actionOnExistingFile = (
- QgsVectorFileWriter.CreateOrOverwriteLayer
- )
- else:
- options.actionOnExistingFile = (
- QgsVectorFileWriter.CreateOrOverwriteFile
- )
- e = QgsVectorFileWriter.writeAsVectorFormat(
- new_layer, self.gpkg_path, options
- )
- if e[0]:
- QgsMessageLog.logMessage(
- "Failed to write layer "
- + layer_name
- + " to geopackage",
- "NLSgpkgloader",
- 2,
- )
- break
- mtk_layer_count += 1
- else:
- layer_name = ""
- # TODO: handle invalid layer error
- # QgsMessageLog.logMessage(
- # "Invalid layer: {} : {}".format(
- # listed_file_name, layer_name
- # ),
- # "NLSgpkgloader",
- # 2,
- # )
- pass
-
- if mtk_layer_count == len(self.products):
- break
- else:
+ percentage = dl_index / float(self.total_download_count) * 100.0
+ self.setProgress(percentage)
+
+ if not dir_path.exists():
QgsMessageLog.logMessage(
- "cannot add the data type "
- + data_type
- + ", listed_file_name: "
- + listed_file_name,
- "NLSgpkgloader",
- 0,
+ "Skipping directory: " + str(dir_path), "NLSgpkgloader", 1
)
- return True
+ continue
+
+ for listed_file_name in dir_path.iterdir():
+ if data_type == "gml" and listed_file_name.suffix == ".xml":
+ all_gml_files.append(listed_file_name)
+
+ if all_gml_files:
+ merge_gmls(all_gml_files, self.gpkg_path)
+
+ return True
+
+ except Exception as e:
+ QgsMessageLog.logMessage(f"Error: {str(e)}", "NLSgpkgloader", 2)
+ return False
def finished(self, result):
if not result:
@@ -154,6 +89,62 @@ def finished(self, result):
)
+def merge_gmls(gmls: list[Path], output: Path) -> None:
+ gdal.UseExceptions()
+
+ try:
+ gpkg_driver = ogr.GetDriverByName("GPKG")
+ if not output.exists():
+ gpkg_driver.CreateDataSource(str(output))
+ creation_options = gdal.VectorTranslateOptions(
+ layerCreationOptions=["FID=id", "GEOMETRY_NAME=geom", "SPATIAL_INDEX=NONE"],
+ mapFieldType="StringList=String",
+ )
+ append_options = gdal.VectorTranslateOptions(
+ accessMode="append",
+ mapFieldType="StringList=String",
+ )
+ for i, gml in enumerate(gmls):
+ QgsMessageLog.logMessage(f"Processing GML file: {gml}", "NLSgpkgloader", 1)
+
+ source_datasource = gdal.OpenEx(
+ str(gml),
+ nOpenFlags=gdal.OF_VECTOR,
+ )
+
+ if source_datasource is None:
+ QgsMessageLog.logMessage(
+ f"Failed to open GML file: {gml}", "NLSgpkgloader", 2
+ )
+ continue
+ QgsMessageLog.logMessage(
+ f"Merging into GeoPackage: {output}", "NLSgpkgloader", 1
+ )
+ options = creation_options if i == 0 else append_options
+
+ result = gdal.VectorTranslate(
+ destNameOrDestDS=str(output),
+ srcDS=source_datasource,
+ options=options,
+ )
+ if not result:
+ QgsMessageLog.logMessage(
+ f"Merge failed for GML file: {gml}", "NLSgpkgloader", 2
+ )
+ else:
+ QgsMessageLog.logMessage(
+ f"Successfully merged GML file: {gml} into {output}",
+ "NLSgpkgloader",
+ 1,
+ )
+
+ except Exception as e:
+ QgsMessageLog.logMessage(
+ f"Error merging GML files: {str(e)}", "NLSgpkgloader", 2
+ )
+ raise e
+
+
class DissolveFeaturesTask(QgsTask):
def __init__(self, description, path):
super().__init__(description, QgsTask.CanCancel)