Updated according to Spatial Extent Refactor changes

helpers.py

@@ -3,6 +3,7 @@
 import csv
 import json
 import subprocess
+import traceback
 
 from datetime import datetime, date
 
@@ -14,6 +15,21 @@
 from shapely.ops import unary_union
 
 
+def apikey_validates(ctx,apikey):
+
+    twdh = ctx.obj['twdh']
+    logecho = ctx.obj['logecho']
+
+    try:
+        results = twdh.action.user_list()
+        logecho('API Key test passed', level='info')
+        return True
+    except Exception as e:
+        logecho('API Key is not valid ', level='error')
+        #print(traceback.format_exc())
+        sys.exit(1)
+
+
 def snapshot(ctx,dest):
 
     twdh = ctx.obj['twdh']
@@ -142,25 +158,38 @@ def snapshot(ctx,dest):
         obj_file = '{}/{}.jsonl'.format(snap_dest, obj_type)
         try:
 
+            """
             command = "ckanapi dump {obj_type} --apikey={apikey} --all -O {obj_file} -r {url}".format( \
                 obj_type=obj_type, \
                 apikey=twdh.apikey, \
                 obj_file=obj_file, \
                 url=twdh.address \
             )
+            """
+
+            command = [
+                "ckanapi",
+                "dump", "{obj_type}".format(obj_type=obj_type),
+                "--apikey={apikey}".format(apikey=twdh.apikey),
+                "--all",
+                "-O", "{obj_file}".format(obj_file=obj_file),
+                "-r", "{url}".format(url=twdh.address)
+            ]
 
+
+            #breakpoint()
             #logecho( command, 'info' )
             logecho( 'Dumping {}...\n'.format(obj_type), 'info' )
-            output = subprocess.getoutput(command)
-            logecho( output, 'info' )
+            subprocess.check_call(command)
             logecho( 'Created snapshot file: {}'.format(obj_file), 'info' )
 
 
-        except FileNotFoundError:
-            logecho( "Unable to write JSONL / Destination not found error", 'error' )
-            sys.exit(1)
+        #except FileNotFoundError:
+            #logecho( "Unable to write JSONL / Destination not found error", 'error' )
+            #sys.exit(1)
         except Exception as e:
             logecho( "An error occurred: {}".format(e), 'error' )
+            print(traceback.format_exc())
             sys.exit(1)
 
 

twdhcli.py

@@ -146,6 +146,7 @@ def logecho(message, level='info'):
         if apikey == None:
             logecho("Cannot continue: --apikey parameter not set and APIKEY not found in .env.secrets","error")
             exit(1)
+
     logecho("apikey set", "detail")
 
     if host == None:
@@ -169,6 +170,11 @@ def logecho(message, level='info'):
     ctx.obj['logecho'] = logecho
     ctx.obj['test_run'] = test_run
 
+    if not h.apikey_validates(ctx,apikey):
+        logecho("Cannot continue: --apikey parameter value is not a valid key","error")
+        exit(1)
+
+
 @twdhcli.command()
 @click.option('--dest',
               type=click.Path(),
@@ -183,6 +189,24 @@ def snapshot(ctx,dest):
     h.snapshot(ctx,dest)
 
 
+@twdhcli.command()
+@click.pass_context
+def list_patch_functions(ctx ):
+    """
+    Patch datasets
+    """
+
+    twdh = ctx.obj['twdh']
+    logecho = ctx.obj['logecho']
+    test_run = ctx.obj['test_run']
+
+    patch_fn_dict = get_patch_functions()
+
+    for patch_fn in patch_fn_dict:
+        logecho( "{}".format(patch_fn), "info" )
+
+
+
 @twdhcli.command()
 @click.option('--patch-fn',
               required=True,
@@ -480,7 +504,7 @@ def patch_fn_clear_spatial_data(ctx,dataset,data):
         if test_run:
             return False
 
-        remote.action.package_patch( id=dataset.get("id"), gazetteer="" )
+        remote.action.package_patch( id=dataset.get("id"), spatial_extent="", spatial_full="" )
 
     except Exception as e:
         if str(e) == 'Not found':
@@ -502,11 +526,7 @@ def patch_fn_clear_spatial_data_full(ctx,dataset,data):
         if test_run:
             return False
 
-        gazetteer = dataset.get('gazetteer', {})
-        if 'spatial_full' in gazetteer:
-            gazetteer['spatial_full'] = ""
-
-        remote.action.package_patch( id=dataset.get("id"), spatial_simp=gazetteer['spatial_simp'], spatial_full=gazetteer['spatial_full'] )
+        remote.action.package_patch( id=dataset.get("id"), spatial_extent="", spatial_full="" )
 
     except Exception as e:
         if str(e) == 'Not found':
@@ -525,16 +545,18 @@ def patch_fn_set_spatial_data(ctx,dataset,data):
     logecho = ctx.obj['logecho']
     test_run = ctx.obj['test_run']
 
+    breakpoint()
+    spatial_extent = data.get('spatial_extent') or data.get('spatial_simp', '{}')
+    spatial_full = data.get('spatial_full', '{}')
+
     try:
-        spatial_simp = data.get('spatial_simp', '{}')
-        parsed_spatial_simp = json.loads(spatial_simp)
+        json.loads(spatial_extent)
 
     except json.JSONDecodeError as e:
-        logecho(f"JSON parsing error on spatial_simp: {e}, value: {spatial_simp}", 'error')
+        logecho(f"JSON parsing error on spatial_extent: {e}, value: {spatial_extent}", 'error')
 
     try:
-        spatial_full = data.get('spatial_full', '{}')
-        parsed_spatial_simp = json.loads(spatial_full)
+        json.loads(spatial_full)
 
     except json.JSONDecodeError as e:
         logecho(f"JSON parsing error on spatial_full: {e}, value: {spatial_full}",'error')
@@ -543,7 +565,7 @@ def patch_fn_set_spatial_data(ctx,dataset,data):
         if test_run:
             return False
 
-        remote.action.package_patch( id=dataset.get("id"), spatial_simp=spatial_simp, spatial_full=spatial_full )
+        remote.action.package_patch( id=dataset.get("id"), spatial_extent=spatial_extent, spatial_full=spatial_full )
 
     except Exception as e:
         if str(e) == 'Not found':
@@ -632,10 +654,14 @@ def patch_fn_set_app_email(ctx,dataset,data):
               default=False,
               is_flag=True,
               help='Confirm each patch operation instead of just once at the start')
+@click.option('--ids',
+              required=False,
+              default=None,
+              help='Space-separated list of dataset ids to patch')
 @click.pass_context
-def restore_spatial(ctx, patch_file, confirm_each):
+def migrate_spatial(ctx, patch_file, confirm_each, ids):
     """
-    Restore spatial data to datasets
+    RestoreMigrate spatial data to remove old GZTR model and use new model with dedicated table for `spatial_full` and move `gazettteer.spatial_simp` to `spatial_extent`
     """
 
     twdh = ctx.obj['twdh']
@@ -656,6 +682,12 @@ def restore_spatial(ctx, patch_file, confirm_each):
         sys.exit(1)
     logecho( "Restoring spatial data from {} ...".format(patch_file), "info" )
 
+    if ids is not None:
+        dataset_filter = ids.split(' ')
+        logecho( "Limiting migration to the following datasets: {}".format(ids), 'info' )
+    else:
+        dataset_filter = []
+
     if not confirm_each:
         logecho( "Hint: Use --confirm-each if you want to confirm one at a time", "note" )
         if click.confirm('🟢 Proceed with all patches from {}? '.format(patch_file)):
@@ -671,35 +703,124 @@ def restore_spatial(ctx, patch_file, confirm_each):
 
         logecho( "", "divider" )
 
-        run_patch = True
+        if len(dataset_filter) == 0 or dataset.get('id') in dataset_filter or dataset.get('name') in dataset_filter:
+            logecho( "Migrating {}".format( dataset['name'] ), 'info' )
 
-        if 'gazetteer' in dataset:
+            run_patch = True
 
-            spatial_full = dataset['gazetteer'].get('spatial_full', None)
-            spatial_simp = dataset['gazetteer'].get('spatial_simp', None)
+            if 'gazetteer' in dataset:
 
-            if spatial_full != None or spatial_simp != None:
+                old_spatial_full = dataset['gazetteer'].get('spatial_full', None)
+                old_spatial_simp = dataset['gazetteer'].get('spatial_simp', None)
 
-                logecho( "Spatial data found for dataset \"{}\"".format(dataset['name']), "info" )
+                if old_spatial_full != None or old_spatial_simp != None:
+
+                    logecho( "Spatial data found for dataset \"{}\"".format(dataset['name']), "info" )
+
+                    if confirm_all:
+                        if click.confirm("🟢 Proceed to patch dataset \"{}\"? ".format(dataset['name']), abort=False, default=True):
+                            run_patch = True
+                        else: 
+                            logecho( "Patch cancelled", "warning" )
+                            run_patch = False
+
+                    if run_patch:
+                        if patch_fn_set_spatial_data( ctx, dataset, {
+                                "spatial_simp": old_spatial_simp,
+                                "spatial_full": old_spatial_full
+                            }):
+                            logecho( "... patched", "info" )
+                        else:
+                            logecho( "Error patching dataset \"{}\"".format(dataset['name']), "info" )
 
-                if confirm_all:
-                    if click.confirm("🟢 Proceed to patch dataset \"{}\"? ".format(dataset['name']), abort=False, default=True):
-                        run_patch = True
-                    else: 
-                        logecho( "Patch cancelled", "warning" )
-                        run_patch = False
+                else:
+                    logecho( "No spatial data found in gazetteer attribute for \"{}\"".format(dataset['name']), "info" )
+            else:
+                logecho( "No gazetteer attribute found", "info" )
 
-                if run_patch:
-                    if patch_fn_set_spatial_data( ctx, dataset, dataset.get('gazetteer', None)):
-                        logecho( "... patched", "info" )
-                    else:
-                        logecho( "Error patching dataset \"{}\"".format(dataset['name']), "info" )
+        else:
+            logecho( "Skipping because not found in filter: \"{}\"".format(dataset['name']), "info" )
 
-            else:
-                logecho( "No spatial data found for \"{}\"".format(dataset['name']), "info" )
+@twdhcli.command()
+@click.option('--patch-file',
+              required=True,
+              default=None,
+              help='JSON file containing patch data')
+@click.option('--confirm-each',
+              default=False,
+              is_flag=True,
+              help='Confirm each patch operation instead of just once at the start')
+@click.pass_context
+def restore_spatial(ctx, patch_file, confirm_each):
+    """
+    Restore spatial data to datasets
+    """
+
+    twdh = ctx.obj['twdh']
+    logecho = ctx.obj['logecho']
+
+    try:
+        with open(patch_file, "r") as file:
+            patch_data = json.load(file)
+    except FileNotFoundError:
+        logecho("Error: The file was not found.", 'error')
+        sys.exit(1)
+    except json.JSONDecodeError as e:
+        logecho(f"Error: Could not decode JSON from '{patch_file}'. Check if the file contains valid JSON.", 'error')
+        logecho( f"{e}", 'error' )
+        sys.exit(1)
+    except Exception as e:
+        logecho(f"An unexpected error occurred: {e}", 'error')
+        sys.exit(1)
+    logecho( "Restoring spatial data from {} ...".format(patch_file), "info" )
+
+    if not confirm_each:
+        logecho( "Hint: Use --confirm-each if you want to confirm one at a time", "note" )
+        if click.confirm('🟢 Proceed with all patches from {}? '.format(patch_file)):
+            logecho( "Proceeding with patches ...", "info" )
+        else: 
+            logecho( "Operation cancelled", "warning" )
+            sys.exit(0)
+        confirm_all = False
+    else:
+        confirm_all = True
+
+    for dataset in patch_data['results']:
+
+        logecho( "", "divider" )
+
+        run_patch = True
+
+        spatial_full = None
+        spatial_extent = None
+        if not spatial_extent:
+            spatial_extent = dataset.get('spatial_extent')
+        if not spatial_full:
+            spatial_full = dataset.get('spatial_full')   
+
+        if spatial_full != None or spatial_extent != None:
+
+            logecho( "Spatial data found for dataset \"{}\"".format(dataset['name']), "info" )
+
+            if confirm_all:
+                if click.confirm("🟢 Proceed to patch dataset \"{}\"? ".format(dataset['name']), abort=False, default=True):
+                    run_patch = True
+                else: 
+                    logecho( "Patch cancelled", "warning" )
+                    run_patch = False
+
+            if run_patch:
+                if patch_fn_set_spatial_data( ctx, dataset, {
+                        "spatial_extent": spatial_extent,
+                        "spatial_full": spatial_full
+                    }):
+                    logecho( "... patched", "info" )
+                else:
+                    logecho( "Error patching dataset \"{}\"".format(dataset['name']), "info" )
 
         else:
-            logecho( "No gazetteer attribute found for \"{}\"".format(dataset['name']), "info" )
+            logecho( "No spatial data found for \"{}\"".format(dataset['name']), "info" )
+
 
 @twdhcli.command()
 @click.option('--new-size',
@@ -760,10 +881,12 @@ def update_spatial_simp(ctx, new_size, ids, confirm_each, allow_enlarge, skip_sn
                 return
 
     for dataset in datasets:
-        gazetteer = dataset.get("gazetteer", {})
-        if 'spatial_full' in gazetteer and gazetteer['spatial_full'] != None:
-            if not allow_enlarge and len(dataset["gazetteer"]["spatial_simp"].encode('utf-8')) < new_size:
-                logecho( "+ {} ({}) spatial_simp = {} already less than {}".format(dataset.get("title"),dataset.get("id"),len(dataset["gazetteer"]["spatial_simp"].encode('utf-8')),new_size), 'info')
+        spatial_full = dataset.get("spatial_full")
+        spatial_extent = dataset.get("spatial_extent")
+
+        if spatial_full:
+            if not allow_enlarge and spatial_extent and len(spatial_extent.encode('utf-8')) < new_size:
+                logecho( "+ {} ({}) spatial_simp = {} already less than {}".format(dataset.get("title"),dataset.get("id"),len(spatial_extent.encode('utf-8')),new_size), 'info')
             else:
 
                 logecho( "About to patch {} ({})".format(dataset.get("title"),dataset.get("id")), 'info')
@@ -774,14 +897,15 @@ def update_spatial_simp(ctx, new_size, ids, confirm_each, allow_enlarge, skip_sn
                         logecho( "Update cancelled", "warning" )
                         continue
                 try:
-                    if len(dataset["gazetteer"]["spatial_full"].encode('utf-8')) < new_size:
-                        logecho( " {} ({}) spatial_full = {} already less than {}, setting spatial_simp = spatial_full".format(dataset.get("title"),dataset.get("id"),len(dataset["gazetteer"]["spatial_simp"].encode('utf-8')),new_size), 'info')
-                        gazetteer['spatial_simp'] = gazetteer['spatial_full']
+                    if len(spatial_full.encode('utf-8')) < new_size:
+                        new_spatial_extent = spatial_full
                     else:
-                        #logecho( " updating {} ({})".format(dataset.get("title"),dataset.get("id")), 'info')
-                        gazetteer['spatial_simp'] = h.simplify_geojson_by_size(ctx,gazetteer['spatial_full'],new_size)
+                        new_spatial_extent = h.simplify_geojson_by_size(spatial_full, new_size)
 
-                    if patch_fn_set_spatial_data(ctx,dataset,gazetteer):
+                    if patch_fn_set_spatial_data(ctx,dataset,{
+                                                    "spatial_extent": new_spatial_extent,
+                                                    "spatial_full": spatial_full
+                                                }):
                         logecho( "Updated spatial_simp on dataset \"{}\"".format(dataset['name']), "info" )
                     else:
                         logecho( "Error updating spatial_simp on dataset \"{}\"".format(dataset['name']), "info" )