Fixup directory locations and chaetognaths

data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Protostomes.PHY

@@ -63,13 +63,13 @@ or with OTTs
 
 ===================Overall tree====================
 
-The Chaetognaths are placed as a outgroup to all other protostomes
+The Chaetognaths are placed within the Gnathifera (see Laumer et al 2019 10.1098/rspb.2019.0831 and Martelaz et al 2019 10.1016/j.cub.2018.11.042)
 
 
-(((((((Nemertea,(((Bryozoa,(Entoprocta,Cycliophora)Polyzoa),Phoronida),Brachiopoda)Lophophorata),Annelida),Mollusca)Trochozoa,(Gastrotricha,Platyhelminthes)),(((Rotifera,Acanthocephala)Syndermata,Micrognathozoa),Gnathostomulida)Gnathifera)Spiralia,((((Onychophora,Arthropoda),Tardigrada),(Nematoda,Nematomorpha)),Scalidophora)Ecdysozoa),Chaetognatha)Protostomia
+((((((Nemertea,(((Bryozoa,(Entoprocta,Cycliophora)Polyzoa),Phoronida),Brachiopoda)Lophophorata),Annelida),Mollusca)Trochozoa,(Gastrotricha,Platyhelminthes)),((((Rotifera,Acanthocephala)Syndermata,Micrognathozoa),Gnathostomulida),Chaetognatha)Gnathifera)Spiralia,((((Onychophora,Arthropoda),Tardigrada),(Nematoda,Nematomorpha)),Scalidophora)Ecdysozoa)Protostomia
 
 
-OpenTree v6 onwards has a slighly improved phylogenetic structure for pancrustacea (including insects), so we simply take this from the OpenTree.
+OpenTree v6 onwards has a slightly improved phylogenetic structure for pancrustacea (including insects), so we simply take that from the OpenTree.
 
 from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Animals]
-(((((((Nemertea_ott445195@,(((Bryozoa_ott442934@,(Entoprocta_ott362916@,Cycliophora_ott41147@))Polyzoa,Phoronida_ott612209@),Brachiopoda_ott826261~-612209@)Lophophorata),Annelida_ott941620@),Mollusca_ott802117@)Trochozoa,(Gastrotricha_ott29723@,Platyhelminthes_ott555379@)),(((Rotifera_ott471706@,Acanthocephala_ott49622@)Syndermata,Micrognathozoa_ott435981@),Gnathostomulida_ott242963@)Gnathifera)Spiralia,((((Onychophora_ott1072433@,Arthropoda_ott632179~-5008315-6293460-4387124@),Tardigrada_ott111438~-2943021@),(Nematoda_ott395057@,Nematomorpha_ott189836@)),Scalidophora_ott434620@)Ecdysozoa),Chaetognatha_ott570366@)Protostomia;
+((((((Nemertea_ott445195@,(((Bryozoa_ott442934@,(Entoprocta_ott362916@,Cycliophora_ott41147@))Polyzoa,Phoronida_ott612209@),Brachiopoda_ott826261~-612209@)Lophophorata),Annelida_ott941620@),Mollusca_ott802117@)Trochozoa,(Gastrotricha_ott29723@,Platyhelminthes_ott555379@)),((((Rotifera_ott471706@,Acanthocephala_ott49622@)Syndermata,Micrognathozoa_ott435981@),Gnathostomulida_ott242963@),Chaetognatha_ott570366@)Gnathifera)Spiralia,((((Onychophora_ott1072433@,Arthropoda_ott632179~-5008315-6293460-4387124@),Tardigrada_ott111438~-2943021@),(Nematoda_ott395057@,Nematomorpha_ott189836@)),Scalidophora_ott434620@)Ecdysozoa)Protostomia;

oz_tree_build/images_and_vernaculars/get_wiki_images.py

@@ -47,6 +47,7 @@
 from .._OZglobals import src_flags
 from ..utilities.db_helper import (
     connect_to_database,
+    default_appconfig,
     get_next_src_id_for_src,
     placeholder,
     read_config,
@@ -60,6 +61,16 @@
 
 logger = logging.getLogger(Path(__file__).name)
 
+default_outdir = os.path.join(
+    os.pardir,
+    os.pardir,
+    os.pardir,
+    "OZtree",
+    "static",
+    "FinalOutputs",
+    "img",
+)
+
 
 # Copied from OZTree/OZprivate/ServerScripts/Utilities/getEOL_crops.py
 def subdir_name(doID):
@@ -502,16 +513,18 @@ def process_leaf(
     s = placeholder(db)
     sql = "SELECT ott,wikidata,name FROM ordered_leaves WHERE "
     if ott_or_taxon.lstrip("-").isnumeric():
+        ott_or_taxon_type = "ott"
         sql += f"ott={s};"
     else:
+        ott_or_taxon_type = "name"
         sql += f"name={s};"
 
     result = db.executesql(sql, (ott_or_taxon,))
     if len(result) > 1:
         logger.error(f"Multiple results for '{ott_or_taxon}'")
         return
     if len(result) == 0:
-        logger.error(f"'{ott_or_taxon}' not found in ordered_leaves table")
+        logger.error(f"{ott_or_taxon_type} '{ott_or_taxon}' not found in ordered_leaves table")
         return
 
     (ott, qid, name) = result[0]
@@ -672,16 +685,7 @@ def process_args(args):
 
     # Default to the static folder in the OZtree repo
     if outdir is None:
-        outdir = os.path.join(
-            os.path.dirname(os.path.abspath(__file__)),
-            os.pardir,
-            os.pardir,
-            os.pardir,
-            "OZtree",
-            "static",
-            "FinalOutputs",
-            "img",
-        )
+        outdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), default_outdir)
     if not os.path.exists(outdir):
         logger.error(f"Output directory '{outdir}' does not exist")
         return
@@ -768,15 +772,15 @@ def add_common_args(parser):
             default=None,
             help=(
                 "The location to save the image files (e.g. 'FinalOutputs/img'). "
-                "Defaults to ../../../static/FinalOutputs/img (relative to the script "
+                f"Defaults to {default_outdir} (relative to the script "
                 "location). Files are saved to output_dir/{src_flag}/{3-digits}/fn.jpg"
             ),
         )
         parser.add_argument(
             "-c",
             "--conf-file",
             default=None,
-            help=("The configuration file to use. " "Defaults to ../../../OZtree/private/appconfig.ini"),
+            help=(f"The configuration file to use. Defaults to {default_appconfig}"),
         )
 
     parser_leaf = subparsers.add_parser("leaf", help="Process a single ott")

oz_tree_build/utilities/db_helper.py

@@ -10,6 +10,8 @@
 
 logger = logging.getLogger(__name__)
 
+default_appconfig = "../../../OZtree/private/appconfig.ini"
+
 
 def is_sqlite(db):
     return db._adapter.driver_name.startswith("sqlite")
@@ -70,7 +72,7 @@ def read_config(conf_file=None):
     if conf_file is None:
         conf_file = os.path.join(
             os.path.dirname(os.path.abspath(__file__)),
-            "../../../OZtree/private/appconfig.ini",
+            default_appconfig,
         )
     if not os.path.exists(conf_file):
         raise ValueError(f"Appconfig file {conf_file} cannot be found.")

oz_tree_build/utilities/generate_filtered_files.py

@@ -34,6 +34,8 @@
 
 __author__ = "David Ebbo"
 
+one_zoom_file_prefix = "OneZoom"
+
 
 def generate_and_cache_filtered_file(original_file, context, processing_function):
     """
@@ -43,7 +45,6 @@ def generate_and_cache_filtered_file(original_file, context, processing_function
     dirname = os.path.dirname(original_file)
     file_name = os.path.basename(original_file)
 
-    one_zoom_file_prefix = "OneZoom"
     filtered_file_prefix = (context.clade or one_zoom_file_prefix) + "_"
     if file_name.startswith(filtered_file_prefix):
         raise Exception(f"Input and output files are the same, with prefix {filtered_file_prefix}")
@@ -495,10 +496,12 @@ def generate_all_filtered_files(
 
     generate_and_cache_filtered_file(eol_id_file, context, generate_filtered_eol_id_file)
 
-    filtered_wikidata_dump_file = generate_and_cache_filtered_file(
-        wikidata_dump_file, context, generate_filtered_wikidata_dump
-    )
-
+    if os.path.basename(wikidata_dump_file).startswith(one_zoom_file_prefix):
+        filtered_wikidata_dump_file = wikidata_dump_file
+    else:
+        filtered_wikidata_dump_file = generate_and_cache_filtered_file(
+            wikidata_dump_file, context, generate_filtered_wikidata_dump
+        )
     read_wikidata_dump(filtered_wikidata_dump_file, context)
 
     generate_and_cache_filtered_file(wikipedia_sql_dump_file, context, generate_filtered_wikipedia_sql_dump)
@@ -555,7 +558,10 @@ def main():
         nargs="?",
         help=(
             "The b2zipped >4GB wikidata JSON dump, "
-            "from https://dumps.wikimedia.org/wikidatawiki/entities/ (latest-all.json.bz2) "
+            "from https://dumps.wikimedia.org/wikidatawiki/entities/ (latest-all.json.bz2). "
+            f"If this starts with '{one_zoom_file_prefix}', it is assumed to be filtered "
+            "already, which is useful if the original dumpfile is missing and additional "
+            "pageview files require filtering."
         ),
     )
     parser.add_argument(