diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Protostomes.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Protostomes.PHY index e32f976..80e05f1 100755 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Protostomes.PHY +++ b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Protostomes.PHY @@ -63,13 +63,13 @@ or with OTTs ===================Overall tree==================== -The Chaetognaths are placed as a outgroup to all other protostomes +The Chaetognaths are placed within the Gnathifera (see Laumer et al 2019 10.1098/rspb.2019.0831 and Martelaz et al 2019 10.1016/j.cub.2018.11.042) -(((((((Nemertea,(((Bryozoa,(Entoprocta,Cycliophora)Polyzoa),Phoronida),Brachiopoda)Lophophorata),Annelida),Mollusca)Trochozoa,(Gastrotricha,Platyhelminthes)),(((Rotifera,Acanthocephala)Syndermata,Micrognathozoa),Gnathostomulida)Gnathifera)Spiralia,((((Onychophora,Arthropoda),Tardigrada),(Nematoda,Nematomorpha)),Scalidophora)Ecdysozoa),Chaetognatha)Protostomia +((((((Nemertea,(((Bryozoa,(Entoprocta,Cycliophora)Polyzoa),Phoronida),Brachiopoda)Lophophorata),Annelida),Mollusca)Trochozoa,(Gastrotricha,Platyhelminthes)),((((Rotifera,Acanthocephala)Syndermata,Micrognathozoa),Gnathostomulida),Chaetognatha)Gnathifera)Spiralia,((((Onychophora,Arthropoda),Tardigrada),(Nematoda,Nematomorpha)),Scalidophora)Ecdysozoa)Protostomia -OpenTree v6 onwards has a slighly improved phylogenetic structure for pancrustacea (including insects), so we simply take this from the OpenTree. +OpenTree v6 onwards has a slightly improved phylogenetic structure for pancrustacea (including insects), so we simply take that from the OpenTree. from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Animals] -(((((((Nemertea_ott445195@,(((Bryozoa_ott442934@,(Entoprocta_ott362916@,Cycliophora_ott41147@))Polyzoa,Phoronida_ott612209@),Brachiopoda_ott826261~-612209@)Lophophorata),Annelida_ott941620@),Mollusca_ott802117@)Trochozoa,(Gastrotricha_ott29723@,Platyhelminthes_ott555379@)),(((Rotifera_ott471706@,Acanthocephala_ott49622@)Syndermata,Micrognathozoa_ott435981@),Gnathostomulida_ott242963@)Gnathifera)Spiralia,((((Onychophora_ott1072433@,Arthropoda_ott632179~-5008315-6293460-4387124@),Tardigrada_ott111438~-2943021@),(Nematoda_ott395057@,Nematomorpha_ott189836@)),Scalidophora_ott434620@)Ecdysozoa),Chaetognatha_ott570366@)Protostomia; +((((((Nemertea_ott445195@,(((Bryozoa_ott442934@,(Entoprocta_ott362916@,Cycliophora_ott41147@))Polyzoa,Phoronida_ott612209@),Brachiopoda_ott826261~-612209@)Lophophorata),Annelida_ott941620@),Mollusca_ott802117@)Trochozoa,(Gastrotricha_ott29723@,Platyhelminthes_ott555379@)),((((Rotifera_ott471706@,Acanthocephala_ott49622@)Syndermata,Micrognathozoa_ott435981@),Gnathostomulida_ott242963@),Chaetognatha_ott570366@)Gnathifera)Spiralia,((((Onychophora_ott1072433@,Arthropoda_ott632179~-5008315-6293460-4387124@),Tardigrada_ott111438~-2943021@),(Nematoda_ott395057@,Nematomorpha_ott189836@)),Scalidophora_ott434620@)Ecdysozoa)Protostomia; diff --git a/oz_tree_build/images_and_vernaculars/get_wiki_images.py b/oz_tree_build/images_and_vernaculars/get_wiki_images.py index ea62d7e..a4c6dcd 100644 --- a/oz_tree_build/images_and_vernaculars/get_wiki_images.py +++ b/oz_tree_build/images_and_vernaculars/get_wiki_images.py @@ -47,6 +47,7 @@ from .._OZglobals import src_flags from ..utilities.db_helper import ( connect_to_database, + default_appconfig, get_next_src_id_for_src, placeholder, read_config, @@ -60,6 +61,16 @@ logger = logging.getLogger(Path(__file__).name) +default_outdir = os.path.join( + os.pardir, + os.pardir, + os.pardir, + "OZtree", + "static", + "FinalOutputs", + "img", +) + # Copied from OZTree/OZprivate/ServerScripts/Utilities/getEOL_crops.py def subdir_name(doID): @@ -502,8 +513,10 @@ def process_leaf( s = placeholder(db) sql = "SELECT ott,wikidata,name FROM ordered_leaves WHERE " if ott_or_taxon.lstrip("-").isnumeric(): + ott_or_taxon_type = "ott" sql += f"ott={s};" else: + ott_or_taxon_type = "name" sql += f"name={s};" result = db.executesql(sql, (ott_or_taxon,)) @@ -511,7 +524,7 @@ def process_leaf( logger.error(f"Multiple results for '{ott_or_taxon}'") return if len(result) == 0: - logger.error(f"'{ott_or_taxon}' not found in ordered_leaves table") + logger.error(f"{ott_or_taxon_type} '{ott_or_taxon}' not found in ordered_leaves table") return (ott, qid, name) = result[0] @@ -672,16 +685,7 @@ def process_args(args): # Default to the static folder in the OZtree repo if outdir is None: - outdir = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - os.pardir, - os.pardir, - os.pardir, - "OZtree", - "static", - "FinalOutputs", - "img", - ) + outdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), default_outdir) if not os.path.exists(outdir): logger.error(f"Output directory '{outdir}' does not exist") return @@ -768,7 +772,7 @@ def add_common_args(parser): default=None, help=( "The location to save the image files (e.g. 'FinalOutputs/img'). " - "Defaults to ../../../static/FinalOutputs/img (relative to the script " + f"Defaults to {default_outdir} (relative to the script " "location). Files are saved to output_dir/{src_flag}/{3-digits}/fn.jpg" ), ) @@ -776,7 +780,7 @@ def add_common_args(parser): "-c", "--conf-file", default=None, - help=("The configuration file to use. " "Defaults to ../../../OZtree/private/appconfig.ini"), + help=(f"The configuration file to use. Defaults to {default_appconfig}"), ) parser_leaf = subparsers.add_parser("leaf", help="Process a single ott") diff --git a/oz_tree_build/utilities/db_helper.py b/oz_tree_build/utilities/db_helper.py index eb652a7..6782231 100644 --- a/oz_tree_build/utilities/db_helper.py +++ b/oz_tree_build/utilities/db_helper.py @@ -10,6 +10,8 @@ logger = logging.getLogger(__name__) +default_appconfig = "../../../OZtree/private/appconfig.ini" + def is_sqlite(db): return db._adapter.driver_name.startswith("sqlite") @@ -70,7 +72,7 @@ def read_config(conf_file=None): if conf_file is None: conf_file = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "../../../OZtree/private/appconfig.ini", + default_appconfig, ) if not os.path.exists(conf_file): raise ValueError(f"Appconfig file {conf_file} cannot be found.") diff --git a/oz_tree_build/utilities/generate_filtered_files.py b/oz_tree_build/utilities/generate_filtered_files.py index 6cd0ee0..7602315 100644 --- a/oz_tree_build/utilities/generate_filtered_files.py +++ b/oz_tree_build/utilities/generate_filtered_files.py @@ -34,6 +34,8 @@ __author__ = "David Ebbo" +one_zoom_file_prefix = "OneZoom" + def generate_and_cache_filtered_file(original_file, context, processing_function): """ @@ -43,7 +45,6 @@ def generate_and_cache_filtered_file(original_file, context, processing_function dirname = os.path.dirname(original_file) file_name = os.path.basename(original_file) - one_zoom_file_prefix = "OneZoom" filtered_file_prefix = (context.clade or one_zoom_file_prefix) + "_" if file_name.startswith(filtered_file_prefix): raise Exception(f"Input and output files are the same, with prefix {filtered_file_prefix}") @@ -495,10 +496,12 @@ def generate_all_filtered_files( generate_and_cache_filtered_file(eol_id_file, context, generate_filtered_eol_id_file) - filtered_wikidata_dump_file = generate_and_cache_filtered_file( - wikidata_dump_file, context, generate_filtered_wikidata_dump - ) - + if os.path.basename(wikidata_dump_file).startswith(one_zoom_file_prefix): + filtered_wikidata_dump_file = wikidata_dump_file + else: + filtered_wikidata_dump_file = generate_and_cache_filtered_file( + wikidata_dump_file, context, generate_filtered_wikidata_dump + ) read_wikidata_dump(filtered_wikidata_dump_file, context) generate_and_cache_filtered_file(wikipedia_sql_dump_file, context, generate_filtered_wikipedia_sql_dump) @@ -555,7 +558,10 @@ def main(): nargs="?", help=( "The b2zipped >4GB wikidata JSON dump, " - "from https://dumps.wikimedia.org/wikidatawiki/entities/ (latest-all.json.bz2) " + "from https://dumps.wikimedia.org/wikidatawiki/entities/ (latest-all.json.bz2). " + f"If this starts with '{one_zoom_file_prefix}', it is assumed to be filtered " + "already, which is useful if the original dumpfile is missing and additional " + "pageview files require filtering." ), ) parser.add_argument(