Config improvements

Snakefile

@@ -9,6 +9,8 @@ min_version("8.0.0")
 
 configfile: "config/configfile.yaml"
 
+include: "shared/vendored/snakemake/config.smk"
+include: "workflow/snakemake_rules/config.smk"
 
 wildcard_constraints:
     a_or_b=r"a|b",

config/configfile.yaml

@@ -61,6 +61,12 @@ filter:
     F: 3000
     F-antibody-escape: 2000
 
+  exclude_where:
+    recent: ["qc.overallStatus=bad"]
+    background: ["qc.overallStatus=bad", "qc.overallStatus=mediocre"]
+
+  missing_data_threshold: 1000
+
 files:
   auspice_config: "config/auspice_config.json"
   auspice_config_additional_colorings: "config/auspice_config_additional_colorings.json"
@@ -84,6 +90,15 @@ cds:
 traits:
   columns: "country region"
 
+frequencies:
+  resolutions:
+    all-time:
+      min_date: 1975-01-01
+    6y:
+      min_date: 6Y
+    3y:
+      min_date: 3Y
+
 nextclade_attributes:
   a:
     name: "RSV-A NextClade using real-time tree"

shared/vendored/.github/workflows/ci.yaml

@@ -11,5 +11,5 @@ jobs:
   shellcheck:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: nextstrain/.github/actions/shellcheck@master

shared/vendored/.github/workflows/pre-commit.yaml

@@ -7,7 +7,7 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
           python-version: "3.12"

shared/vendored/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/nextstrain/shared
 	branch = main
-	commit = bfbbb6875b084e22920712d89704ccb259f8950e
-	parent = 401279d49d2c703f5f79a584c4530dec736fbaad                                        
+	commit = 37cf39c2a4f4c42474046e70c856acb6a2031e2d
+	parent = f1f17cec6fe39fa2dd06429a44bcd925a441e47f
 	method = merge
-	cmdver = 0.4.6
+	cmdver = 0.4.9

shared/vendored/snakemake/config.smk

@@ -11,6 +11,45 @@ from typing import Optional
 from textwrap import dedent, indent
 
 
+# Set search paths for Augur
+if "AUGUR_SEARCH_PATHS" in os.environ:
+    print(dedent(f"""\
+        Using existing search paths in AUGUR_SEARCH_PATHS:
+
+            {os.environ["AUGUR_SEARCH_PATHS"]!r}
+        """), file=sys.stderr)
+else:
+    # Note that this differs from the search paths used in
+    # resolve_config_path().
+    # This is the preferred default moving forwards, and the plan is to
+    # eventually update resolve_config_path() to use AUGUR_SEARCH_PATHS.
+    search_paths = [
+        # User analysis directory
+        Path.cwd(),
+
+        # Workflow defaults folder
+        Path(workflow.basedir) / "defaults",
+
+        # Workflow root (contains Snakefile)
+        Path(workflow.basedir),
+    ]
+
+    # This should work for majority of workflows, but we could consider doing a
+    # more thorough search for the nextstrain-pathogen.yaml. This would likely
+    # replicate how CLI searches for the root.¹
+    # ¹ <https://github.com/nextstrain/cli/blob/d5e184c5/nextstrain/cli/command/build.py#L413-L420>
+    repo_root = Path(workflow.basedir) / ".."
+    if (repo_root / "nextstrain-pathogen.yaml").is_file():
+        search_paths.extend([
+            # Pathogen repo root
+            repo_root,
+        ])
+
+    search_paths = [path.resolve() for path in search_paths if path.is_dir()]
+
+    os.environ["AUGUR_SEARCH_PATHS"] = ":".join(map(str, search_paths))
+
+
 class InvalidConfigError(Exception):
     pass
 
@@ -147,15 +186,40 @@ def resolve_config_path(path: str, defaults_dir: Optional[str] = None) -> Callab
     return _resolve_config_path
 
 
-def write_config(path):
+def write_config(path, section=None):
     """
-    Write Snakemake's 'config' variable to a file.
+    Write Snakemake's 'config' variable, or a section of it, to a file.
+
+    *section* is an optional list of keys to navigate to a specific section of
+    config. If provided, only that section will be written.
     """
     global config
 
     os.makedirs(os.path.dirname(path), exist_ok=True)
 
+    data = config
+    section_str = "config"
+
+    if section:
+        # Navigate to the specified section
+        for key in section:
+            # Error if key doesn't exist
+            if key not in data:
+                raise Exception(f"ERROR: Key {key!r} not found in {section_str!r}.")
+
+            data = data[key]
+            section_str += f".{key}"
+
+            # Error if value is not a mapping
+            if not isinstance(data, dict):
+                raise Exception(f"ERROR: {section_str!r} is not a mapping of key/value pairs.")
+
     with open(path, 'w') as f:
-        yaml.dump(config, f, sort_keys=False)
+        yaml.dump(data, f, sort_keys=False, Dumper=NoAliasDumper)
+
+    print(f"Saved {section_str!r} to {path!r}.", file=sys.stderr)
+
 
-    print(f"Saved current run config to {path!r}.", file=sys.stderr)
+class NoAliasDumper(yaml.SafeDumper):
+    def ignore_aliases(self, data):
+        return True

workflow/snakemake_rules/chores.smk

@@ -6,7 +6,7 @@ rule update_example_data_wildcards:
 
     The subset of data is generated by an augur filter call which:
     - sets the subsampling size to 50
-    - applies the grouping from the config
+    - groups by year and country
     """
     message:
         "Update example data"
@@ -18,14 +18,13 @@ rule update_example_data_wildcards:
         metadata = "example_data/{a_or_b}/metadata.tsv",
     params:
         strain_id=config["strain_id_field"],
-        group_by=config["filter"]["group_by"],
     shell:
         """
         augur filter \
             --metadata {input.metadata} \
             --metadata-id-columns {params.strain_id} \
             --sequences {input.sequences} \
-            --group-by {params.group_by} \
+            --group-by year country \
             --subsample-max-sequences 50 \
             --subsample-seed 0 \
             --output-metadata {output.metadata} \

workflow/snakemake_rules/config.smk

@@ -0,0 +1,8 @@
+"""
+This part of the workflow deals with configuration.
+
+OUTPUTS:
+
+    results/run_config.yaml
+"""
+write_config("results/run_config.yaml")

workflow/snakemake_rules/core.smk

@@ -55,7 +55,6 @@ rule filter_recent:
         """
     input:
         sequences="results/{a_or_b}/sequences.fasta",
-        reference="config/{a_or_b}reference.gbk",
         metadata="results/{a_or_b}/metadata.tsv",
         sequence_index=rules.index_sequences.output,
         exclude=config["exclude"],
@@ -71,6 +70,8 @@ rule filter_recent:
         ][w.build_name],
         strain_id=config["strain_id_field"],
         min_date=lambda w: config["filter"]["resolutions"][w.resolution]["min_date"],
+        exclude_where=config["filter"]["exclude_where"]["recent"],
+        missing_data_threshold=config["filter"]["missing_data_threshold"],
     shell:
         """
         augur filter \
@@ -79,13 +80,13 @@ rule filter_recent:
             --metadata {input.metadata} \
             --metadata-id-columns {params.strain_id} \
             --exclude {input.exclude} \
-            --exclude-where 'qc.overallStatus=bad' \
+            --exclude-where {params.exclude_where:q} \
             --min-date {params.min_date} \
             --min-length {params.min_length} \
             --output {output.sequences} \
             --group-by {params.group_by} \
             --subsample-max-sequences {params.subsample_max_sequences} \
-            --query '({params.min_coverage}) & missing_data<1000'
+            --query '({params.min_coverage}) & missing_data<{params.missing_data_threshold}'
         """
 
 
@@ -96,7 +97,6 @@ rule filter_background:
         """
     input:
         sequences="results/{a_or_b}/sequences.fasta",
-        reference="config/{a_or_b}reference.gbk",
         metadata="results/{a_or_b}/metadata.tsv",
         sequence_index=rules.index_sequences.output,
         include="config/include_{a_or_b}.txt",
@@ -119,6 +119,8 @@ rule filter_background:
         min_date=lambda w: config["filter"]["resolutions"][w.resolution][
             "background_min_date"
         ],
+        exclude_where=config["filter"]["exclude_where"]["background"],
+        missing_data_threshold=config["filter"]["missing_data_threshold"],
     shell:
         """
         augur filter \
@@ -128,15 +130,15 @@ rule filter_background:
             --metadata-id-columns {params.strain_id} \
             --include {input.include} \
             --exclude {input.exclude} \
-            --exclude-where 'qc.overallStatus=bad' 'qc.overallStatus=mediocre' \
+            --exclude-where {params.exclude_where:q}  \
             --min-date {params.min_date} \
             --max-date {params.max_date} \
             --min-length {params.min_length} \
             --output-sequences {output.sequences} \
             --output-metadata {output.metadata} \
             --group-by {params.group_by} \
             --subsample-max-sequences {params.subsample_max_sequences} \
-            --query '({params.min_coverage}) & missing_data<1000'
+            --query '({params.min_coverage}) & missing_data<{params.missing_data_threshold}'
         """
 
 rule exclude_preduplication:
@@ -681,7 +683,7 @@ rule frequencies:
     output:
         frequencies = build_dir + "/{a_or_b}/{build_name}/{resolution}/frequencies.json"
     params:
-        min_date_arg = lambda w: f"--min-date {config['filter']['resolutions'][w.resolution]['min_date']}",
+        min_date_arg = lambda w: f"--min-date {config['frequencies']['resolutions'][w.resolution]['min_date']}",
     shell:
         """
         augur frequencies \