From f8f4820a4b3756951651e2d5cc099b09914c8bee Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Tue, 3 Mar 2026 10:16:00 -0800 Subject: [PATCH 1/6] git subrepo pull (merge) shared/vendored subrepo: subdir: "shared/vendored" merged: "37cf39c" upstream: origin: "https://github.com/nextstrain/shared" branch: "main" commit: "37cf39c" git-subrepo: version: "0.4.9" origin: "https://github.com/ingydotnet/git-subrepo" commit: "4f60dd7" --- shared/vendored/.github/workflows/ci.yaml | 2 +- .../.github/workflows/pre-commit.yaml | 2 +- shared/vendored/.gitrepo | 6 +- shared/vendored/snakemake/config.smk | 72 +++++++++++++++++-- 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/shared/vendored/.github/workflows/ci.yaml b/shared/vendored/.github/workflows/ci.yaml index 94d3054..6dba82b 100644 --- a/shared/vendored/.github/workflows/ci.yaml +++ b/shared/vendored/.github/workflows/ci.yaml @@ -11,5 +11,5 @@ jobs: shellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: nextstrain/.github/actions/shellcheck@master diff --git a/shared/vendored/.github/workflows/pre-commit.yaml b/shared/vendored/.github/workflows/pre-commit.yaml index bea15f6..c63b890 100644 --- a/shared/vendored/.github/workflows/pre-commit.yaml +++ b/shared/vendored/.github/workflows/pre-commit.yaml @@ -7,7 +7,7 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/shared/vendored/.gitrepo b/shared/vendored/.gitrepo index c561adb..3723d3d 100644 --- a/shared/vendored/.gitrepo +++ b/shared/vendored/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/nextstrain/shared branch = main - commit = bfbbb6875b084e22920712d89704ccb259f8950e - parent = 401279d49d2c703f5f79a584c4530dec736fbaad + commit = 37cf39c2a4f4c42474046e70c856acb6a2031e2d + parent = f1f17cec6fe39fa2dd06429a44bcd925a441e47f method = merge - cmdver = 0.4.6 + cmdver = 0.4.9 diff --git a/shared/vendored/snakemake/config.smk b/shared/vendored/snakemake/config.smk index 15727b0..2f0df30 100644 --- a/shared/vendored/snakemake/config.smk +++ b/shared/vendored/snakemake/config.smk @@ -11,6 +11,45 @@ from typing import Optional from textwrap import dedent, indent +# Set search paths for Augur +if "AUGUR_SEARCH_PATHS" in os.environ: + print(dedent(f"""\ + Using existing search paths in AUGUR_SEARCH_PATHS: + + {os.environ["AUGUR_SEARCH_PATHS"]!r} + """), file=sys.stderr) +else: + # Note that this differs from the search paths used in + # resolve_config_path(). + # This is the preferred default moving forwards, and the plan is to + # eventually update resolve_config_path() to use AUGUR_SEARCH_PATHS. + search_paths = [ + # User analysis directory + Path.cwd(), + + # Workflow defaults folder + Path(workflow.basedir) / "defaults", + + # Workflow root (contains Snakefile) + Path(workflow.basedir), + ] + + # This should work for majority of workflows, but we could consider doing a + # more thorough search for the nextstrain-pathogen.yaml. This would likely + # replicate how CLI searches for the root.¹ + # ¹ + repo_root = Path(workflow.basedir) / ".." + if (repo_root / "nextstrain-pathogen.yaml").is_file(): + search_paths.extend([ + # Pathogen repo root + repo_root, + ]) + + search_paths = [path.resolve() for path in search_paths if path.is_dir()] + + os.environ["AUGUR_SEARCH_PATHS"] = ":".join(map(str, search_paths)) + + class InvalidConfigError(Exception): pass @@ -147,15 +186,40 @@ def resolve_config_path(path: str, defaults_dir: Optional[str] = None) -> Callab return _resolve_config_path -def write_config(path): +def write_config(path, section=None): """ - Write Snakemake's 'config' variable to a file. + Write Snakemake's 'config' variable, or a section of it, to a file. + + *section* is an optional list of keys to navigate to a specific section of + config. If provided, only that section will be written. """ global config os.makedirs(os.path.dirname(path), exist_ok=True) + data = config + section_str = "config" + + if section: + # Navigate to the specified section + for key in section: + # Error if key doesn't exist + if key not in data: + raise Exception(f"ERROR: Key {key!r} not found in {section_str!r}.") + + data = data[key] + section_str += f".{key}" + + # Error if value is not a mapping + if not isinstance(data, dict): + raise Exception(f"ERROR: {section_str!r} is not a mapping of key/value pairs.") + with open(path, 'w') as f: - yaml.dump(config, f, sort_keys=False) + yaml.dump(data, f, sort_keys=False, Dumper=NoAliasDumper) + + print(f"Saved {section_str!r} to {path!r}.", file=sys.stderr) + - print(f"Saved current run config to {path!r}.", file=sys.stderr) +class NoAliasDumper(yaml.SafeDumper): + def ignore_aliases(self, data): + return True From 9243fdfe73aff5c3176ca50787905e18dfb0b308 Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Mon, 22 Sep 2025 13:31:39 -0700 Subject: [PATCH 2/6] Write config to a file Useful for debugging. --- Snakefile | 2 ++ workflow/snakemake_rules/config.smk | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100644 workflow/snakemake_rules/config.smk diff --git a/Snakefile b/Snakefile index 1ac3c23..d2ffb97 100644 --- a/Snakefile +++ b/Snakefile @@ -9,6 +9,8 @@ min_version("8.0.0") configfile: "config/configfile.yaml" +include: "shared/vendored/snakemake/config.smk" +include: "workflow/snakemake_rules/config.smk" wildcard_constraints: a_or_b=r"a|b", diff --git a/workflow/snakemake_rules/config.smk b/workflow/snakemake_rules/config.smk new file mode 100644 index 0000000..199b4a9 --- /dev/null +++ b/workflow/snakemake_rules/config.smk @@ -0,0 +1,8 @@ +""" +This part of the workflow deals with configuration. + +OUTPUTS: + + results/run_config.yaml +""" +write_config("results/run_config.yaml") From be77edd4d6a26b900fec8d6d72ba60e00b1dfecb Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Mon, 22 Sep 2025 12:29:16 -0700 Subject: [PATCH 3/6] Remove unused input augur filter has never taken a reference file as input, so my guess is this was originally copied by mistake. --- workflow/snakemake_rules/core.smk | 2 -- 1 file changed, 2 deletions(-) diff --git a/workflow/snakemake_rules/core.smk b/workflow/snakemake_rules/core.smk index 154e851..8ee57ea 100644 --- a/workflow/snakemake_rules/core.smk +++ b/workflow/snakemake_rules/core.smk @@ -55,7 +55,6 @@ rule filter_recent: """ input: sequences="results/{a_or_b}/sequences.fasta", - reference="config/{a_or_b}reference.gbk", metadata="results/{a_or_b}/metadata.tsv", sequence_index=rules.index_sequences.output, exclude=config["exclude"], @@ -96,7 +95,6 @@ rule filter_background: """ input: sequences="results/{a_or_b}/sequences.fasta", - reference="config/{a_or_b}reference.gbk", metadata="results/{a_or_b}/metadata.tsv", sequence_index=rules.index_sequences.output, include="config/include_{a_or_b}.txt", From a8e39f03c9421f92b200daa600d652fc51c3bd0f Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Mon, 22 Sep 2025 13:32:49 -0700 Subject: [PATCH 4/6] Decouple example data grouping columns from config Preparing to make changes to the config that would have broken this. It seems fine to hardcode along with the already hardcoded sample size. --- workflow/snakemake_rules/chores.smk | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/workflow/snakemake_rules/chores.smk b/workflow/snakemake_rules/chores.smk index eae7d72..234f8a6 100644 --- a/workflow/snakemake_rules/chores.smk +++ b/workflow/snakemake_rules/chores.smk @@ -6,7 +6,7 @@ rule update_example_data_wildcards: The subset of data is generated by an augur filter call which: - sets the subsampling size to 50 - - applies the grouping from the config + - groups by year and country """ message: "Update example data" @@ -18,14 +18,13 @@ rule update_example_data_wildcards: metadata = "example_data/{a_or_b}/metadata.tsv", params: strain_id=config["strain_id_field"], - group_by=config["filter"]["group_by"], shell: """ augur filter \ --metadata {input.metadata} \ --metadata-id-columns {params.strain_id} \ --sequences {input.sequences} \ - --group-by {params.group_by} \ + --group-by year country \ --subsample-max-sequences 50 \ --subsample-seed 0 \ --output-metadata {output.metadata} \ From a87eebdee51e58f7ca5df9654591307fb480883d Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Mon, 22 Sep 2025 13:51:38 -0700 Subject: [PATCH 5/6] Move all filter parameters to config This will make it easier to compare changes in the switch to augur subsample which will define all parameters in config. --- config/configfile.yaml | 6 ++++++ workflow/snakemake_rules/core.smk | 12 ++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/config/configfile.yaml b/config/configfile.yaml index 87271ce..8288f4a 100644 --- a/config/configfile.yaml +++ b/config/configfile.yaml @@ -61,6 +61,12 @@ filter: F: 3000 F-antibody-escape: 2000 + exclude_where: + recent: ["qc.overallStatus=bad"] + background: ["qc.overallStatus=bad", "qc.overallStatus=mediocre"] + + missing_data_threshold: 1000 + files: auspice_config: "config/auspice_config.json" auspice_config_additional_colorings: "config/auspice_config_additional_colorings.json" diff --git a/workflow/snakemake_rules/core.smk b/workflow/snakemake_rules/core.smk index 8ee57ea..fa801fb 100644 --- a/workflow/snakemake_rules/core.smk +++ b/workflow/snakemake_rules/core.smk @@ -70,6 +70,8 @@ rule filter_recent: ][w.build_name], strain_id=config["strain_id_field"], min_date=lambda w: config["filter"]["resolutions"][w.resolution]["min_date"], + exclude_where=config["filter"]["exclude_where"]["recent"], + missing_data_threshold=config["filter"]["missing_data_threshold"], shell: """ augur filter \ @@ -78,13 +80,13 @@ rule filter_recent: --metadata {input.metadata} \ --metadata-id-columns {params.strain_id} \ --exclude {input.exclude} \ - --exclude-where 'qc.overallStatus=bad' \ + --exclude-where {params.exclude_where:q} \ --min-date {params.min_date} \ --min-length {params.min_length} \ --output {output.sequences} \ --group-by {params.group_by} \ --subsample-max-sequences {params.subsample_max_sequences} \ - --query '({params.min_coverage}) & missing_data<1000' + --query '({params.min_coverage}) & missing_data<{params.missing_data_threshold}' """ @@ -117,6 +119,8 @@ rule filter_background: min_date=lambda w: config["filter"]["resolutions"][w.resolution][ "background_min_date" ], + exclude_where=config["filter"]["exclude_where"]["background"], + missing_data_threshold=config["filter"]["missing_data_threshold"], shell: """ augur filter \ @@ -126,7 +130,7 @@ rule filter_background: --metadata-id-columns {params.strain_id} \ --include {input.include} \ --exclude {input.exclude} \ - --exclude-where 'qc.overallStatus=bad' 'qc.overallStatus=mediocre' \ + --exclude-where {params.exclude_where:q} \ --min-date {params.min_date} \ --max-date {params.max_date} \ --min-length {params.min_length} \ @@ -134,7 +138,7 @@ rule filter_background: --output-metadata {output.metadata} \ --group-by {params.group_by} \ --subsample-max-sequences {params.subsample_max_sequences} \ - --query '({params.min_coverage}) & missing_data<1000' + --query '({params.min_coverage}) & missing_data<{params.missing_data_threshold}' """ rule exclude_preduplication: From 0b2218513525469da029b5f78c131f9069aac724 Mon Sep 17 00:00:00 2001 From: Victor Lin <13424970+victorlin@users.noreply.github.com> Date: Mon, 22 Sep 2025 15:22:47 -0700 Subject: [PATCH 6/6] Add separate frequencies config This shouldn't rely on config from another rule. --- config/configfile.yaml | 9 +++++++++ workflow/snakemake_rules/core.smk | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/config/configfile.yaml b/config/configfile.yaml index 8288f4a..3bd9a4c 100644 --- a/config/configfile.yaml +++ b/config/configfile.yaml @@ -90,6 +90,15 @@ cds: traits: columns: "country region" +frequencies: + resolutions: + all-time: + min_date: 1975-01-01 + 6y: + min_date: 6Y + 3y: + min_date: 3Y + nextclade_attributes: a: name: "RSV-A NextClade using real-time tree" diff --git a/workflow/snakemake_rules/core.smk b/workflow/snakemake_rules/core.smk index fa801fb..88ef71c 100644 --- a/workflow/snakemake_rules/core.smk +++ b/workflow/snakemake_rules/core.smk @@ -683,7 +683,7 @@ rule frequencies: output: frequencies = build_dir + "/{a_or_b}/{build_name}/{resolution}/frequencies.json" params: - min_date_arg = lambda w: f"--min-date {config['filter']['resolutions'][w.resolution]['min_date']}", + min_date_arg = lambda w: f"--min-date {config['frequencies']['resolutions'][w.resolution]['min_date']}", shell: """ augur frequencies \