diff --git a/Snakefile b/Snakefile index 1ac3c23..d2ffb97 100644 --- a/Snakefile +++ b/Snakefile @@ -9,6 +9,8 @@ min_version("8.0.0") configfile: "config/configfile.yaml" +include: "shared/vendored/snakemake/config.smk" +include: "workflow/snakemake_rules/config.smk" wildcard_constraints: a_or_b=r"a|b", diff --git a/config/configfile.yaml b/config/configfile.yaml index 87271ce..3bd9a4c 100644 --- a/config/configfile.yaml +++ b/config/configfile.yaml @@ -61,6 +61,12 @@ filter: F: 3000 F-antibody-escape: 2000 + exclude_where: + recent: ["qc.overallStatus=bad"] + background: ["qc.overallStatus=bad", "qc.overallStatus=mediocre"] + + missing_data_threshold: 1000 + files: auspice_config: "config/auspice_config.json" auspice_config_additional_colorings: "config/auspice_config_additional_colorings.json" @@ -84,6 +90,15 @@ cds: traits: columns: "country region" +frequencies: + resolutions: + all-time: + min_date: 1975-01-01 + 6y: + min_date: 6Y + 3y: + min_date: 3Y + nextclade_attributes: a: name: "RSV-A NextClade using real-time tree" diff --git a/shared/vendored/.github/workflows/ci.yaml b/shared/vendored/.github/workflows/ci.yaml index 94d3054..6dba82b 100644 --- a/shared/vendored/.github/workflows/ci.yaml +++ b/shared/vendored/.github/workflows/ci.yaml @@ -11,5 +11,5 @@ jobs: shellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: nextstrain/.github/actions/shellcheck@master diff --git a/shared/vendored/.github/workflows/pre-commit.yaml b/shared/vendored/.github/workflows/pre-commit.yaml index bea15f6..c63b890 100644 --- a/shared/vendored/.github/workflows/pre-commit.yaml +++ b/shared/vendored/.github/workflows/pre-commit.yaml @@ -7,7 +7,7 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/shared/vendored/.gitrepo b/shared/vendored/.gitrepo index c561adb..3723d3d 100644 --- a/shared/vendored/.gitrepo +++ b/shared/vendored/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/nextstrain/shared branch = main - commit = bfbbb6875b084e22920712d89704ccb259f8950e - parent = 401279d49d2c703f5f79a584c4530dec736fbaad + commit = 37cf39c2a4f4c42474046e70c856acb6a2031e2d + parent = f1f17cec6fe39fa2dd06429a44bcd925a441e47f method = merge - cmdver = 0.4.6 + cmdver = 0.4.9 diff --git a/shared/vendored/snakemake/config.smk b/shared/vendored/snakemake/config.smk index 15727b0..2f0df30 100644 --- a/shared/vendored/snakemake/config.smk +++ b/shared/vendored/snakemake/config.smk @@ -11,6 +11,45 @@ from typing import Optional from textwrap import dedent, indent +# Set search paths for Augur +if "AUGUR_SEARCH_PATHS" in os.environ: + print(dedent(f"""\ + Using existing search paths in AUGUR_SEARCH_PATHS: + + {os.environ["AUGUR_SEARCH_PATHS"]!r} + """), file=sys.stderr) +else: + # Note that this differs from the search paths used in + # resolve_config_path(). + # This is the preferred default moving forwards, and the plan is to + # eventually update resolve_config_path() to use AUGUR_SEARCH_PATHS. + search_paths = [ + # User analysis directory + Path.cwd(), + + # Workflow defaults folder + Path(workflow.basedir) / "defaults", + + # Workflow root (contains Snakefile) + Path(workflow.basedir), + ] + + # This should work for majority of workflows, but we could consider doing a + # more thorough search for the nextstrain-pathogen.yaml. This would likely + # replicate how CLI searches for the root.¹ + # ¹ + repo_root = Path(workflow.basedir) / ".." + if (repo_root / "nextstrain-pathogen.yaml").is_file(): + search_paths.extend([ + # Pathogen repo root + repo_root, + ]) + + search_paths = [path.resolve() for path in search_paths if path.is_dir()] + + os.environ["AUGUR_SEARCH_PATHS"] = ":".join(map(str, search_paths)) + + class InvalidConfigError(Exception): pass @@ -147,15 +186,40 @@ def resolve_config_path(path: str, defaults_dir: Optional[str] = None) -> Callab return _resolve_config_path -def write_config(path): +def write_config(path, section=None): """ - Write Snakemake's 'config' variable to a file. + Write Snakemake's 'config' variable, or a section of it, to a file. + + *section* is an optional list of keys to navigate to a specific section of + config. If provided, only that section will be written. """ global config os.makedirs(os.path.dirname(path), exist_ok=True) + data = config + section_str = "config" + + if section: + # Navigate to the specified section + for key in section: + # Error if key doesn't exist + if key not in data: + raise Exception(f"ERROR: Key {key!r} not found in {section_str!r}.") + + data = data[key] + section_str += f".{key}" + + # Error if value is not a mapping + if not isinstance(data, dict): + raise Exception(f"ERROR: {section_str!r} is not a mapping of key/value pairs.") + with open(path, 'w') as f: - yaml.dump(config, f, sort_keys=False) + yaml.dump(data, f, sort_keys=False, Dumper=NoAliasDumper) + + print(f"Saved {section_str!r} to {path!r}.", file=sys.stderr) + - print(f"Saved current run config to {path!r}.", file=sys.stderr) +class NoAliasDumper(yaml.SafeDumper): + def ignore_aliases(self, data): + return True diff --git a/workflow/snakemake_rules/chores.smk b/workflow/snakemake_rules/chores.smk index eae7d72..234f8a6 100644 --- a/workflow/snakemake_rules/chores.smk +++ b/workflow/snakemake_rules/chores.smk @@ -6,7 +6,7 @@ rule update_example_data_wildcards: The subset of data is generated by an augur filter call which: - sets the subsampling size to 50 - - applies the grouping from the config + - groups by year and country """ message: "Update example data" @@ -18,14 +18,13 @@ rule update_example_data_wildcards: metadata = "example_data/{a_or_b}/metadata.tsv", params: strain_id=config["strain_id_field"], - group_by=config["filter"]["group_by"], shell: """ augur filter \ --metadata {input.metadata} \ --metadata-id-columns {params.strain_id} \ --sequences {input.sequences} \ - --group-by {params.group_by} \ + --group-by year country \ --subsample-max-sequences 50 \ --subsample-seed 0 \ --output-metadata {output.metadata} \ diff --git a/workflow/snakemake_rules/config.smk b/workflow/snakemake_rules/config.smk new file mode 100644 index 0000000..199b4a9 --- /dev/null +++ b/workflow/snakemake_rules/config.smk @@ -0,0 +1,8 @@ +""" +This part of the workflow deals with configuration. + +OUTPUTS: + + results/run_config.yaml +""" +write_config("results/run_config.yaml") diff --git a/workflow/snakemake_rules/core.smk b/workflow/snakemake_rules/core.smk index 154e851..88ef71c 100644 --- a/workflow/snakemake_rules/core.smk +++ b/workflow/snakemake_rules/core.smk @@ -55,7 +55,6 @@ rule filter_recent: """ input: sequences="results/{a_or_b}/sequences.fasta", - reference="config/{a_or_b}reference.gbk", metadata="results/{a_or_b}/metadata.tsv", sequence_index=rules.index_sequences.output, exclude=config["exclude"], @@ -71,6 +70,8 @@ rule filter_recent: ][w.build_name], strain_id=config["strain_id_field"], min_date=lambda w: config["filter"]["resolutions"][w.resolution]["min_date"], + exclude_where=config["filter"]["exclude_where"]["recent"], + missing_data_threshold=config["filter"]["missing_data_threshold"], shell: """ augur filter \ @@ -79,13 +80,13 @@ rule filter_recent: --metadata {input.metadata} \ --metadata-id-columns {params.strain_id} \ --exclude {input.exclude} \ - --exclude-where 'qc.overallStatus=bad' \ + --exclude-where {params.exclude_where:q} \ --min-date {params.min_date} \ --min-length {params.min_length} \ --output {output.sequences} \ --group-by {params.group_by} \ --subsample-max-sequences {params.subsample_max_sequences} \ - --query '({params.min_coverage}) & missing_data<1000' + --query '({params.min_coverage}) & missing_data<{params.missing_data_threshold}' """ @@ -96,7 +97,6 @@ rule filter_background: """ input: sequences="results/{a_or_b}/sequences.fasta", - reference="config/{a_or_b}reference.gbk", metadata="results/{a_or_b}/metadata.tsv", sequence_index=rules.index_sequences.output, include="config/include_{a_or_b}.txt", @@ -119,6 +119,8 @@ rule filter_background: min_date=lambda w: config["filter"]["resolutions"][w.resolution][ "background_min_date" ], + exclude_where=config["filter"]["exclude_where"]["background"], + missing_data_threshold=config["filter"]["missing_data_threshold"], shell: """ augur filter \ @@ -128,7 +130,7 @@ rule filter_background: --metadata-id-columns {params.strain_id} \ --include {input.include} \ --exclude {input.exclude} \ - --exclude-where 'qc.overallStatus=bad' 'qc.overallStatus=mediocre' \ + --exclude-where {params.exclude_where:q} \ --min-date {params.min_date} \ --max-date {params.max_date} \ --min-length {params.min_length} \ @@ -136,7 +138,7 @@ rule filter_background: --output-metadata {output.metadata} \ --group-by {params.group_by} \ --subsample-max-sequences {params.subsample_max_sequences} \ - --query '({params.min_coverage}) & missing_data<1000' + --query '({params.min_coverage}) & missing_data<{params.missing_data_threshold}' """ rule exclude_preduplication: @@ -681,7 +683,7 @@ rule frequencies: output: frequencies = build_dir + "/{a_or_b}/{build_name}/{resolution}/frequencies.json" params: - min_date_arg = lambda w: f"--min-date {config['filter']['resolutions'][w.resolution]['min_date']}", + min_date_arg = lambda w: f"--min-date {config['frequencies']['resolutions'][w.resolution]['min_date']}", shell: """ augur frequencies \