diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8cf05e0725..cf88e0d39d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,7 @@ repos: hooks: - id: ruff-check # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix + priority: 10 - id: ruff-format # formatter - repo: https://github.com/pre-commit/mirrors-prettier rev: "v3.1.0" @@ -17,6 +18,7 @@ repos: hooks: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] + priority: 10 exclude: | (?x)^( .*\.snap$| @@ -25,6 +27,7 @@ repos: tests/pipelines/__snapshots__/.* )$ - id: end-of-file-fixer + priority: 10 exclude: | (?x)^( .*\.snap$| @@ -37,6 +40,7 @@ repos: hooks: - id: mypy args: [--ignore-missing-imports, --scripts-are-modules, -n4] + priority: 20 additional_dependencies: - types-PyYAML - types-requests @@ -48,10 +52,12 @@ repos: rev: 0.11.19 hooks: - id: uv-lock + priority: 20 - repo: local hooks: - id: lint-test-docstrings name: Check lint test names are documented in docstrings language: system entry: python3 docs/api/check_lint_docstrings.py + priority: 20 files: ^nf_core/(modules|subworkflows)/lint/.*\.py$ diff --git a/CHANGELOG.md b/CHANGELOG.md index 622987cc5a..33399c7b72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,11 @@ ### General -- Update pre-commit hook pre-commit/mirrors-mypy to v2 ([#4270](https://github.com/nf-core/tools/pull/4270)) - container configs: use correct key for conda configs ([#4269](https://github.com/nf-core/tools/pull/4269)) - Coerce launch params_out to Path to fix AttributeError (#4299) ([#4317](https://github.com/nf-core/tools/pull/4317)) - generate valid Markdown when piping schema docs to file ([#4319](https://github.com/nf-core/tools/pull/4319)) - fix API docs bug in pydantic autodoc ([#4320](https://github.com/nf-core/tools/pull/4320)) -- Update pre-commit npm dependencies to v3.8.4 ([#4335](https://github.com/nf-core/tools/pull/4335)) +- Improve CLI startup speed ([#4383](https://github.com/nf-core/tools/pull/4383)) ### Linting @@ -21,12 +20,10 @@ ### Modules - `modules info`: handle one element channels correctly ([#4268](https://github.com/nf-core/tools/pull/4268)) -- feat: module lint for module name granularity ([#4325](https://github.com/nf-core/tools/pull/4325)) +- `module` lint for module name matching against nf-core module naming specifications ([#4325](https://github.com/nf-core/tools/pull/4325)) - Clarify execution context of nf-core modules test command ([#4346](https://github.com/nf-core/tools/pull/4346)) - Add `nf-core modules containers create` to build module containers via Seqera Wave and record them in the `containers:` section of `meta.yml` ([#3954](https://github.com/nf-core/tools/pull/3954)) -- Container linting now validates the `containers:` section of `meta.yml` (Wave modules). The legacy `main.nf` container checks (URL reachability, registry prefix, docker/singularity tag match) have been removed; modules still using the old container syntax get a `deprecated_container_syntax` warning instead ([#3954](https://github.com/nf-core/tools/pull/3954)) - -### Subworkflows +- Add a log hint in `modules create` for `create containers` and switch `modules bump-versions` to use Seqera containers ([#4374](https://github.com/nf-core/tools/pull/4374)) ### Template @@ -45,6 +42,9 @@ #### Version updates +- Update pre-commit hook pre-commit/mirrors-mypy to v2 ([#4270](https://github.com/nf-core/tools/pull/4270)) +- Update pre-commit npm dependencies to v3.8.4 ([#4335](https://github.com/nf-core/tools/pull/4335)) + ## [v4.0.2 - Bold Boa Patch 2](https://github.com/nf-core/tools/releases/tag/4.0.2) - [2026-04-30] ### General diff --git a/nf_core/__main__.py b/nf_core/__main__.py index e05bfa0b43..bdf60b8cda 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -6,96 +6,55 @@ import sys from pathlib import Path -import requests -import rich.console -import rich.logging -import rich.panel -import rich.traceback import rich_click as click import rich_click.rich_click as rc -from trogon import tui from nf_core import __version__ -from nf_core.commands_modules import ( - modules_bump_versions, - modules_containers_conda_lock, - modules_containers_create, - modules_containers_list, - modules_create, - modules_info, - modules_install, - modules_lint, - modules_list_local, - modules_list_remote, - modules_patch, - modules_remove, - modules_test, - modules_update, -) -from nf_core.commands_pipelines import ( - pipelines_bump_version, - pipelines_create, - pipelines_create_logo, - pipelines_create_params_file, - pipelines_download, - pipelines_launch, - pipelines_lint, - pipelines_list, - pipelines_rocrate, - pipelines_schema_build, - pipelines_schema_docs, - pipelines_schema_lint, - pipelines_schema_validate, - pipelines_sync, -) -from nf_core.commands_subworkflows import ( - subworkflows_create, - subworkflows_info, - subworkflows_install, - subworkflows_lint, - subworkflows_list_local, - subworkflows_list_remote, - subworkflows_remove, - subworkflows_test, - subworkflows_update, -) -from nf_core.commands_test_datasets import test_datasets_list_branches, test_datasets_list_remote, test_datasets_search -from nf_core.components.components_completion import autocomplete_modules, autocomplete_subworkflows from nf_core.components.constants import NF_CORE_MODULES_REMOTE -from nf_core.pipelines.download.download import DownloadError -from nf_core.pipelines.list import autocomplete_pipelines -from nf_core.utils import check_if_outdated, nfcore_logo, rich_force_colors, setup_nfcore_dir +from nf_core.utils import check_if_outdated, nfcore_logo, stderr + + +def autocomplete_pipelines(ctx, param, incomplete): + from nf_core.pipelines.list import autocomplete_pipelines as complete + + return complete(ctx, param, incomplete) + + +def autocomplete_modules(ctx, param, incomplete): + from nf_core.components.components_completion import autocomplete_modules as complete + + return complete(ctx, param, incomplete) + + +def autocomplete_subworkflows(ctx, param, incomplete): + from nf_core.components.components_completion import autocomplete_subworkflows as complete + + return complete(ctx, param, incomplete) + # Set up logging as the root logger # Submodules should all traverse back to this log = logging.getLogger() -# Set up .nfcore directory for storing files between sessions -setup_nfcore_dir() - # Set up nicer formatting of click cli help messages rc.MAX_WIDTH = 100 rc.USE_RICH_MARKUP = True rc.COMMANDS_BEFORE_OPTIONS = True -# Set up rich stderr console -stderr = rich.console.Console(stderr=True, force_terminal=rich_force_colors()) -stdout = rich.console.Console(force_terminal=rich_force_colors()) - -# Set up the rich traceback -rich.traceback.install(console=stderr, width=200, word_wrap=True, extra_lines=1) - - # Define exceptions for which no traceback should be printed, # because they are actually preliminary, but intended program terminations. # (Custom exceptions are cleaner than `sys.exit(1)`, which we used before) def selective_traceback_hook(exctype, value, traceback): + from nf_core.pipelines.download.download import DownloadError + if exctype in {DownloadError, UserWarning, ValueError}: # extend set as needed log.error(value) else: + from rich.traceback import Traceback + # print the colored traceback for all other exceptions with rich as usual - stderr.print(rich.traceback.Traceback.from_exception(exctype, value, traceback)) + stderr.print(Traceback.from_exception(exctype, value, traceback, width=200, extra_lines=1)) sys.excepthook = selective_traceback_hook @@ -120,21 +79,17 @@ def run_nf_core(): f"\n[grey39] nf-core/tools version {__version__} - [link=https://nf-co.re]https://nf-co.re[/]", highlight=False, ) - try: - is_outdated, _, remote_vers = check_if_outdated() - if is_outdated: - stderr.print( - f"[bold bright_yellow] There is a new version of nf-core/tools available! ({remote_vers})", - highlight=False, - ) - except requests.exceptions.RequestException as e: - log.debug(f"Could not check latest version: {e}") + is_outdated, _, remote_vers = check_if_outdated() + if is_outdated: + stderr.print( + f"[bold bright_yellow] There is a new version of nf-core/tools available! ({remote_vers})", + highlight=False, + ) stderr.print("\n") # Launch the click cli nf_core_cli(auto_envvar_prefix="NFCORE") -@tui(command="interface", help="Launch the nf-core interface") @click.group(context_settings={"help_option_names": ["-h", "--help"]}) @click.version_option(__version__) @click.option( @@ -164,10 +119,12 @@ def nf_core_cli(ctx, verbose, hide_progress, log_file): # Set up logs to the console - use the shared console so log output # coordinates with progress spinners (avoids output on the same line) - from nf_core.pipelines.lint_utils import console as shared_console + from rich.logging import RichHandler + + from nf_core.utils import stdout as shared_console log.addHandler( - rich.logging.RichHandler( + RichHandler( level=logging.DEBUG if verbose else logging.INFO, console=shared_console, show_time=False, @@ -197,6 +154,15 @@ def nf_core_cli(ctx, verbose, hide_progress, log_file): } +# nf-core interface (registered by hand so trogon is only imported when run) +@nf_core_cli.command("interface", help="Launch the nf-core interface") +@click.pass_context +def interface(ctx): + from trogon import Trogon + + Trogon(nf_core_cli, app_name=None, command_name="interface", click_context=ctx).run() + + # nf-core pipelines subcommands @nf_core_cli.group(aliases=["p", "pipeline"]) @click.command_panel("For users", commands=["download", "create-params-file", "launch", "list"]) @@ -238,6 +204,8 @@ def command_pipelines_create(ctx, name, description, author, version, force, out """ Create a new pipeline using the nf-core template. """ + from nf_core.commands_pipelines import pipelines_create + pipelines_create(ctx, name, description, author, version, force, outdir, template_yaml, organisation) @@ -321,6 +289,8 @@ def command_pipelines_lint( """ Check pipeline code against nf-core guidelines. """ + from nf_core.commands_pipelines import pipelines_lint + pipelines_lint( ctx, directory, release, fix, key, show_passed, fail_ignored, fail_warned, markdown, json, sort_by, plain_text ) @@ -420,6 +390,8 @@ def command_pipelines_download( """ Download a pipeline, nf-core/configs and pipeline singularity images. """ + from nf_core.commands_pipelines import pipelines_download + pipelines_download( ctx, pipeline, @@ -469,6 +441,8 @@ def command_pipelines_create_params_file(ctx, pipeline, revision, output, force, """ Build a parameter file for a pipeline. """ + from nf_core.commands_pipelines import pipelines_create_params_file + pipelines_create_params_file(ctx, pipeline, revision, output, force, show_hidden, no_prompts) @@ -541,6 +515,8 @@ def command_pipelines_launch( """ Launch a pipeline using a web GUI or command line prompts. """ + from nf_core.commands_pipelines import pipelines_launch + pipelines_launch( ctx, pipeline, launch_id, revision, command_only, params_in, params_out, save_all, show_hidden, url, no_prompts ) @@ -563,6 +539,8 @@ def command_pipelines_list(ctx, keywords, sort, json, show_archived): """ List available nf-core pipelines with local info. """ + from nf_core.commands_pipelines import pipelines_list + pipelines_list(ctx, keywords, sort, json, show_archived) @@ -601,6 +579,8 @@ def rocrate( """ Make an Research Object Crate """ + from nf_core.commands_pipelines import pipelines_rocrate + pipelines_rocrate(ctx, pipeline_dir, json_path, zip_path, pipeline_version) @@ -654,6 +634,8 @@ def command_pipelines_sync( """ Sync a pipeline [cyan i]TEMPLATE[/] branch with the nf-core template. """ + from nf_core.commands_pipelines import pipelines_sync + pipelines_sync( ctx, directory, @@ -691,6 +673,8 @@ def command_pipelines_bump_version(ctx, new_version, directory, nextflow): """ Update nf-core pipeline version number with `nf-core pipelines bump-version`. """ + from nf_core.commands_pipelines import pipelines_bump_version + pipelines_bump_version(ctx, new_version, directory, nextflow) @@ -737,6 +721,8 @@ def command_pipelines_create_logo(logo_text, directory, name, theme, width, img_ """ Generate a logo with the nf-core logo template. """ + from nf_core.commands_pipelines import pipelines_create_logo + pipelines_create_logo(logo_text, directory, name, theme, width, img_format, force) @@ -777,6 +763,8 @@ def command_pipelines_schema_validate(directory, pipeline, params): # this is a local pipeline pipeline = Path(directory, pipeline) + from nf_core.commands_pipelines import pipelines_schema_validate + pipelines_schema_validate(pipeline, params) @@ -810,6 +798,8 @@ def command_pipelines_schema_build(directory, no_prompts, web_only, url): """ Interactively build a pipeline schema from Nextflow params. """ + from nf_core.commands_pipelines import pipelines_schema_build + pipelines_schema_build(directory, no_prompts, web_only, url) @@ -833,6 +823,8 @@ def command_pipelines_schema_lint(directory, schema_file): """ Check that a given pipeline schema is valid. """ + from nf_core.commands_pipelines import pipelines_schema_lint + pipelines_schema_lint(Path(directory, schema_file)) @@ -881,6 +873,8 @@ def command_pipelines_schema_docs(directory, schema_file, output, output_format, """ Outputs parameter documentation for a pipeline schema. """ + from nf_core.commands_pipelines import pipelines_schema_docs + pipelines_schema_docs(Path(directory, schema_file), output, output_format, force, columns) @@ -943,6 +937,8 @@ def command_modules_list_remote(ctx, keywords, json): """ List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ + from nf_core.commands_modules import modules_list_remote + modules_list_remote(ctx, keywords, json) @@ -963,6 +959,8 @@ def command_modules_list_local(ctx, keywords, json, directory): # pylint: disab """ List modules installed locally in a pipeline """ + from nf_core.commands_modules import modules_list_local + modules_list_local(ctx, keywords, json, directory) @@ -1004,6 +1002,8 @@ def command_modules_install(ctx, tool, directory, prompt, force, sha): """ Install DSL2 modules within a pipeline. """ + from nf_core.commands_modules import modules_install + modules_install(ctx, tool, directory, prompt, force, sha) @@ -1095,6 +1095,8 @@ def command_modules_update( """ Update DSL2 modules within a pipeline. """ + from nf_core.commands_modules import modules_update + modules_update( ctx, tool, directory, force, prompt, sha, install_all, preview, save_diff, update_deps, limit_output, skip_deps ) @@ -1124,6 +1126,8 @@ def command_modules_patch(ctx, tool, directory, remove): """ Create a patch file for minor changes in a module """ + from nf_core.commands_modules import modules_patch + modules_patch(ctx, tool, directory, remove) @@ -1157,6 +1161,8 @@ def command_modules_remove(ctx, directory, tool, force): """ Remove a module from a pipeline. """ + from nf_core.commands_modules import modules_remove + modules_remove(ctx, directory, tool, force) @@ -1237,6 +1243,8 @@ def command_modules_create( """ Create a new DSL2 module from the nf-core template. """ + from nf_core.commands_modules import modules_create + modules_create( ctx, tool, @@ -1305,6 +1313,8 @@ def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile """ if verbose: ctx.obj["verbose"] = verbose + from nf_core.commands_modules import modules_test + modules_test(ctx, tool, directory, no_prompts, update, once, profile) @@ -1372,6 +1382,8 @@ def command_modules_lint( """ Lint one or more modules in a directory. """ + from nf_core.commands_modules import modules_lint + modules_lint( ctx, tool, @@ -1412,6 +1424,8 @@ def command_modules_info(ctx, tool, directory): """ Show developer usage information about a given module. """ + from nf_core.commands_modules import modules_info + modules_info(ctx, tool, directory) @@ -1442,6 +1456,8 @@ def command_modules_bump_versions(ctx, tool, directory, all_modules, show_all, d Bump versions for one or more modules in a clone of the nf-core/modules repo. """ + from nf_core.commands_modules import modules_bump_versions + modules_bump_versions(ctx, tool, directory, all_modules, show_all, dry_run) @@ -1449,8 +1465,10 @@ def command_modules_bump_versions(ctx, tool, directory, all_modules, show_all, d @click.pass_context def modules_containers(ctx): """Manage module container builds and metadata [yellow](beta)[/].""" + from rich.panel import Panel + stderr.print( - rich.panel.Panel( + Panel( "The [bold]nf-core modules containers[/] commands are still in beta.\n" "Their behaviour and output may change in future releases.\n" "Best to always use the latest dev version.", @@ -1492,6 +1510,8 @@ def command_modules_containers_create(ctx, module: str, directory: Path, force: """ Build docker and singularity container files for linux/arm64 and linux/amd64 with wave from environment.yml and create container config file. """ + from nf_core.commands_modules import modules_containers_create + modules_containers_create(ctx, module, directory, force) @@ -1509,6 +1529,8 @@ def command_modules_containers_conda_lock(ctx, module: str) -> None: """ Build a Docker linux/arm64 container and fetch the conda lock file for a module. """ + from nf_core.commands_modules import modules_containers_conda_lock + modules_containers_conda_lock(ctx, module) @@ -1534,6 +1556,8 @@ def command_modules_containers_list(ctx, module, plain_text): """ Print containers defined in a module meta.yml. """ + from nf_core.commands_modules import modules_containers_list + modules_containers_list(ctx, module, plain_text) @@ -1600,6 +1624,8 @@ def command_subworkflows_create(ctx, subworkflow, directory, author, force): """ Create a new subworkflow from the nf-core template. """ + from nf_core.commands_subworkflows import subworkflows_create + subworkflows_create(ctx, subworkflow, directory, author, force) @@ -1647,6 +1673,8 @@ def command_subworkflows_test(ctx, subworkflow, directory, no_prompts, update, o """ Run nf-test for a subworkflow. """ + from nf_core.commands_subworkflows import subworkflows_test + subworkflows_test(ctx, subworkflow, directory, no_prompts, update, once, profile) @@ -1669,6 +1697,8 @@ def command_subworkflows_list_remote(ctx, keywords, json): """ List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ + from nf_core.commands_subworkflows import subworkflows_list_remote + subworkflows_list_remote(ctx, keywords, json) @@ -1689,6 +1719,8 @@ def command_subworkflows_list_local(ctx, keywords, json, directory): # pylint: """ List subworkflows installed locally in a pipeline """ + from nf_core.commands_subworkflows import subworkflows_list_local + subworkflows_list_local(ctx, keywords, json, directory) @@ -1751,6 +1783,8 @@ def command_subworkflows_lint( """ Lint one or more subworkflows in a directory. """ + from nf_core.commands_subworkflows import subworkflows_lint + subworkflows_lint( ctx, subworkflow, @@ -1790,6 +1824,8 @@ def command_subworkflows_info(ctx, subworkflow, directory): """ Show developer usage information about a given subworkflow. """ + from nf_core.commands_subworkflows import subworkflows_info + subworkflows_info(ctx, subworkflow, directory) @@ -1843,6 +1879,8 @@ def command_subworkflows_install(ctx, subworkflow, directory, prompt, force, sha """ Install DSL2 subworkflow within a pipeline. """ + from nf_core.commands_subworkflows import subworkflows_install + subworkflows_install(ctx, subworkflow, directory, prompt, force, sha, skip_deps) @@ -1921,6 +1959,8 @@ def command_subworkflows_remove(ctx, directory, subworkflow, force): """ Remove a subworkflow from a pipeline. """ + from nf_core.commands_subworkflows import subworkflows_remove + subworkflows_remove(ctx, directory, subworkflow, force) @@ -2019,6 +2059,8 @@ def command_subworkflows_update( """ Update DSL2 subworkflow within a pipeline. """ + from nf_core.commands_subworkflows import subworkflows_update + subworkflows_update( ctx, subworkflow, @@ -2071,6 +2113,8 @@ def command_test_dataset_search(ctx, branch, generate_nf_path, generate_dl_url, Search files filtered by QUERY on a specified branch in the nf-core/test-datasets repository. If no QUERY is given or QUERY is ambiguous, an auto-completion form is shown. """ + from nf_core.commands_test_datasets import test_datasets_search + test_datasets_search(ctx, branch, generate_nf_path, generate_dl_url, query) @@ -2096,6 +2140,8 @@ def command_test_dataset_list_remote(ctx, branch, generate_nf_path, generate_dl_ """ List files on a specified branch in the nf-core/test-datasets repository. """ + from nf_core.commands_test_datasets import test_datasets_list_remote + test_datasets_list_remote(ctx, branch, generate_nf_path, generate_dl_url) @@ -2106,6 +2152,8 @@ def command_test_datasets_list_branches(ctx): """ List remote branches with test data in the nf-core/test-dataset repository. """ + from nf_core.commands_test_datasets import test_datasets_list_branches + test_datasets_list_branches(ctx) diff --git a/nf_core/commands_pipelines.py b/nf_core/commands_pipelines.py index 535ed074ca..1384540f6d 100644 --- a/nf_core/commands_pipelines.py +++ b/nf_core/commands_pipelines.py @@ -5,7 +5,6 @@ import rich import nf_core.utils -from nf_core.pipelines.params_file import ParamsFileBuilder from nf_core.utils import rich_force_colors log = logging.getLogger(__name__) @@ -226,6 +225,8 @@ def pipelines_create_params_file(ctx, pipeline, revision, output, force, show_hi Run using a remote pipeline name (such as GitHub `user/repo` or a URL), a local pipeline directory. """ + from nf_core.pipelines.params_file import ParamsFileBuilder + builder = ParamsFileBuilder(pipeline, revision, no_prompts) if not builder.write_params_file(Path(output), show_hidden=show_hidden, force=force): diff --git a/nf_core/commands_test_datasets.py b/nf_core/commands_test_datasets.py index b9a2688cd3..a6b94ae7d1 100644 --- a/nf_core/commands_test_datasets.py +++ b/nf_core/commands_test_datasets.py @@ -3,8 +3,6 @@ import click import rich -from nf_core.test_datasets.list import list_dataset_branches, list_datasets -from nf_core.test_datasets.search import search_datasets from nf_core.utils import rich_force_colors log = logging.getLogger(__name__) @@ -17,6 +15,8 @@ def test_datasets_list_branches(ctx: click.Context) -> None: Only lists test data and module test data based on the curated list of pipeline names [on the website](https://raw.githubusercontent.com/nf-core/website/refs/heads/main/public/pipeline_names.json). """ + from nf_core.test_datasets.list import list_dataset_branches + list_dataset_branches() @@ -25,6 +25,8 @@ def test_datasets_list_remote(ctx: click.Context, branch: str, generate_nf_path: List all files on a given branch in the remote nf-core/testdatasets repository on github. The resulting files can be parsed as a nextflow path or a url for downloading. """ + from nf_core.test_datasets.list import list_datasets + list_datasets(branch, generate_nf_path, generate_dl_url) @@ -37,4 +39,6 @@ def test_datasets_search( Specifying a branch is required. The resulting file can optionally be parsed as a nextflow path or a url for downloading """ + from nf_core.test_datasets.search import search_datasets + search_datasets(branch, generate_nf_path=generate_nf_path, generate_dl_url=generate_dl_url, query=query) diff --git a/nf_core/components/components_completion.py b/nf_core/components/components_completion.py index 8abecb4649..4f2179c837 100644 --- a/nf_core/components/components_completion.py +++ b/nf_core/components/components_completion.py @@ -1,13 +1,11 @@ import sys -import git from click.shell_completion import CompletionItem -from nf_core.modules.list import ModuleList -from nf_core.subworkflows.list import SubworkflowList - def autocomplete_components(ctx, param, incomplete: str, component_type: str, list_class): + import git + # Defaults modules_repo_url = "https://github.com/nf-core/modules" modules_repo_branch = "master" @@ -31,8 +29,12 @@ def autocomplete_components(ctx, param, incomplete: str, component_type: str, li def autocomplete_modules(ctx, param, incomplete: str): + from nf_core.modules.list import ModuleList + return autocomplete_components(ctx, param, incomplete, "modules", ModuleList) def autocomplete_subworkflows(ctx, param, incomplete: str): + from nf_core.subworkflows.list import SubworkflowList + return autocomplete_components(ctx, param, incomplete, "subworkflows", SubworkflowList) diff --git a/nf_core/components/components_differ.py b/nf_core/components/components_differ.py index b028b55626..5024478d48 100644 --- a/nf_core/components/components_differ.py +++ b/nf_core/components/components_differ.py @@ -235,16 +235,16 @@ def append_modules_json_diff(diff_path, old_modules_json, new_modules_json, modu @staticmethod def print_diff( - component, - repo_path, - from_dir, - to_dir, - current_version=None, - new_version=None, - dsp_from_dir=None, - dsp_to_dir=None, - limit_output=False, - ): + component: str, + repo_path: str, + from_dir: str | Path, + to_dir: str | Path, + current_version: str | None = None, + new_version: str | None = None, + dsp_from_dir: str | Path | None = None, + dsp_to_dir: str | Path | None = None, + limit_output: bool = False, + ) -> None: """ Prints the diffs between two component versions to the terminal diff --git a/nf_core/github_api.py b/nf_core/github_api.py new file mode 100644 index 0000000000..2228c7d33f --- /dev/null +++ b/nf_core/github_api.py @@ -0,0 +1,189 @@ +"""GitHub API session handling for the nf-core python package. + +Kept separate from nf_core.utils so that requests_cache is only +imported when the GitHub API is actually used. +""" + +import json +import logging +import os +import random +import re +import sys +import time +from pathlib import Path + +import requests.auth +import requests_cache +import rich.console +import rich.markup +import yaml + +from nf_core.utils import rich_force_colors, setup_requests_cachedir + +log = logging.getLogger(__name__) + + +class GitHubAPISession(requests_cache.CachedSession): + """ + Class to provide a single session for interacting with the GitHub API for a run. + Inherits the requests_cache.CachedSession and adds additional functionality, + such as automatically setting up GitHub authentication if we can. + """ + + def __init__(self) -> None: + self.auth_mode: str | None = None + self.return_ok: list[int] = [200, 201] + self.return_retry: list[int] = [403] + self.return_unauthorised: list[int] = [401] + self.has_init: bool = False + + def lazy_init(self) -> None: + """ + Initialise the object. + + Only do this when it's actually being used (due to global import) + """ + log.debug("Initialising GitHub API requests session") + cache_config = setup_requests_cachedir() + super().__init__(**cache_config) + self.setup_github_auth() + self.has_init = True + + def setup_github_auth(self, auth=None): + """ + Try to automatically set up GitHub authentication + """ + if auth is not None: + self.auth = auth + self.auth_mode = "supplied to function" + + # Class for Bearer token authentication + # https://stackoverflow.com/a/58055668/713980 + class BearerAuth(requests.auth.AuthBase): + def __init__(self, token): + self.token = token + + def __call__(self, r): + r.headers["authorization"] = f"Bearer {self.token}" + return r + + # Default auth if we're running and the gh CLI tool is installed + gh_cli_config_fn = Path.home() / ".config" / "gh" / "hosts.yml" + if self.auth is None and gh_cli_config_fn.exists(): + try: + with open(gh_cli_config_fn) as fh: + gh_cli_config = yaml.safe_load(fh) + self.auth = requests.auth.HTTPBasicAuth( + gh_cli_config["github.com"]["user"], + gh_cli_config["github.com"]["oauth_token"], + ) + self.auth_mode = f"gh CLI config: {gh_cli_config['github.com']['user']}" + except (OSError, KeyError, yaml.YAMLError): + ex_type, ex_value, _ = sys.exc_info() + if ex_type is not None: + output = rich.markup.escape(f"{ex_type.__name__}: {ex_value}") + log.debug(f"Couldn't auto-auth with GitHub CLI auth from '{gh_cli_config_fn}': [red]{output}") + + # Default auth if we have a GitHub Token (eg. GitHub Actions CI) + if os.environ.get("GITHUB_TOKEN") is not None and self.auth is None: + self.auth_mode = "Bearer token with GITHUB_TOKEN" + self.auth = BearerAuth(os.environ["GITHUB_TOKEN"]) + else: + log.warning("Could not find GitHub authentication token. Some API requests may fail.") + + log.debug(f"Using GitHub auth: {self.auth_mode}") + + def log_content_headers(self, request, post_data=None): + """ + Try to dump everything to the console, useful when things go wrong. + """ + log.debug(f"Requested URL: {request.url}") + log.debug(f"From requests cache: {request.from_cache}") + log.debug(f"Request status code: {request.status_code}") + log.debug(f"Request reason: {request.reason}") + if post_data is None: + post_data = {} + try: + log.debug(json.dumps(dict(request.headers), indent=4)) + log.debug(json.dumps(request.json(), indent=4)) + log.debug(json.dumps(post_data, indent=4)) + except (json.JSONDecodeError, TypeError) as e: + log.debug(f"Could not parse JSON response from GitHub API! {e}") + log.debug(request.headers) + log.debug(request.content) + log.debug(post_data) + + def safe_get(self, url): + """ + Run a GET request, raise a nice exception with lots of logging if it fails. + """ + if not self.has_init: + self.lazy_init() + request = self.get(url) + if request.status_code in self.return_retry: + stderr = rich.console.Console(stderr=True, force_terminal=rich_force_colors()) + try: + r = self.request_retry(url) + except Exception as e: + stderr.print_exception() + raise e + else: + return r + elif request.status_code in self.return_unauthorised: + raise RuntimeError("GitHub API PR failed, probably due to an expired GITHUB_TOKEN.") + + return request + + def get(self, url, **kwargs): + """ + Initialise the session if we haven't already, then call the superclass get method. + """ + if not self.has_init: + self.lazy_init() + return super().get(url, **kwargs) + + def request_retry(self, url, post_data=None): + """ + Try to fetch a URL, keep retrying if we get a certain return code. + + Used in nf-core pipelines sync code because we get 403 errors: too many simultaneous requests + See https://github.com/nf-core/tools/issues/911 + """ + if not self.has_init: + self.lazy_init() + + # Start the loop for a retry mechanism + while True: + # GET request + if post_data is None: + log.debug(f"Sending GET request to {url}") + r = self.get(url=url) + # POST request + else: + log.debug(f"Sending POST request to {url}") + r = self.post(url=url, json=post_data) + + # Failed but expected - try again + if r.status_code in self.return_retry: + self.log_content_headers(r, post_data) + log.debug(f"GitHub API PR failed - got return code {r.status_code}") + wait_time = float(re.sub("[^0-9]", "", str(r.headers.get("Retry-After", 0)))) + if wait_time == 0: + log.debug("Couldn't find 'Retry-After' header, guessing a length of time to wait") + wait_time = random.randrange(10, 60) + log.warning(f"Got API return code {r.status_code}. Trying again after {wait_time} seconds..") + time.sleep(wait_time) + + # Unexpected error - raise + elif r.status_code not in self.return_ok: + self.log_content_headers(r, post_data) + raise RuntimeError(f"GitHub API PR failed - got return code {r.status_code} from {url}") + + # Success! + else: + return r + + +# Single session object to use for entire codebase. Not sure if there's a better way to do this? +gh_api = GitHubAPISession() diff --git a/nf_core/pipelines/__init__.py b/nf_core/pipelines/__init__.py index bc981c449f..55c151c56f 100644 --- a/nf_core/pipelines/__init__.py +++ b/nf_core/pipelines/__init__.py @@ -1 +1,9 @@ -from .create import PipelineCreateApp +from nf_core.utils import lazy_attrs + +# Lazy imports to keep CLI start-up fast +__getattr__, __dir__ = lazy_attrs( + globals(), + { + "PipelineCreateApp": "nf_core.pipelines.create", + }, +) diff --git a/nf_core/pipelines/download/download.py b/nf_core/pipelines/download/download.py index 5006847d23..ef3b84dfcf 100644 --- a/nf_core/pipelines/download/download.py +++ b/nf_core/pipelines/download/download.py @@ -20,6 +20,7 @@ import nf_core import nf_core.pipelines.list import nf_core.utils +from nf_core.github_api import gh_api from nf_core.pipelines.download.container_fetcher import ContainerFetcher from nf_core.pipelines.download.docker import DockerFetcher from nf_core.pipelines.download.singularity import SINGULARITY_CACHE_DIR_ENV_VAR, SingularityFetcher @@ -29,7 +30,6 @@ NF_INSPECT_MIN_NF_VERSION, NFCORE_VER_LAST_WITHOUT_NF_INSPECT, check_nextflow_version, - gh_api, pretty_nf_version, run_cmd, set_wd_tempdir, diff --git a/nf_core/pipelines/download/singularity.py b/nf_core/pipelines/download/singularity.py index c0ecb5f8f5..98d4b5981a 100644 --- a/nf_core/pipelines/download/singularity.py +++ b/nf_core/pipelines/download/singularity.py @@ -60,6 +60,24 @@ def get_container_library_dir(container_system: str) -> str: raise KeyError(f"Container engine: {container_system} is unknown.") +class SingularityCacheFilePathValidator(questionary.Validator): + """ + Validator for file path specified as --singularity-cache-index argument in nf-core pipelines download + """ + + def validate(self, value): + if len(value.text): + if Path(value.text).is_file(): + return True + else: + raise questionary.ValidationError( + message="Invalid remote cache index file", + cursor_position=len(value.text), + ) + else: + return True + + class SingularityProgress(ContainerProgress): def get_task_types_and_columns(self): task_types_and_columns = super().get_task_types_and_columns() @@ -385,7 +403,7 @@ def prompt_cachedir_remote(cache_dir_env_var: str = SINGULARITY_CACHE_DIR_ENV_VA while cachedir_index is None: prompt_cachedir_index = questionary.path( "Specify a list of the container images that are already present on the remote system:", - validate=nf_core.utils.SingularityCacheFilePathValidator, + validate=SingularityCacheFilePathValidator, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() if prompt_cachedir_index == "": diff --git a/nf_core/pipelines/launch.py b/nf_core/pipelines/launch.py index 7329985c1b..8b4aba1a81 100644 --- a/nf_core/pipelines/launch.py +++ b/nf_core/pipelines/launch.py @@ -26,17 +26,17 @@ class Launch: def __init__( self, - pipeline=None, - revision=None, - command_only=False, - params_in=None, - params_out=None, - save_all=False, - show_hidden=False, - url=None, - web_id=None, - no_prompts=False, - ): + pipeline: str | None = None, + revision: str | None = None, + command_only: bool = False, + params_in: str | Path | None = None, + params_out: str | Path | None = None, + save_all: bool = False, + show_hidden: bool = False, + url: str | None = None, + web_id: str | None = None, + no_prompts: bool = False, + ) -> None: """Initialise the Launcher class Args: @@ -94,8 +94,8 @@ def __init__( }, } } - self.nxf_flags = {} - self.params_user = {} + self.nxf_flags: dict[str, str | bool] = {} + self.params_user: dict[str, str] = {} self.cli_launch = True def launch_pipeline(self): diff --git a/nf_core/pipelines/lint_utils.py b/nf_core/pipelines/lint_utils.py index d2dc56f67a..65aafdb293 100644 --- a/nf_core/pipelines/lint_utils.py +++ b/nf_core/pipelines/lint_utils.py @@ -7,19 +7,16 @@ import git import rich import yaml -from rich.console import Console from rich.table import Table import nf_core.utils from nf_core import __version__ from nf_core.utils import plural_s as _s +from nf_core.utils import stdout as console from nf_core.utils import strip_ansi_codes log = logging.getLogger(__name__) -# Create a console used by all lint tests -console = Console(force_terminal=nf_core.utils.rich_force_colors()) - def print_results_plain_text(results_list, directory=None, component_type=None): """Print lint results in plain text format. diff --git a/nf_core/pipelines/list.py b/nf_core/pipelines/list.py index bc9bf30205..cb5d22d444 100644 --- a/nf_core/pipelines/list.py +++ b/nf_core/pipelines/list.py @@ -8,8 +8,6 @@ from datetime import datetime from pathlib import Path -import git -import requests import rich.table from click.shell_completion import CompletionItem @@ -52,6 +50,8 @@ def _resolve_wf_path(path: Path) -> Path: bare_dir = path / "bare" if clones_dir.is_dir(): if bare_dir.is_dir(): + import git + try: sha = git.Repo(bare_dir).head.commit.hexsha clone = clones_dir / sha @@ -169,6 +169,8 @@ def get_remote_workflows(self): Remote workflows are stored in :attr:`self.remote_workflows` list. """ + import requests + # List all repositories at nf-core log.debug("Fetching list of nf-core workflows") nfcore_url = "https://nf-co.re/pipelines.json" @@ -421,6 +423,8 @@ def get_local_nf_workflow_details(self): # Pull information from the local git repository if self.local_path is not None: + import git + log.debug(f"Pulling git info from {self.local_path}") try: repo = git.Repo(self.local_path) diff --git a/nf_core/pipelines/sync.py b/nf_core/pipelines/sync.py index 003880491d..1cd1a51794 100644 --- a/nf_core/pipelines/sync.py +++ b/nf_core/pipelines/sync.py @@ -18,6 +18,7 @@ import nf_core import nf_core.pipelines.create.create import nf_core.utils +from nf_core.github_api import gh_api from nf_core.pipelines.lint_utils import dump_yaml_with_prettier log = logging.getLogger(__name__) @@ -120,7 +121,7 @@ def __init__( ) # Set up the API auth if supplied on the command line - self.gh_api = nf_core.utils.gh_api + self.gh_api = gh_api self.gh_api.lazy_init() if self.gh_username and "GITHUB_AUTH_TOKEN" in os.environ: log.debug(f"Authenticating sync as {self.gh_username}") diff --git a/nf_core/pydantic_models.py b/nf_core/pydantic_models.py new file mode 100644 index 0000000000..ffdcebc0fb --- /dev/null +++ b/nf_core/pydantic_models.py @@ -0,0 +1,211 @@ +"""Pydantic models for the `.nf-core.yml` configuration file. + +Kept separate from nf_core.utils so that pydantic is only imported +when the config file is actually parsed. +""" + +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class NFCoreTemplateConfig(BaseModel): + """Template configuration schema""" + + org: str | None = None + """ Organisation name """ + name: str | None = None + """ Pipeline name """ + description: str | None = None + """ Pipeline description """ + author: str | None = None + """ Pipeline author """ + version: str | None = None + """ Pipeline version """ + force: bool | None = True + """ Force overwrite of existing files """ + outdir: str | Path | None = None + """ Output directory """ + skip_features: list | None = None + """ Skip features. See https://nf-co.re/docs/nf-core-tools/pipelines/create for a list of features. """ + is_nfcore: bool | None = None + """ Whether the pipeline is an nf-core pipeline. """ + + # convert outdir to str + @field_validator("outdir") + @classmethod + def outdir_to_str(cls, v: str | Path | None) -> str | None: + if v is not None: + v = str(v) + return v + + def __getitem__(self, item: str) -> Any: + if self is None: + return None + return getattr(self, item) + + def get(self, item: str, default: Any = None) -> Any: + return getattr(self, item, default) + + +class NFCoreYamlLintConfig(BaseModel): + """ + schema for linting config in `.nf-core.yml` should cover: + + .. code-block:: yaml + + files_unchanged: + - .github/workflows/branch.yml + modules_config: False + modules_config: + - fastqc + # merge_markers: False + merge_markers: + - docs/my_pdf.pdf + nextflow_config: False + nextflow_config: + - manifest.name + - config_defaults: + - params.annotation_db + - params.multiqc_comment_headers + - params.custom_table_headers + # multiqc_config: False + multiqc_config: + - report_section_order + - report_comment + files_exist: + - CITATIONS.md + template_strings: False + template_strings: + - docs/my_pdf.pdf + nfcore_components: False + # nf_test_content: False + nf_test_content: + - tests/.nf.test + - tests/nextflow.config + - nf-test.config + """ + + files_unchanged: bool | list[str] | None = None + """ List of files that should not be changed """ + modules_config: bool | list[str] | None = None + """ List of modules that should not be changed """ + merge_markers: bool | list[str] | None = None + """ List of files that should not contain merge markers """ + nextflow_config: bool | list[str | dict[str, list[str]]] | None = None + """ List of Nextflow config files that should not be changed """ + nf_test_content: bool | list[str] | None = None + """ List of nf-test content that should not be changed """ + multiqc_config: bool | list[str] | None = None + """ List of MultiQC config options that be changed """ + files_exist: bool | list[str] | None = None + """ List of files that can not exist """ + template_strings: bool | list[str] | None = None + """ List of files that can contain template strings """ + readme: bool | list[str] | None = None + """ Lint the README.md file """ + nfcore_components: bool | None = None + """ Lint all required files to use nf-core modules and subworkflows """ + actions_nf_test: bool | None = None + """ Lint all required files to use GitHub Actions CI """ + actions_awstest: bool | None = None + """ Lint all required files to run tests on AWS """ + actions_awsfulltest: bool | None = None + """ Lint all required files to run full tests on AWS """ + pipeline_todos: bool | None = None + """ Lint for TODOs statements""" + pipeline_if_empty_null: bool | None = None + """ Lint for ifEmpty(null) statements""" + plugin_includes: bool | None = None + """ Lint for nextflow plugin """ + pipeline_name_conventions: bool | None = None + """ Lint for pipeline name conventions """ + schema_lint: bool | None = None + """ Lint nextflow_schema.json file""" + schema_params: bool | None = None + """ Lint schema for all params """ + system_exit: bool | None = None + """ Lint for System.exit calls in groovy/nextflow code """ + schema_description: bool | None = None + """ Check that every parameter in the schema has a description. """ + actions_schema_validation: bool | None = None + """ Lint GitHub Action workflow files with schema""" + modules_json: bool | None = None + """ Lint modules.json file """ + modules_structure: bool | None = None + """ Lint modules structure """ + base_config: bool | None = None + """ Lint base.config file """ + nfcore_yml: bool | None = None + """ Lint nf-core.yml """ + version_consistency: bool | None = None + """ Lint for version consistency """ + included_configs: bool | None = None + """ Lint for included configs """ + local_component_structure: bool | None = None + """ Lint local components use correct structure mirroring remote""" + container_configs: bool | None = None + """ Lint that container configuration files in conf/ are up to date """ + rocrate_readme_sync: bool | None = None + """ Lint for README.md and rocrate.json sync """ + + def __getitem__(self, item: str) -> Any: + return getattr(self, item) + + def get(self, item: str, default: Any = None) -> Any: + if getattr(self, item, default) is None: + return default + return getattr(self, item, default) + + def __setitem__(self, item: str, value: Any) -> None: + setattr(self, item, value) + + +class NFCoreYamlConfig(BaseModel): + """.nf-core.yml configuration file schema""" + + model_config = ConfigDict(populate_by_name=True) + + repository_type: Literal["pipeline", "modules"] | None = None + """ Type of repository """ + nf_core_version: str | None = None + """ Version of nf-core/tools used to create/update the pipeline """ + org_path: str | None = None + """ Path to the organisation's modules repository (used for modules repo_type only) """ + lint: NFCoreYamlLintConfig | None = None + """ Pipeline linting configuration, see https://nf-co.re/docs/nf-core-tools/pipelines/lint#linting-config for examples and documentation """ + template: NFCoreTemplateConfig | None = None + """ Pipeline template configuration """ + bump_version: dict[str, bool] | None = None + """ Disable bumping of the version for a module/subworkflow (when repository_type is modules). See https://nf-co.re/docs/nf-core-tools/modules/bump-versions for more information. """ + update: dict[str, str | bool | dict[str, str | dict[str, str | bool]]] | None = None + """ Disable updating specific modules/subworkflows (when repository_type is pipeline). See https://nf-co.re/docs/nf-core-tools/modules/update for more information. """ + container_registry: list[str] | None = Field(default=None, alias="container-registry") + """ Additional container registry prefixes allowed when linting container directives. """ + + def __getitem__(self, item: str) -> Any: + return getattr(self, item) + + def get(self, item: str, default: Any = None) -> Any: + return getattr(self, item, default) + + def __setitem__(self, item: str, value: Any) -> None: + setattr(self, item, value) + + def model_dump(self, **kwargs) -> dict[str, Any]: + # Get the initial data + config = super().model_dump(**kwargs) + + if self.repository_type == "modules": + # Fields to exclude for modules + fields_to_exclude = ["template", "update"] + else: # pipeline + # Fields to exclude for pipeline + fields_to_exclude = ["bump_version", "org_path"] + + # Remove the fields based on repository_type + for field in fields_to_exclude: + config.pop(field, None) + + return config diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py index 8e3c85a271..3ca23d0f30 100644 --- a/nf_core/subworkflows/__init__.py +++ b/nf_core/subworkflows/__init__.py @@ -1,8 +1,16 @@ -from .create import SubworkflowCreate -from .info import SubworkflowInfo -from .install import SubworkflowInstall -from .lint import SubworkflowLint -from .list import SubworkflowList -from .patch import SubworkflowPatch -from .remove import SubworkflowRemove -from .update import SubworkflowUpdate +from nf_core.utils import lazy_attrs + +# Lazy imports to keep CLI start-up fast +__getattr__, __dir__ = lazy_attrs( + globals(), + { + "SubworkflowCreate": "nf_core.subworkflows.create", + "SubworkflowInfo": "nf_core.subworkflows.info", + "SubworkflowInstall": "nf_core.subworkflows.install", + "SubworkflowLint": "nf_core.subworkflows.lint", + "SubworkflowList": "nf_core.subworkflows.list", + "SubworkflowPatch": "nf_core.subworkflows.patch", + "SubworkflowRemove": "nf_core.subworkflows.remove", + "SubworkflowUpdate": "nf_core.subworkflows.update", + }, +) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 72b5cf34b3..dc79e9a759 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -227,36 +227,22 @@ def checkout_branch(self): """ Checks out the specified branch of the repository """ - try: - self.repo.git.checkout(self.branch) - except GitCommandError as e: - if ( - self.fullname - and "modules" in self.fullname - and "Your local changes to the following files would be overwritten by checkout" in str(e) - ): - log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") - self.repo.git.checkout(self.branch, force=True) - else: - raise e + self.checkout(self.branch) def checkout(self, commit): """ - Checks out the repository at the requested commit + Checks out the repository at the requested commit (or branch) Args: - commit (str): Git SHA of the commit + commit (str): Git SHA of the commit, or a branch name """ try: self.repo.git.checkout(commit) except GitCommandError as e: - if ( - self.fullname - and "modules" in self.fullname - and "Your local changes to the following files would be overwritten by checkout" in str(e) - ): - log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") - self.repo.git.checkout(self.branch, force=True) + if self.fullname and "modules" in self.fullname and "would be overwritten by checkout" in str(e): + log.debug(f"Discarding local changes in '{self.local_repo_dir}'") + self.repo.git.clean("-df") + self.repo.git.checkout(commit, force=True) else: raise e @@ -305,7 +291,8 @@ def install_component(self, component_name: str, install_dir: str | Path, commit # Check out the repository at the requested ref try: self.checkout(commit) - except git.GitCommandError: + except git.GitCommandError as e: + log.error(f"Could not check out '{commit}' in the modules cache '{self.local_repo_dir}':\n{e}") return False # Check if the module/subworkflow exists in the branch diff --git a/nf_core/utils.py b/nf_core/utils.py index 0e751d4c52..e53744bb88 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -3,48 +3,32 @@ """ import ast -import concurrent.futures import datetime import errno import fnmatch -import hashlib +import functools +import importlib import io import json import logging -import mimetypes import os -import random import re -import shlex -import subprocess import sys -import tempfile import time from collections.abc import Callable, Generator from contextlib import contextmanager, suppress from enum import Enum from functools import lru_cache from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal - -import git -import git.exc -import prompt_toolkit.styles -import questionary -import requests.auth -import requests_cache -import rich -import rich.markup -import yaml -from packaging.version import Version -from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator -from rich.live import Live -from rich.spinner import Spinner +from typing import TYPE_CHECKING, Any + +from rich.console import Console import nf_core if TYPE_CHECKING: from nf_core.pipelines.schema import PipelineSchema + from nf_core.pydantic_models import NFCoreYamlConfig log = logging.getLogger(__name__) @@ -57,38 +41,83 @@ r"[green] `._,._,'", ] -# Custom style for questionary -nfcore_question_style = prompt_toolkit.styles.Style( - [ - ("qmark", "fg:ansiblue bold"), # token in front of the question - ("question", "bold"), # question text - ( - "answer", - "fg:ansigreen nobold bg:", - ), # submitted answer text behind the question - ( - "pointer", - "fg:ansiyellow bold", - ), # pointer used in select and checkbox prompts - ( - "highlighted", - "fg:ansiblue bold", - ), # pointed-at choice in select and checkbox prompts - ( - "selected", - "fg:ansiyellow noreverse bold", - ), # style for a selected item of a checkbox - ("separator", "fg:ansiblack"), # separator in lists - ("instruction", ""), # user instructions for select, rawselect, checkbox - ("text", ""), # plain text - ( - "disabled", - "fg:gray italic", - ), # disabled choices for select and checkbox prompts - ("choice-default", "fg:ansiblack"), - ("choice-default-changed", "fg:ansiyellow"), - ("choice-required", "fg:ansired"), - ] + +def lazy_attrs(module_globals: dict, mapping: dict[str, "str | Callable"]): + """Build module-level ``__getattr__`` and ``__dir__`` for lazy attributes (PEP 562). + + Each attribute maps either to the dotted path of the module to import it + from, or to a zero-argument callable that builds it. Resolved attributes + are cached in the module's globals, so the hook only fires once per name. + + Usage:: + + __getattr__, __dir__ = lazy_attrs(globals(), {"PipelineCreateApp": "nf_core.pipelines.create"}) + """ + + def module_getattr(name): + target = mapping.get(name) + if target is None: + raise AttributeError(f"module {module_globals['__name__']!r} has no attribute {name!r}") + value = target() if callable(target) else getattr(importlib.import_module(target), name) + module_globals[name] = value + return value + + def module_dir(): + return sorted(set(module_globals) | set(mapping)) + + return module_getattr, module_dir + + +# Custom style for questionary (built lazily to keep CLI start-up fast) +@functools.cache +def _nfcore_question_style(): + import prompt_toolkit.styles + + return prompt_toolkit.styles.Style( + [ + ("qmark", "fg:ansiblue bold"), # token in front of the question + ("question", "bold"), # question text + ( + "answer", + "fg:ansigreen nobold bg:", + ), # submitted answer text behind the question + ( + "pointer", + "fg:ansiyellow bold", + ), # pointer used in select and checkbox prompts + ( + "highlighted", + "fg:ansiblue bold", + ), # pointed-at choice in select and checkbox prompts + ( + "selected", + "fg:ansiyellow noreverse bold", + ), # style for a selected item of a checkbox + ("separator", "fg:ansiblack"), # separator in lists + ("instruction", ""), # user instructions for select, rawselect, checkbox + ("text", ""), # plain text + ( + "disabled", + "fg:gray italic", + ), # disabled choices for select and checkbox prompts + ("choice-default", "fg:ansiblack"), + ("choice-default-changed", "fg:ansiyellow"), + ("choice-required", "fg:ansired"), + ] + ) + + +# Lazy imports to keep CLI start-up fast +__getattr__, __dir__ = lazy_attrs( + globals(), + { + "nfcore_question_style": _nfcore_question_style, + "GitHubAPISession": "nf_core.github_api", + "gh_api": "nf_core.github_api", + "NFCoreTemplateConfig": "nf_core.pydantic_models", + "NFCoreYamlLintConfig": "nf_core.pydantic_models", + "NFCoreYamlConfig": "nf_core.pydantic_models", + }, ) @@ -98,7 +127,7 @@ def is_interactive() -> bool: NFCORE_CACHE_DIR = Path( - os.environ.get("XDG_CACHE_HOME", Path(os.getenv("HOME") or "", ".cache")), + os.environ.get("XDG_CACHE_HOME") or Path(os.getenv("HOME") or "") / ".cache", "nfcore", ) NFCORE_DIR = Path( @@ -149,10 +178,60 @@ def unquote(s: str) -> str: return s -def fetch_remote_version(source_url): - response = requests.get(source_url, timeout=3) - remote_version = re.sub(r"[^0-9\.]", "", response.text) - return remote_version +# The version check never fetches on the hot path (update-notifier pattern): +# it only reads the cached remote version, and refreshes the cache in a +# detached background process so that no run ever blocks on the network. +REMOTE_VERSION_CACHE = Path(NFCORE_CACHE_DIR, "latest_version.json") +REMOTE_VERSION_CACHE_EXPIRY = 60 * 60 * 24 # refresh at most once a day +REMOTE_VERSION_REFRESH_BACKOFF = 60 * 10 # wait at least this long between refresh attempts + + +def _load_version_cache() -> dict: + try: + with open(REMOTE_VERSION_CACHE) as fh: + cached = json.load(fh) + if isinstance(cached, dict): + return cached + except (OSError, ValueError): + pass + return {} + + +def _load_cached_remote_version() -> str | None: + from packaging.version import InvalidVersion, Version + + cached = _load_version_cache() + try: + if time.time() - cached["timestamp"] < REMOTE_VERSION_CACHE_EXPIRY: + Version(cached["version"]) # guard against a corrupted cache + return cached["version"] + except (KeyError, TypeError, InvalidVersion): + pass + return None + + +def _spawn_remote_version_refresh(source_url: str) -> None: + """Kick off a detached background process to refresh the remote version cache.""" + import subprocess + + try: + cached = _load_version_cache() + if time.time() - float(cached.get("attempted_at") or 0) < REMOTE_VERSION_REFRESH_BACKOFF: + return + # Record the attempt up front, so failed refreshes back off instead of respawning every run + cached["attempted_at"] = time.time() + setup_nfcore_cachedir() + with open(REMOTE_VERSION_CACHE, "w") as fh: + json.dump(cached, fh) + subprocess.Popen( + [sys.executable, "-m", "nf_core.version_updater", source_url, str(REMOTE_VERSION_CACHE)], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + except (OSError, ValueError, TypeError, subprocess.SubprocessError) as e: + log.debug(f"Could not start background version check: {e}") def check_if_outdated( @@ -165,7 +244,10 @@ def check_if_outdated( """ # Exit immediately if disabled via ENV var if os.environ.get("NFCORE_NO_VERSION_CHECK", False): - return (True, "", "") + return (False, "", "") + + from packaging.version import Version + # Set and clean up the current version string if current_version is None: current_version = nf_core.__version__ @@ -176,12 +258,10 @@ def check_if_outdated( # check if we have a newer version without blocking the rest of the script is_outdated = False if remote_version is None: # we set it manually for tests - try: - with concurrent.futures.ThreadPoolExecutor() as executor: - future = executor.submit(fetch_remote_version, source_url) - remote_version = future.result() - except requests.exceptions.RequestException as e: - log.debug(f"Could not check for nf-core updates: {e}") + remote_version = _load_cached_remote_version() + if remote_version is None: + # No fresh cache - refresh it in the background for the next run + _spawn_remote_version_refresh(source_url) if remote_version is not None and Version(remote_version) > Version(current_version): is_outdated = True return (is_outdated, current_version, remote_version) @@ -196,6 +276,10 @@ def rich_force_colors(): return None +stdout = Console(force_terminal=rich_force_colors()) +stderr = Console(stderr=True, force_terminal=rich_force_colors()) + + class Pipeline: """Object to hold information about a local pipeline. @@ -216,6 +300,8 @@ class Pipeline: def __init__(self, wf_path: Path) -> None: """Initialise pipeline object""" + import git + self.conda_config: dict = {} self.conda_package_info: dict = {} self.nf_config: dict = {} @@ -248,6 +334,8 @@ def _load(self) -> bool: def _load_conda_environment(self) -> bool: """Try to load the pipeline environment.yml file, if it exists""" + import yaml + try: with open(Path(self.wf_path, "environment.yml")) as fh: self.conda_config = yaml.safe_load(fh) @@ -262,6 +350,8 @@ def _fp(self, fn: str | Path) -> Path: def list_files(self) -> list[Path]: """Get a list of all files in the pipeline""" + import subprocess + files = [] try: # First, try to get the list of files using git @@ -336,6 +426,8 @@ def pretty_nf_version(version: tuple[int, int, int, bool]) -> str: @lru_cache(maxsize=1) def get_nf_version() -> tuple[int, int, int, bool] | None: """Get the version of Nextflow installed on the system. Cached for the lifetime of the process.""" + import subprocess + try: cmd_out = run_cmd("nextflow", "-v") if cmd_out is None: @@ -412,6 +504,8 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict: dict: Workflow configuration settings. """ + import hashlib + log.debug(f"Got '{wf_path}' as path") wf_path = Path(wf_path) config: dict[str, Any] = {} @@ -503,6 +597,9 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict: def run_cmd(executable: str, cmd: str) -> tuple[bytes, bytes] | None: """Run a specified command and capture the output. Handle errors nicely.""" + import shlex + import subprocess + full_cmd = f"{executable} {cmd}" log.debug(f"Running command: {full_cmd}") try: @@ -529,16 +626,6 @@ def run_cmd(executable: str, cmd: str) -> tuple[bytes, bytes] | None: return (proc.stdout, proc.stderr) -def setup_nfcore_dir() -> bool: - """Creates a directory for files that need to be kept between sessions - - Currently only used for keeping local copies of modules repos - """ - if not NFCORE_DIR.exists(): - NFCORE_DIR.mkdir(parents=True) - return True - - def setup_requests_cachedir() -> dict[str, Path | datetime.timedelta | str]: """Sets up local caching for faster remote HTTP requests. @@ -560,14 +647,16 @@ def setup_requests_cachedir() -> dict[str, Path | datetime.timedelta | str]: return config -def setup_nfcore_cachedir(cache_fn: str | Path) -> Path: - """Sets up local caching for caching files between sessions.""" +def setup_nfcore_cachedir(cache_fn: str | Path | None = None) -> Path: + """Sets up local caching for caching files between sessions. - cachedir = Path(NFCORE_CACHE_DIR, cache_fn) + Creates and returns a subdirectory of the nf-core cache directory, + or the cache directory itself if no subdirectory is given. + """ + cachedir = Path(NFCORE_CACHE_DIR, cache_fn) if cache_fn else NFCORE_CACHE_DIR try: - if not Path(cachedir).exists(): - Path(cachedir).mkdir(parents=True) + cachedir.mkdir(parents=True, exist_ok=True) except PermissionError: log.warning(f"Could not create cache directory: {cachedir}") @@ -587,6 +676,9 @@ def wait_cli_function(poll_func: Callable[[], bool], refresh_per_second: int = 2 Returns: None. Just sits in an infinite loop until the function returns True. """ + from rich.live import Live + from rich.spinner import Spinner + try: spinner = Spinner("dots2", "Use ctrl+c to stop waiting and force exit.") with Live(spinner, refresh_per_second=refresh_per_second): @@ -606,6 +698,9 @@ def poll_nfcore_web_api(api_url: str, post_data: dict | None = None) -> dict: Expects API response to be valid JSON and contain a top-level 'status' key. """ + import requests + import requests_cache + # Run without requests_cache so that we get the updated statuses with requests_cache.disabled(): try: @@ -647,171 +742,6 @@ def poll_nfcore_web_api(api_url: str, post_data: dict | None = None) -> dict: return web_response -class GitHubAPISession(requests_cache.CachedSession): - """ - Class to provide a single session for interacting with the GitHub API for a run. - Inherits the requests_cache.CachedSession and adds additional functionality, - such as automatically setting up GitHub authentication if we can. - """ - - def __init__(self) -> None: - self.auth_mode: str | None = None - self.return_ok: list[int] = [200, 201] - self.return_retry: list[int] = [403] - self.return_unauthorised: list[int] = [401] - self.has_init: bool = False - - def lazy_init(self) -> None: - """ - Initialise the object. - - Only do this when it's actually being used (due to global import) - """ - log.debug("Initialising GitHub API requests session") - cache_config = setup_requests_cachedir() - super().__init__(**cache_config) # type: ignore[arg-type] - self.setup_github_auth() - self.has_init = True - - def setup_github_auth(self, auth=None): - """ - Try to automatically set up GitHub authentication - """ - if auth is not None: - self.auth = auth - self.auth_mode = "supplied to function" - - # Class for Bearer token authentication - # https://stackoverflow.com/a/58055668/713980 - class BearerAuth(requests.auth.AuthBase): - def __init__(self, token): - self.token = token - - def __call__(self, r): - r.headers["authorization"] = f"Bearer {self.token}" - return r - - # Default auth if we're running and the gh CLI tool is installed - gh_cli_config_fn = Path.home() / ".config" / "gh" / "hosts.yml" - if self.auth is None and gh_cli_config_fn.exists(): - try: - with open(gh_cli_config_fn) as fh: - gh_cli_config = yaml.safe_load(fh) - self.auth = requests.auth.HTTPBasicAuth( - gh_cli_config["github.com"]["user"], - gh_cli_config["github.com"]["oauth_token"], - ) - self.auth_mode = f"gh CLI config: {gh_cli_config['github.com']['user']}" - except (OSError, KeyError, yaml.YAMLError): - ex_type, ex_value, _ = sys.exc_info() - if ex_type is not None: - output = rich.markup.escape(f"{ex_type.__name__}: {ex_value}") - log.debug(f"Couldn't auto-auth with GitHub CLI auth from '{gh_cli_config_fn}': [red]{output}") - - # Default auth if we have a GitHub Token (eg. GitHub Actions CI) - if os.environ.get("GITHUB_TOKEN") is not None and self.auth is None: - self.auth_mode = "Bearer token with GITHUB_TOKEN" - self.auth = BearerAuth(os.environ["GITHUB_TOKEN"]) - else: - log.warning("Could not find GitHub authentication token. Some API requests may fail.") - - log.debug(f"Using GitHub auth: {self.auth_mode}") - - def log_content_headers(self, request, post_data=None): - """ - Try to dump everything to the console, useful when things go wrong. - """ - log.debug(f"Requested URL: {request.url}") - log.debug(f"From requests cache: {request.from_cache}") - log.debug(f"Request status code: {request.status_code}") - log.debug(f"Request reason: {request.reason}") - if post_data is None: - post_data = {} - try: - log.debug(json.dumps(dict(request.headers), indent=4)) - log.debug(json.dumps(request.json(), indent=4)) - log.debug(json.dumps(post_data, indent=4)) - except (json.JSONDecodeError, TypeError) as e: - log.debug(f"Could not parse JSON response from GitHub API! {e}") - log.debug(request.headers) - log.debug(request.content) - log.debug(post_data) - - def safe_get(self, url): - """ - Run a GET request, raise a nice exception with lots of logging if it fails. - """ - if not self.has_init: - self.lazy_init() - request = self.get(url) - if request.status_code in self.return_retry: - stderr = rich.console.Console(stderr=True, force_terminal=rich_force_colors()) - try: - r = self.request_retry(url) - except Exception as e: - stderr.print_exception() - raise e - else: - return r - elif request.status_code in self.return_unauthorised: - raise RuntimeError("GitHub API PR failed, probably due to an expired GITHUB_TOKEN.") - - return request - - def get(self, url, params=None, **kwargs): - """ - Initialise the session if we haven't already, then call the superclass get method. - """ - if not self.has_init: - self.lazy_init() - return super().get(url, params=params, **kwargs) - - def request_retry(self, url, post_data=None): - """ - Try to fetch a URL, keep retrying if we get a certain return code. - - Used in nf-core pipelines sync code because we get 403 errors: too many simultaneous requests - See https://github.com/nf-core/tools/issues/911 - """ - if not self.has_init: - self.lazy_init() - - # Start the loop for a retry mechanism - while True: - # GET request - if post_data is None: - log.debug(f"Sending GET request to {url}") - r = self.get(url=url) - # POST request - else: - log.debug(f"Sending POST request to {url}") - r = self.post(url=url, json=post_data) - - # Failed but expected - try again - if r.status_code in self.return_retry: - self.log_content_headers(r, post_data) - log.debug(f"GitHub API PR failed - got return code {r.status_code}") - wait_time = float(re.sub("[^0-9]", "", str(r.headers.get("Retry-After", 0)))) - if wait_time == 0: - log.debug("Couldn't find 'Retry-After' header, guessing a length of time to wait") - wait_time = random.randrange(10, 60) - log.warning(f"Got API return code {r.status_code}. Trying again after {wait_time} seconds..") - time.sleep(wait_time) - - # Unexpected error - raise - elif r.status_code not in self.return_ok: - self.log_content_headers(r, post_data) - raise RuntimeError(f"GitHub API PR failed - got return code {r.status_code} from {url}") - - # Success! - else: - return r - - -# Single session object to use for entire codebase. Not sure if there's a better way to do this? -gh_api = GitHubAPISession() - - def anaconda_package(dep, dep_channels=None): """Query conda package information. @@ -825,6 +755,7 @@ def anaconda_package(dep, dep_channels=None): A LookupError, if the connection fails or times out or gives an unexpected status code A ValueError, if the package name can not be found (404) """ + import requests if dep_channels is None: dep_channels = ["conda-forge", "bioconda"] @@ -910,6 +841,8 @@ def pip_package(dep): A LookupError, if the connection fails or times out A ValueError, if the package name can not be found """ + import requests + pip_depname, _ = dep.split("=", 1) pip_api_url = f"https://pypi.python.org/pypi/{pip_depname}/json" try: @@ -938,6 +871,8 @@ def get_biocontainer_tag(package, version): A ValueError, if the package name can not be found (404) """ + import requests + biocontainers_api_url = f"https://api.biocontainers.pro/ga4gh/trs/v2/tools/{package}/versions/{package}-{version}" def get_tag_date(tag_date): @@ -1002,6 +937,7 @@ def get_tag_date(tag_date): def custom_yaml_dumper(): """Overwrite default PyYAML output to make Prettier YAML linting happy""" + import yaml class CustomDumper(yaml.Dumper): def represent_dict_preserve_order(self, data): @@ -1034,6 +970,8 @@ def write_line_break(self, data=None): def is_file_binary(path): """Check file path to see if it is a binary file""" + import mimetypes + binary_ftypes = ["image", "application/java-archive", "application/x-java-archive"] binary_extensions = [".jpeg", ".jpg", ".png", ".zip", ".gz", ".jar", ".tar"] @@ -1060,13 +998,17 @@ def prompt_remote_pipeline_name(wfs): Raises: AssertionError, if pipeline cannot be found """ + import questionary + import requests + + from nf_core.github_api import gh_api if not is_interactive(): raise UserWarning("No pipeline name provided and session is not interactive (no TTY detected).") pipeline = questionary.autocomplete( "Pipeline name:", choices=[wf.name for wf in wfs.remote_workflows], - style=nfcore_question_style, + style=_nfcore_question_style(), ).unsafe_ask() # Check nf-core repos @@ -1101,6 +1043,8 @@ def prompt_pipeline_release_branch( Returns: choice (questionary.Choice or bool): Selected release / branch or False if no releases / branches available """ + import questionary + # Prompt user for release tag, tag_set will contain all available. choices: list[questionary.Choice] = [] tag_set: list[str] = [] @@ -1132,30 +1076,21 @@ def prompt_pipeline_release_branch( if multiple: return ( - questionary.checkbox("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask(), + questionary.checkbox( + "Select release / branch:", choices=choices, style=_nfcore_question_style() + ).unsafe_ask(), tag_set, ) else: return ( - questionary.select("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask(), + questionary.select( + "Select release / branch:", choices=choices, style=_nfcore_question_style() + ).unsafe_ask(), tag_set, ) -class SingularityCacheFilePathValidator(questionary.Validator): - """ - Validator for file path specified as --singularity-cache-index argument in nf-core pipelines download - """ - - def validate(self, document) -> None: - if len(document.text) and not Path(document.text).is_file(): - raise questionary.ValidationError( - message="Invalid remote cache index file", - cursor_position=len(document.text), - ) - - def get_repo_releases_branches(pipeline, wfs): """Fetches details of a nf-core workflow to download. @@ -1169,6 +1104,7 @@ def get_repo_releases_branches(pipeline, wfs): Raises: LockupError, if the pipeline can not be found. """ + from nf_core.github_api import gh_api wf_releases = [] wf_branches = {} @@ -1247,6 +1183,7 @@ def get_repo_commit(pipeline, commit_id): Returns: commit_id: String or None """ + from nf_core.github_api import gh_api commit_response = gh_api.get( f"https://api.github.com/repos/{pipeline}/commits/{commit_id}", headers={"Accept": "application/vnd.github.sha"} @@ -1261,208 +1198,7 @@ def get_repo_commit(pipeline, commit_id): DEPRECATED_CONFIG_PATHS = [".nf-core-lint.yml", ".nf-core-lint.yaml"] -class NFCoreTemplateConfig(BaseModel): - """Template configuration schema""" - - org: str | None = None - """ Organisation name """ - name: str | None = None - """ Pipeline name """ - description: str | None = None - """ Pipeline description """ - author: str | None = None - """ Pipeline author """ - version: str | None = None - """ Pipeline version """ - force: bool | None = True - """ Force overwrite of existing files """ - outdir: str | Path | None = None - """ Output directory """ - skip_features: list | None = None - """ Skip features. See https://nf-co.re/docs/nf-core-tools/pipelines/create for a list of features. """ - is_nfcore: bool | None = None - """ Whether the pipeline is an nf-core pipeline. """ - - # convert outdir to str - @field_validator("outdir") - @classmethod - def outdir_to_str(cls, v: str | Path | None) -> str | None: - if v is not None: - v = str(v) - return v - - def __getitem__(self, item: str) -> Any: - if self is None: - return None - return getattr(self, item) - - def get(self, item: str, default: Any = None) -> Any: - return getattr(self, item, default) - - -class NFCoreYamlLintConfig(BaseModel): - """ - schema for linting config in `.nf-core.yml` should cover: - - .. code-block:: yaml - - files_unchanged: - - .github/workflows/branch.yml - modules_config: False - modules_config: - - fastqc - # merge_markers: False - merge_markers: - - docs/my_pdf.pdf - nextflow_config: False - nextflow_config: - - manifest.name - - config_defaults: - - params.annotation_db - - params.multiqc_comment_headers - - params.custom_table_headers - # multiqc_config: False - multiqc_config: - - report_section_order - - report_comment - files_exist: - - CITATIONS.md - template_strings: False - template_strings: - - docs/my_pdf.pdf - nfcore_components: False - # nf_test_content: False - nf_test_content: - - tests/.nf.test - - tests/nextflow.config - - nf-test.config - """ - - files_unchanged: bool | list[str] | None = None - """ List of files that should not be changed """ - modules_config: bool | list[str] | None = None - """ List of modules that should not be changed """ - merge_markers: bool | list[str] | None = None - """ List of files that should not contain merge markers """ - nextflow_config: bool | list[str | dict[str, list[str]]] | None = None - """ List of Nextflow config files that should not be changed """ - nf_test_content: bool | list[str] | None = None - """ List of nf-test content that should not be changed """ - multiqc_config: bool | list[str] | None = None - """ List of MultiQC config options that be changed """ - files_exist: bool | list[str] | None = None - """ List of files that can not exist """ - template_strings: bool | list[str] | None = None - """ List of files that can contain template strings """ - readme: bool | list[str] | None = None - """ Lint the README.md file """ - nfcore_components: bool | None = None - """ Lint all required files to use nf-core modules and subworkflows """ - actions_nf_test: bool | None = None - """ Lint all required files to use GitHub Actions CI """ - actions_awstest: bool | None = None - """ Lint all required files to run tests on AWS """ - actions_awsfulltest: bool | None = None - """ Lint all required files to run full tests on AWS """ - pipeline_todos: bool | None = None - """ Lint for TODOs statements""" - pipeline_if_empty_null: bool | None = None - """ Lint for ifEmpty(null) statements""" - plugin_includes: bool | None = None - """ Lint for nextflow plugin """ - pipeline_name_conventions: bool | None = None - """ Lint for pipeline name conventions """ - schema_lint: bool | None = None - """ Lint nextflow_schema.json file""" - schema_params: bool | None = None - """ Lint schema for all params """ - system_exit: bool | None = None - """ Lint for System.exit calls in groovy/nextflow code """ - schema_description: bool | None = None - """ Check that every parameter in the schema has a description. """ - actions_schema_validation: bool | None = None - """ Lint GitHub Action workflow files with schema""" - modules_json: bool | None = None - """ Lint modules.json file """ - modules_structure: bool | None = None - """ Lint modules structure """ - base_config: bool | None = None - """ Lint base.config file """ - nfcore_yml: bool | None = None - """ Lint nf-core.yml """ - version_consistency: bool | None = None - """ Lint for version consistency """ - included_configs: bool | None = None - """ Lint for included configs """ - local_component_structure: bool | None = None - """ Lint local components use correct structure mirroring remote""" - container_configs: bool | None = None - """ Lint that container configuration files in conf/ are up to date """ - rocrate_readme_sync: bool | None = None - """ Lint for README.md and rocrate.json sync """ - - def __getitem__(self, item: str) -> Any: - return getattr(self, item) - - def get(self, item: str, default: Any = None) -> Any: - if getattr(self, item, default) is None: - return default - return getattr(self, item, default) - - def __setitem__(self, item: str, value: Any) -> None: - setattr(self, item, value) - - -class NFCoreYamlConfig(BaseModel): - """.nf-core.yml configuration file schema""" - - model_config = ConfigDict(populate_by_name=True) - - repository_type: Literal["pipeline", "modules"] | None = None - """ Type of repository """ - nf_core_version: str | None = None - """ Version of nf-core/tools used to create/update the pipeline """ - org_path: str | None = None - """ Path to the organisation's modules repository (used for modules repo_type only) """ - lint: NFCoreYamlLintConfig | None = None - """ Pipeline linting configuration, see https://nf-co.re/docs/nf-core-tools/pipelines/lint#linting-config for examples and documentation """ - template: NFCoreTemplateConfig | None = None - """ Pipeline template configuration """ - bump_version: dict[str, bool] | None = None - """ Disable bumping of the version for a module/subworkflow (when repository_type is modules). See https://nf-co.re/docs/nf-core-tools/modules/bump-versions for more information. """ - update: dict[str, str | bool | dict[str, str | dict[str, str | bool]]] | None = None - """ Disable updating specific modules/subworkflows (when repository_type is pipeline). See https://nf-co.re/docs/nf-core-tools/modules/update for more information. """ - container_registry: list[str] | None = Field(default=None, alias="container-registry") - """ Additional container registry prefixes allowed when linting container directives. """ - - def __getitem__(self, item: str) -> Any: - return getattr(self, item) - - def get(self, item: str, default: Any = None) -> Any: - return getattr(self, item, default) - - def __setitem__(self, item: str, value: Any) -> None: - setattr(self, item, value) - - def model_dump(self, **kwargs) -> dict[str, Any]: - # Get the initial data - config = super().model_dump(**kwargs) - - if self.repository_type == "modules": - # Fields to exclude for modules - fields_to_exclude = ["template", "update"] - else: # pipeline - # Fields to exclude for pipeline - fields_to_exclude = ["bump_version", "org_path"] - - # Remove the fields based on repository_type - for field in fields_to_exclude: - config.pop(field, None) - - return config - - -def load_tools_config(directory: str | Path = ".") -> tuple[Path | None, NFCoreYamlConfig | None]: +def load_tools_config(directory: str | Path = ".") -> "tuple[Path | None, NFCoreYamlConfig | None]": """ Parse the nf-core.yml configuration file @@ -1473,6 +1209,11 @@ def load_tools_config(directory: str | Path = ".") -> tuple[Path | None, NFCoreY Returns the loaded config dict or False, if the file couldn't be loaded """ + import yaml + from pydantic import ValidationError + + from nf_core.pydantic_models import NFCoreTemplateConfig, NFCoreYamlConfig + tools_config = {} config_fn = get_first_available_path(directory, CONFIG_PATHS) @@ -1626,6 +1367,8 @@ def file_md5(fname): fname (str): Path to a local file. """ + import hashlib + # Calculate the md5 for the file on disk hash_md5 = hashlib.md5() with open(fname, "rb") as f: @@ -1717,6 +1460,8 @@ def set_wd_tempdir(base_dir: Path | None = None) -> Generator[Path, None, None]: Args: base_dir: Directory in which to create the tempdir. Defaults to the system temp location. """ + import tempfile + with tempfile.TemporaryDirectory(dir=base_dir) as tmp, set_wd(Path(tmp)): yield Path(tmp) diff --git a/nf_core/version_updater.py b/nf_core/version_updater.py new file mode 100644 index 0000000000..817cc379f8 --- /dev/null +++ b/nf_core/version_updater.py @@ -0,0 +1,38 @@ +"""Background updater for the cached nf-core version.""" + +import json +import re +import sys +import tempfile +import time +import urllib.request +from pathlib import Path + +from packaging.version import Version + + +def refresh_version_cache(source_url: str, cache_path: Path) -> None: + """Fetch and atomically cache the latest nf-core version.""" + with urllib.request.urlopen(source_url, timeout=10) as response: + remote_version = re.sub(r"[^0-9.]", "", response.read().decode()) + + # Do not replace a valid cache with a malformed response. + Version(remote_version) + + cache_path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=cache_path.parent) as temporary_dir: + temporary_path = Path(temporary_dir, cache_path.name) + temporary_path.write_text(json.dumps({"version": remote_version, "timestamp": time.time()})) + temporary_path.replace(cache_path) + + +def main() -> None: + """Run the background version updater.""" + if len(sys.argv) != 3: + raise SystemExit("usage: version_updater ") + + refresh_version_cache(sys.argv[1], Path(sys.argv[2])) + + +if __name__ == "__main__": + main() diff --git a/tests/pipelines/test_launch.py b/tests/pipelines/test_launch.py index 9cbee9ece7..419c618ac0 100644 --- a/tests/pipelines/test_launch.py +++ b/tests/pipelines/test_launch.py @@ -17,7 +17,7 @@ class TestLaunch(TestPipelines): def setUp(self) -> None: super().setUp() self.nf_params_fn = Path(self.pipeline_dir, "nf-params.json") - self.launcher = nf_core.pipelines.launch.Launch(self.pipeline_dir, params_out=self.nf_params_fn) + self.launcher = nf_core.pipelines.launch.Launch(str(self.pipeline_dir), params_out=self.nf_params_fn) @mock.patch.object(nf_core.pipelines.launch.Launch, "prompt_web_gui", side_effect=[True]) @mock.patch.object(nf_core.pipelines.launch.Launch, "launch_web_gui") diff --git a/tests/subworkflows/test_patch.py b/tests/subworkflows/test_patch.py index 884af41e83..62a39b3cb0 100644 --- a/tests/subworkflows/test_patch.py +++ b/tests/subworkflows/test_patch.py @@ -196,7 +196,7 @@ def test_create_patch_update_success(self): ) == Path("subworkflows", GITLAB_REPO, "bam_sort_stats_samtools", patch_fn) # Update the subworkflow - update_obj = nf_core.subworkflows.update.SubworkflowUpdate( + update_obj = nf_core.subworkflows.SubworkflowUpdate( self.pipeline_dir, sha=OLD_SHA, show_diff=False, @@ -256,7 +256,7 @@ def test_create_patch_update_fail(self): with open(swf_path / patch_fn) as fh: patch_contents = fh.read() - update_obj = nf_core.subworkflows.update.SubworkflowUpdate( + update_obj = nf_core.subworkflows.SubworkflowUpdate( self.pipeline_dir, sha=FAIL_SHA, show_diff=False, diff --git a/tests/test_version_updater.py b/tests/test_version_updater.py new file mode 100644 index 0000000000..5d938f5b20 --- /dev/null +++ b/tests/test_version_updater.py @@ -0,0 +1,63 @@ +"""Tests for the detached remote-version cache updater.""" + +import json +import subprocess +import sys +from unittest import mock + +import pytest +from packaging.version import InvalidVersion + +import nf_core.utils +from nf_core.version_updater import refresh_version_cache + + +def test_refresh_version_cache(tmp_path): + """A valid remote version is written to the cache atomically.""" + remote_version_path = tmp_path / "remote_version" + remote_version_path.write_text("4.1.0\n") + cache_path = tmp_path / "latest_version.json" + + refresh_version_cache(remote_version_path.as_uri(), cache_path) + + cached = json.loads(cache_path.read_text()) + assert cached["version"] == "4.1.0" + assert isinstance(cached["timestamp"], float) + + +def test_refresh_version_cache_rejects_invalid_version(tmp_path): + """A malformed response does not replace an existing valid cache.""" + remote_version_path = tmp_path / "remote_version" + remote_version_path.write_text("not a version") + cache_path = tmp_path / "latest_version.json" + original_cache = '{"version": "4.0.0", "timestamp": 123}' + cache_path.write_text(original_cache) + + with pytest.raises(InvalidVersion): + refresh_version_cache(remote_version_path.as_uri(), cache_path) + + assert cache_path.read_text() == original_cache + + +def test_spawn_remote_version_refresh_uses_updater_module(tmp_path, monkeypatch): + """The parent process launches the updater without embedding Python source.""" + cache_path = tmp_path / "latest_version.json" + monkeypatch.setattr(nf_core.utils, "NFCORE_CACHE_DIR", tmp_path) + monkeypatch.setattr(nf_core.utils, "REMOTE_VERSION_CACHE", cache_path) + + with mock.patch("subprocess.Popen") as popen: + nf_core.utils._spawn_remote_version_refresh("https://example.com/tools_version?v=4.0.0") + + popen.assert_called_once_with( + [ + sys.executable, + "-m", + "nf_core.version_updater", + "https://example.com/tools_version?v=4.0.0", + str(cache_path), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + )