diff --git a/README.rst b/README.rst index 2be32d5..41ec3d6 100644 --- a/README.rst +++ b/README.rst @@ -29,7 +29,7 @@ If you already have all requirements, you can install the packages using pip:: pip install sequana_mapper --upgrade -You will need third-party software such as fastqc. Please see below for details. +You will need third-party software such as bwa and samtools. Please see below for details. Usage ~~~~~ @@ -52,7 +52,7 @@ For capture-seq projects (feature counting):: This creates a ``mapper/`` directory with the pipeline and configuration file. Execute the pipeline locally:: cd mapper - sh mapper.sh + bash mapper.sh See ``.sequana/profile/config.yaml`` to tune Snakemake behaviour (cores, cluster settings, etc.). @@ -70,7 +70,7 @@ Images are downloaded in the working directory but you can store them in a share and then:: cd mapper - sh mapper.sh + bash mapper.sh Requirements @@ -83,6 +83,7 @@ This pipeline requires the following executables (install via bioconda/conda): - **bowtie2** — alternative short-read aligner - **samtools** / **sambamba** — BAM processing - **bamtools** — BAM statistics +- **fastqc** — raw-read quality control (optional, ``--do-fastqc``) - **deeptools** — bigwig generation (``bamCoverage``) - **bedtools** — genome arithmetic - **subread** — feature counting (``featureCounts``, capture-seq only) @@ -117,6 +118,7 @@ filtered BAM files, a MultiQC HTML report, and optionally coverage tracks and fe **Optional outputs**: - ``--do-coverage`` — runs ``sequana_coverage`` for depth-of-coverage analysis (prokaryotes) +- ``--do-fastqc`` — runs ``fastqc`` on the raw input reads (included in the MultiQC report) - ``--create-bigwig`` — generates bigwig files via ``bamCoverage`` (deeptools) - ``--capture-annotation-file`` — enables ``featureCounts`` for capture-seq efficiency metrics @@ -135,6 +137,7 @@ Changelog ========= ====================================================================== Version Description ========= ====================================================================== +1.5.0 * add optional FastQC on raw reads (--do-fastqc), included in MultiQC 1.4.1 * update to use wrappers.shells and wrappers.snippets drop wrappers usage 1.4.0 * update wrappers to v24.8.29 diff --git a/doc/conf.py b/doc/conf.py index 6c7dbc5..5729e98 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -11,10 +11,12 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import os +import sys + import sphinx -sys.path.insert(0, os.path.abspath('sphinxext')) +sys.path.insert(0, os.path.abspath("sphinxext")) # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -24,9 +26,11 @@ # This is for ReadTheDoc import matplotlib -matplotlib.use('Agg') + +matplotlib.use("Agg") import pkg_resources + version = pkg_resources.require(pkg_name)[0].version import matplotlib @@ -36,55 +40,56 @@ author = "Thomas Cokelaer" title = "Sequana mapper pipeline" copyright = author + ", 2016-2019" -project = 'Sequana mapper pipeline' +project = "Sequana mapper pipeline" import easydev from easydev import get_path_sphinx_themes - # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - 'sphinx.ext.autodoc', - ('sphinx.ext.imgmath' # only available for sphinx >= 1.4 - if sphinx.version_info[:2] >= (1, 4) - else 'sphinx.ext.pngmath'), - 'sphinx.ext.coverage', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', + "sphinx.ext.autodoc", + ( + "sphinx.ext.imgmath" # only available for sphinx >= 1.4 + if sphinx.version_info[:2] >= (1, 4) + else "sphinx.ext.pngmath" + ), + "sphinx.ext.coverage", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", "numpydoc.numpydoc", - 'matplotlib.sphinxext.plot_directive', - 'sphinx.ext.autosummary', - "sequana.sphinxext.snakemakerule" - ] + "matplotlib.sphinxext.plot_directive", + "sphinx.ext.autosummary", + "sequana.sphinxext.snakemakerule", +] # note that the numpy directives is buggy. Example: class and init are not recognised as two entities for the autoclass_content=both here below math_number_all = False -todo_include_todos=True +todo_include_todos = True jscopybutton_path = "copybutton.js" -autoclass_content = 'both' +autoclass_content = "both" # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. project = project @@ -95,34 +100,34 @@ # built documents. # # The short X.Y version. -version = 'Current version: ' + str(version) +version = "Current version: " + str(version) # The full version, including alpha/beta/rc tags. release = release # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. -#unused_docs = [] +# unused_docs = [] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_trees = ['_build'] +exclude_trees = ["_build"] exclude_patterns = [] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). @@ -133,7 +138,7 @@ show_authors = True # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. modindex_common_prefix = ["sequana."] @@ -150,22 +155,17 @@ def touch_example_backreferences(app, what, name, obj, options, lines): # generate empty examples files, so that we don't get # inclusion errors if there are no examples for a class / module - examples_path = os.path.join(app.srcdir, "modules", "generated", - "%s.examples" % name) + examples_path = os.path.join(app.srcdir, "modules", "generated", "%s.examples" % name) if not os.path.exists(examples_path): # touch file - open(examples_path, 'w').close() - + open(examples_path, "w").close() # Add the 'copybutton' javascript, to hide/show the prompt in code # examples def setup(app): - app.add_javascript('copybutton.js') - app.connect('autodoc-process-docstring', touch_example_backreferences) - - - + app.add_javascript("copybutton.js") + app.connect("autodoc-process-docstring", touch_example_backreferences) # -- Options for HTML output --------------------------------------------------- @@ -175,7 +175,8 @@ def setup(app): on_rtd = os.environ.get("READTHEDOCS", None) == "True" if not on_rtd: import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' + + html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] else: html_theme = "default" @@ -184,53 +185,52 @@ def setup(app): # further. For a list of options available for each theme, see the # documentation. # the user theme contains the otpions 'homepage', which is populated here -#html_theme_options = {'homepage': init_sphinx.url} +# html_theme_options = {'homepage': init_sphinx.url} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [get_path_sphinx_themes()] - +# html_theme_path = [get_path_sphinx_themes()] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. html_short_title = "sequana" # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = "../../share/data/images/crx_logo.png" +# html_logo = "../../share/data/images/crx_logo.png" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = "../../share/data/images/crx_logo.ico" +# html_favicon = "../../share/data/images/crx_logo.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = "%b %d, %Y" # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -html_index = 'index.html' +html_index = "index.html" -#Custom sidebar templates, maps page names to templates. -#html_sidebars = { -# 'index': [ 'indexsidebar.html'], +# Custom sidebar templates, maps page names to templates. +# html_sidebars = { +# 'index': [ 'indexsidebar.html'], # 'contents':'indexsidebar.html', -#} +# } # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = { 'index': 'index.html'} +# html_additional_pages = { 'index': 'index.html'} # If false, no module index is generated. @@ -250,18 +250,18 @@ def setup(app): html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'doc' +htmlhelp_basename = "doc" # -- Options for LaTeX output -------------------------------------------------- @@ -270,52 +270,48 @@ def setup(app): pngmath_use_preview = True # The font size ('10pt', '11pt' or '12pt'). -latex_font_size = '10pt' +latex_font_size = "10pt" # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'main.tex', title, - author, 'manual'), + ("index", "main.tex", title, author, "manual"), ] -latex_elements = { 'inputenc': '\\usepackage[utf8]{inputenc}' } +latex_elements = {"inputenc": "\\usepackage[utf8]{inputenc}"} # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Additional stuff for the LaTeX preamble. # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ('index', project, project, - [author], 1) -] +man_pages = [("index", project, project, [author], 1)] # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - "python": ('http://docs.python.org/', None), + "python": ("http://docs.python.org/", None), } diff --git a/doc/index.rst b/doc/index.rst index 675d27e..13fb184 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -36,6 +36,3 @@ What is Sequana ? To join the project, please let us know on `github `_. For more information, please see `github `_. - - - diff --git a/environment.yml b/environment.yml index 3489b98..76e2120 100644 --- a/environment.yml +++ b/environment.yml @@ -11,6 +11,7 @@ dependencies: - deeptools - bedtools - bamtools +- fastqc - mosdepth - sambamba - subread diff --git a/pyproject.toml b/pyproject.toml index 6ac41ab..fdd3576 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "sequana-mapper" -version = "1.4.1" +version = "1.5.0" description = "A multi-sample mapper to map reads onto a reference" authors = ["Sequana Team"] license = "BSD-3" diff --git a/sequana_pipelines/mapper/__init__.py b/sequana_pipelines/mapper/__init__.py index 71b8c2e..1746818 100644 --- a/sequana_pipelines/mapper/__init__.py +++ b/sequana_pipelines/mapper/__init__.py @@ -1,5 +1,6 @@ import importlib.metadata as metadata + def get_package_version(package_name): try: version = metadata.version(package_name) @@ -7,4 +8,5 @@ def get_package_version(package_name): except metadata.PackageNotFoundError: return f"{package_name} not found" + version = get_package_version("sequana-mapper") diff --git a/sequana_pipelines/mapper/config.yaml b/sequana_pipelines/mapper/config.yaml index c26571f..d28e0b2 100644 --- a/sequana_pipelines/mapper/config.yaml +++ b/sequana_pipelines/mapper/config.yaml @@ -21,16 +21,17 @@ general: apptainers: bamtools: https://zenodo.org/record/10211475/files/bamtools_2.5.2.img - bowtie2: https://zenodo.org/record/8037162/files/sequana_minimal_23.6.13.img + bowtie2: https://zenodo.org/record/19127803/files/sequana_minimal_26.3.19.img bwa: https://zenodo.org/record/8037162/files/sequana_minimal_23.6.13.img - deeptools: https://zenodo.org/record/18257162/files/sequana_tools_26.1.14.img + deeptools: https://zenodo.org/record/19187268/files/deeptools_3.5.6.img + fastqc: https://zenodo.org/record/7015004/files/fastqc_0.11.9-py3.img graphviz: https://zenodo.org/record/7928262/files/graphviz_7.0.5.img - minimap2: https://zenodo.org/record/18257162/files/sequana_tools_26.1.14.img + minimap2: https://zenodo.org/record/19127803/files/sequana_minimal_26.3.19.img multiqc: https://zenodo.org/record/10205070/files/multiqc_1.16.0.img samtools: https://zenodo.org/record/7437898/files/samtools_1.16.1.img - sequana_coverage: https://zenodo.org/record/13946414/files/sequana_coverage_0.18.0.img + sequana_coverage: https://zenodo.org/record/20467327/files/sequana_0.22.0.img seqkit: https://zenodo.org/record/7821924/files/seqkit_2.4.0.img - subread: https://zenodo.org/record/18257162/files/sequana_tools_26.1.14.img + subread: https://zenodo.org/record/8037099/files/subread_2.0.3.img mosdepth: https://zenodo.org/record/12805460/files/mosdepth_0.3.8.img @@ -161,6 +162,21 @@ sequana_coverage: mem: 8G +############################################################################# +## FastQC computes quality-control metrics on the raw input FastQ files. +## +## :Parameters: +## +## - do: set to true to run FastQC on the raw reads +## - options: any options recognised by fastqc (e.g. --nogroup) +fastqc: + do: false + options: --nogroup + threads: 4 + resources: + mem: 4G + + ############################################################################# ## MultiQC aggregates results from bioinformatics analyses across many ## samples into a single report. diff --git a/sequana_pipelines/mapper/main.py b/sequana_pipelines/mapper/main.py index adce2d8..e02627f 100755 --- a/sequana_pipelines/mapper/main.py +++ b/sequana_pipelines/mapper/main.py @@ -32,6 +32,7 @@ "--capture-annotation-file", "--create-bigwig", "--do-coverage", + "--do-fastqc", "--nanopore", "--pacbio", "--reference-file", @@ -57,6 +58,7 @@ @click.option("--reference-file", required=True, help="You input reference file in fasta format") @click.option("--annotation-file", help="Used by the sequana_coverage tool if provided") @click.option("--do-coverage", is_flag=True, help="Use sequana_coverage (prokaryotes)") +@click.option("--do-fastqc", is_flag=True, help="Run FastQC on the raw input reads (included in MultiQC report)") @click.option( "--pacbio", is_flag=True, @@ -99,6 +101,9 @@ def main(**options): if options.do_coverage: cfg.sequana_coverage.do = True + if options.do_fastqc: + cfg.fastqc.do = True + if options.create_bigwig: cfg.general.create_bigwig = True diff --git a/sequana_pipelines/mapper/mapper.rules b/sequana_pipelines/mapper/mapper.rules index cf9d2ef..20e1127 100644 --- a/sequana_pipelines/mapper/mapper.rules +++ b/sequana_pipelines/mapper/mapper.rules @@ -28,6 +28,8 @@ manager = PipelineManager("mapper", config) def get_multiqc_inputs(): inputs = [] inputs += expand("{sample}/bamtools_stats/sequana_bamtools_stats_{sample}.txt", sample=manager.samples) + if config["fastqc"]["do"]: + inputs += expand("{sample}/fastqc/fastqc.done", sample=manager.samples) if config["feature_counts"]["do"]: inputs += expand("{sample}/feature_counts/{sample}_feature.out", sample=manager.samples) if config["sequana_coverage"]["do"]: @@ -87,6 +89,8 @@ if config['general']['mapper'] == "bwa": fai="reference/genome.fasta.fai", reference="reference/genome.fasta" output: + sam=temp("{sample}/{sample}.sam"), + bam=temp("{sample}/{sample}.bam"), sorted="{sample}/{sample}.sorted.bam" log: "{sample}/bwa/{sample}.log" @@ -99,7 +103,7 @@ if config['general']['mapper'] == "bwa": container: config['apptainers']['bwa'] shell: - manager.get_shell("bwa/align", "v1") + manager.get_shell("bwa/align", "v2") elif config['general']['mapper'] == 'bwa_split': @@ -115,23 +119,36 @@ elif config['general']['mapper'] == 'bwa_split': seqkit split2 --by-size {params.nreads} -O {wildcards.sample}/split -1 {input[0]} -2 {input[1]} """ - def get_bwa_input(): + def _seqkit_stem(filename): + # seqkit split2 names parts .part_NNN.fastq.gz where is + # the input basename minus the fastq/fq[.gz] extension. Derive it from + # the real raw filenames so we do not assume a fixed readtag suffix. + base = os.path.basename(filename) + for ext in (".fastq.gz", ".fq.gz", ".fastq", ".fq"): + if base.endswith(ext): + return base[: -len(ext)] + return base + + def get_bwa_input(wildcards): + files = manager.samples[wildcards.sample] if manager.paired is True: return ( - "{sample}/split/{sample}_R1_001.{splitid}.fastq.gz", - "{sample}/split/{sample}_R2_001.{splitid}.fastq.gz" + f"{wildcards.sample}/split/{_seqkit_stem(files[0])}.{wildcards.splitid}.fastq.gz", + f"{wildcards.sample}/split/{_seqkit_stem(files[1])}.{wildcards.splitid}.fastq.gz" ) else: - return "{sample}/split/{sample}_R1_001.{splitid}.fastq.gz" + return f"{wildcards.sample}/split/{_seqkit_stem(files[0])}.{wildcards.splitid}.fastq.gz" # intermediate rule bwa_intermediate: input: - fastq=get_bwa_input(), + fastq=get_bwa_input, bwa_bwt="reference/genome.fasta.bwt", fai="reference/genome.fasta.fai", reference="reference/genome.fasta" output: + sam=temp("{sample}/split/{sample}.{splitid}.sam"), + bam=temp("{sample}/split/{sample}.{splitid}.bam"), sorted=temp("{sample}/split/{sample}.sorted.{splitid}.bam") log: "{sample}/split/{sample}.{splitid}.log" @@ -144,7 +161,7 @@ elif config['general']['mapper'] == 'bwa_split': container: config['apptainers']['bwa'] shell: - manager.get_shell("bwa/align", "v1") + manager.get_shell("bwa/align", "v2") def aggregate_bwa(wildcards): checkpoint_output = checkpoints.split_fasta.get(**wildcards).output[0] @@ -256,6 +273,27 @@ rule filtering_bam: """ +if config["fastqc"]["do"]: + rule fastqc: + input: + manager.getrawdata() + output: + done="{sample}/fastqc/fastqc.done" + params: + options=config["fastqc"]["options"], + working_directory="{sample}/fastqc/" + threads: + config["fastqc"]["threads"] + log: + "{sample}/fastqc/fastqc.log" + resources: + **config["fastqc"]["resources"] + container: + config['apptainers']['fastqc'] + shell: + manager.get_shell("fastqc/run", "v1") + + # sample should be included at the end of the filename for the multiqc to work rule bamtools_stats: input: @@ -379,7 +417,7 @@ rule multiqc: options=config['multiqc']['options'], input_directory=config['multiqc']['input_directory'], config_file=config['multiqc']['config_file'], - modules=config['multiqc']['modules'] + modules=config['multiqc']['modules'] + (" fastqc" if config["fastqc"]["do"] else "") log: "multiqc/multiqc.log" resources: diff --git a/sequana_pipelines/mapper/schema.yaml b/sequana_pipelines/mapper/schema.yaml index f56cc04..c2ca184 100644 --- a/sequana_pipelines/mapper/schema.yaml +++ b/sequana_pipelines/mapper/schema.yaml @@ -64,6 +64,23 @@ mapping: "mem": type: str + "fastqc": + type: map + mapping: + "do": + type: bool + default: False + "options": + type: str + "threads": + type: int + range: { min: 1 } + "resources": + type: map + mapping: + "mem": + type: str + "multiqc": type: map mapping: diff --git a/sequana_pipelines/mapper/tools.txt b/sequana_pipelines/mapper/tools.txt index a22c1d9..33864cb 100644 --- a/sequana_pipelines/mapper/tools.txt +++ b/sequana_pipelines/mapper/tools.txt @@ -3,6 +3,7 @@ bedtools bowtie2 bwa deeptools +fastqc featureCounts minimap2 sambamba diff --git a/test/test_main.py b/test/test_main.py index e8a2e06..ea8db2d 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -1,3 +1,4 @@ +import glob import os import subprocess import sys @@ -79,6 +80,25 @@ def test_standalone_script_saf(tmpdir): assert results.exit_code == 0 +def test_standalone_script_fastqc(tmpdir): + with tempfile.TemporaryDirectory() as directory: + runner = CliRunner() + results = runner.invoke( + main, + [ + "--input-directory", + sharedir, + "--reference-file", + sharedir + "/measles.fa", + "--working-directory", + directory, + "--force", + "--do-fastqc", + ], + ) + assert results.exit_code == 0 + + def test_full(): with tempfile.TemporaryDirectory() as directory: @@ -86,7 +106,7 @@ def test_full(): wk = directory cmd = "sequana_mapper --input-directory {} " - cmd += " --working-directory {} --force " + cmd += " --working-directory {} --force --do-fastqc " cmd += " --reference-file " + sharedir + "/measles.fa" cmd = cmd.format(sharedir, wk) subprocess.call(cmd.split()) @@ -95,6 +115,11 @@ def test_full(): assert os.path.exists(wk + "/multiqc/multiqc_report.html") + # --do-fastqc must produce per-sample FastQC output + fastqc_done = glob.glob(wk + "/*/fastqc/fastqc.done") + assert len(fastqc_done) > 0 + assert len(glob.glob(wk + "/*/fastqc/*_fastqc.zip")) > 0 + def test_version(): cmd = "sequana_mapper --version"