From d757db5669e6baea778757e7e4298b41fb6750d2 Mon Sep 17 00:00:00 2001 From: Thomas Cokelaer Date: Mon, 29 Aug 2022 15:36:10 +0200 Subject: [PATCH 01/12] Update pipeline to set singularity - refactorise the code to make it more readable - update CI --- .github/workflows/main.yml | 13 +- .github/workflows/pypi.yml | 8 +- README.rst | 5 +- environment.yml | 18 +++ requirements.txt | 4 +- sequana_pipelines/ribofinder/main.py | 5 - sequana_pipelines/ribofinder/requirements.txt | 2 + sequana_pipelines/ribofinder/ribofinder.rules | 127 +++++++++++------- setup.py | 23 +--- test/test_main.py | 9 +- 10 files changed, 125 insertions(+), 89 deletions(-) create mode 100644 environment.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0b3a5f4..75f3f83 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,9 +3,12 @@ name: Tests on: push: branches: - - master + - main + - dev pull_request: branches-ignore: [] + schedule: + - cron: '0 0 * * SUN' jobs: build-linux: @@ -13,9 +16,10 @@ jobs: strategy: max-parallel: 5 matrix: - python: [3.7,3.8,3.9] + python: [3.7, 3.8, 3.9] fail-fast: false + steps: - name: install graphviz @@ -34,12 +38,10 @@ jobs: run: | # $CONDA is an environment variable pointing to the root of the miniconda directory echo $CONDA/bin >> $GITHUB_PATH - conda update ruamel_yaml - name: conda run: | - conda install -c conda-forge --quiet mamba python=${{ matrix.python }} - mamba install -c bioconda -c conda-forge --quiet -y bowtie samtools bamtools pigz bedtools + conda install -c conda-forge -c bioconda -y python=${{ matrix.python }} 'bowtie>=1.3.0' samtools bamtools pigz bedtools - name: Install dependencies run: | @@ -48,6 +50,7 @@ jobs: - name: install package itself run: | pip install . + pip install requests --upgrade - name: testing run: | diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index a13767b..18a194d 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -8,9 +8,9 @@ on: jobs: build-n-publish: name: Build and publish to PyPI and TestPyPI - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@master + - uses: actions/checkout@main - name: Set up Python 3.7 uses: actions/setup-python@v1 with: @@ -26,14 +26,14 @@ jobs: python setup.py sdist - name: Publish distribution to Test PyPI - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository_url: https://test.pypi.org/legacy/ - name: Publish distribution to PyPI if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/README.rst b/README.rst index 6f0d89d..0a54f22 100644 --- a/README.rst +++ b/README.rst @@ -62,8 +62,10 @@ Requirements This pipelines requires the following executable(s): -- bowtie1 +- bowtie1 >= 1.3.0 +- bedtools - samtools +- bamtools - pigz .. image:: https://raw.githubusercontent.com/sequana/ribofinder/master/sequana_pipelines/ribofinder/dag.png @@ -102,6 +104,7 @@ Changelog ========= ==================================================================== Version Description ========= ==================================================================== +0.12.0 * set singularity containers 0.11.1 * Fix config file (removing hard-coded path) 0.11.0 * Fix multiqc plot using same fix as in sequna_rnaseq pipelines * add utility plot to check rate of ribosomal per sequence and also diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..bbb7fbc --- /dev/null +++ b/environment.yml @@ -0,0 +1,18 @@ +name: sequana_variant_calling + +channels: + - conda-forge + - bioconda + - defaults + +dependencies: + - bowtie>=1.3.0 + - samtools + - bamtools + - pigz + - bedtools + - pip + - pip: + - sequana + + diff --git a/requirements.txt b/requirements.txt index bf9e35a..7d6b785 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -sequana>=0.12.7 -sequana_pipetools>=0.7.1 +sequana>=0.14.1 +sequana_pipetools>=0.9.2 diff --git a/sequana_pipelines/ribofinder/main.py b/sequana_pipelines/ribofinder/main.py index 6cb5b6e..ad9ab2d 100755 --- a/sequana_pipelines/ribofinder/main.py +++ b/sequana_pipelines/ribofinder/main.py @@ -1,12 +1,8 @@ -# -*- coding: utf-8 -*- # # This file is part of Sequana software # # Copyright (c) 2016 - Sequana Development Team # -# File author(s): -# Thomas Cokelaer -# # Distributed under the terms of the 3-clause BSD license. # The full license is in the LICENSE file, distributed with this software. # @@ -17,7 +13,6 @@ import sys import os import argparse -import shutil import subprocess from sequana_pipetools.options import * diff --git a/sequana_pipelines/ribofinder/requirements.txt b/sequana_pipelines/ribofinder/requirements.txt index dddf2a1..e5c02cc 100644 --- a/sequana_pipelines/ribofinder/requirements.txt +++ b/sequana_pipelines/ribofinder/requirements.txt @@ -1,3 +1,5 @@ bowtie samtools +bedtools +bamtools pigz diff --git a/sequana_pipelines/ribofinder/ribofinder.rules b/sequana_pipelines/ribofinder/ribofinder.rules index 8171ee6..443ccad 100644 --- a/sequana_pipelines/ribofinder/ribofinder.rules +++ b/sequana_pipelines/ribofinder/ribofinder.rules @@ -12,13 +12,10 @@ # Documentation: https://github.com/sequana/fastqc/README.rst ############################################################################## """Ribofinder pipeline""" -import glob -import os import shutil -from os.path import join +import os from collections import Counter -import sequana from sequana_pipetools import PipelineManager from sequana_pipetools import snaketools as sm @@ -29,86 +26,107 @@ from sequana.gff3 import GFF3 configfile: "config.yaml" manager = PipelineManager("ribofinder", config) -manager.setup(globals(), mode="warning") -__data__input = manager.getrawdata() - rule pipeline: - input: + input: ".sequana/rulegraph.svg", "outputs/proportions.png", "outputs/RPKM.png", "multiqc/multiqc_report.html" -__fasta_file__ = config['general']['reference_file'] -if config['general']['genbank_file']: - __annot_file__ = config['general']['genbank_file'] -elif config['general']['gff_file']: - __annot_file__ = config['general']['gff_file'] -elif config['general']['rRNA_file']: - __annot_file__ = config['general']['rRNA_file'] -__prefix_name__ = "indexing/features" +def get_annot_file(): + if config['general']['genbank_file']: + return config['general']['genbank_file'] + elif config['general']['gff_file']: + return config['general']['gff_file'] + elif config['general']['rRNA_file']: + return config['general']['rRNA_file'] + raise ValueError("You must provide a genbank of gff or file with rRNA sequences.") + + +input_fasta = config['general']['reference_file'] if manager.config.general.rRNA_file: - __bowtie1_index_rna__fasta = config["general"]["rRNA_file"] - if os.path.exists(__bowtie1_index_rna__fasta) is False: - log.error(f"File {__bowtie1_index_rna__fasta} does not exists. Check your config file") + user_file = config["general"]["rRNA_file"] + if os.path.exists(user_file) is False: + log.error(f"File {user_file} does not exists. Check your config file") sys.exit(1) -elif __annot_file__.endswith(".gbk"): + os.makedirs("build_feature_fasta", exist_ok=True) + shutil.copy(user_file, "build_feature_fasta/feature.fasta") +elif get_annot_file().endswith(".gbk"): # This is for genbank input - __build_feature_fasta__output = "build_feature_fasta/feature.fasta" - for this in [__fasta_file__, __annot_file__]: + for this in [input_fasta, get_annot_file()]: if os.path.exists(this) is False: - raise IOError("File {} not found".format(__fasta_file__)) + raise IOError("File {} not found".format(input_fasta)) + rule build_feature_fasta: input: - fasta = __fasta_file__, - annot = __annot_file__ - output: __build_feature_fasta__output + fasta = input_fasta, + annot = get_annot_file() + output: "build_feature_fasta/feature.fasta" run: from sequana.genbank import GenBank gg = GenBank(input.annot) sequence = gg.extract_fasta(input.fasta, features=['rRNA']) with open(output[0], "w") as fout: fout.write(sequence) - __bowtie1_index_rna__fasta = __build_feature_fasta__output -elif __annot_file__.endswith(".gff") or __annot_file__.endswith(".gff3"): - for this in [__fasta_file__, __annot_file__]: + +elif get_annot_file().endswith(".gff") or get_annot_file().endswith(".gff3"): + for this in [input_fasta, get_annot_file()]: if os.path.exists(this) is False: - raise IOError("File {} not found".format(__fasta_file__)) - # extract rRNA feature from GFF and get corresponding fasta - # and gff. if no match for rRNA, save empty fasta as AAAAAAAAAAA - __build_feature_fasta__output = "build_feature_fasta/feature.fasta" + raise IOError("File {} not found".format(input_fasta)) - __extract_fasta_from_bed__input = __fasta_file__ - __extract_fasta_from_bed__gff = __annot_file__ - __extract_fasta_from_bed__feature = config["general"]["rRNA_feature"] - __extract_fasta_from_bed__output = __build_feature_fasta__output - __extract_fasta_from_bed__output_features = __prefix_name__ + "_rRNA.gff" - __extract_fasta_from_bed__log = "indexing/get_rRNA.log" - # ---------------------------------------------------------------------------- - include: sm.modules["extract_fasta_from_bed"] - __bowtie1_index_rna__fasta = __extract_fasta_from_bed__output + + rule build_feature_fasta: + """ extract rRNA feature from GFF and get corresponding fasta and gff. + + if no match for rRNA, save empty fasta as AAAAAAAAAAA + """ + input: + fasta = input_fasta, + gff = get_annot_file() + output: + fasta = "build_feature_fasta/feature.fasta", + fai = "build_feature_fasta/feature.fasta.fai", + gff = "build_feature_fasta/feature_rRNA.gff" + params: + feature = config['general']['rRNA_feature'] + container: + "https://zenodo.org/record/7031863/files/sequana_tools_0.14.2.img" + shell: + """ + # used to be gawk but awk is more generic. + awk '{{ if ($3=="{params.feature}") print }}' {input.gff} > {output.gff} + if [ -s {output.gff} ] + then + bedtools getfasta -fi {input.fasta} -bed {output.gff} -fo {output.fasta} + else : + echo -e ">empty\\nAAAAAAAAAAAAAA" > {output.fasta} + fi + samtools faidx {output.fasta} + """ -__bowtie1_reference__ = __bowtie1_index_rna__fasta.rsplit(".",1)[0] + "_rRNA.1.ebwt" rule bowtie1_indexing: input: - reference= __bowtie1_index_rna__fasta + reference= "build_feature_fasta/feature.fasta" output: - __bowtie1_reference__ + "build_feature_fasta/feature_rRNA.1.ebwt" log: "indexing/bowtie_rRNA.log" params: options="" - threads: 2 + threads: + 2 + container: + "https://zenodo.org/record/7031863/files/sequana_tools_0.14.2.img" wrapper: "main/wrappers/bowtie1/build" @@ -126,6 +144,8 @@ rule unpigz: input: manager.getrawdata() output: temp("{sample}/unpigz/{sample}.fastq") threads: 4 + container: + "https://zenodo.org/record/7031863/files/sequana_tools_0.14.2.img" shell: """ unpigz -p {threads} -fk --stdout {input[0]} > {output} @@ -137,7 +157,7 @@ rule unpigz: rule bowtie1_mapping_rna: input: fastq="{sample}/unpigz/{sample}.fastq", - index=__bowtie1_reference__ + index="build_feature_fasta/feature_rRNA.1.ebwt" output: bam="{sample}/bowtie1/{sample}.bam", sorted="{sample}/bowtie1/{sample}.sorted.bam" @@ -145,7 +165,10 @@ rule bowtie1_mapping_rna: "{sample}/bowtie1/{sample}.log" params: options="" - threads: config['bowtie1_mapping_rna']['threads'] + threads: + config['bowtie1_mapping_rna']['threads'] + container: + "https://zenodo.org/record/7031863/files/sequana_tools_0.14.2.img" wrapper: "main/wrappers/bowtie1/align" @@ -178,13 +201,13 @@ rule fix_bowtie1_log: rule plotting: input: bam_files=expand("{sample}/bowtie1/{sample}.sorted.bam", sample=manager.samples), - fasta_file=__bowtie1_index_rna__fasta + fasta_file= "build_feature_fasta/feature.fasta" output: png="outputs/proportions.png", rpkm="outputs/RPKM.png" run: import pandas as pd - from sequana import BAM + from sequana import BAM, FastA from pylab import tight_layout, savefig, xlabel, ylabel, clf results = [] @@ -214,7 +237,7 @@ rule plotting: savefig(output['png']) # now, we shown the RPKM (read count normalised by gene length) - from sequana import FastA + f = FastA(input['fasta_file']) df = df.T L = [ f.get_lengths_as_dict()[x] for x in df.index] @@ -235,6 +258,8 @@ rule bam_indexing: "{sample}/bowtie1/{sample}.sorted.bam" output: "{sample}/bowtie1/{sample}.sorted.bam.bai" + container: + "https://zenodo.org/record/7031863/files/sequana_tools_0.14.2.img" shell: """ bamtools index -in {input} diff --git a/setup.py b/setup.py index eda7af5..cd05fb9 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,11 @@ -# -*- coding: utf-8 -*- -# License: 3-clause BSD from setuptools import setup, find_namespace_packages from setuptools.command.develop import develop from setuptools.command.install import install import subprocess _MAJOR = 0 -_MINOR = 11 -_MICRO = 1 +_MINOR = 12 +_MICRO = 0 version = '%d.%d.%d' % (_MAJOR, _MINOR, _MICRO) release = '%d.%d' % (_MAJOR, _MINOR) @@ -39,20 +37,6 @@ NAME = "ribofinder" -class Install(install): - def run(self): - cmd = "sequana_completion --name {} --force ".format(NAME) - try: subprocess.run(cmd.split()) - except:pass - install.run(self) - -class Develop(develop): - def run(self): - cmd = "sequana_completion --name {} --force ".format(NAME) - try:subprocess.run(cmd.split()) - except:pass - develop.run(self) - setup( name = "sequana_{}".format(NAME), version = version, @@ -76,8 +60,7 @@ def run(self): # This is recursive include of data files exclude_package_data = {"": ["__pycache__"]}, package_data = { - '': ['*.yaml', "*.rules", "*.json", "requirements.txt", "*png"], - 'sequana_pipelines.ribofinder.data' : ['*.*'], + '': ['*.yaml', "*.rules", "*.json", "requirements.txt", "*png", "*yml", "*smk"] }, zip_safe=False, diff --git a/test/test_main.py b/test/test_main.py index 87f2dc2..e4d73f9 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -39,7 +39,14 @@ def test_full_rRNA_extract(): cmd += f"--working-directory {wk} --force --reference-file {sharedir}/Lepto.fa --gff-file {sharedir}/Lepto.gff" subprocess.call(cmd.split()) stat = subprocess.call("sh ribofinder.sh".split(), cwd=wk) - assert os.path.exists(wk + "/summary.html") + + + if os.path.exists(wk + "/summary.html"): + pass + else: + with open(f"{wk}/indexing/bowtie_rRNA.log", "r") as fout: + print(fout.read()) + raise IOError def test_version(): cmd = "sequana_ribofinder --version" From b027e3d84438e93b6e3013fb6aed5b647e375bec Mon Sep 17 00:00:00 2001 From: Thomas Cokelaer Date: Tue, 22 Nov 2022 18:58:32 +0100 Subject: [PATCH 02/12] update apptainers fix apptainer CI fix typo --- .github/workflows/apptainer.yml | 67 +++++++++++++++++++ .github/workflows/main.yml | 25 +++---- README.rst | 1 + environment.yml | 24 +++---- sequana_pipelines/ribofinder/config.yaml | 10 ++- sequana_pipelines/ribofinder/ribofinder.rules | 35 ++++++---- sequana_pipelines/ribofinder/schema.yaml | 2 + setup.py | 2 +- 8 files changed, 120 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/apptainer.yml diff --git a/.github/workflows/apptainer.yml b/.github/workflows/apptainer.yml new file mode 100644 index 0000000..fdd7f5b --- /dev/null +++ b/.github/workflows/apptainer.yml @@ -0,0 +1,67 @@ +name: Apptainer Run + +on: + push: + branches: + - main + - dev + pull_request: + branches-ignore: [] + schedule: + - cron: '0 0 * * SUN' + +jobs: + build-linux: + runs-on: ubuntu-latest + strategy: + max-parallel: 5 + matrix: + python: [3.7, 3.8, 3.9] + fail-fast: false + + + steps: + + - name: precleanup + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - name: install graphviz + run: | + sudo apt-get install -y graphviz + + - name: checkout git repo + uses: actions/checkout@v2 + + - name: Set up Python 3.X + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH + + - name: conda + run: | + conda install -c conda-forge -c bioconda --quiet -y python=${{ matrix.python }} 'singularity>3' + + - name: Install dependencies + run: | + pip install .[testing] + + - name: install package itself + run: | + pip install . + + - name: testing + run: | + + sequana_ribofinder --input-directory test/data --reference-file test/data/Lepto.fa --gff-file test/data/Lepto.gff --use-apptainer && cd ribofinder && sh ribofinder.sh + + + + + + diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 75f3f83..8f53ccd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,34 +29,31 @@ jobs: - name: checkout git repo uses: actions/checkout@v2 - - name: Set up Python 3.X - uses: actions/setup-python@v2 + - name: conda/mamba + uses: mamba-org/provision-with-micromamba@main with: - python-version: ${{ matrix.python }} + environment-file: environment.yml + extra-specs: | + python=${{ matrix.python }} - - name: Add conda to system path + - name: install package itself + shell: bash -l {0} run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH + pip install . - - name: conda - run: | - conda install -c conda-forge -c bioconda -y python=${{ matrix.python }} 'bowtie>=1.3.0' samtools bamtools pigz bedtools - name: Install dependencies + shell: bash -l {0} run: | pip install coveralls pytest-cov pytest pytest-xdist - - name: install package itself - run: | - pip install . - pip install requests --upgrade - - name: testing + shell: bash -l {0} run: | pytest -v --cov-report term-missing --cov=sequana_pipelines.ribofinder - name: coveralls + shell: bash -l {0} run: | echo $COVERALLS_REPO_TOKEN coveralls --service=github diff --git a/README.rst b/README.rst index 0a54f22..7d6af2f 100644 --- a/README.rst +++ b/README.rst @@ -104,6 +104,7 @@ Changelog ========= ==================================================================== Version Description ========= ==================================================================== +0.13.0 * add final apptainers and update CI actions 0.12.0 * set singularity containers 0.11.1 * Fix config file (removing hard-coded path) 0.11.0 * Fix multiqc plot using same fix as in sequna_rnaseq pipelines diff --git a/environment.yml b/environment.yml index bbb7fbc..e30f4d3 100644 --- a/environment.yml +++ b/environment.yml @@ -1,18 +1,14 @@ -name: sequana_variant_calling +name: sequana_ribofinder channels: - - conda-forge - - bioconda - - defaults +- conda-forge +- bioconda +- defaults +- r dependencies: - - bowtie>=1.3.0 - - samtools - - bamtools - - pigz - - bedtools - - pip - - pip: - - sequana - - +- 'bowtie>=1.3.0' +- samtools>1.7 +- bamtools +- bedtools +- pigz diff --git a/sequana_pipelines/ribofinder/config.yaml b/sequana_pipelines/ribofinder/config.yaml index 979ab16..306e670 100644 --- a/sequana_pipelines/ribofinder/config.yaml +++ b/sequana_pipelines/ribofinder/config.yaml @@ -37,10 +37,14 @@ general: rRNA_file: '' rRNA_feature: rRNA genbank_file: '' - gff_file: - reference_file: - + gff_file: + reference_file: +apptainers: + sequana_ribofinder: "https://zenodo.org/record/7348115/files/sequana_ribofinder_0.12.0.img" + pigz: "https://zenodo.org/record/7346805/files/pigz_2.4.0.img" + bedtools: "https://zenodo.org/record/7346774/files/bedtools_2.30.0.img" + samtools: "https://zenodo.org/record/7215278/files/samtools_1.15.0.img" ############################################################################# # bowtie1_mapping_rna used to align reads against ribosomal RNA diff --git a/sequana_pipelines/ribofinder/ribofinder.rules b/sequana_pipelines/ribofinder/ribofinder.rules index 443ccad..df4a160 100644 --- a/sequana_pipelines/ribofinder/ribofinder.rules +++ b/sequana_pipelines/ribofinder/ribofinder.rules @@ -82,24 +82,23 @@ elif get_annot_file().endswith(".gff") or get_annot_file().endswith(".gff3"): rule build_feature_fasta: - """ extract rRNA feature from GFF and get corresponding fasta and gff. + """ extract rRNA feature from GFF and get corresponding fasta and gff. - if no match for rRNA, save empty fasta as AAAAAAAAAAA + if there is no match for rRNA, we can use polyA AAAAAAAAAAA """ input: fasta = input_fasta, gff = get_annot_file() output: fasta = "build_feature_fasta/feature.fasta", - fai = "build_feature_fasta/feature.fasta.fai", gff = "build_feature_fasta/feature_rRNA.gff" params: feature = config['general']['rRNA_feature'] container: - "https://zenodo.org/record/7031863/files/sequana_tools_0.14.2.img" + config["apptainers"]["bedtools"] shell: """ - # used to be gawk but awk is more generic. + # used to be gawk but awk is more generic and available in the container. awk '{{ if ($3=="{params.feature}") print }}' {input.gff} > {output.gff} if [ -s {output.gff} ] then @@ -107,11 +106,19 @@ elif get_annot_file().endswith(".gff") or get_annot_file().endswith(".gff3"): else : echo -e ">empty\\nAAAAAAAAAAAAAA" > {output.fasta} fi - samtools faidx {output.fasta} """ - - + rule samtools_faidx: + input: + fasta = "build_feature_fasta/feature.fasta" + output: + fai = "build_feature_fasta/feature.fasta.fai" + container: + config["apptainers"]["samtools"] + shell: + """ + samtools faidx {input.fasta} + """ rule bowtie1_indexing: @@ -123,10 +130,10 @@ rule bowtie1_indexing: "indexing/bowtie_rRNA.log" params: options="" - threads: + threads: 2 container: - "https://zenodo.org/record/7031863/files/sequana_tools_0.14.2.img" + config['apptainers']['sequana_ribofinder'] wrapper: "main/wrappers/bowtie1/build" @@ -145,7 +152,7 @@ rule unpigz: output: temp("{sample}/unpigz/{sample}.fastq") threads: 4 container: - "https://zenodo.org/record/7031863/files/sequana_tools_0.14.2.img" + config['apptainers']['pigz'] shell: """ unpigz -p {threads} -fk --stdout {input[0]} > {output} @@ -165,10 +172,10 @@ rule bowtie1_mapping_rna: "{sample}/bowtie1/{sample}.log" params: options="" - threads: + threads: config['bowtie1_mapping_rna']['threads'] container: - "https://zenodo.org/record/7031863/files/sequana_tools_0.14.2.img" + config['apptainers']['sequana_ribofinder'] wrapper: "main/wrappers/bowtie1/align" @@ -259,7 +266,7 @@ rule bam_indexing: output: "{sample}/bowtie1/{sample}.sorted.bam.bai" container: - "https://zenodo.org/record/7031863/files/sequana_tools_0.14.2.img" + config['apptainers']['sequana_ribofinder'] shell: """ bamtools index -in {input} diff --git a/sequana_pipelines/ribofinder/schema.yaml b/sequana_pipelines/ribofinder/schema.yaml index f8c1715..0e46eb8 100644 --- a/sequana_pipelines/ribofinder/schema.yaml +++ b/sequana_pipelines/ribofinder/schema.yaml @@ -12,6 +12,8 @@ mapping: "input_pattern": type: str required: True + "apptainers": + type: any "general": type: map diff --git a/setup.py b/setup.py index cd05fb9..7480792 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import subprocess _MAJOR = 0 -_MINOR = 12 +_MINOR = 13 _MICRO = 0 version = '%d.%d.%d' % (_MAJOR, _MINOR, _MICRO) release = '%d.%d' % (_MAJOR, _MINOR) From 80828c033c1d50d9ac3afa6ba5eade9715e19528 Mon Sep 17 00:00:00 2001 From: Thomas Cokelaer Date: Thu, 2 Feb 2023 10:05:29 +0100 Subject: [PATCH 03/12] Fixes CI apptainer --- .github/workflows/apptainer.yml | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/apptainer.yml b/.github/workflows/apptainer.yml index fdd7f5b..a541ebd 100644 --- a/.github/workflows/apptainer.yml +++ b/.github/workflows/apptainer.yml @@ -16,7 +16,7 @@ jobs: strategy: max-parallel: 5 matrix: - python: [3.7, 3.8, 3.9] + python: [3.8, 3.9, '3.10'] fail-fast: false @@ -28,25 +28,20 @@ jobs: sudo rm -rf "$AGENT_TOOLSDIRECTORY" - name: install graphviz run: | - sudo apt-get install -y graphviz + sudo apt update + sudo apt-get install -y graphviz software-properties-common + sudo add-apt-repository -y ppa:apptainer/ppa + sudo apt update + sudo apt install -y apptainer - name: checkout git repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.X - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python }} - - name: Add conda to system path - run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH - - - name: conda - run: | - conda install -c conda-forge -c bioconda --quiet -y python=${{ matrix.python }} 'singularity>3' - - name: Install dependencies run: | pip install .[testing] From 33c1f023f773a4734917b29f3a5b314199d8e4fe Mon Sep 17 00:00:00 2001 From: Thomas Cokelaer Date: Mon, 22 May 2023 15:04:16 +0200 Subject: [PATCH 04/12] update pipeline to use latest apptainers --- .github/workflows/apptainer.yml | 1 - .github/workflows/main.yml | 4 +- README.rst | 1 + sequana_pipelines/ribofinder/config.yaml | 1 + sequana_pipelines/ribofinder/ribofinder.rules | 48 ++++++++++++++++--- setup.py | 6 +-- 6 files changed, 48 insertions(+), 13 deletions(-) diff --git a/.github/workflows/apptainer.yml b/.github/workflows/apptainer.yml index a541ebd..399461c 100644 --- a/.github/workflows/apptainer.yml +++ b/.github/workflows/apptainer.yml @@ -59,4 +59,3 @@ jobs: - diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8f53ccd..ac8818e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ on: pull_request: branches-ignore: [] schedule: - - cron: '0 0 * * SUN' + - cron: '0 0 2 * *' jobs: build-linux: @@ -16,7 +16,7 @@ jobs: strategy: max-parallel: 5 matrix: - python: [3.7, 3.8, 3.9] + python: [3.8, 3.9, '3.10'] fail-fast: false diff --git a/README.rst b/README.rst index 7d6af2f..581c3bc 100644 --- a/README.rst +++ b/README.rst @@ -104,6 +104,7 @@ Changelog ========= ==================================================================== Version Description ========= ==================================================================== +1.0.0 * use graphviz apptainer and latest wrappers 0.13.0 * add final apptainers and update CI actions 0.12.0 * set singularity containers 0.11.1 * Fix config file (removing hard-coded path) diff --git a/sequana_pipelines/ribofinder/config.yaml b/sequana_pipelines/ribofinder/config.yaml index 306e670..ba9399f 100644 --- a/sequana_pipelines/ribofinder/config.yaml +++ b/sequana_pipelines/ribofinder/config.yaml @@ -45,6 +45,7 @@ apptainers: pigz: "https://zenodo.org/record/7346805/files/pigz_2.4.0.img" bedtools: "https://zenodo.org/record/7346774/files/bedtools_2.30.0.img" samtools: "https://zenodo.org/record/7215278/files/samtools_1.15.0.img" + graphviz: "https://zenodo.org/record/7928262/files/graphviz_7.0.5.img" ############################################################################# # bowtie1_mapping_rna used to align reads against ribosomal RNA diff --git a/sequana_pipelines/ribofinder/ribofinder.rules b/sequana_pipelines/ribofinder/ribofinder.rules index df4a160..b591a1d 100644 --- a/sequana_pipelines/ribofinder/ribofinder.rules +++ b/sequana_pipelines/ribofinder/ribofinder.rules @@ -26,7 +26,7 @@ from sequana.gff3 import GFF3 configfile: "config.yaml" manager = PipelineManager("ribofinder", config) - +sequana_wrapper_branch = "main" rule pipeline: @@ -135,7 +135,7 @@ rule bowtie1_indexing: container: config['apptainers']['sequana_ribofinder'] wrapper: - "main/wrappers/bowtie1/build" + f"{sequana_wrapper_branch}/wrappers/bowtie1/build" @@ -177,7 +177,7 @@ rule bowtie1_mapping_rna: container: config['apptainers']['sequana_ribofinder'] wrapper: - "main/wrappers/bowtie1/align" + f"{sequana_wrapper_branch}/wrappers/bowtie1/align" # ======================================================== Fix bowtie log @@ -277,12 +277,46 @@ rule bam_indexing: # sequana_multiqc_input = expand("{sample}/bowtie1/{sample}.sorted.bam.bai", sample=sorted(manager.samples)) sequana_multiqc_input += ["logs/fix_bowtie1_log"] -include: sm.modules["multiqc/2.0"] + + +rule multiqc: + input: + sequana_multiqc_input + output: + "multiqc/multiqc_report.html" + params: + options=config['multiqc']['options'], + input_directory=config['multiqc']['input_directory'], + config_file=config['multiqc']['config_file'], + modules=config['multiqc']['modules'] + log: + "multiqc/multiqc.log" + wrapper: + f"{sequana_wrapper_branch}/wrappers/multiqc" + # ========================================================== rulegraph -# -sequana_rulegraph_mapper = {"multiqc": "../multiqc/multiqc_report.html"} -include: sm.modules['rulegraph'] +rule rulegraph: + input: str(manager.snakefile) + output: + svg = "rulegraph/rulegraph.dot" + params: + mapper = {"multiqc": "../multiqc/multiqc_report.html"}, + configname = "config.yaml" + wrapper: + f"{sequana_wrapper_branch}/wrappers/rulegraph" + + +rule dot2svg: + input: + "rulegraph/rulegraph.dot" + output: + ".sequana/rulegraph.svg" + container: + config['apptainers']['graphviz'] + shell: + """dot -Tsvg {input} -o {output}""" + diff --git a/setup.py b/setup.py index 7480792..c562073 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,8 @@ from setuptools.command.install import install import subprocess -_MAJOR = 0 -_MINOR = 13 +_MAJOR = 1 +_MINOR = 0 _MICRO = 0 version = '%d.%d.%d' % (_MAJOR, _MINOR, _MICRO) release = '%d.%d' % (_MAJOR, _MINOR) @@ -25,9 +25,9 @@ 'Intended Audience :: Science/Research', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Scientific/Engineering :: Bio-Informatics', 'Topic :: Scientific/Engineering :: Information Analysis', From fe2fc11ada34b4c34be84cfc6dbdc512b4caa11f Mon Sep 17 00:00:00 2001 From: Thomas Cokelaer Date: Mon, 22 May 2023 16:21:05 +0200 Subject: [PATCH 05/12] Update README.rst --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 581c3bc..74bf5d6 100644 --- a/README.rst +++ b/README.rst @@ -9,6 +9,11 @@ .. image:: https://github.com/sequana/ribofinder/actions/workflows/main.yml/badge.svg :target: https://github.com/sequana/ribofinder/actions/workflows/main.yml +.. image:: https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C3.10-blue.svg + :target: https://pypi.python.org/pypi/sequana + :alt: Python 3.8 | 3.9 | 3.10 + + This is is the **ribofinder** pipeline from the `Sequana `_ project From c29b4fe1cb29c48942c1117996382f9ba9df0ae3 Mon Sep 17 00:00:00 2001 From: Thomas Cokelaer Date: Tue, 6 Jun 2023 11:38:49 +0200 Subject: [PATCH 06/12] Remove py3.9 (aiohttp failure) --- .github/workflows/apptainer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/apptainer.yml b/.github/workflows/apptainer.yml index 399461c..7cf8cab 100644 --- a/.github/workflows/apptainer.yml +++ b/.github/workflows/apptainer.yml @@ -16,7 +16,7 @@ jobs: strategy: max-parallel: 5 matrix: - python: [3.8, 3.9, '3.10'] + python: [3.8, '3.10'] fail-fast: false From f6b6181075d05e0b56895c5967f2e7c3b3117d28 Mon Sep 17 00:00:00 2001 From: Thomas Cokelaer Date: Thu, 3 Aug 2023 13:31:38 +0200 Subject: [PATCH 07/12] Use sequana_wrappers from the config file --- README.rst | 1 + requirements.txt | 2 +- sequana_pipelines/ribofinder/config.yaml | 1 + sequana_pipelines/ribofinder/ribofinder.rules | 13 +++++-------- setup.py | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 74bf5d6..711dcff 100644 --- a/README.rst +++ b/README.rst @@ -109,6 +109,7 @@ Changelog ========= ==================================================================== Version Description ========= ==================================================================== +1.0.1 * add sequana_wrappers in the config/pipeline 1.0.0 * use graphviz apptainer and latest wrappers 0.13.0 * add final apptainers and update CI actions 0.12.0 * set singularity containers diff --git a/requirements.txt b/requirements.txt index 7d6b785..a566416 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ sequana>=0.14.1 -sequana_pipetools>=0.9.2 +sequana_pipetools>=0.12.5 diff --git a/sequana_pipelines/ribofinder/config.yaml b/sequana_pipelines/ribofinder/config.yaml index ba9399f..b5c4695 100644 --- a/sequana_pipelines/ribofinder/config.yaml +++ b/sequana_pipelines/ribofinder/config.yaml @@ -7,6 +7,7 @@ # If input_directory provided, use it otherwise if input_pattern provided, # use it, otherwise use input_samples. # ============================================================================ +sequana_wrappers: "v0.15.1" input_directory: input_readtag: _R[12]_ input_pattern: '*fastq.gz' diff --git a/sequana_pipelines/ribofinder/ribofinder.rules b/sequana_pipelines/ribofinder/ribofinder.rules index b591a1d..be61ad5 100644 --- a/sequana_pipelines/ribofinder/ribofinder.rules +++ b/sequana_pipelines/ribofinder/ribofinder.rules @@ -18,15 +18,12 @@ from collections import Counter from sequana_pipetools import PipelineManager -from sequana_pipetools import snaketools as sm -from sequana import logger as log - from sequana.gff3 import GFF3 +from sequana import logger as log configfile: "config.yaml" manager = PipelineManager("ribofinder", config) -sequana_wrapper_branch = "main" rule pipeline: @@ -135,7 +132,7 @@ rule bowtie1_indexing: container: config['apptainers']['sequana_ribofinder'] wrapper: - f"{sequana_wrapper_branch}/wrappers/bowtie1/build" + f"{manager.wrappers}/wrappers/bowtie1/build" @@ -177,7 +174,7 @@ rule bowtie1_mapping_rna: container: config['apptainers']['sequana_ribofinder'] wrapper: - f"{sequana_wrapper_branch}/wrappers/bowtie1/align" + f"{manager.wrappers}/wrappers/bowtie1/align" # ======================================================== Fix bowtie log @@ -292,7 +289,7 @@ rule multiqc: log: "multiqc/multiqc.log" wrapper: - f"{sequana_wrapper_branch}/wrappers/multiqc" + f"{manager.wrappers}/wrappers/multiqc" # ========================================================== rulegraph @@ -304,7 +301,7 @@ rule rulegraph: mapper = {"multiqc": "../multiqc/multiqc_report.html"}, configname = "config.yaml" wrapper: - f"{sequana_wrapper_branch}/wrappers/rulegraph" + f"{manager.wrappers}/wrappers/rulegraph" rule dot2svg: diff --git a/setup.py b/setup.py index c562073..089b145 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ _MAJOR = 1 _MINOR = 0 -_MICRO = 0 +_MICRO = 1 version = '%d.%d.%d' % (_MAJOR, _MINOR, _MICRO) release = '%d.%d' % (_MAJOR, _MINOR) From 9730dca9f7048bfcf3892090c153bd8523e4139c Mon Sep 17 00:00:00 2001 From: Thomas Cokelaer Date: Sun, 3 Dec 2023 22:43:30 +0100 Subject: [PATCH 08/12] Use new sequana_pipetools framework(click) --- .github/workflows/main.yml | 1 + .github/workflows/pypi.yml | 14 +- MANIFEST.in | 5 - README.rst | 1 + pyproject.toml | 52 +++++ requirements.txt | 2 - sequana_pipelines/ribofinder/config.yaml | 4 +- sequana_pipelines/ribofinder/main.py | 184 +++++++++--------- sequana_pipelines/ribofinder/ribofinder.rules | 16 +- sequana_pipelines/ribofinder/schema.yaml | 3 + .../{requirements.txt => tools.txt} | 0 setup.cfg | 13 -- setup.py | 72 ------- test/test_main.py | 13 +- 14 files changed, 176 insertions(+), 204 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 pyproject.toml delete mode 100644 requirements.txt rename sequana_pipelines/ribofinder/{requirements.txt => tools.txt} (100%) delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ac8818e..773e125 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,6 +5,7 @@ on: branches: - main - dev + workflow_dispatch: pull_request: branches-ignore: [] schedule: diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 18a194d..435f810 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -11,20 +11,20 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@main - - name: Set up Python 3.7 - uses: actions/setup-python@v1 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.8 - - name: Install package + - name: Install package run: | - pip install build + pip install build poetry - name: Build source tarball run: | rm -rf dist; - python setup.py sdist - + poetry build + - name: Publish distribution to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 5219b3c..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -recursive-exclude * __pycache__ -recursive-exclude * *pyc -recursive-include * *rules -include requirements*txt -include README.rst diff --git a/README.rst b/README.rst index 711dcff..24846f4 100644 --- a/README.rst +++ b/README.rst @@ -109,6 +109,7 @@ Changelog ========= ==================================================================== Version Description ========= ==================================================================== +1.1.0 * Uses click (refactoring of sequana_pipetools) 1.0.1 * add sequana_wrappers in the config/pipeline 1.0.0 * use graphviz apptainer and latest wrappers 0.13.0 * add final apptainers and update CI actions diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dea6c01 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,52 @@ +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "sequana-ribofinder" +version = "1.1.0" +description = "A multi-sample identification of ribosomal content" +authors = ["Sequana Team"] +license = "BSD-3" +repository = "https://github.com/sequana/ribofinder" +readme = "README.rst" +keywords = ["snakemake", "ribosomal", "rRNA", "sequana"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Education", + "Intended Audience :: End Users/Desktop", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Scientific/Engineering :: Bio-Informatics", + "Topic :: Scientific/Engineering :: Information Analysis", +] + +packages = [ + {include = "sequana_pipelines"} +] + + +[tool.poetry.dependencies] +python = ">=3.8,<4.0" +sequana = ">=0.15.0" +sequana_pipetools = ">=0.16.0" +click-completion = "^0.5.2" + + +[tool.poetry.scripts] +sequana_ribofinder = "sequana_pipelines.ribofinder.main:main" + + +[tool.poetry.group.dev.dependencies] +black = "^23.7.0" +pytest = "^7.4.0" +mock = "^5.1.0" +pytest-mock = "^3.11.1" +pytest-cov = "^4.1.0" + diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a566416..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -sequana>=0.14.1 -sequana_pipetools>=0.12.5 diff --git a/sequana_pipelines/ribofinder/config.yaml b/sequana_pipelines/ribofinder/config.yaml index b5c4695..4c7abe8 100644 --- a/sequana_pipelines/ribofinder/config.yaml +++ b/sequana_pipelines/ribofinder/config.yaml @@ -7,7 +7,8 @@ # If input_directory provided, use it otherwise if input_pattern provided, # use it, otherwise use input_samples. # ============================================================================ -sequana_wrappers: "v0.15.1" +sequana_wrappers: "v23.11.18" + input_directory: input_readtag: _R[12]_ input_pattern: '*fastq.gz' @@ -47,6 +48,7 @@ apptainers: bedtools: "https://zenodo.org/record/7346774/files/bedtools_2.30.0.img" samtools: "https://zenodo.org/record/7215278/files/samtools_1.15.0.img" graphviz: "https://zenodo.org/record/7928262/files/graphviz_7.0.5.img" + multiqc: "https://zenodo.org/record/10205070/files/multiqc_1.16.0.img" ############################################################################# # bowtie1_mapping_rna used to align reads against ribosomal RNA diff --git a/sequana_pipelines/ribofinder/main.py b/sequana_pipelines/ribofinder/main.py index ad9ab2d..4f19366 100755 --- a/sequana_pipelines/ribofinder/main.py +++ b/sequana_pipelines/ribofinder/main.py @@ -12,139 +12,141 @@ ############################################################################## import sys import os -import argparse import subprocess +import shutil +from pathlib import Path + +import rich_click as click +import click_completion + +click_completion.init() from sequana_pipetools.options import * -from sequana_pipetools.options import before_pipeline -from sequana_pipetools.misc import Colors -from sequana_pipetools.info import sequana_epilog, sequana_prolog from sequana_pipetools import SequanaManager -from sequana import logger -col = Colors() NAME = "ribofinder" - -class Options(argparse.ArgumentParser): - def __init__(self, prog=NAME, epilog=None): - usage = col.purple(sequana_prolog.format(**{"name": NAME})) - super(Options, self).__init__(usage=usage, prog=prog, description="", - epilog=epilog, - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - # add a new group of options to the parser - so = SlurmOptions() - so.add_options(self) - - # add a snakemake group of options to the parser - so = SnakemakeOptions(working_directory=NAME) - so.add_options(self) - - so = InputOptions() - so.add_options(self) - - so = GeneralOptions() - so.add_options(self) - - pipeline_group = self.add_argument_group("pipeline_general") - pipeline_group.add_argument("--aligner", dest="aligner", - choices=['bowtie1'], default='bowtie1', - help= "a mapper in bowtie") - pipeline_group.add_argument("--rRNA-feature", - default="rRNA", - help="""Feature name corresponding to the rRNA to be identified for QCs""") - pipeline_group.add_argument("--rRNA-file", - default=None, - help="""If you already have the rRNA file, just provide it""") - pipeline_group.add_argument("--genbank-file", - default=None, - help="""genbank. if provided, do not provide gff""") - pipeline_group.add_argument("--gff-file", - default=None, - help="""If provided, do not provide genbank""") - pipeline_group.add_argument("--reference-file", - default=None, - help="""The required referenceto fetch features into""") - - - self.add_argument("--run", default=False, action="store_true", - help="execute the pipeline directly") - - def parse_args(self, *args): - args_list = list(*args) - if "--from-project" in args_list: - if len(args_list)>2: - msg = "WARNING [sequana]: With --from-project option, " + \ - "pipeline and data-related options will be ignored." - print(col.error(msg)) - for action in self._actions: - if action.required is True: - action.required = False - options = super(Options, self).parse_args(*args) - return options - - -def main(args=None): - - if args is None: - args = sys.argv - - # whatever needs to be called by all pipeline before the options parsing - before_pipeline(NAME) - - # option parsing including common epilog - options = Options(NAME, epilog=sequana_epilog).parse_args(args[1:]) - +help = init_click( + NAME, + groups={ + "Pipeline Specific": [ + "--aligner", + "--genbank-file", + "--gff-file", + "--rRNA-feature", + "--rRNA-file", + "--reference-file", + ], + }, +) + + +@click.command(context_settings=help) +@include_options_from(ClickSnakemakeOptions, working_directory=NAME) +@include_options_from(ClickSlurmOptions) +@include_options_from(ClickInputOptions) +@include_options_from(ClickGeneralOptions) +@click.option( + "--aligner", "aligner", type=click.Choice(["bowtie1"]), default="bowtie1", help="the alignement tool (bowtie1)" +) +@click.option( + "--rRNA-feature", + "rRNA_feature", + default="rRNA", + type=click.STRING, + help="""Feature name corresponding to the rRNA to be identified for QCs""", +) +@click.option( + "--rRNA-file", + "rRNA_file", + default=None, + type=click.Path(file_okay=True, dir_okay=False), + help="""If you already have the rRNA file, just provide it""", +) +@click.option("--genbank-file", default=None, help="""genbank. if provided, do not provide gff""") +@click.option("--gff-file", default=None, help="""If provided, do not provide genbank""") +@click.option("--reference-file", default=None, help="""The required referenceto fetch features into""") +def main(**options): # the real stuff is here manager = SequanaManager(options, NAME) # create the beginning of the command and the working directory manager.setup() + # aliases + options = manager.options + cfg = manager.config.config + + # fills input_data, input_directory, input_readtag + # needs an update of sequana_pipetools to exclude readtag + manager.fill_data_options() + # fill the config file with input parameters - if options.from_project is None: - cfg = manager.config.config - # --------------------------------------------------------- general + # --------------------------------------------------------- general + def fill_aligner(): cfg.general.aligner = options.aligner + def fill_rRNA_feature(): cfg.general.rRNA_feature = options.rRNA_feature + + def fill_rRNA_file(): if options.rRNA_file: cfg.general.rRNA_file = os.path.abspath(options.rRNA_file) + + def fill_genbank_file(): if options.genbank_file: cfg.general.genbank_file = os.path.abspath(options.genbank_file) + + def fill_gff_file(): if options.gff_file: cfg.general.gff_file = os.path.abspath(options.gff_file) + + def fill_reference_file(): if options.reference_file: cfg.general.reference_file = os.path.abspath(options.reference_file) + if options["from_project"]: + if "--aligner" in sys.argv: + fill_aligner() + if "--rRNA-feature" in sys.argv: + fill_rRNA_feature() + if "--rRNA-file" in sys.argv: + fill_rRNA_file() + if "--genbank-file" in sys.argv: + fill_genbank_file() + if "--gff-file" in sys.argv: + fill_gff_file() + if "--reference-fle" in sys.argv: + fill_reference_file() + + else: + fill_aligner() + fill_rRNA_feature() + fill_rRNA_file() + fill_genbank_file() + fill_gff_file() + fill_reference_file() + if options.reference_file is None and options.rRNA_file is None: - logger.error("You must provide a rRNA file or a reference_file") + click.echo("You must provide a rRNA file or a reference_file", err=True) sys.exit(1) if options.reference_file: - logger.info("checking your input GFF file and rRNA feature if provided") + click.echo("checking your input GFF file and rRNA feature if provided") if options.genbank_file: from sequana.genbank import GenBank + gbk = GenBank(options.genbank_file) if options.genbank_file is None and options.gff_file is None: - logger.error("Most probably you want to provide an annotation (genbank-file or gff-file)") + click.echo("Most probably you want to provide an annotation (genbank-file or gff-file)", err=True) sys.exit(0) - # ---------------------------------------------------- others - cfg.input_pattern = options.input_pattern - cfg.input_directory = os.path.abspath(options.input_directory) - cfg.input_readtag = options.input_readtag - - manager.exists(cfg.input_directory) # finalise the command and save it; copy the snakemake. update the config # file and save it. manager.teardown() - if options.run: - subprocess.Popen(["sh", '{}.sh'.format(NAME)], cwd=options.workdir) if __name__ == "__main__": main() diff --git a/sequana_pipelines/ribofinder/ribofinder.rules b/sequana_pipelines/ribofinder/ribofinder.rules index be61ad5..bc7c88a 100644 --- a/sequana_pipelines/ribofinder/ribofinder.rules +++ b/sequana_pipelines/ribofinder/ribofinder.rules @@ -231,7 +231,7 @@ rule plotting: df1 = df.copy() df1.columns = [x[0:30] for x in df1.columns ] - + # first plot shown percentage of hits on each sequence import seaborn as sns sns.violinplot(data=df1, orient='h') @@ -288,6 +288,8 @@ rule multiqc: modules=config['multiqc']['modules'] log: "multiqc/multiqc.log" + container: + config["apptainers"]["multiqc"] wrapper: f"{manager.wrappers}/wrappers/multiqc" @@ -328,7 +330,7 @@ onsuccess: manager.teardown() - from sequana.modules_report.summary import SummaryModule2 + from sequana.modules_report.summary import SequanaReport from sequana_pipelines import ribofinder data = { "name": manager.name, @@ -360,16 +362,12 @@ onsuccess: except Exception: pass - s = SummaryModule2(data, intro=HTML) + data = manager.getmetadata() + s = SequanaReport(data, intro=HTML) shell("chmod -R g+w .") shell("rm -rf rulegraph") onerror: - print("An error occurred. See message above.") - -onerror: - from sequana_pipetools.errors import PipeError - p = PipeError("ribofinder") - p.status() + manager.onerror() diff --git a/sequana_pipelines/ribofinder/schema.yaml b/sequana_pipelines/ribofinder/schema.yaml index 0e46eb8..fb2ebb7 100644 --- a/sequana_pipelines/ribofinder/schema.yaml +++ b/sequana_pipelines/ribofinder/schema.yaml @@ -3,6 +3,9 @@ type: map mapping: + "sequana_wrappers": + type: str + required: true "input_directory": type: str required: True diff --git a/sequana_pipelines/ribofinder/requirements.txt b/sequana_pipelines/ribofinder/tools.txt similarity index 100% rename from sequana_pipelines/ribofinder/requirements.txt rename to sequana_pipelines/ribofinder/tools.txt diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 692dbab..0000000 --- a/setup.cfg +++ /dev/null @@ -1,13 +0,0 @@ -# a setup configuration -[build_sphinx] -source_dir = doc/source -build_dir = doc/build -all_files = 1 - - -[aliases] -test=pytest - -[tool:pytest] -# do not use --cov because it interfers with travis command -addopts= --durations=10 --verbose -n 1 --cov sequana_pipelines.ribofinder --cov-report term-missing diff --git a/setup.py b/setup.py deleted file mode 100644 index 089b145..0000000 --- a/setup.py +++ /dev/null @@ -1,72 +0,0 @@ -from setuptools import setup, find_namespace_packages -from setuptools.command.develop import develop -from setuptools.command.install import install -import subprocess - -_MAJOR = 1 -_MINOR = 0 -_MICRO = 1 -version = '%d.%d.%d' % (_MAJOR, _MINOR, _MICRO) -release = '%d.%d' % (_MAJOR, _MINOR) - -metainfo = { - 'authors': {"main": ("thomas cokelaer", "thomas.cokelaer@pasteur.fr")}, - 'version': version, - 'license' : 'new BSD', - 'url' : "https://github.com/sequana/", - 'description': "NGS, ribosomal, rRNA, snakemake, sequana" , - 'platforms' : ['Linux', 'Unix', 'MacOsX', 'Windows'], - 'keywords' : ['NGS, ribosomal, rRNA, snakemake, sequana'], - 'classifiers' : [ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Education', - 'Intended Audience :: End Users/Desktop', - 'Intended Audience :: Developers', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Scientific/Engineering :: Bio-Informatics', - 'Topic :: Scientific/Engineering :: Information Analysis', - 'Topic :: Scientific/Engineering :: Mathematics', - 'Topic :: Scientific/Engineering :: Physics'] - } - -NAME = "ribofinder" - -setup( - name = "sequana_{}".format(NAME), - version = version, - maintainer = metainfo['authors']['main'][0], - maintainer_email = metainfo['authors']['main'][1], - author = metainfo['authors']['main'][0], - author_email = metainfo['authors']['main'][1], - long_description = open("README.rst").read(), - keywords = metainfo['keywords'], - description = metainfo['description'], - license = metainfo['license'], - platforms = metainfo['platforms'], - url = metainfo['url'], - classifiers = metainfo['classifiers'], - - # package installation - packages = ["sequana_pipelines.ribofinder"], - - install_requires = open("requirements.txt").read(), - - # This is recursive include of data files - exclude_package_data = {"": ["__pycache__"]}, - package_data = { - '': ['*.yaml', "*.rules", "*.json", "requirements.txt", "*png", "*yml", "*smk"] - }, - - zip_safe=False, - - entry_points = {'console_scripts':[ - 'sequana_ribofinder=sequana_pipelines.ribofinder.main:main'] - } - -) diff --git a/test/test_main.py b/test/test_main.py index e4d73f9..f2b3ffa 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -3,6 +3,9 @@ import tempfile import subprocess import sys +from click.testing import CliRunner +from sequana_pipelines.ribofinder.main import main + from . import test_dir @@ -17,10 +20,12 @@ def test_standalone_subprocess(): def test_standalone_script(): directory = tempfile.TemporaryDirectory() - import sequana_pipelines.ribofinder.main as m - sys.argv = ["test", "--input-directory", sharedir, - "--working-directory", directory.name, "--force", "--rRNA-file", sharedir+"feature.fasta"] - m.main() + + + runner = CliRunner() + results = runner.invoke(main, [ "--input-directory", sharedir, + "--working-directory", directory.name, "--force", "--rRNA-file", sharedir+"feature.fasta"]) + assert results.exit_code == 0 def test_full_rRNA_file(): From bd91da9cabb4314baf678899ce91e0946c39e88e Mon Sep 17 00:00:00 2001 From: Thomas Cokelaer Date: Tue, 5 Dec 2023 10:43:35 +0100 Subject: [PATCH 09/12] hotfix to use on HPC clusters --- README.rst | 1 + pyproject.toml | 2 +- sequana_pipelines/ribofinder/__init__.py | 15 ++++++++++----- sequana_pipelines/ribofinder/ribofinder.rules | 4 ++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 24846f4..6c691f6 100644 --- a/README.rst +++ b/README.rst @@ -109,6 +109,7 @@ Changelog ========= ==================================================================== Version Description ========= ==================================================================== +1.1.1 * hotfix for running on HPC (slurm) 1.1.0 * Uses click (refactoring of sequana_pipetools) 1.0.1 * add sequana_wrappers in the config/pipeline 1.0.0 * use graphviz apptainer and latest wrappers diff --git a/pyproject.toml b/pyproject.toml index dea6c01..6e39ead 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "sequana-ribofinder" -version = "1.1.0" +version = "1.1.1" description = "A multi-sample identification of ribosomal content" authors = ["Sequana Team"] license = "BSD-3" diff --git a/sequana_pipelines/ribofinder/__init__.py b/sequana_pipelines/ribofinder/__init__.py index 132e999..6d5f263 100644 --- a/sequana_pipelines/ribofinder/__init__.py +++ b/sequana_pipelines/ribofinder/__init__.py @@ -1,6 +1,11 @@ -import pkg_resources -try: - version = pkg_resources.require("sequana_ribofinder")[0].version -except: - version = ">=0.8.0" +import importlib.metadata as metadata + +def get_package_version(package_name): + try: + version = metadata.version(package_name) + return version + except metadata.PackageNotFoundError: + return f"{package_name} not found" + +version = get_package_version("sequana-ribofinder") diff --git a/sequana_pipelines/ribofinder/ribofinder.rules b/sequana_pipelines/ribofinder/ribofinder.rules index bc7c88a..3fc0753 100644 --- a/sequana_pipelines/ribofinder/ribofinder.rules +++ b/sequana_pipelines/ribofinder/ribofinder.rules @@ -184,7 +184,7 @@ rule fix_bowtie1_log: input: expand("{sample}/bowtie1/{sample}.log", sample=manager.samples) output: - "logs/fix_bowtie1_log" + "logs/fix_bowtie1_log/done" run: for filename in input: # we read the file @@ -273,7 +273,7 @@ rule bam_indexing: # ========================================================== multiqc # sequana_multiqc_input = expand("{sample}/bowtie1/{sample}.sorted.bam.bai", sample=sorted(manager.samples)) -sequana_multiqc_input += ["logs/fix_bowtie1_log"] +sequana_multiqc_input += ["logs/fix_bowtie1_log/done"] rule multiqc: From bae54fb477900c0181362c30ffaf7e9102584ee8 Mon Sep 17 00:00:00 2001 From: Etienne Kornobis Date: Wed, 3 Sep 2025 14:56:54 +0200 Subject: [PATCH 10/12] Fix error reporting issue when gff file is missing --- sequana_pipelines/ribofinder/ribofinder.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequana_pipelines/ribofinder/ribofinder.rules b/sequana_pipelines/ribofinder/ribofinder.rules index 3fc0753..170ba19 100644 --- a/sequana_pipelines/ribofinder/ribofinder.rules +++ b/sequana_pipelines/ribofinder/ribofinder.rules @@ -75,7 +75,7 @@ elif get_annot_file().endswith(".gbk"): elif get_annot_file().endswith(".gff") or get_annot_file().endswith(".gff3"): for this in [input_fasta, get_annot_file()]: if os.path.exists(this) is False: - raise IOError("File {} not found".format(input_fasta)) + raise IOError(f"File {this} not found") rule build_feature_fasta: From 264f26dbf6a6aea11300d1cb076d31addbe59cf5 Mon Sep 17 00:00:00 2001 From: Thomas Cokelaer Date: Tue, 7 Apr 2026 20:32:35 +0200 Subject: [PATCH 11/12] Fix CI: update mamba-org/setup-micromamba to v2 --- .github/workflows/main.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 773e125..bd3788a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ jobs: strategy: max-parallel: 5 matrix: - python: [3.8, 3.9, '3.10'] + python: ['3.11', '3.12'] fail-fast: false @@ -25,24 +25,26 @@ jobs: - name: install graphviz run: | + sudo apt-get update -qq sudo apt-get install -y graphviz - name: checkout git repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: conda/mamba - uses: mamba-org/provision-with-micromamba@main + uses: mamba-org/setup-micromamba@v2 with: environment-file: environment.yml - extra-specs: | + create-args: >- python=${{ matrix.python }} + cache-environment: true + cache-downloads: true - name: install package itself shell: bash -l {0} run: | pip install . - - name: Install dependencies shell: bash -l {0} run: | @@ -54,6 +56,7 @@ jobs: pytest -v --cov-report term-missing --cov=sequana_pipelines.ribofinder - name: coveralls + continue-on-error: true shell: bash -l {0} run: | echo $COVERALLS_REPO_TOKEN From 86e2f1e56993cc592adbaf1e84de8564e2b4d35f Mon Sep 17 00:00:00 2001 From: Thomas Cokelaer Date: Tue, 7 Apr 2026 22:41:38 +0200 Subject: [PATCH 12/12] update apptainer --- .github/workflows/pypi.yml | 17 +++++++++-------- README.rst | 7 +++---- sequana_pipelines/ribofinder/main.py | 0 test/test_main.py | 13 +++---------- 4 files changed, 15 insertions(+), 22 deletions(-) mode change 100755 => 100644 sequana_pipelines/ribofinder/main.py diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 435f810..5b1e408 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -8,11 +8,13 @@ on: jobs: build-n-publish: name: Build and publish to PyPI and TestPyPI - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + permissions: + id-token: write steps: - - uses: actions/checkout@main - - name: Set up Python 3.8 - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - name: Set up Python 3.7 + uses: actions/setup-python@v5 with: python-version: 3.8 @@ -26,14 +28,13 @@ jobs: poetry build - name: Publish distribution to Test PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master with: - user: __token__ password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ + repository-url: https://test.pypi.org/legacy/ - name: Publish distribution to PyPI if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 with: - user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/README.rst b/README.rst index 6c691f6..64e3cd0 100644 --- a/README.rst +++ b/README.rst @@ -9,10 +9,9 @@ .. image:: https://github.com/sequana/ribofinder/actions/workflows/main.yml/badge.svg :target: https://github.com/sequana/ribofinder/actions/workflows/main.yml -.. image:: https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C3.10-blue.svg - :target: https://pypi.python.org/pypi/sequana - :alt: Python 3.8 | 3.9 | 3.10 - +.. image:: https://img.shields.io/badge/python-3.11%20%7C%203.12-blue.svg + :target: https://pypi.python.org/pypi/sequana_ribofinder + :alt: Python 3.11 | 3.12 This is is the **ribofinder** pipeline from the `Sequana `_ project diff --git a/sequana_pipelines/ribofinder/main.py b/sequana_pipelines/ribofinder/main.py old mode 100755 new mode 100644 diff --git a/test/test_main.py b/test/test_main.py index f2b3ffa..091b00c 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -34,7 +34,7 @@ def test_full_rRNA_file(): cmd = f"sequana_ribofinder --input-directory {sharedir} " cmd += f"--working-directory {wk} --force --rRNA-file {sharedir}/feature.fasta" subprocess.call(cmd.split()) - stat = subprocess.call("sh ribofinder.sh".split(), cwd=wk) + stat = subprocess.call("bash ribofinder.sh".split(), cwd=wk) assert os.path.exists(wk + "/summary.html") def test_full_rRNA_extract(): @@ -43,15 +43,8 @@ def test_full_rRNA_extract(): cmd = f"sequana_ribofinder --input-directory {sharedir} " cmd += f"--working-directory {wk} --force --reference-file {sharedir}/Lepto.fa --gff-file {sharedir}/Lepto.gff" subprocess.call(cmd.split()) - stat = subprocess.call("sh ribofinder.sh".split(), cwd=wk) - - - if os.path.exists(wk + "/summary.html"): - pass - else: - with open(f"{wk}/indexing/bowtie_rRNA.log", "r") as fout: - print(fout.read()) - raise IOError + stat = subprocess.call("bash ribofinder.sh".split(), cwd=wk) + assert os.path.exists(wk + "/summary.html") def test_version(): cmd = "sequana_ribofinder --version"