diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 079bdb4..c95db53 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ fish=3.3.1+ds-3 \ fuse-overlayfs=1.7.1-1 \ - openssh-server=1:8.9p1-3ubuntu0.11 \ + openssh-server=1:8.9p1-3ubuntu0.13 \ silversearcher-ag=2.2.0+git20200805-1 \ && echo "LC_ALL=en_US.UTF-8" > /etc/default/locale \ && locale-gen en_US.UTF-8 \ @@ -25,8 +25,8 @@ RUN apt-get update \ && rm -Rf /usr/share/doc && rm -Rf /usr/share/man \ && rm -rf /var/lib/apt/lists/* # convert gitpod to local user -RUN groupmod -n ${USERNAME} -g ${USER_GID} gitpod \ - && usermod -md /home/${USERNAME} -l ${USERNAME} -u ${USER_UID} -s /usr/bin/fish -aG docker gitpod \ +RUN groupmod -n ${USERNAME} -g ${USER_GID:-1000} gitpod \ + && usermod -md /home/${USERNAME} -l ${USERNAME} -u ${USER_UID:-1000} -s /usr/bin/fish -aG docker gitpod \ && echo ${USERNAME} ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/${USERNAME} \ && chmod 0440 /etc/sudoers.d/${USERNAME} \ && find / -uid 33333 -not -path "/proc/*" -exec chown ${USERNAME}:${USERNAME} {} + @@ -35,4 +35,4 @@ ENV HOME=/home/${USERNAME} \ USER ${USERNAME} RUN git config --global pull.rebase false \ && git config --global user.email "${MAIL}" \ - && git config --global user.name "${FULLNAME}" + && git config --global user.name "${FULLNAME}" \ No newline at end of file diff --git a/.gitignore b/.gitignore index deba4a3..d3a21ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ .nextflow* +assets/stub_data/tso500_root_output_directory/200103_NS500643_0003_AHG5MGBGYM_pipeline_info +assets/stub_data/tso500_root_output_directory/200103_NS500643_0003_AHG5MGBGYM_TSO_500_LocalApp_results +assets/stub_data/tso500_root_output_directory/metrics_plots_NODE_all +assets/stub_data/tso500_root_output_directory/metrics_plots_NODE_latest nohup.out null test_stub diff --git a/.nf-core.yml b/.nf-core.yml index a44986f..16bd97d 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -59,6 +59,7 @@ lint: multiqc_config: false nextflow_config: false pipeline_name_conventions: false + schema_lint: false readme: - nextflow_badge diff --git a/assets/example.samplesheet.csv b/assets/example.samplesheet.csv deleted file mode 100644 index b21bc58..0000000 --- a/assets/example.samplesheet.csv +++ /dev/null @@ -1,3 +0,0 @@ -dataset_id,sample_id,molecule,sample_type,tumor_site,tumor_content,run_id,path_to_run,path_to_samplesheet,barcode -IPH0001,IPH0001-D01-T01-A19,dna,tumor,10,50,200501_NS500643_0793_AHG5MGBGYM,assets/200501_NS500643_0793_AHG5MGBGYM,,UDP0029 -IPH0001,IPH0001-D01-N01-A19,dna,normal,XX,0,200501_NS500643_0793_AHG5MGBGYM,assets/200501_NS500643_0793_AHG5MGBGYM,,UDP0030 diff --git a/assets/schema_input.json b/assets/schema_input.json index 72a4d1d..b617efe 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "inpred/tso500_nxf_workflow - params.input schema", - "description": "Schema for the file provided with params.input (ideally a samplesheet)", + "title": "inpred/tso500_nxf_workflow - params.nxf_samplesheet schema", + "description": "Schema for the file provided with params.nxf_samplesheet", "type": "array", "items": { "type": "object", diff --git a/assets/200501_NS500643_0793_AHG5MGBGYM/.gitkeep b/assets/stub_data/decoy_RunCompletionStatus.xml similarity index 100% rename from assets/200501_NS500643_0793_AHG5MGBGYM/.gitkeep rename to assets/stub_data/decoy_RunCompletionStatus.xml diff --git a/assets/stub_data/nxf_samplesheet.csv b/assets/stub_data/nxf_samplesheet.csv new file mode 100644 index 0000000..224d1d3 --- /dev/null +++ b/assets/stub_data/nxf_samplesheet.csv @@ -0,0 +1,3 @@ +dataset_id,sample_id,molecule,sample_type,tumor_site,tumor_content,run_id,path_to_run,path_to_samplesheet,barcode +IPH0001,IPH0001-D01-T01-A19,dna,tumor,10,50,200103_NS500643_0003_AHG5MGBGYM,assets/stub_data/raw_data_root_directory/200103_NS500643_0003_AHG5MGBGYM,,UDP0029 +IPH0001,IPH0001-D01-N01-A19,dna,normal,XX,0,200103_NS500643_0003_AHG5MGBGYM,assets/stub_data/raw_data_root_directory/200103_NS500643_0003_AHG5MGBGYM,,UDP0030 diff --git a/assets/example.tso500_resource_folder/.gitkeep b/assets/stub_data/raw_data_root_directory/200101_NS500643_0001_AHG5MGBGYM/RunCompletionStatus.xml similarity index 100% rename from assets/example.tso500_resource_folder/.gitkeep rename to assets/stub_data/raw_data_root_directory/200101_NS500643_0001_AHG5MGBGYM/RunCompletionStatus.xml diff --git a/assets/stub_data/raw_data_root_directory/200102_NS500643_0002_AHG5MGBGYM/RunCompletionStatus.xml b/assets/stub_data/raw_data_root_directory/200102_NS500643_0002_AHG5MGBGYM/RunCompletionStatus.xml new file mode 100644 index 0000000..e69de29 diff --git a/assets/stub_data/raw_data_root_directory/200103_NS500643_0003_AHG5MGBGYM/RunCompletionStatus.xml b/assets/stub_data/raw_data_root_directory/200103_NS500643_0003_AHG5MGBGYM/RunCompletionStatus.xml new file mode 100644 index 0000000..e69de29 diff --git a/assets/stub_data/resources/.gitkeep b/assets/stub_data/resources/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/assets/stub_data/tso500_root_output_directory/200101_NS500643_0001_AHG5MGBGYM/200101_NS500643_0001_AHG5MGBGYM_LocalApp_results/Results/MetricsOutput.tsv b/assets/stub_data/tso500_root_output_directory/200101_NS500643_0001_AHG5MGBGYM/200101_NS500643_0001_AHG5MGBGYM_LocalApp_results/Results/MetricsOutput.tsv new file mode 100644 index 0000000..e69de29 diff --git a/assets/stub_data/tso500_root_output_directory/200102_NS500643_0002_AHG5MGBGYM_TSO_500_LocalApp_results/Results/MetricsOutput.tsv b/assets/stub_data/tso500_root_output_directory/200102_NS500643_0002_AHG5MGBGYM_TSO_500_LocalApp_results/Results/MetricsOutput.tsv new file mode 100644 index 0000000..e69de29 diff --git a/conf/modules.config b/conf/modules.config index 12e22b0..c424d24 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -17,24 +17,48 @@ process { mode: params.publish_dir_mode ] - withName: LOCAL_APP_DEMULTIPLEX { + withName: GATHER_RESULTS_WORKFLOW { publishDir = [ - enabled: false + path: { "${params.tso500_root_output_directory}/${id}_TSO_500_LocalApp_results" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - withName: LOCAL_APP_TSO500 { + withName: LOCAL_APP_PREPPER { publishDir = [ - enabled: false + path: { "${params.tso500_root_output_directory}/${id}_pipeline_info" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - withName: LOCAL_APP_PREPPER { + withName: PROCESS_METRICS_FILES_ALL { + ext.args = '--create_plots False' publishDir = [ - path: { "${params.outdir}/pipeline_info" }, + path: { "${params.tso500_root_output_directory}/${params.metrics_plots_all_output_directory}" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } + withName: PROCESS_METRICS_FILES_N { + publishDir = [ + path: { "${params.tso500_root_output_directory}/${params.metrics_plots_latest_output_directory}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: TSO500_WORKFLOW_DEMULTIPLEX { + publishDir = [ + enabled: false + ] + } + + withName: TSO500_WORKFLOW { + publishDir = [ + enabled: false + ] + } } diff --git a/conf/test_stub.config b/conf/test_stub.config index 429b4f4..9551fd5 100644 --- a/conf/test_stub.config +++ b/conf/test_stub.config @@ -23,7 +23,12 @@ params { config_profile_description = 'Simply stubbing the tasks in the pipeline instead of running them' // Input data - input = 'assets/example.samplesheet.csv' - outdir = 'test_stub' - tso500_resource_folder = 'assets/example.tso500_resource_folder' + //decoy_run_completion_status_xml = 'assets/stub_data/decoy_RunCompletionStatus.xml' + metrics_plots_all_output_directory = 'metrics_plots_NODE_all' + metrics_plots_latest_output_directory = 'metrics_plots_NODE_latest' + nxf_samplesheet = 'assets/stub_data/nxf_samplesheet.csv' + outdir = 'test_stub' + raw_data_root_directory = 'assets/stub_data/raw_data_root_directory' + tso500_resources_directory = 'assets/stub_data/resources' + tso500_root_output_directory = 'assets/stub_data/tso500_root_output_directory' } diff --git a/main.nf b/main.nf index d539f42..b6d675b 100644 --- a/main.nf +++ b/main.nf @@ -21,7 +21,7 @@ include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_full workflow TSO500_PIPELINE { take: - samplesheet // channel: samplesheet read in from --input + samplesheet // channel: samplesheet read in from --nxf_samplesheet main: @@ -50,7 +50,7 @@ workflow { params.monochrome_logs, args, params.outdir, - params.input + params.nxf_samplesheet ) // diff --git a/modules/local/local_app/local_app.nf b/modules/local/acadia_500_wdl_workflow/main.nf similarity index 98% rename from modules/local/local_app/local_app.nf rename to modules/local/acadia_500_wdl_workflow/main.nf index 5f3a91b..31e5abf 100644 --- a/modules/local/local_app/local_app.nf +++ b/modules/local/acadia_500_wdl_workflow/main.nf @@ -1,4 +1,4 @@ -process LOCAL_APP { +process TSO500_WORKFLOW { tag "$id" label 'process_medium' @@ -46,6 +46,7 @@ process LOCAL_APP { output = 'localapp_' + id """ mkdir $output + cat <<-END_VERSIONS > versions.yml "${task.process}": cromwell: stub @@ -55,7 +56,7 @@ process LOCAL_APP { """ } -process GATHER { +process GATHER_RESULTS_WORKFLOW { tag "$id" label 'process_medium' @@ -101,6 +102,7 @@ process GATHER { """ touch inputs.json mkdir cromwell-executions cromwell-workflow-logs Results + cat <<-END_VERSIONS > versions.yml "${task.process}": cromwell: stub diff --git a/modules/local/add_variant_summary/main.nf b/modules/local/add_variant_summary/main.nf new file mode 100644 index 0000000..5646591 --- /dev/null +++ b/modules/local/add_variant_summary/main.nf @@ -0,0 +1,44 @@ +process ADD_VARIANT_SUMMARY { + tag "$run_id" + label 'process_low' + + container "inpred/tsoppi_main:v0.3.2" + containerOptions "-v \$(pwd):/inpred/data" + + input: + tuple val(run_id), path(analysis_results_dir) + + output: + path "${run_id}_variant_summary.tsv", emit: tsv + path 'versions.yml' , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + """ + python ${params.user_scripts_dir}/summarize_run_variants.py \\ + -r \$(pwd)/${analysis_results_dir} \\ + -o \$(pwd)/${run_id}_variant_summary.tsv \\ + -s \$(pwd) \\ + $args + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + summarize_run_variants: \$(python ${params.user_scripts_dir}/summarize_run_variants.py --version 2>&1 | grep -oP 'version \\K[0-9.:-]+' || echo "0.3.2:22-06-07") + python: \$(python --version | sed 's/Python //g') + END_VERSIONS + """ + + stub: + """ + touch ${run_id}_variant_summary.tsv + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + summarize_run_variants: stub + python: stub + END_VERSIONS + """ +} diff --git a/modules/local/local_app_prepper/local_app_prepper.nf b/modules/local/local_app_prepper/main.nf similarity index 92% rename from modules/local/local_app_prepper/local_app_prepper.nf rename to modules/local/local_app_prepper/main.nf index 4e3b05e..867e4ba 100644 --- a/modules/local/local_app_prepper/local_app_prepper.nf +++ b/modules/local/local_app_prepper/main.nf @@ -2,7 +2,7 @@ process LOCAL_APP_PREPPER { tag "$id" label 'process_low' - container "inpred/local_app_prepper:latest" + container "inpred/local_app_prepper:1.0.0" input: tuple val(id), path(runfolder), val(sample_list) @@ -25,7 +25,7 @@ process LOCAL_APP_PREPPER { cat <<-END_VERSIONS > versions.yml "${task.process}": - local_app_prepper: latest + local_app_prepper: 1.0.0 python: \$(python --version | sed 's/Python //g') END_VERSIONS """ @@ -33,6 +33,7 @@ process LOCAL_APP_PREPPER { stub: """ touch demultiplex.json gather.json tso500_IPH0001-D01-T01-A1.json tso500_IPH0001-D01-N01-A1.json + cat <<-END_VERSIONS > versions.yml "${task.process}": local_app_prepper: stub diff --git a/modules/local/process_metrics_files/main.nf b/modules/local/process_metrics_files/main.nf new file mode 100644 index 0000000..5c85e05 --- /dev/null +++ b/modules/local/process_metrics_files/main.nf @@ -0,0 +1,48 @@ +process PROCESS_METRICS_FILES { + tag "${id.size()}_runs" + label 'process_low' + + container "inpred/tsoppi_main:v0.3.2" + containerOptions = "-v \$(pwd):/workdir -v \$(pwd):/inpred/data" + + input: + tuple val(id), path(tsv, stageAs: "?/*"), path(xml, stageAs: "?/*") // stage file with index as folder name + + output: + path "intermediate_metrics_files/master_metrics_table.tsv" , emit: 'tsv' + path "TSO500_run_metrics.pdf" , emit: 'pdf', optional: true + path 'versions.yml' , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + for (int i = 0; i < id.size(); i++) { + args = args + " -m \$(pwd)/${tsv[i]} -r \$(pwd)/${xml[i]} -l ${id[i]}" // due to tsoppi container replacing absolute input mount paths we need to create absolute paths for staged files + } + """ + bash /inpred/user_scripts/process_metrics_files.sh \\ + --output_directory \$(pwd) \\ + --host_system_mounting_directory \$(pwd) \\ + $args + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + process_metrics_files: \$(bash /inpred/user_scripts/process_metrics_files.sh --version | grep process_metrics_files.py | awk '{print \$2}') + python: \$(python --version | sed 's/Python //g') + END_VERSIONS + """ + + stub: + """ + mkdir intermediate_metrics_files + touch TSO500_run_metrics.pdf intermediate_metrics_files/master_metrics_table.tsv + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + process_metrics_files: stub + python: stub + END_VERSIONS + """ +} diff --git a/nextflow.config b/nextflow.config index 24f5d86..44157ae 100644 --- a/nextflow.config +++ b/nextflow.config @@ -10,8 +10,14 @@ params { // Input options - input = null - tso500_resource_folder = null + decoy_run_completion_status_xml = null + metrics_plots_all_output_directory = null + metrics_plots_latest_output_directory = null + nxf_samplesheet = null + process_metrics_files_n = 10 + raw_data_root_directory = null + tso500_resources_directory = null + tso500_root_output_directory = null // Boilerplate options outdir = null @@ -145,7 +151,7 @@ validation { monochromeLogs = params.monochrome_logs help { enabled = true - command = "nextflow run InPreD/tso500_nxf_workflow -profile --input samplesheet.csv --outdir " + command = "nextflow run InPreD/tso500_nxf_workflow -profile --nxf_samplesheet samplesheet.csv --outdir " fullParameter = "help_full" showHiddenParameter = "show_hidden" } diff --git a/nextflow_schema.json b/nextflow_schema.json index fcbeeaf..80e642c 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -10,9 +10,40 @@ "type": "object", "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", - "required": ["input", "outdir", "tso500_resource_folder"], + "required": [ + "metrics_plots_all_output_directory", + "metrics_plots_latest_output_directory", + "nxf_samplesheet", + "process_metrics_files_n", + "outdir", + "raw_data_root_directory", + "tso500_resources_directory", + "tso500_root_output_directory" + ], "properties": { - "input": { + "decoy_run_completion_status_xml": { + "type": "string", + "format": "file-path", + "exists": true, + "mimetype": "text/xml", + "pattern": "RunCompletionStatus.xml$", + "description": "Path to xml file containing acting as decoy if sequencing was done on NovaSeq.", + "help_text": "The decoy file can be used for any run that did not produce a RunCompletionStatus.xml.", + "fa_icon": "fas fa-file-xml" + }, + "metrics_plots_all_output_directory": { + "type": "string", + "format": "directory-path", + "description": "The output directory where the results from process_metrics_files (all runs) should be saved", + "fa_icon": "fas fa-folder-open" + }, + "metrics_plots_latest_output_directory": { + "type": "string", + "format": "directory-path", + "description": "The output directory where the results from process_metrics_files (last n runs) should be saved", + "fa_icon": "fas fa-folder-open" + }, + "nxf_samplesheet": { "type": "string", "format": "file-path", "exists": true, @@ -29,13 +60,35 @@ "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", "fa_icon": "fas fa-folder-open" }, - "tso500_resource_folder": { + "process_metrics_files_n": { + "type": "integer", + "description": "Number of sequencing runs to consider.", + "help_text": "Specify the number of sequencing runs to include in plotting of the latest n metrics. For examples, process_metrics_files_n = 10 will create a pdf file containing metrics plotted for last 10 sequencing runs.", + "fa_icon": "fas fa-hashtag" + }, + "raw_data_root_directory": { + "type": "string", + "format": "directory-path", + "exists": true, + "description": "Path to root directory where all TSO500 sequencing data results are stored.", + "help_text": "The root directory is usually contains all results generated by the sequencing runs.", + "fa_icon": "fas fa-folder-open" + }, + "tso500_resources_directory": { "type": "string", "format": "directory-path", "exists": true, "description": "Path to directory where all reference files, required by the TSO500 LocalApp, are stored.", "help_text": "The resource directory is usually obtained together with the LocalApp image.", "fa_icon": "fas fa-folder-open" + }, + "tso500_root_output_directory": { + "type": "string", + "format": "directory-path", + "exists": true, + "description": "Path to root directory where all TSO500 LocalApp results are stored.", + "help_text": "The root directory is usually contains all run results generated by LocalApp (all dirs called _LocalApp should be contained, might even be subdirectories).", + "fa_icon": "fas fa-folder-open" } } }, diff --git a/subworkflows/local/utils_nfcore_full_pipeline/main.nf b/subworkflows/local/utils_nfcore_full_pipeline/main.nf index 6881361..67bf686 100644 --- a/subworkflows/local/utils_nfcore_full_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_full_pipeline/main.nf @@ -29,7 +29,7 @@ workflow PIPELINE_INITIALISATION { monochrome_logs // boolean: Do not use coloured log outputs nextflow_cli_args // array: List of positional nextflow CLI args outdir // string: The output directory where the results will be saved - input // string: Path to input samplesheet + nxf_samplesheet // string: Path to input samplesheet main: @@ -62,11 +62,11 @@ workflow PIPELINE_INITIALISATION { ) // - // Create channel from input file provided through params.input + // Create channel from input file provided through params.nxf_samplesheet // Channel - .fromList(samplesheetToList(params.input, "${projectDir}/assets/schema_input.json")) + .fromList(samplesheetToList(params.nxf_samplesheet, "${projectDir}/assets/schema_input.json")) .set { ch_samplesheet } emit: diff --git a/workflows/main.nf b/workflows/main.nf index 912662f..0fc6218 100644 --- a/workflows/main.nf +++ b/workflows/main.nf @@ -4,14 +4,16 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { GATHER } from '../modules/local/local_app/local_app' -include { LOCAL_APP as LOCAL_APP_DEMULTIPLEX } from '../modules/local/local_app/local_app' -include { LOCAL_APP as LOCAL_APP_TSO500 } from '../modules/local/local_app/local_app' -include { LOCAL_APP_PREPPER } from '../modules/local/local_app_prepper/local_app_prepper' -include { paramsSummaryMap } from 'plugin/nf-schema' -include { samplesheetToList } from 'plugin/nf-schema' -include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' -include { validateParameters } from 'plugin/nf-schema' +include { GATHER_RESULTS_WORKFLOW } from '../modules/local/acadia_500_wdl_workflow/main' +include { TSO500_WORKFLOW as TSO500_WORKFLOW_DEMULTIPLEX } from '../modules/local/acadia_500_wdl_workflow/main' +include { TSO500_WORKFLOW } from '../modules/local/acadia_500_wdl_workflow/main' +include { LOCAL_APP_PREPPER } from '../modules/local/local_app_prepper/main' +include { PROCESS_METRICS_FILES as PROCESS_METRICS_FILES_ALL } from '../modules/local/process_metrics_files/main' +include { PROCESS_METRICS_FILES as PROCESS_METRICS_FILES_N } from '../modules/local/process_metrics_files/main' +include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { samplesheetToList } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -29,63 +31,152 @@ workflow MAIN { // empty channel to store all software versions versions = Channel.empty() - // channel holding information about run id, path to the run folder and a list of sample ids - local_app_prepper_input = samplesheet - .map{ it -> return [ it[6], it[7], it[1] ] } - .groupTuple( by: [ 0, 1 ] ) + /* + MODULE: LocalApp prepper + */ - // channel holding information about run id, path to run folder and samplesheet - run_folders = samplesheet - .map{ it -> return [ it[6], it[7], it[8], [] ] } - .unique() + // prepare channel from samplesheet holding information about run id, path to the run folder and a list of sample ids + local_app_prepper_input = samplesheet // Channel: [ [ dataset_id, sample_id, molecule, sample_type, tumor_site, tumor_content, run_id, run, samplesheet, barcode ] ] + .map{ it -> return [ it[6], it[7], it[1] ] } // Channel: [ [ run_id, run, sample_id ] ] + .groupTuple( by: [ 0, 1 ] ) // Channel: [ [ run_id, run, [ sample_id, sample_id, ... ] ] ] - // MODULE: Prepare inputs.json for LocalApp + // run process LOCAL_APP_PREPPER ( local_app_prepper_input ) + + // add versions to versions channel versions = versions.mix(LOCAL_APP_PREPPER.out.versions.first()) + /* + MODULE: TSO500 demultiplex workflow (LocalApp) + */ + + // prepare channel from samplesheet holding information about run id, path to run folder and samplesheet + run_folders = samplesheet // Channel: [ [ dataset_id, sample_id, molecule, sample_type, tumor_site, tumor_content, run_id, run, samplesheet, barcode ] ] + .map{ it -> return [ it[6], it[7], it[8], [] ] } // Channel: [ [ run_id, run, samplesheet, [] ] ] + .unique() // Channel: [ [ run_id, run, samplesheet, [] ] ] + // attach the json to the correct run folder information - local_app_demultiplex_input = run_folders.join(LOCAL_APP_PREPPER.out.demultiplex) + tso500_workflow_demultiplex_input = run_folders // Channel: [ [ run_id, run, samplesheet, [] ] ] + .join(LOCAL_APP_PREPPER.out.demultiplex) // Channel: [ [ run_id, run, samplesheet, [], json ] ] - // MODULE: Run LocalApp demultiplex workflow - LOCAL_APP_DEMULTIPLEX ( - local_app_demultiplex_input, - file(params.tso500_resource_folder) + // run process + TSO500_WORKFLOW_DEMULTIPLEX ( + tso500_workflow_demultiplex_input, + file(params.tso500_resources_directory) ) - versions = versions.mix(LOCAL_APP_DEMULTIPLEX.out.versions.first()) - - // construct a channel for each sample - local_app_tso500_input = run_folders - .join(LOCAL_APP_DEMULTIPLEX.out.results) - .combine(LOCAL_APP_PREPPER.out.tso500.transpose(), by: 0) - .map{ it -> return [ get_sample_id(it[5]), it[1], it[2], it[4], it[5] ] } - - // MODULE: Run LocalApp TSO500 workflow - LOCAL_APP_TSO500 ( - local_app_tso500_input, - file(params.tso500_resource_folder) + + // add versions to versions channel + versions = versions.mix(TSO500_WORKFLOW_DEMULTIPLEX.out.versions.first()) + + /* + MODULE: TSO500 workflow (LocalApp) + */ + + // prepare channel for each sample + tso500_workflow_input = run_folders // Channel: [ [ run_id, run, samplesheet, [] ] ] + .join(TSO500_WORKFLOW_DEMULTIPLEX.out.results) // Channel: [ [ run_id, run, samplesheet, [], results ] ] + .combine(LOCAL_APP_PREPPER.out.tso500.transpose(), by: 0) // Channel: [ [ run_id, run, samplesheet, [], results, json ] ] + .map{ it -> return [ get_sample_id(it[5]), it[1], it[2], it[4], it[5] ] } // Channel: [ [ sample_id, run, samplesheet, results, json ] ] + + // run process + TSO500_WORKFLOW ( + tso500_workflow_input, + file(params.tso500_resources_directory) ) - versions = versions.mix(LOCAL_APP_TSO500.out.versions.first()) - - // merge all local app output folders in tuple and construct input channel for gather - gather_inputfolders = LOCAL_APP_PREPPER.out.tso500.transpose() - .map{ it -> return [ get_sample_id(it[1]), it[0] ] } - .join(LOCAL_APP_TSO500.out.results) - .map{ it -> return [ it[1], it[2] ] } - .concat(LOCAL_APP_DEMULTIPLEX.out.results) - .groupTuple() - gather_input = run_folders - .map{ it -> return [ it[0], it[1], it[2] ] } - .join(gather_inputfolders) - .join(LOCAL_APP_PREPPER.out.gather) - - // MODULE: Run LocalApp Gather workflow - GATHER ( - gather_input, - file(params.tso500_resource_folder) + + // add versions to versions channel + versions = versions.mix(TSO500_WORKFLOW.out.versions.first()) + + /* + MODULE: Gather results workflow (LocalApp) + */ + + // merge all LocalApp output directories in tuple + results_directories = LOCAL_APP_PREPPER.out.tso500 // Channel: [ run_id, [ json ] ] + .transpose() // Channel: [ [ run_id, json ] ] + .map{ it -> return [ get_sample_id(it[1]), it[0] ] } // Channel: [ [ sample_id, run_id ] ] + .join(TSO500_WORKFLOW.out.results) // Channel: [ [ sample_id, run_id, results ] ] + .map{ it -> return [ it[1], it[2] ] } // Channel: [ [ run_id, results ] ] + .concat(TSO500_WORKFLOW_DEMULTIPLEX.out.results) // Channel: [ [ run_id, results ] ] + .groupTuple() // Channel: [ [ run_id, [ results, results, ... ] ] ] + + // prepare channel holding run directories, results directories and gather json + gather_results_workflow_input = run_folders // Channel: [ [ run_id, run, samplesheet, [] ] ] + .map{ it -> return [ it[0], it[1], it[2] ] } // Channel: [ [ run_id, run, samplesheet ] ] + .join(results_directories) // Channel: [ [ run_id, run, samplesheet, [ results, results, ... ] ] ] + .join(LOCAL_APP_PREPPER.out.gather) // Channel: [ [ run_id, run, samplesheet, [ results, results, ... ], json ] ] + + // run process + GATHER_RESULTS_WORKFLOW ( + gather_results_workflow_input, + file(params.tso500_resources_directory) ) - versions = versions.mix(GATHER.out.versions.first()) + + // add versions to versions channel + versions = versions.mix(GATHER_RESULTS_WORKFLOW.out.versions.first()) + + /* + MODULE: Process metrics files (All runs) + */ + + // gather all relevant MetricsOutput.tsv files + metrics_output_tsv = channel.fromPath("${params.tso500_root_output_directory}/**_LocalApp_results/Results/MetricsOutput.tsv") // Channel: [ tsv ] + .map { file -> + def run_id = (file.toString() =~ /(\d{6}_\D{1,3}\d{5,6}(_RUO)?_\d{4}_\w{10})(_TSO_500)?_LocalApp_results/)[0][1] + return [ run_id, file ] + } // Channel: [ [ run_id, tsv ] ] + + // join with (decoy) RunCompletionStatus.xml with MetricsOutput.tsv + if (params.decoy_run_completion_status_xml) { + metrics_output_tsv_run_completion_status_xml = metrics_output_tsv // Channel: [ [ run_id, tsv ] ] + .map { run_id, file -> return [ run_id, file, params.decoy_run_completion_status_xml ] } // Channel: [ [ run_id, tsv, xml ] ] + } else { + run_completion_status_xml = channel.fromPath("${params.raw_data_root_directory}/*/RunCompletionStatus.xml") // Channel: [ xml ] + .map { file -> + def run_id = (file.toString() =~ /(\d{6}_\D{1,3}\d{5,6}(_RUO)?_\d{4}_\w{10})/)[0][1] + return [ run_id, file ] + } // Channel: [ [ run_id, xml ] ] + metrics_output_tsv_run_completion_status_xml = metrics_output_tsv // Channel: [ [ run_id, tsv ] ] + .join(run_completion_status_xml) // Channel: [ [ run_id, tsv, xml ] ] + } + + // sort MetricsOutput.tsv and RunCompletionStatus.xml by run_id descending + metrics_output_tsv_run_completion_status_xml_sorted = metrics_output_tsv_run_completion_status_xml + .toSortedList{ a, b -> b[0] <=> a[0] } // Channel: [ [ [ run_id, tsv, xml ], [ run_id, tsv, xml ], ... ] ] sorted by run_id descending + + // prepare channel holding list of run_ids, list of MetricsOutput.tsv files and list of RunCompletionStatus.xml files + process_metrics_files_all_input = metrics_output_tsv_run_completion_status_xml_sorted // Channel: [ [ run_id, tsv, xml ], [run_id, tsv, xml], ... ] + .transpose() // Channel: [ [ run_id, run_id, ... ], [ tsv, tsv, ... ], [ xml, xml, ... ] ] + .collect(flat: false) // Channel: [ [ [ run_id, run_id, ... ], [ tsv, tsv, ... ], [ xml, xml, ... ] ] ] + + // run process + PROCESS_METRICS_FILES_ALL(process_metrics_files_all_input) + + // add versions to versions channel + versions = versions.mix(PROCESS_METRICS_FILES_ALL.out.versions.first()) + + /* + MODULE: Process metrics files (Last N runs) + */ + + // calculate index in metrics tuples + idx = params.process_metrics_files_n - 1 + metrics_output_tsv.count().map{ it -> + if( it < params.process_metrics_files_n ) { + idx = it - 1 // the scope in here is weird as it is the same as in the next mapping function so idx is set correctly but printing idx outside will show the original value + } + } + // select last n runs + process_metrics_files_n_input = process_metrics_files_all_input // Channel: [ [ [ run_id, run_id, ... ], [ tsv, tsv, ... ], [ xml, xml, ... ] ] ] + .map { it -> return [ it[0][0..idx], it[1][0..idx], it[2][0..idx] ] } // Channel: [ [ [ run_id, run_id, ... ], [ tsv, tsv, ... ], [ xml, xml, ... ] ] ] select last n items + + // run process + PROCESS_METRICS_FILES_N(process_metrics_files_n_input) + + // add versions to versions channel + versions = versions.mix(PROCESS_METRICS_FILES_N.out.versions.first()) // collate and save software versions softwareVersionsToYAML(versions) @@ -95,7 +186,6 @@ workflow MAIN { sort: true, newLine: true ).set { ch_collated_versions } - emit: versions = versions