12 add variant summary module

.devcontainer/Dockerfile

@@ -17,16 +17,16 @@ RUN apt-get update \
     && apt-get install -y --no-install-recommends \
         fish=3.3.1+ds-3 \
         fuse-overlayfs=1.7.1-1 \
-        openssh-server=1:8.9p1-3ubuntu0.11 \
+        openssh-server=1:8.9p1-3ubuntu0.13 \
         silversearcher-ag=2.2.0+git20200805-1 \
     && echo "LC_ALL=en_US.UTF-8" > /etc/default/locale \
     && locale-gen en_US.UTF-8 \
     && apt-get clean \
     && rm -Rf /usr/share/doc && rm -Rf /usr/share/man \
     && rm -rf /var/lib/apt/lists/*
 # convert gitpod to local user
-RUN groupmod -n ${USERNAME} -g ${USER_GID} gitpod \
-    && usermod -md /home/${USERNAME} -l ${USERNAME} -u ${USER_UID} -s /usr/bin/fish -aG docker gitpod \
+RUN groupmod -n ${USERNAME} -g ${USER_GID:-1000} gitpod \
+    && usermod -md /home/${USERNAME} -l ${USERNAME} -u ${USER_UID:-1000} -s /usr/bin/fish -aG docker gitpod \
     && echo ${USERNAME} ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/${USERNAME} \
     && chmod 0440 /etc/sudoers.d/${USERNAME} \
     && find / -uid 33333 -not -path "/proc/*" -exec chown ${USERNAME}:${USERNAME} {} +
@@ -35,4 +35,4 @@ ENV HOME=/home/${USERNAME} \
 USER ${USERNAME}
 RUN git config --global pull.rebase false \
     && git config --global user.email "${MAIL}" \
-    && git config --global user.name "${FULLNAME}"
+    && git config --global user.name "${FULLNAME}"

.gitignore

@@ -1,4 +1,8 @@
 .nextflow*
+assets/stub_data/tso500_root_output_directory/200103_NS500643_0003_AHG5MGBGYM_pipeline_info
+assets/stub_data/tso500_root_output_directory/200103_NS500643_0003_AHG5MGBGYM_TSO_500_LocalApp_results
+assets/stub_data/tso500_root_output_directory/metrics_plots_NODE_all
+assets/stub_data/tso500_root_output_directory/metrics_plots_NODE_latest
 nohup.out
 null
 test_stub

.nf-core.yml

@@ -59,6 +59,7 @@ lint:
   multiqc_config: false
   nextflow_config: false
   pipeline_name_conventions: false
+  schema_lint: false
   readme:
     - nextflow_badge
 

assets/schema_input.json

@@ -1,7 +1,7 @@
 {
     "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "title": "inpred/tso500_nxf_workflow - params.input schema",
-    "description": "Schema for the file provided with params.input (ideally a samplesheet)",
+    "title": "inpred/tso500_nxf_workflow - params.nxf_samplesheet schema",
+    "description": "Schema for the file provided with params.nxf_samplesheet",
     "type": "array",
     "items": {
         "type": "object",

assets/stub_data/nxf_samplesheet.csv

@@ -0,0 +1,3 @@
+dataset_id,sample_id,molecule,sample_type,tumor_site,tumor_content,run_id,path_to_run,path_to_samplesheet,barcode
+IPH0001,IPH0001-D01-T01-A19,dna,tumor,10,50,200103_NS500643_0003_AHG5MGBGYM,assets/stub_data/raw_data_root_directory/200103_NS500643_0003_AHG5MGBGYM,,UDP0029
+IPH0001,IPH0001-D01-N01-A19,dna,normal,XX,0,200103_NS500643_0003_AHG5MGBGYM,assets/stub_data/raw_data_root_directory/200103_NS500643_0003_AHG5MGBGYM,,UDP0030

conf/modules.config

@@ -17,24 +17,48 @@ process {
         mode: params.publish_dir_mode
     ]
 
-    withName: LOCAL_APP_DEMULTIPLEX {
+    withName: GATHER_RESULTS_WORKFLOW {
         publishDir = [
-            enabled: false
+            path: { "${params.tso500_root_output_directory}/${id}_TSO_500_LocalApp_results" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
 
-    withName: LOCAL_APP_TSO500 {
+    withName: LOCAL_APP_PREPPER {
         publishDir = [
-            enabled: false
+            path: { "${params.tso500_root_output_directory}/${id}_pipeline_info" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
 
-    withName: LOCAL_APP_PREPPER {
+    withName: PROCESS_METRICS_FILES_ALL {
+        ext.args = '--create_plots False'
         publishDir = [
-            path: { "${params.outdir}/pipeline_info" },
+            path: { "${params.tso500_root_output_directory}/${params.metrics_plots_all_output_directory}" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
 
+    withName: PROCESS_METRICS_FILES_N {
+        publishDir = [
+            path: { "${params.tso500_root_output_directory}/${params.metrics_plots_latest_output_directory}" },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+        ]
+    }
+
+    withName: TSO500_WORKFLOW_DEMULTIPLEX {
+        publishDir = [
+            enabled: false
+        ]
+    }
+
+    withName: TSO500_WORKFLOW {
+        publishDir = [
+            enabled: false
+        ]
+    }
 }

conf/test_stub.config

@@ -23,7 +23,12 @@ params {
     config_profile_description = 'Simply stubbing the tasks in the pipeline instead of running them'
 
     // Input data
-    input                  = 'assets/example.samplesheet.csv'
-    outdir                 = 'test_stub'
-    tso500_resource_folder = 'assets/example.tso500_resource_folder'
+    //decoy_run_completion_status_xml            = 'assets/stub_data/decoy_RunCompletionStatus.xml'
+    metrics_plots_all_output_directory         = 'metrics_plots_NODE_all'
+    metrics_plots_latest_output_directory      = 'metrics_plots_NODE_latest'
+    nxf_samplesheet                            = 'assets/stub_data/nxf_samplesheet.csv'
+    outdir                                     = 'test_stub'
+    raw_data_root_directory                    = 'assets/stub_data/raw_data_root_directory'
+    tso500_resources_directory                 = 'assets/stub_data/resources'
+    tso500_root_output_directory               = 'assets/stub_data/tso500_root_output_directory'
 }

main.nf

@@ -21,7 +21,7 @@ include { PIPELINE_COMPLETION     } from './subworkflows/local/utils_nfcore_full
 workflow TSO500_PIPELINE {
 
     take:
-    samplesheet // channel: samplesheet read in from --input
+    samplesheet // channel: samplesheet read in from --nxf_samplesheet
 
     main:
 
@@ -50,7 +50,7 @@ workflow {
         params.monochrome_logs,
         args,
         params.outdir,
-        params.input
+        params.nxf_samplesheet
     )
 
     //

modules/local/local_app/local_app.nf → modules/local/acadia_500_wdl_workflow/main.nf

@@ -1,4 +1,4 @@
-process LOCAL_APP {
+process TSO500_WORKFLOW {
     tag "$id"
     label 'process_medium'
 
@@ -46,6 +46,7 @@ process LOCAL_APP {
     output = 'localapp_' + id
     """
     mkdir $output
+
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         cromwell: stub
@@ -55,7 +56,7 @@ process LOCAL_APP {
     """
 }
 
-process GATHER {
+process GATHER_RESULTS_WORKFLOW {
     tag "$id"
     label 'process_medium'
 
@@ -101,6 +102,7 @@ process GATHER {
     """
     touch inputs.json
     mkdir cromwell-executions cromwell-workflow-logs Results
+
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         cromwell: stub

modules/local/add_variant_summary/main.nf

@@ -0,0 +1,44 @@
+process ADD_VARIANT_SUMMARY {
+    tag "$run_id"
+    label 'process_low'
+
+    container "inpred/tsoppi_main:v0.3.2"
+    containerOptions "-v \$(pwd):/inpred/data"
+
+    input:
+    tuple val(run_id), path(analysis_results_dir)
+
+    output:
+    path "${run_id}_variant_summary.tsv", emit: tsv
+    path 'versions.yml'                  , emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    def args = task.ext.args ?: ''
+    """
+    python ${params.user_scripts_dir}/summarize_run_variants.py \\
+        -r \$(pwd)/${analysis_results_dir} \\
+        -o \$(pwd)/${run_id}_variant_summary.tsv \\
+        -s \$(pwd) \\
+        $args
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        summarize_run_variants: \$(python ${params.user_scripts_dir}/summarize_run_variants.py --version 2>&1 | grep -oP 'version \\K[0-9.:-]+' || echo "0.3.2:22-06-07")
+        python: \$(python --version | sed 's/Python //g')
+    END_VERSIONS
+    """
+
+    stub:
+    """
+    touch ${run_id}_variant_summary.tsv
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        summarize_run_variants: stub
+        python: stub
+    END_VERSIONS
+    """
+}

modules/local/local_app_prepper/local_app_prepper.nf → modules/local/local_app_prepper/main.nf

@@ -2,7 +2,7 @@ process LOCAL_APP_PREPPER {
     tag "$id"
     label 'process_low'
 
-    container "inpred/local_app_prepper:latest"
+    container "inpred/local_app_prepper:1.0.0"
 
     input:
     tuple val(id), path(runfolder), val(sample_list)
@@ -25,14 +25,15 @@ process LOCAL_APP_PREPPER {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        local_app_prepper: latest
+        local_app_prepper: 1.0.0
         python: \$(python --version | sed 's/Python //g')
     END_VERSIONS
     """
 
     stub:
     """
     touch demultiplex.json gather.json tso500_IPH0001-D01-T01-A1.json tso500_IPH0001-D01-N01-A1.json
+
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         local_app_prepper: stub

modules/local/process_metrics_files/main.nf

@@ -0,0 +1,48 @@
+process PROCESS_METRICS_FILES {
+    tag "${id.size()}_runs"
+    label 'process_low'
+
+    container "inpred/tsoppi_main:v0.3.2"
+    containerOptions = "-v \$(pwd):/workdir -v \$(pwd):/inpred/data"
+
+    input:
+    tuple val(id), path(tsv, stageAs: "?/*"), path(xml, stageAs: "?/*") // stage file with index as folder name
+
+    output:
+    path "intermediate_metrics_files/master_metrics_table.tsv" , emit: 'tsv'
+    path "TSO500_run_metrics.pdf"                              , emit: 'pdf', optional: true
+    path 'versions.yml'                                        , emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    def args = task.ext.args ?: ''
+    for (int i = 0; i < id.size(); i++) {
+        args = args + " -m \$(pwd)/${tsv[i]} -r \$(pwd)/${xml[i]} -l ${id[i]}" // due to tsoppi container replacing absolute input mount paths we need to create absolute paths for staged files
+    }
+    """
+    bash /inpred/user_scripts/process_metrics_files.sh \\
+        --output_directory \$(pwd) \\
+        --host_system_mounting_directory \$(pwd) \\
+        $args
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        process_metrics_files: \$(bash /inpred/user_scripts/process_metrics_files.sh --version | grep process_metrics_files.py | awk '{print \$2}')
+        python: \$(python --version | sed 's/Python //g')
+    END_VERSIONS
+    """
+
+    stub:
+    """
+    mkdir intermediate_metrics_files
+    touch TSO500_run_metrics.pdf intermediate_metrics_files/master_metrics_table.tsv
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        process_metrics_files: stub
+        python: stub
+    END_VERSIONS
+    """
+}

nextflow.config

@@ -10,8 +10,14 @@
 params {
 
     // Input options
-    input                      = null
-    tso500_resource_folder     = null
+    decoy_run_completion_status_xml       = null
+    metrics_plots_all_output_directory    = null
+    metrics_plots_latest_output_directory = null
+    nxf_samplesheet                       = null
+    process_metrics_files_n               = 10
+    raw_data_root_directory               = null
+    tso500_resources_directory            = null
+    tso500_root_output_directory          = null
 
     // Boilerplate options
     outdir                       = null
@@ -145,7 +151,7 @@ validation {
     monochromeLogs = params.monochrome_logs
     help {
         enabled = true
-        command = "nextflow run InPreD/tso500_nxf_workflow -profile <docker/singularity/.../institute> --input samplesheet.csv --outdir <OUTDIR>"
+        command = "nextflow run InPreD/tso500_nxf_workflow -profile <docker/singularity/.../institute> --nxf_samplesheet samplesheet.csv --outdir <OUTDIR>"
         fullParameter = "help_full"
         showHiddenParameter = "show_hidden"
     }