Update master to dev (release 2.5.0)

.gitignore

@@ -1,2 +1,18 @@
+/tools/
+/data/
+/pipeline/config.groovy
+/tasks/nectar/current.manifest.json
+samples.db
+pipeline_id
+swift_credentials.sh
+**.doit.db
+.idea
+.bpipe
 *.pyc
 *.swp
+tmpdata
+batches
+!designs/ALL
+**.log
+**.bak
+lib/*.egg-info/

Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu
+
+# Start using bash
+SHELL ["/bin/bash", "-c"]
+
+# Install basic linux tools that we need to make python
+RUN apt-get update && apt-get install -y curl make build-essential libssl-dev zlib1g-dev libsqlite3-dev libreadline-dev git libbz2-dev libz-dev libssl-dev gfortran xorg-dev libcurl4-openssl-dev ncurses-dev openjdk-8-jdk
+
+# Copy in the repository
+ADD . /opt/cpipe
+
+# Move into the cpipe dir
+WORKDIR /opt/cpipe
+
+# Run the install script
+RUN ./install.sh --noninteractive
+
+# Run the main script
+ENTRYPOINT ["./cpipe"]

README.md

@@ -1,16 +1,40 @@
-Cpipe  
-=======================
+# Cpipe  
 
 Cpipe is a clinically focused exome sequencing pipeline developed
 by the Melbourne Genomics Health Alliance. Cpipe offers an industry
 standard variant calling pipeline with a suite of additional features 
 needed by diagnostic laboratories added on top.
 
+A simplified workflow is presented here. For more specific information, refer to the [documentation](docs/index.md).
+
+## Installation
+
 To set up Cpipe, clone this repository and then run the install script:
 
     git clone https://github.com/MelbourneGenomics/cpipe.git
     cd cpipe
-    ./pipeline/scripts/install.sh
+    ./install.sh
+
+Ensure you follow any instructions given by the install script. For example, non MGHA users must install their own
+version of GATK since their license forbids us from redistributing it.
+
+For more detailed instructions, have a look at the [installation documentation](docs/install.md).
+
+## Creating your analysis batch
+* First, rename your fastqs to ensure they fit the following pattern:
+`sampleID_<anything>_L[0-9]*_R[0-9].fastq.gz`
+* Tell Cpipe to create a new batch using your fastq files
+   ```bash
+   ./cpipe batch create MyBatch --data path/to/samples/*.fastq.gz --exome path/to/exons.bed
+   ```
+  * `MyBatch` is the batch name 
+  * `path/to/exons.bed` is the filepath to a capture regions bed file specified by your sequencer
+  * `path/to/samples/*.fastq.gz` is the filepath to the sample fastqs you want to put in your batch
+
+For more information about this stage, refer to the [batches documentation](docs/batches.md).
+
+## Running the Pipeline
 
-For further instructions, take a look at the [User Guide](https://melbournegenomics.github.io/docs/Cpipe_User_Guide.pdf).
+Now, all you need to do is run `./cpipe run MyBatch`
 
+The run command is documented in the [command documentation](docs/commands.md#run).

_env

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Local variables
+
+# Util functions
+function join() {
+    local IFS=$1
+    shift
+    echo "$*"
+}
+
+function load_config {
+    CONFIG=`sed 's/\/\/.*$//' $CONFIG_FILE`
+    eval "$CONFIG"
+}
+
+function set_config_variable {
+    NAME="$1"
+    VALUE="$2"
+    cp "$CONFIG_FILE" "$CONFIG_FILE.tmp"
+    sed 's,'^[\s]*$NAME'=\("\?\).*$,'$NAME'=\1'$VALUE'\1,g' ${CONFIG_FILE}.tmp > "$CONFIG_FILE"
+    rm "${CONFIG_FILE}.tmp"
+    load_config
+}
+
+# Work out the directory name
+if [ -n "$ZSH_VERSION" ]; then
+    ROOT=$(readlink -f $(dirname $0))
+else
+    ROOT=$(readlink -f $(dirname $BASH_SOURCE))
+fi
+
+# Set root variables
+export CPIPE_ROOT=${ROOT}
+CONFIG_FILE=${CPIPE_ROOT}/pipeline/config.groovy
+
+# Load config groovy
+if [[ -f ${CONFIG_FILE} ]] ; then
+    load_config
+fi
+
+# Add all tool directories and bin folders to PATH
+export SYS_JAVA=`which -a java | grep -v -m1 'cpipe'` # Export the old system java before we override it
+export PATH=${TOOLS}/bin:${TOOLS}/maven/bin:${TOOLS}/bpipe/bin:${PATH}
+export HTSLIB_DIR=$TOOLS/lib
+export PERL5LIB=$TOOLS/perl_lib/lib/perl5:$TOOLS/perl/lib:$TOOLS/vep_plugins:${TOOLS}/vep_libs
+export CPATH="${TOOLS}/include"
+export CFLAGS="$CFLAGS -I${TOOLS}/include"
+export CPPFLAGS="$CPPFLAGS -I${TOOLS}/include"
+export LDFLAGS="$LDFLAGS -L${TOOLS}/lib"
+export LD_LIBRARY_PATH=${TOOLS}/lib:${LD_LIBRARY_PATH}
+export JAVA_OPTS #Pass JAVA_OPTS to the script in c_libs/bin/java
+export TMPDIR #TMPDIR is set in config.groovy.
+
+# Load virtualenv
+source ${CPIPE_ROOT}/tools/python/bin/activate
+

build.gradle

@@ -0,0 +1,42 @@
+apply plugin:'java'
+apply plugin: 'maven'
+group = 'com.github.MelbourneGenomics'
+
+repositories {
+    mavenCentral()
+    maven { url 'https://jitpack.io' }
+    maven { 
+        name "biojava"
+        url "http://www.biojava.org/download/maven/" 
+    }
+    maven {
+      name "gral" 
+      url "http://mvn.erichseifert.de/maven2"
+    }
+    mavenLocal()
+    maven {
+        url "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/" //for htsjdk snapshots
+    }
+}
+
+//configurations.all {
+//    transitive false
+//}
+
+dependencies {
+    compile 'com.github.MelbourneGenomics:groovy-hts-sample-info:562ba2a64e8@jar'
+    compile 'com.github.ssadedin:groovy-ngs-utils:d6fdc538d9b@jar'
+    compile 'com.github.barrypitman:JUnitXmlFormatter:9893370f4112@jar'
+    compile 'io.takari.junit:takari-cpsuite:1.2.7@jar'
+    //compile 'com.github.broadinstitute:picard:2.9.0'
+    //compile 'com.github.ssadedin:bpipe:0.9.9.2@jar'
+    //compile 'com.github.broadgsa:gatk-protected:3.6'
+}
+
+task copyDeps(type: Copy) {
+    // Use `gradle  copyDeps -Pdir=/some/dir to specify where to puts the jars`
+    from configurations.runtime
+    exclude '**/*picard.jar', '**/*groovy-*-2.*.*.jar', '**/*groovy-2.*.*.jar'
+    into project.hasProperty('dir') ? project.property('dir') : 'tools/java_libs'
+    //rename '(.*)-[a-z0-9\\.]+.jar', '$1.jar'
+}

cpanfile

@@ -0,0 +1,9 @@
+requires 'HTTP::Tiny', '== 0.056';
+requires 'LWP::Simple', '== 6.15';
+requires 'Archive::Zip', '== 1.56';
+requires 'DBI', '== 1.636';
+requires 'LWP::Protocol::https', '== 6.04';
+requires 'File::Copy::Recursive', '== 0.38';
+requires 'Archive::Extract', '== 0.78';
+requires 'CGI', '== 4.32';
+requires 'Bio::DB::HTS', '== 2.4';

cpipe

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+REMAINDER="${@:1}"
+ROOT=`dirname $0`
+ENV_SCRIPT="$ROOT/_env"
+
+if [[ -n $REMAINDER ]]; then
+    # If the user provided arguments, run the cpipe command using those arguments
+    source $ENV_SCRIPT
+    cpipe $REMAINDER
+else
+    # If the user provided no arguments, start a new subshell, set variables in that, and then give control back to the user. They can then quit using Ctrl+D
+    bash --init-file "${ROOT}/_env"
+fi