get_data.sh

mkdir -p data/{genomes,rawdata,results,indexes,gtf_files,bed_files,fasta_files,vcf_files}

pushd data/vcf_files
#1KGP biallelic phased VCF, chr21, and random subset of 100K variants
wget "http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/data_collections/1000_genomes_project/release/20181203_biallelic_SNV/ALL.chr21.shapeit2_integrated_v1a.GRCh38.20181129.phased.vcf.gz" .
gunzip ALL.chr21.shapeit2_integrated_v1a.GRCh38.20181129.phased.vcf.gz
grep "#" ALL.chr21.shapeit2_integrated_v1a.GRCh38.20181129.phased.vcf > random_snippet.vcf
grep -v "#" ALL.chr21.shapeit2_integrated_v1a.GRCh38.20181129.phased.vcf | shuf -n 10000 | sort -k2,1n >> random_snippet.vcf
gzip ALL.chr21.shapeit2_integrated_v1a.GRCh38.20181129.phased.vcf

#dbSNP chr21 annotation vcf
wget "https://ftp.ncbi.nlm.nih.gov/snp/latest_release/VCF/GCF_000001405.39.gz" .
gunzip GCF_000001405.39.gz
head -n 38 GCF_000001405.39 > dbSNP_chr21_GRCh38.vcf
awk '{if ($1 == "NC_000021.9") {sub(/NC_000021.9/,"chr21"); print}}' GCF_000001405.39 >> dbSNP_chr21_GRCh38.vcf
gzip dbSNP_chr21_GRCh38.vcf
rm GCF_000001405.39.gz

popd

pushd data/gtf_files

#gencode gtf
wget "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_41/gencode.v41.annotation.gtf.gz" .
gunzip gencode.v41.annotation.gtf.gz

grep "#" gencode.v41.annotation.gtf > gencode.v41.chr21.gtf
awk '{if ($1 == "chr21") {print}}' gencode.v41.annotation.gtf >> gencode.v41.chr21.gtf

grep "#" gencode.v41.chr21.gtf > gencode.v41.chr21.genes.gtf
awk '{if ($3 == "gene") {print}}' gencode.v41.chr21.gtf >> gencode.v41.chr21.genes.gtf

awk 'BEGIN{OFS="\t"}{if ($3=="gene" && $12=="\"protein_coding\";") {gsub(/[";]/,"");print $1,$4-1,$5,$14}}' gencode.v41.annotation.gtf > ~/data/bed_files/genes.bed
awk 'BEGIN{OFS="\t"}{if ($1=="chr21" && $3=="exon" && $14=="\"protein_coding\";") {gsub(/[";]/,"");print $1,$4-1,$5,$16}}' gencode.v41.annotation.gtf > ~/data/bed_files/exons.chr21.genename.bed
gzip gencode.v41.annotation.gtf

popd

pushd data/metadata_and_txt_files
#1KGP sample metadata
wget "http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/release/20130502/integrated_call_samples_v3.20200731.ALL.ped" .
wget "http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/phase3/20131219.populations.tsv" .

Rscript data_prep_scripts/make_sample_metadata.R

rm integrated_call_samples_v3.20200731.ALL.ped 20131219.populations.tsv

#chromosome sizes
wget "https://hgdownload.soe.ucsc.edu/goldenPath/hg38/bigZips/hg38.chrom.sizes" .
head -n 24 hg38.chrom.sizes > hg38.chrom.sizes_main.txt
rm hg38.chrom.sizes

#RNA-seq human blood cells
wget "https://usevision.org/data/hg38/RNA/Oct2021/tpmFeb21v3.batchCorrected.tab" .

Rscript data_prep_scripts/subset_RNA.R

#chromHMM segmentation
wget "https://egg2.wustl.edu/roadmap/data/byFileType/chromhmmSegmentations/ChmmModels/coreMarks/jointModel/final/browserlabelmap_15_coreMarks.tab" .

popd

pushd data/bed_files
wget "https://egg2.wustl.edu/roadmap/data/byFileType/chromhmmSegmentations/ChmmModels/coreMarks/jointModel/final/E116_15_coreMarks_hg38lift_stateno.bed.gz" .
awk '{if ($1 == "chr21") {print}}' E116_15_coreMarks_hg38lift_stateno.bed > chromHMM.E116_15_coreMarks_hg38lift_stateno.chr21.bed

#Histone bedgrph files
#naive B cells
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/B15_50/naive_B_cell/ChIP-Seq/NCMLS/S0159LH1.ERX1007461.H3K4me1.bwa.GRCh38.20151029.bw" --output-document H3K4me1.naive_b_cell.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/B15_50/naive_B_cell/ChIP-Seq/NCMLS/S0159LH1.ERX1007462.H3K9me3.bwa.GRCh38.20151029.bw" --output-document H3K9me3.naive_b_cell.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/B15_50/naive_B_cell/ChIP-Seq/NCMLS/S0159LH1.ERX1007481.H3K27ac.bwa.GRCh38.20151029.bw" --output-document H3K27ac.naive_b_cell.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/B15_50/naive_B_cell/ChIP-Seq/NCMLS/S0159LH1.ERX1007482.H3K36me3.bwa.GRCh38.20151029.bw" --output-document H3K36me3.naive_b_cell.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/B15_50/naive_B_cell/ChIP-Seq/NCMLS/S0159LH1.ERX1007483.H3K27me3.bwa.GRCh38.20151029.bw" --output-document H3K27me3.naive_b_cell.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/B15_50/naive_B_cell/ChIP-Seq/NCMLS/S0159LH1.ERX1122444.H3K4me3.bwa.GRCh38.20151029.bw" --output-document H3K4me3.naive_b_cell.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/B15_50/naive_B_cell/ChIP-Seq/NCMLS/S0159LH1.ERX1122482.Input.bwa.GRCh38.20151207.bw" --output-document Input.naive_b_cell.bwa.GRCh38.bw

#CD8 T cells
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/C005DF/CD8-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/C005DFH1.ERX240833.Input.bwa.GRCh38.20150528.bw" --output-document Input.cd8.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/C005DF/CD8-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/C005DFH1.ERX240834.H3K4me3.bwa.GRCh38.20150528.bw" --output-document H3K4me3.cd8.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/C005DF/CD8-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/C005DFH1.ERX337016.H3K9me3.bwa.GRCh38.20150528.bw" --output-document H3K9me3.cd8.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/C005DF/CD8-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/C005DFH1.ERX337035.H3K36me3.bwa.GRCh38.20150528.bw" --output-document H3K36me3.cd8.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/C005DF/CD8-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/C005DFH1.ERX337036.H3K4me1.bwa.GRCh38.20150528.bw" --output-document H3K4me1.cd8.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/cord_blood/C0066P/CD8-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/C0066PH1.ERX240846.H3K27me3.bwa.GRCh38.20150528.bw" --output-document H3K27me3.cd8.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/cord_blood/C0066P/CD8-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/C0066PH1.ERX547957.H3K27ac.bwa.GRCh38.20150528.bw" --output-document H3K27ac.cd8.bwa.GRCh38.bw

#CD4 T cells
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/S008H1/CD4-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/S008H1H1.ERX337051.H3K36me3.bwa.GRCh38.20150528.bw" --output-document H3K36me3.cd4.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/S008H1/CD4-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/S008H1H1.ERX337052.H3K4me1.bwa.GRCh38.20150528.bw" --output-document H3K4me1.cd4.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/S008H1/CD4-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/S008H1H1.ERX337053.H3K27me3.bwa.GRCh38.20150528.bw" --output-document H3K27me3.cd4.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/S008H1/CD4-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/S008H1H1.ERX337054.H3K9me3.bwa.GRCh38.20150528.bw" --output-document H3K9me3.cd4.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/S008H1/CD4-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/S008H1H1.ERX547925.Input.bwa.GRCh38.20150528.bw" --output-document Input.cd4.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/S008H1/CD4-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/S008H1H1.ERX547940.H3K27ac.bwa.GRCh38.20150528.bw" --output-document H3K27ac.cd4.bwa.GRCh38.bw
wget "ftp://ftp.ebi.ac.uk/pub/databases/blueprint/data/homo_sapiens/GRCh38/venous_blood/S008H1/CD4-positive_alpha-beta_T_cell/ChIP-Seq/NCMLS/S008H1H1.ERX547958.H3K4me3.bwa.GRCh38.20150528.bw" --output-document H3K4me3.cd4.bwa.GRCh38.bw

bash data_prep_scripts/run_bigwigtobedgraph.sh
rm *.bw

# gene, exon, lncRNA, etc bed files

grep "protein_coding" ~/data/gtf_files/gencode.v41.chr21.genes.gtf | awk 'BEGIN { OFS = "\t"} {print $1,$4-1,$5}' > genes.chr21.bed
grep "protein_coding" ~/data/gtf_files/gencode.v41.chr21.gtf | awk 'BEGIN {OFS = "\t"}{if ($3 == "exon"){print $1,$4-1,$5}}' > exons.chr21.bed
grep "lncRNA" ~/data/gtf_files/gencode.v41.chr21.genes.gtf | awk 'BEGIN {OFS = "\t"} {print $1, $4-1, $5}' > lncRNA.chr21.bed
grep "processed_pseudogene" ~/data/gtf_files/gencode.v41.chr21.genes.gtf | awk 'BEGIN {OFS = "\t"} {print $1, $4-1, $5}' > processed_pseudogene.chr21.bed
bedtools intersect -a ~/data/vcf_files/random_snippet.vcf -b exons.chr21.bed > hg38.chr21.exon_1KGPrandom_snippet.intersection.bed

bedtools makewindows -g ~/data/metadata_and_txt_files/hg38.chrom.sizes_main.txt -w 500000 > windows.0pt5Mbp.bed
awk '{if ($1 == "chr21") {print}}' windows.0pt5Mbp.bed > windows.0pt5Mbp.chr21.bed
rm windows.0pt5Mbp.bed
bedtools makewindows -g ~/data/metadata_and_txt_files/hg38.chrom.sizes_main.txt -w 50000 > windows.50Kbp.bed
awk '{if ($1 == "chr21") {print}}' windows.50Kbp.bed > windows.50Kbp.chr21.bed
rm windows.50Kbp.bed

popd

pushd data/fasta_files

#fasta files for chr21 and bins on chr21
wget "https://hgdownload.soe.ucsc.edu/goldenPath/hg38/chromosomes/chr21.fa.gz" .
gunzip chr21.fa.gz

bedtools getfasta -fi chr21.fa -bed ~/data/bed_files/windows.0pt5Mbp.chr21.bed -fo hg38chr21.0pt5MbpWindows.fa
bedtools getfasta -fi chr21.fa -bed ~/data/bed_files/windows.50Kbp.chr21.bed -fo hg38chr21.50KbpWindows.fa

popd

pushd data/genomes
wget -r -np -nH --cut-dirs=2 ftp://ftp.flybase.net/releases/FB2017_03/dmel_r6.16/

wget "http://hgdownload.soe.ucsc.edu/goldenPath/sacCer3/bigZips/chromFa.tar.gz"
tar -xvzf chromFa.tar.gz
cat chr*.fa > sacCer3.fa
rm chr*.fa chromFa.tar.gz

wget "http://hgdownload.soe.ucsc.edu/goldenPath/hg38/bigZips/hg38.chromFa.tar.gz"
tar -xvzf hg38.chromFa.tar.gz
cat chroms/chr*.fa > hg38.fa
rm -r chroms hg38.chromFa.tar.gz

wget "http://hgdownload.soe.ucsc.edu/goldenPath/mm10/bigZips/chromFa.tar.gz"
tar -xvzf chromFa.tar.gz
cat chr*.fa > mm10.fa
rm chr*.fa chromFa.tar.gz

wget http://datacache.galaxyproject.org/indexes/dm6/seq/dm6.fa
wget http://datacache.galaxyproject.org/indexes/ce10/seq/ce10.fa
popd

pushd data/indexes
mkdir bowtie2 bwa
mkdir bowtie2/hg38 bwa/hg38
wget https://genome-idx.s3.amazonaws.com/bt/GRCh38_noalt_as.zip
unzip GRCh38_noalt_as.zip
for I in GRCh38_noalt_as/*; do mv ${I} bowtie2/${I/GRCh38_noalt_as/hg38}; done
rm -r GRCh38_noalt_as
ln -s /Users/cmdb/data/genomes/hg38.fa bwa/hg38/
for I in amb ann bwt pac sa;
    do wget http://datacache.galaxyproject.org/indexes/hg38/hg38full/bwa_index/hg38full.fa.${I};
    mv hg38full.fa.${I} bwa/hg38/hg38.fa.${I};
done

mkdir bowtie2/mm10 bwa/mm10
wget https://genome-idx.s3.amazonaws.com/bt/mm10.zip
unzip bowtie2/mm10.zip
mv mm10.* bowtie2/mm10/
ln -s /Users/cmdb/data/genomes/mm10.fa bwa/mm10/
for I in amb ann bwt pac sa;
    do wget http://datacache.galaxyproject.org/indexes/mm10/bwa_index/mm10.fa.${I};
    mv mm10.fa.${I} bwa/mm10/;
done

mkdir bowtie2/dm6 bwa/dm6
wget https://genome-idx.s3.amazonaws.com/bt/BDGP6.zip
unzip BDGP6.zip
for I in BDGP6/*; do mv ${I} bowtie2/${I/BDGP6/dm6}; done
rm -r BDGP6
ln -s /Users/cmdb/data/genomes/dm6.fa bwa/dm6/
for I in amb ann bwt pac sa;
    do wget http://datacache.galaxyproject.org/indexes/dm6/bwa_index/dm6.fa.${I};
    mv dm6.fa.${I} bwa/dm6/;
done

mkdir bowtie2/ce10 bwa/ce10
for I in 1.bt2 2.bt2 3.bt2 4.bt2 rev.1.bt2 rev.2.bt2;
    do wget http://datacache.galaxyproject.org/indexes/ce10/bowtie2_index/ce10.${I};
    mv ce10.${I} bowtie2/ce10/;
done
ln -s /Users/cmdb/data/genomes/ce10.fa bwa/ce10/
for I in amb ann bwt pac sa;
    do wget http://datacache.galaxyproject.org/indexes/dm10/bwa_index/dm10.fa.${I};
    mv dm10.fa.${I} bwa/dm10/;
done
popd

pushd data/rawdata
fastq-dump SRR072893 # male cycle 10 - 848 MB
fastq-dump SRR072903 # male cycle 14D - 730 MB
fastq-dump SRR072905 # female cycle 10 - 985 MB
fastq-dump SRR072915 # female cycle 14D - 860 MB

# GTEX PCA data
wget https://storage.googleapis.com/gtex_analysis_pilot_v3/rna_seq_data/GTEx_Analysis_RNA-seq_RNA-SeQCv1.1.8_gene_rpkm__Pilot_V3_patch1.gct.gz
wget https://ftp.1000genomes.ebi.ac.uk/vol1/ftp/release/20130502/integrated_call_samples_v3.20130502.ALL.panel

# Linear regression
wget http://dl.dropboxusercontent.com/s/rjefaxqmud51nfg/penguins.csv

# Multiple testing ASE
wget http://dl.dropboxusercontent.com/s/rvwz50pons1l8qm/ase_table.txt

# Fasta parsing
wget https://bx.bio.jhu.edu/track-hubs/cmdb/SRR072893.sam

# 1K genome SNP subset
wget http://dl.dropboxusercontent.com/s/p2ef992kl2jmmis/matrix_1kg.txt

# Mapping Identifiers
wget http://www.uniprot.org/docs/fly.txt

# Intersecting genomic features
wget https://bx.bio.jhu.edu/track-hubs/cmdb/SRR072893.bam

# Generalized linear regression
wget http://dl.dropboxusercontent.com/s/g47l2r2kmjfzst2/aau1043_dnm.tsv
wget http://dl.dropboxusercontent.com/s/vxc4tw1qv7j4s4h/aau1043_parental_age.tsv

# GWAS
wget http://dl.dropboxusercontent.com/s/6e0vhffv6mcfy9l/genotypes.vcf
wget http://dl.dropboxusercontent.com/s/cvznmgbip40plkb/GS451_IC50.txt

# Multi-sample variant calling - Missing
#wget http://67.207.142.119/outgoing/BYxRM_subset.tar.xv

# GWAS - Missing
#wget http://67.207.142.119/outgoing/BYxRM_segs_saccer3.bam.simplified.vcf.xz
#wget http://67.207.142.119/outgoing/BYxRM_PhenoData.txt

# 3D genome
wget https://bx.bio.jhu.edu/data/cmdb-lab/3DGenomeData.tar.gz

# Genome Assembly - missing
#wget http://67.207.142.119/outgoing/reads_low_1.fastq
#wget http://67.207.142.119/outgoing/reads_low_2.fastq
#wget http://67.207.142.119/outgoing/reference.fasta
#wget http://67.207.142.119/outgoing/MAP006.subset.fa


# ChIP-Seq and Motif finding
wget http://bx.bio.jhu.edu/data/msauria/cmdb-lab/g1e.tar.xz
wget https://raw.githubusercontent.com/bxlab/qbb2020/master/week5/Mus_musculus.GRCm38.94_features.bed
wget https://meme-suite.org/meme/meme-software/Databases/motifs/motif_databases.12.21.tgz
mkdir -p ../meme_db/motifs
mv motif_databases.12.21.tgz ../meme_db/motifs/
wget https://meme-suite.org/meme/meme-software/Databases/gomo/gomo_databases.3.2.tgz
mkdir ../meme_db/gomo
mv gomo_databases.3.2.tgz ../meme_db/gomo/
wget https://meme-suite.org/meme/meme-software/Databases/tgene/tgene_databases.1.0.tgz
mkdir ../meme/tgene
mv tgene_databases.1.0.tgz ../meme/tgene/
wget https://www.ebi.ac.uk/~emily/Online%20courses/NGS/ChIP-seq.zip
wget https://raw.githubusercontent.com/bxlab/qbb2019/master/data/ER4_peaks.narrowPeak

# Methylation
wget https://bx.bio.jhu.edu/data/msauria/cmdb-lab/mm10_refseq_genes_chr6_50M_60M.bed
wget https://bx.bio.jhu.edu/data/msauria/cmdb-lab/methylation_fastq.tar.gz
wget http://hgdownload.soe.ucsc.edu/goldenPath/mm10/chromosomes/chr6.fa.gz

# Gene Expression
wget http://dl.dropboxusercontent.com/s/hxjvua05abz8zqb/all_annotated.csv
wget http://dl.dropboxusercontent.com/s/7hxh7f61756vc4k/hema_data.txt

# scRNA-seq
wget https://bx.bio.jhu.edu/data/cmdb-lab/scrnaseq/neuron_10k_v3_filtered_feature_bc_matrix.h5

# Metagenomics
wget https://bx.bio.jhu.edu/data/cmdb-lab/week13_data.tar
wget https://bx.bio.jhu.edu/data/cmdb-lab/bins.tar
wget https://bx.bio.jhu.edu/data/cmdb-lab/abundance_table.tab

wget http://dl.dropboxusercontent.com/s/6e0vhffv6mcfy9l/genotypes.vcf
wget http://dl.dropboxusercontent.com/s/cvznmgbip40plkb/GS451_IC50.txt
popd

pushd data/results
wget http://dl.dropboxusercontent.com/u/169950361/SRP004442.stringtie.tar.gz
popd