diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ab5b672..368af11a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,13 @@ Removed features goes here Miscellaneous goes here +## 1.15.0 - 2026-02-23 + +### Added + +* Added `OPERA` and `SMR` tools to `gwas.sif` container, with corresponding tests. +* Added Python and R packages to `python3.sif` and `r.sif` containers, with corresponding tests. + ## 1.14.0 - 2025-12-05 ### Updated diff --git a/docker/README.md b/docker/README.md index bb6cb606..e674fc43 100644 --- a/docker/README.md +++ b/docker/README.md @@ -17,6 +17,23 @@ The build process is defined in the corresponding [.github/workflows/docker_buil In general, a created PR will trigger the build process and run the unit tests automatically. Resulting container files will be uploaded to the [GitHub packages](https://github.com/orgs/comorment/packages?repo_name=containers) when new Git tags are pushed on the main branch. +### Offline builds + +It should be possible to build the Docker containers offline, using the provided Dockerfiles. +Suggested steps for building the containers offline are as follows: + +```{bash} +docker build --platform=linux/amd64 -t ghcr.io/comorment/ -f dockerfiles//Dockerfile . +``` + +To successfully build the `r` container, a GITHUB_PAT file with a valid GitHub Personal Access Token (PAT) is required, which can be created in the GitHub account settings and should have at least `read:packages` scope. +This is needed to access a number of R package GitHub repositories. + +```{bash} +docker build --secret id=github_pat,src=~/secrets/GITHUB_PAT --platform=linux/amd64 -t ghcr.io/comorment/r -f dockerfiles/r/Dockerfile . +``` + + ## Note about NREC machine (old) We use NREC machine to develop and build containers. diff --git a/docker/dockerfiles/gwas/Dockerfile b/docker/dockerfiles/gwas/Dockerfile index 98da4252..69afd7cb 100644 --- a/docker/dockerfiles/gwas/Dockerfile +++ b/docker/dockerfiles/gwas/Dockerfile @@ -202,4 +202,18 @@ COPY /scripts/install_liftover.sh /tmp/liftover/ RUN bash /tmp/liftover/install_liftover.sh && \ rm -rf /tmp/liftover +# OPERA +WORKDIR /tmp/opera +RUN wget --progress=dot:giga https://github.com/wuyangf7/OPERA/blob/main/opera_Linux.zip?raw=true -O opera_Linux.zip && \ + unzip opera_Linux.zip && \ + rm opera_Linux.zip && \ + mv opera_Linux/opera_Linux /usr/bin/opera + +# SMR +WORKDIR /tmp/smr +RUN wget --progress=dot:giga https://yanglab.westlake.edu.cn/software/smr/download/smr-1.4.0-linux-x86_64.zip && \ + unzip smr-1.4.0-linux-x86_64.zip && \ + rm smr-1.4.0-linux-x86_64.zip && \ + mv smr-1.4.0-linux-x86_64/smr /usr/bin/smr + WORKDIR /tools diff --git a/docker/dockerfiles/python3/Dockerfile b/docker/dockerfiles/python3/Dockerfile index c9564a17..97061ec1 100644 --- a/docker/dockerfiles/python3/Dockerfile +++ b/docker/dockerfiles/python3/Dockerfile @@ -22,14 +22,21 @@ RUN conda install mamba -n base -c conda-forge && \ # keep the list of packages sorted alphabetically # https://www.online-utility.org/text/sort.jsp RUN mamba install python=3.12.12 \ + BeautifulSoup4=4.14.3 \ + bottleneck=1.6.0 \ configparser=7.2.0 \ dask=2025.11.0 \ fastparquet=2024.11.0 \ + fsspec=2026.2.0 \ + gcsfs=2026.2.0 \ graphviz=14.0.5 \ "h5py=3.15.1=nompi*" \ + html5lib=1.1 \ + Jinja2=3.1.6 \ jupyterlab=4.5.0 \ lightgbm=4.6.0 \ lifelines=0.30.0 \ + lxml=6.0.2 \ imbalanced-learn=0.14.0 \ intervaltree=3.1.0 \ matplotlib-venn=1.1.2 \ @@ -40,18 +47,29 @@ RUN mamba install python=3.12.12 \ networkx=3.6 \ numdifftools=0.9.41 \ numba=0.62.1 \ + numexpr=2.14.1 \ numpy=2.3.5 \ + odfpy=1.4.1 \ openpyxl=3.1.5 \ pandas=2.3.3 \ + pandas-gbq=0.28.0 \ pandas-plink=2.3.2 \ paramiko=4.0.0 \ psutil=7.1.3 \ + psycopg2=2.9.11 \ pyarrow=22.0.0 \ pydot=4.0.1 \ + pymysql=1.1.2 \ + pyqt=5.15.11 \ + pyreadr=0.5.4 \ pyreadstat=1.3.2 \ pytables=3.10.2 \ python-graphviz=0.21 \ + python-blosc2=4.0.0 \ + pyxlsb=1.0.10 \ pyyaml=6.0.3 \ + qtpy=2.4.3 \ + s3fs=2026.2.0 \ scikit-learn=1.7.2 \ scikit-survival=0.25.0 \ scipy=1.16.3 \ @@ -59,8 +77,12 @@ RUN mamba install python=3.12.12 \ semantic_version=2.10.0 \ shap=0.48.0 \ statsmodels=0.14.5 \ + SQLAlchemy=2.0.46 \ + tabulate=0.9.0 \ + xarray=2026.2.0 \ xgboost=3.1.2 \ xlrd=2.0.2 \ + xlsxwriter=3.2.9 \ xmltodict=1.0.2 \ --yes && \ mamba clean -a -y diff --git a/docker/scripts/R/bioconductor.R b/docker/scripts/R/bioconductor.R index 1cd71e08..2bccb724 100644 --- a/docker/scripts/R/bioconductor.R +++ b/docker/scripts/R/bioconductor.R @@ -23,6 +23,7 @@ packages <- c( 'GWASTools', 'limma', 'org.Hs.eg.db', + 'rhdf5', 'rtracklayer', 'snpStats', 'survcomp', diff --git a/docker/scripts/R/cran.R b/docker/scripts/R/cran.R index 6f74d61e..ca732476 100644 --- a/docker/scripts/R/cran.R +++ b/docker/scripts/R/cran.R @@ -72,6 +72,8 @@ packages <- list( 'lightgbm', 'lmerTest', 'magrittr', + 'matlab', + 'matlab2r', 'MatrixEQTL', 'matrixStats', 'MendelianRandomization', @@ -97,6 +99,8 @@ packages <- list( 'remotes', 'RiskScorescvd', 'rjags', + 'rmatio', + 'R.matlab', 'ROCR', 'rmarkdown', 'rstan', @@ -120,6 +124,7 @@ packages <- list( 'vctrs', 'xgboost', 'zlib') + # install package from CRAN and quit with error if installation fails # for (package in names(packages)) { diff --git a/docker/scripts/R/github.R b/docker/scripts/R/github.R index 6d762fdc..e5f69a58 100644 --- a/docker/scripts/R/github.R +++ b/docker/scripts/R/github.R @@ -26,6 +26,7 @@ packages <- list( "qingyuanzhao/mr.raps" = "27b96f31e26ae97a395422bd757f514a7f96cc85", "MRCIEU/TwoSampleMR" = "578c68fa754c57d764553812bf85d69ecf43b011", "MRCIEU/gwasglue" = "c2d5660eed389e1a9b3e04406b88731d642243f1", + # misc "noahlorinczcomi/MRBEE" = "6295549a1f5a158c6701eb793646d60c8aef11ca", "norment/normentR" = "dfa1fbae9587db6c3613b0405df4f9cfa98ee0e1", "psychgen/phenotools" = "62dd11e111d8d952837c9f207557e9b297ba56bc", @@ -38,7 +39,9 @@ packages <- list( "JBPG/Gsens" = "6cac02ba1ccaf38870e2526076f0306c0cf0dc0a", "MRCIEU/genetics.binaRies" = "2fcd3ee3088b729c7eb34cf2aac9dc2e04fe4412", "mrcieu/gwasglue2" = "c93b3a1fca7d2eae5d40bd62117091b1ad57f0fa", - "gqi/MRMix" = "56afdb2bc96760842405396f5d3f02e60e305039") + "gqi/MRMix" = "56afdb2bc96760842405396f5d3f02e60e305039", + "JianYang-Lab/gsmr2" = "e7953cc136ad1cfae14674facde21e90caddea5f", + "n-mounier/MRlap" = "660f026864f8bfbbad5a8206bdff7d58f5d5d05b") # install package from GitHub and quit with error if installation fails for (package in names(packages)) { diff --git a/docker/scripts/apt_get_essential.sh b/docker/scripts/apt_get_essential.sh index 9c182203..a5594476 100644 --- a/docker/scripts/apt_get_essential.sh +++ b/docker/scripts/apt_get_essential.sh @@ -20,7 +20,7 @@ apt-get update && apt-get install -y --no-install-recommends \ less=590-2ubuntu2.1 \ libatlas-base-dev=3.10.3-13ubuntu1 \ libcurl4-openssl-dev=8.5.0-2ubuntu10.6 \ - libgomp1=14.2.0-4ubuntu2~24.04 \ + libgomp1=14.2.0-4ubuntu2~24.04.1 \ libgsl-dev=2.7.1+dfsg-6ubuntu2 \ libnss3=2:3.98-1build1 \ libpcre2-dev=10.42-4ubuntu2.1 \ diff --git a/docker/scripts/install_eagle.sh b/docker/scripts/install_eagle.sh index b5dae4b8..957b5aed 100644 --- a/docker/scripts/install_eagle.sh +++ b/docker/scripts/install_eagle.sh @@ -2,7 +2,7 @@ set -euo pipefail apt-get update && apt-get install --no-install-recommends \ - libopenblas-dev=0.3.26+ds-1 \ + libopenblas-dev=0.3.26+ds-1ubuntu0.1 \ -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/scripts/install_htslib.sh b/docker/scripts/install_htslib.sh index ed325cf5..f6d4a830 100644 --- a/docker/scripts/install_htslib.sh +++ b/docker/scripts/install_htslib.sh @@ -5,7 +5,7 @@ set -euo pipefail apt-get update && apt-get install --no-install-recommends \ libbz2-dev=1.0.8-5.1build0.1 \ liblzma-dev=5.6.1+really5.4.5-1ubuntu0.2 \ - libssl-dev=3.0.13-0ubuntu3.6 \ + libssl-dev=3.0.13-0ubuntu3.7 \ -y apt-get clean && \ diff --git a/docker/scripts/install_switchError.sh b/docker/scripts/install_switchError.sh index 8780733b..7a5a7d11 100644 --- a/docker/scripts/install_switchError.sh +++ b/docker/scripts/install_switchError.sh @@ -3,7 +3,7 @@ set -euo pipefail apt-get update && apt-get install --no-install-recommends \ r-mathlib=4.3.3-2build2 \ - libboost-iostreams1.83-dev=1.83.0-2.1ubuntu3.1 \ + libboost-iostreams1.83-dev=1.83.0-2.1ubuntu3.2 \ -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/docs/containers/gwas.md b/docs/containers/gwas.md index 4ae4a0b7..2cffc886 100644 --- a/docs/containers/gwas.md +++ b/docs/containers/gwas.md @@ -48,6 +48,7 @@ List of software included in the container: | liftOver[^liftover] | latest | [permissive](https://genome-store.ucsc.edu) | metal[^metal] | 2020-05-05 | - | minimac4[^minimac4] | v4.1.6 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | OPERA[^opera] | v1.18 | - | plink[^plink19] | v1.9.0-b.7.11 64-bit (19 Aug 2025) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | plink2[^plink2] | v2.0.0-a.6.23LM 64-bit Intel (24 Aug 2025) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | plink2_avx2[^plink2] | v2.0.0-a.6.23LM AVX2 Intel (24 Aug 2025) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) @@ -57,6 +58,7 @@ List of software included in the container: | samtools[^bcftools] | v1.19.2 | [MIT/ExpatD](https://github.com/samtools/samtools/blob/develop/LICENSE) | shapeit4.2[^shapeit4] | v4.2.2 | [MIT](https://opensource.org/licenses/MIT) | simu_linux[^simu] | v0.9.4 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | SMR[^smr] | v1.3.2 | [GPLv2](https://www.gnu.org/licenses/gpl-2.0.html) | snptest[^snptest] | v2.5.6 | [permissive](https://www.chg.ox.ac.uk/~gav/snptest/#download) | switchError[^switcherror] | 6e688b1 | [MIT](https://opensource.org/licenses/MIT) | vcftools[^vcftools] | 0.1.17 (git SHA: d511f469e) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) @@ -101,6 +103,8 @@ List of software included in the container: [^minimac4]: [https://genome.sph.umich.edu/wiki/Minimac4](https://genome.sph.umich.edu/wiki/Minimac4) +[^opera]: Wu Y., Qi T., Wray N.R., Visscher P.M., Zeng J. & Yang J. (2023) Joint analysis of GWAS and multi-omics QTL summary statistics reveals a large fraction of GWAS signals shared with molecular phenotypes. Cell Genomics. + [^plink19]: Christopher C Chang, Carson C Chow, Laurent CAM Tellier, Shashaank Vattikuti, Shaun M Purcell, James J Lee, Second-generation PLINK: rising to the challenge of larger and richer datasets, GigaScience, Volume 4, Issue 1, December 2015, s13742–015–0047–8, [^plink2]: [https://www.cog-genomics.org/plink/2.0/](https://www.cog-genomics.org/plink/2.0/) @@ -115,6 +119,8 @@ List of software included in the container: [^simu]: [https://github.com/precimed/simu](https://github.com/precimed/simu) +[^smr]: Zhu Z, Zhang F, Hu H, Bakshi A, Robinson MR, Powell JE, Montgomery GW, Goddard ME, Wray NR, Visscher PM & Yang J (2016) Integration of summary data from GWAS and eQTL studies predicts complex trait gene targets. Nature Genetics, 48:481-487. + [^snptest]: [https://www.chg.ox.ac.uk/~gav/snptest/#download](https://www.chg.ox.ac.uk/~gav/snptest/#download) [^switcherror]: O'Connell J, Gurdasani D, Delaneau O, Pirastu N, Ulivi S, et al. (2014) A General Approach for Haplotype Phasing across the Full Spectrum of Relatedness. PLOS Genetics 10(4): e1004234. diff --git a/tests/extras/r.R b/tests/extras/r.R index 79e76fab..91887a94 100644 --- a/tests/extras/r.R +++ b/tests/extras/r.R @@ -106,6 +106,7 @@ libraries_to_check <- c( "gplots", "gridExtra", "Gsens", + "gsmr2", "gwasglue", "gwasglue2", "gwasurvivr", @@ -126,12 +127,15 @@ libraries_to_check <- c( "lubridate", "kinship2", "MASS", + "matlab", + "matlab2r", "MatrixEQTL", "matrixStats", "MendelianRandomization", "mgcv", "miniCRAN", "MRBEE", + "MRlap", "MVMR", "modelr", "moments", @@ -150,6 +154,9 @@ libraries_to_check <- c( "pracma", "PredictABEL", "psych", + "rhdf5", + "R.matlab", + "rmatio", "RColorBrewer", "Rcpp", "readxl", @@ -177,6 +184,7 @@ libraries_to_check <- c( "tree", "viridis") + # Define a test test_that("Required libraries are installed", { missing_libraries <- setdiff(libraries_to_check, installed.packages()[, "Package"]) diff --git a/tests/test_gwas.py b/tests/test_gwas.py index b3f2fe8b..030a3949 100644 --- a/tests/test_gwas.py +++ b/tests/test_gwas.py @@ -37,7 +37,6 @@ f'--mount type=bind,source={cwd},target={cwd} ' + '{custom_mount}' + '-w /home/ ' + - # '--platform linux/amd64 ' + 'ghcr.io/comorment/gwas') except FileNotFoundError as err: # neither singularity nor docker found, fall back to plain python @@ -339,3 +338,16 @@ def test_gwas_duohmm(): call = f'{PREFIX} duohmm' out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 + + +def test_gwas_opera(): + """test opera""" + call = f'{PREFIX} opera' + out = subprocess.run(call.split(' '), shell=True, check=False, capture_output=True) + assert out.returncode == 0 + +def test_gwas_smr(): + """test smr""" + call = f'{PREFIX} smr' + out = subprocess.run(call.split(' '), shell=True, check=False, capture_output=True) + assert out.returncode == 0 diff --git a/tests/test_python3.py b/tests/test_python3.py index 399089b2..3f456348 100644 --- a/tests/test_python3.py +++ b/tests/test_python3.py @@ -106,42 +106,63 @@ def test_python3_miniwdl(): def test_python3_packages(): packages = [ + 'blosc2', + 'bottleneck', + 'bs4', # BeautifulSoup4 'configparser', 'crc32c', 'dask', - 'fastparquet', 'dxpy', + 'fastparquet', + 'fsspec', + 'gcsfs', 'graphviz', 'h5py', + 'html5lib', 'imblearn', 'intervaltree', 'ldpred', 'lifelines', 'lightgbm', + 'lxml', 'matplotlib', 'matplotlib_venn', 'numba', 'numdifftools', 'numpy', + 'numexpr', + 'odf', 'openpyxl', 'pandas', + 'pandas_gbq', 'pandas_plink', 'paramiko', 'plinkio', + 'psycopg2', 'pyarrow', 'pydot', 'pyliftover', + 'pymysql', + 'PyQt5', + 'pyreadr', 'pyreadstat', + 'pyxlsb', + 'qtpy', 'redcap', # pycap 'scipy', + 's3fs', 'seaborn', 'semantic_version', 'shap', 'sklearn', 'sksurv', + 'sqlalchemy', 'statsmodels', 'tables', + 'tabulate', + 'xarray', 'xlrd', + 'xlsxwriter', 'xmltodict', 'xgboost', 'yaml', # pyyaml diff --git a/version/version.py b/version/version.py index 2ca129dd..3db9b64c 100644 --- a/version/version.py +++ b/version/version.py @@ -1,5 +1,5 @@ _MAJOR = "1" -_MINOR = "14" +_MINOR = "15" # On main and in a nightly release the patch should be one ahead of the last # released build. _PATCH = "0"