Add Docker-based bit-compare CI workflow

.github/workflows/bit-compare-docker.yml

@@ -0,0 +1,135 @@
+name: bit-compare-docker
+
+# Controls when the action will run
+on:
+
+  # Trigger on pushes to main only. Feature branches and cycle LTS branches
+  # are validated via their pull request — running on push as well would
+  # double-spend minutes for internal PRs, and LTS branches are protected.
+  push:
+    branches:
+    - main
+
+  # Trigger the workflow on all pull requests
+  pull_request: ~
+
+  # Allow workflow to be dispatched on demand
+  workflow_dispatch: ~
+
+concurrency:
+  group: bit-compare-docker-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  CACHE_SUFFIX: v1         # Increase to force new cache to be created
+
+jobs:
+  ci:
+    name: ci
+
+    strategy:
+      fail-fast: false  # false: try to complete all jobs
+
+      matrix:
+        name:
+          - linux gcc-12
+
+        include:
+
+          - name: linux gcc-12
+            os: ubuntu-22.04
+            gcc: '12'
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 180
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+    - name: Derive cycle from bundle.yml
+      run: |
+        set -eu
+        version=$(awk '/^name[[:space:]]*:[[:space:]]*ifs-bundle/{found=1} found && /^version[[:space:]]*:/{print $NF; exit}' bundle.yml)
+        IFS=. read -r MAJ MIN PAT <<< "$version"
+        OIFS_VERSION="${MAJ}r${MIN}"
+        CYCLE_BRANCH="openifs-lts/CY${MAJ}R${MIN}.${PAT}"
+        # Fall back to main when the cycle LTS branch has not been cut yet
+        # (e.g. a new cycle has landed on main but its openifs-lts/CY* branch
+        # does not yet exist on the remote).
+        if [ -z "$(git ls-remote --heads origin "$CYCLE_BRANCH")" ]; then
+          echo "Cycle branch ${CYCLE_BRANCH} not found on remote — falling back to main"
+          CYCLE_BRANCH=main
+        fi
+        echo "OIFS_VERSION=${OIFS_VERSION}"  >> $GITHUB_ENV
+        echo "CYCLE_BRANCH=${CYCLE_BRANCH}"  >> $GITHUB_ENV
+
+    - name: Environment
+      env:
+        PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
+      run: |
+        set -eu
+        # WORK_DIR sits outside github.workspace because the test phase copies
+        # the workspace into WORK_DIR/build_dir_test/...; a destination inside
+        # the source would recurse into itself in shutil.copytree.
+        WORK_DIR="${{ runner.temp }}/_oifs_docker_ci"
+        echo "CI_DIR=${{ github.workspace }}/scripts/ci/docker_ci" >> $GITHUB_ENV
+        echo "WORK_DIR=${WORK_DIR}"                                >> $GITHUB_ENV
+        echo "NORMS_DIR=${WORK_DIR}/control_saved_norms"           >> $GITHUB_ENV
+        echo "REPORTS_DIR=${WORK_DIR}/ci_reports"                  >> $GITHUB_ENV
+        CONTROL_BRANCH="${PR_BASE_REF:-$CYCLE_BRANCH}"
+        echo "CONTROL_BRANCH=${CONTROL_BRANCH}"                    >> $GITHUB_ENV
+
+    - name: Cache control NORMs
+      uses: actions/cache@v4
+      with:
+        path: ${{ env.NORMS_DIR }}
+        key: control-norms-gcc${{ matrix.gcc }}-${{ env.OIFS_VERSION }}-${{ github.event.pull_request.base.sha || github.sha }}-${{ env.CACHE_SUFFIX }}
+
+    - name: Set up Python venv
+      run: |
+        set -eu
+        python3 -m venv "${WORK_DIR}/venv"
+        source "${WORK_DIR}/venv/bin/activate"
+        python3 -m pip install --upgrade pip
+        python3 -m pip install gitpython pyyaml
+
+    - name: Render CI config
+      run: |
+        set -eu
+        mkdir -p "${WORK_DIR}"
+        cat > "${WORK_DIR}/ci_test_docker.yml" <<EOF
+        openifs_version                : "${OIFS_VERSION}"
+        openifs_repo_url               : "https://github.com/${{ github.repository }}.git"
+        control_branch                 : "${CONTROL_BRANCH}"
+        test_branch                    : "${{ github.workspace }}"
+        base_docker_image              : "${{ matrix.gcc }}"
+        docker_template                : "./Dockerfile.ci"
+        include_openifs_data_downloads : False
+        openifs_build_docker_dir       : "${WORK_DIR}"
+        ci_reports                     : "${REPORTS_DIR}"
+        control_saved_norms_dir        : "${NORMS_DIR}"
+        force_reclone                  : True
+        reuse_control_if_present       : True
+        openifs_test_extra_flags       : "--without-single-precision --cmake=BUILD_ifsbench=OFF --clean"
+        EOF
+
+    - name: Build & Test
+      id: build-test
+      run: |
+        set -eu
+        cd "$CI_DIR"
+        source "${WORK_DIR}/venv/bin/activate"
+        echo "::group::ci-oifs-docker.py"
+        python3 ci-oifs-docker.py -c "${WORK_DIR}/ci_test_docker.yml"
+        echo "::endgroup::"
+
+    - name: Upload reports
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: bit-compare-reports-gcc${{ matrix.gcc }}
+        path: ${{ env.REPORTS_DIR }}/
+        if-no-files-found: warn

.gitignore

@@ -21,4 +21,4 @@ __pycache__/
 .DS_Store
 
 # User-specific Docker config
-scripts/docker/config/my_config.yml
+scripts/bootstrap/docker/config/my_config.yml

README.md

@@ -148,11 +148,11 @@ END ifstest on OpenIFS build
 
 The previous section, [Installing and Building OpenIFS](#installing-and-building-openifs), describes the pre-requisites and build process for OpenIFS on a generic Linux based system.
 
-[create-oifs-docker.py](scripts/docker/create-oifs-docker.py) and associated scripts and configuration automates the process described in section [Installing and Building OpenIFS](#installing-and-building-openifs), by creating a Docker container, installing OpenIFS and dependencies and then building OpenIFS and running the test.
+[create-oifs-docker.py](scripts/bootstrap/docker/create-oifs-docker.py) and associated scripts and configuration automates the process described in section [Installing and Building OpenIFS](#installing-and-building-openifs), by creating a Docker container, installing OpenIFS and dependencies and then building OpenIFS and running the test.
 
-* Please go to [OpenIFS Docker Builder](scripts/docker/README.md) for details about the Docker install.
+* Please go to [OpenIFS Docker Builder](scripts/bootstrap/docker/README.md) for details about the Docker install.
 
-[create-oifs-docker.py](scripts/docker/create-oifs-docker.py) and the resulting Docker development has been tested on macOS but it can be applied to other systems, as long as Docker is installed and the appropriate python dependencies are available.
+[create-oifs-docker.py](scripts/bootstrap/docker/create-oifs-docker.py) and the resulting Docker development has been tested on macOS but it can be applied to other systems, as long as Docker is installed and the appropriate python dependencies are available.
 
 ## Install the static input data files for OpenIFS
 

oifs-config.edit_me.sh

@@ -67,8 +67,6 @@ export OIFS_TEST="${OIFS_HOME}/scripts/build_test"
 export OIFS_LOGFILE="${OIFS_HOME}/openifs-test.log"
 #---Path to dir containing scripts to run OpenIFS experiment
 export OIFS_RUN_SCRIPT="${OIFS_HOME}/scripts/exp_3d"
-#---Path to OpenIFS docker scripts and yaml config for docker
-export OIFS_DOCKER="${OIFS_HOME}/scripts/docker"
 
 alias oenv="env -0 | sort -z | tr '\0' '\n' | grep -a OIFS_"
 

scripts/docker/Dockerfile → scripts/bootstrap/docker/Dockerfile

@@ -30,23 +30,24 @@ RUN apt update && \
     apt install -y git cmake python3 python3-ruamel.yaml python3-yaml python3-venv \
         libomp-dev libboost-dev libboost-date-time-dev libboost-filesystem-dev \
         libboost-serialization-dev libboost-program-options-dev netcdf-bin \
-        libnetcdf-dev libnetcdff-dev liblapack-dev libeigen3-dev vim emacs \
+        libnetcdf-dev libnetcdff-dev liblapack-dev libeigen3-dev vim less \
         wget bc ca-certificates && \
     update-ca-certificates && \
     cd /tmp && \
     wget https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.10.tar.gz && \
     tar -xvf openmpi-5.0.10.tar.gz && \
     cd openmpi-5.0.10 && \
     ./configure --disable-libxml2 --prefix=/usr/local && \
-    make -j4 all && \
+    make -j"$(nproc)" all && \
     make install && \
     ldconfig && \
     cd / && \
     rm -rf /tmp/openmpi-5.0.10* && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Create user and directory structure
+# Create user and directory structure.
+# Container security: run as unprivileged uid-1000 'openifs' user.
 RUN groupadd --gid 1000 openifs && \
     useradd --uid 1000 --gid openifs --shell /bin/bash --create-home openifs && \
     mkdir -p /home/openifs/${OPENIFS_DIR} && \
@@ -76,7 +77,10 @@ RUN sed -i 's|export OIFS_HOME="${HOME}/.*"|export OIFS_HOME="${HOME}/'${OPENIFS
     cd $OIFS_EXPT/ab7z/2016092500 && \
     cp $OIFS_RUN_SCRIPT/oifs-run .
 
-# Download and extract all OpenIFS data files in one layer
+# Download and extract all OpenIFS data files in a separate RUN layer so that
+# the source-copy layer above can be cached and re-used across rebuilds. The
+# host bash installer fetches the same archives in a single shell function
+# since there is no layer cache to optimise for.
 WORKDIR /home/openifs/${OPENIFS_DIR}/openifs-data
 RUN mkdir -p ifsdata rtables ${CLIMATE_VERSION} && \
     cd rtables && \

scripts/docker/README.md → scripts/bootstrap/docker/README.md

@@ -15,7 +15,7 @@ Automated Docker container creation for the stand-alone OpenIFS model.
 It's recommended to use a virtual environment to install the required Python packages:
 
 ```bash
-cd scripts/docker
+cd scripts/bootstrap/docker
 
 # Create a virtual environment
 python3 -m venv openifs-env
@@ -75,19 +75,19 @@ The configuration file, `config/create_openifs_docker.yml`, controls the build o
 # OpenIFS version (used for directory naming and image tagging)
 openifs_version: "48r1"
 
-# Git branch to extract from repository
-openifs_branch: "main"
+# Where to get the OpenIFS source tree. Three modes:
+#   a branch name    -> clone that branch from openifs_repo_url (default)
+#   a directory path -> copy that local checkout into the Docker build directory
+#   empty / not set  -> auto-detect the checkout containing this script
+openifs_source: "main"
 
-# Repository URL (requires SSH access)
-openifs_repo_url: "git@github.com:ecmwf-ifs/openifs.git"
+# Repository URL
+openifs_repo_url: "https://github.com/ecmwf-ifs/openifs.git"
 
 # SCM experiment data URL (tar.gz or tar file)
 scm_url: https://openifs.ecmwf.int/data/scm/48r1/scm_openifs_48r1.tar.gz
 
-# Clone repository (True) or use existing directory (False)
-clone_openifs: True
-
-# Force removal of existing clone without prompting
+# Force removal of existing source directory before re-staging
 force_reclone: False
 
 # Run openifs build command after building image
@@ -159,11 +159,15 @@ This is a clean container in which `source oifs-config.edit_me.sh` is run upon s
 - Validates base Docker image is from official sources (security)
 - Checks if base image exists locally, pulls if needed
 
-### Step 2: Repository Setup
+### Step 2: Source Setup
+
+Resolves `openifs_source` using the same three-mode convention as the CI driver:
+
+- **Branch name** (default, e.g. `"main"`) — shallow-clones from `openifs_repo_url` into the build directory
+- **Directory path** (e.g. `"~/src/openifs"`) — copies that local checkout into the build directory, skipping transient artefacts (`.git`, `build/`, `__pycache__`, etc.)
+- **Empty** — auto-detects the OpenIFS checkout that contains this script (useful when running from inside the repository)
 
-- Shallow clones OpenIFS from specified branch (if `clone_openifs: True`)
-- Copies SCM experiment data to build directory
-- Updates configuration files with correct paths
+`force_reclone: True` removes and re-stages the source regardless of mode.
 
 ### Step 3: Docker Build
 
@@ -185,10 +189,10 @@ This is a clean container in which `source oifs-config.edit_me.sh` is run upon s
 - Set `force_rebuild: True` to rebuild
 - Or manually remove the image
 
-### Clone Directory Exists
+### Source Directory Exists
 
-- Set `force_reclone: True` to remove and re-clone
-- Or set `clone_openifs: False` to use existing directory
+- Set `force_reclone: True` to remove and re-stage the source
+- Or change `openifs_source` to a different branch or path
 
 ### Base Image Not Found
 

scripts/docker/config/create_openifs_docker.yml → scripts/bootstrap/docker/config/create_openifs_docker.yml

@@ -9,45 +9,50 @@
 # nor does it submit to any jurisdiction.
 #
 
-# Openifs version
+# OpenIFS version (used for directory naming, image tagging, and as a
+# component of every data-archive URL below).
 openifs_version : "48r1"
 
-# The OpenIFS branch to extract from repo
-openifs_branch : "main"
-
-# Climate data version
+# Climate data version (embedded in the climate tarball filename).
 climate_version : "climate.v020"
 
-# Base URL for OpenIFS data files
+# OpenIFS repository (must be accessible from the machine running this script).
+openifs_repo_url : "https://github.com/ecmwf-ifs/openifs.git"
+
+# Base URL for OpenIFS data files.  Per-version ifsdata, rtables and
+# climate tarballs are derived from this in docker_lib.modify_dockerfile().
 openifs_data_base_url : "https://sites.ecmwf.int/openifs/openifs-download/ifsdata"
 
-# SCM experiment data URL (tar.gz or tar file)
+# SCM experiment data archive.
 scm_url : "https://openifs.ecmwf.int/data/scm/48r1/scm_openifs_48r1.tar.gz"
 
-# OpenIFS experiment package (low res)
+# Low-resolution 3-D experiment package.
 openifs_expt_url : "https://sites.ecmwf.int/openifs/openifs-download/experiments/48r1/2016-09-25_Karl/ab7z.tar.gz"
 
+# The OpenIFS branch to build inside the Docker image.
+# openifs_source controls where the OpenIFS tree comes from.
+# Three modes — same convention as the CI test_branch key:
+#   empty / not set  -> auto-detect the OpenIFS checkout containing this script
+#   a directory path -> copy that local checkout into the Docker build directory
+#                       (transient artefacts .git, build/, __pycache__ are skipped)
+#   a branch name    -> clone that branch from openifs_repo_url
+openifs_source : ""
+
 # Docker image - the following images are known to work.
 #base_docker_image : "12.3.0-bullseye"
-base_docker_image : "13"
+base_docker_image : "14"
 
 # Path for the Docker file template
 docker_template : "./Dockerfile"
 
 # Directory to store the OpenIFS Dockerfile and base OpenIFS version and files
 openifs_build_docker_dir : "~/oifs_docker_create_dir"
 
-# The URL for the OpenIFS repository, where the branch will be extracted
-openifs_repo_url : "git@github.com:ecmwf-ifs/openifs.git" 
-
-# True to clone OpenIFS repository, False to use existing
-clone_openifs : True
-
-# Force removal of existing clone directory without prompting
+# Force removal of existing clone/staged directory before re-staging
 force_reclone : True
 
 # Force rebuild of Docker image even if it exists
-force_rebuild : False
+force_rebuild : True
 
 # Run tests after building image
 run_tests : True
@@ -60,4 +65,3 @@ remove_test_container : False
 
 # Skip URL validation (useful if behind firewall or for offline builds)
 skip_url_validation : False
-