diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..4ecb75c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ + version: 2 + updates: + - package-ecosystem: "github-actions" + directory: "/.github/workflows" + schedule: + interval: "monthly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fda3d4e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,106 @@ +name: ci + +on: + push: + branches: + - devel + pull_request: + branches: + - devel + +permissions: + contents: read + +concurrency: + group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} + cancel-in-progress: true + +jobs: + spindle-serial-ubuntu: + name: Testsuite (Serial, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + + - name: Setup Docker Compose + uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746 + with: + version: latest + + - name: Build spindle-serial-ubuntu image + id: serial-ubuntu-build + run: | + cd containers/spindle-serial-ubuntu + docker compose --progress=plain build + + - name: Bring spindle-serial-ubuntu up + id: serial-ubuntu-up + run: | + cd containers/spindle-serial-ubuntu + docker compose up -d + + - name: Verify munge works in spindle-serial-ubuntu + id: serial-ubuntu-munge + run: | + docker exec spindlenode bash -c 'munge -n | unmunge' + + - name: Run spindle-serial-ubuntu testsuite + id: serial-ubuntu-testsuite + run: | + docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./runTests' + + - name: Bring spindle-serial-ubuntu down + id: serial-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-serial-ubuntu + docker compose down + + spindle-flux-ubuntu: + name: Testsuite (Flux, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + + - name: Setup Docker Compose + uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746 + with: + version: latest + + - name: Build spindle-flux-ubuntu image + id: flux-ubuntu-build + run: | + cd containers/spindle-flux-ubuntu + docker compose --progress=plain build + + - name: Bring spindle-flux-ubuntu up + id: flux-ubuntu-up + run: | + cd containers/spindle-flux-ubuntu + docker compose up -d --wait --wait-timeout 60 + + - name: Verify munge works in spindle-flux-ubuntu + id: flux-ubuntu-munge + run: | + docker exec node-1 bash -c 'munge -n | unmunge' + + - name: Run spindle-flux-ubuntu testsuite + id: flux-ubuntu-testsuite + run: | + docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./runTests --nodes=${workers} --tasks-per-node=3' + + - name: Bring spindle-flux-ubuntu down + id: flux-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-flux-ubuntu + docker compose down + diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 0000000..c27dca0 --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,30 @@ +name: Build & Push Slurm Base Container +on: + workflow_dispatch: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build & Push Slurm Base Image + uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 + with: + context: ./containers/spindle-slurm-ubuntu/base + platforms: linux/amd64 + push: true + tags: ghcr.io/paratoolsinc/spindle-slurm-base:latest + cache-from: type=registry,ref=ghcr.io/paratoolsinc/spindle-slurm-base:buildcache + cache-to: type=registry,ref=ghcr.io/paratoolsinc/spindle-slurm-base:buildcache,mode=max diff --git a/containers/spindle-flux-ubuntu/Dockerfile b/containers/spindle-flux-ubuntu/Dockerfile new file mode 100644 index 0000000..3af607a --- /dev/null +++ b/containers/spindle-flux-ubuntu/Dockerfile @@ -0,0 +1,67 @@ +# This is based on the Flux Container Tutorial +# See https://flux-framework.readthedocs.io/en/latest/tutorials/containers +ARG flux_sched_version=noble +FROM fluxrm/flux-sched:${flux_sched_version} AS builder +ARG replicas=4 +ENV workers=${replicas} +USER root + +RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ + && apt-get -qq install -y --no-install-recommends \ + autotools-dev \ + autoconf \ + automake \ + cmake \ + git \ + python3 \ + openssh-server \ + openssh-client \ + libdb-dev \ + apt-utils \ + dnsutils \ + iputils-ping \ + python3-pip \ + libgcrypt20 \ + libgcrypt20-dev \ + gdb \ + software-properties-common + +ARG USER=fluxuser +ARG CONFIG_ROOT=containers/spindle-flux-ubuntu + +# Allow fluxuser to run as other users so it can start munged +RUN sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers" + +# Configure flux +ENV STATE_DIR=/var/lib/flux +RUN mkdir -p ${STATE_DIR} /etc/flux/system /etc/flux/system/cron.d /etc/flux/config /run/flux /etc/flux/imp/conf.d +COPY ${CONFIG_ROOT}/flux/imp.toml /etc/flux/imp/conf.d/ +COPY ${CONFIG_ROOT}/flux/broker.toml /etc/flux/config/ +RUN mkdir -p /etc/flux/system/cron.d && \ + mkdir -p /mnt/curve && \ + flux keygen /mnt/curve/curve.cert && \ + flux R encode --hosts="node-[1-${workers}]" > /etc/flux/system/R && \ + chown -R ${USER}:${USER} /run/flux ${STATE_DIR} /mnt/curve/curve.cert + +# Build Spindle +WORKDIR /home/${USER} +# Copy the whole git repo into the container. +COPY . /home/${USER}/Spindle +COPY ${CONFIG_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN ./build_spindle.sh + +RUN chown -R ${USER}:${USER} /home/fluxuser && \ + chown -R ${USER}:${USER} /run/flux + +USER ${USER} +COPY ${CONFIG_ROOT}/scripts/flux_healthcheck.sh ./ +COPY ${CONFIG_ROOT}/scripts/entrypoint.sh ./ +ENV PATH /home/${USER}/Spindle-inst/bin:${PATH} +# Make libfabric work with fork. +ENV RDMAV_FORK_SAFE 1 +# Silence warning from hwloc about unsupported PCI device +# on GitHub-hosted runners. +ENV HWLOC_HIDE_ERRORS 2 + +ENTRYPOINT /bin/bash ./entrypoint.sh + diff --git a/containers/spindle-flux-ubuntu/docker-compose.yml b/containers/spindle-flux-ubuntu/docker-compose.yml new file mode 100644 index 0000000..e587e1d --- /dev/null +++ b/containers/spindle-flux-ubuntu/docker-compose.yml @@ -0,0 +1,70 @@ +# 4-node Flux cluster +# For information on running Flux in containers, see +# https://flux-framework.readthedocs.io/en/latest/tutorials/containers + +# `replicas` must match the number of nodes defined in the services section +x-shared-workers: + &workers + replicas: 4 + +# Ubuntu version to use (noble = 24.04) +x-shared-build-args: &shared-build-args + flux_sched_version: noble + <<: *workers + +# Docker prohibits copying files from outside of the build context. +# In order to be able to copy the whole repo into the container, +# we have to set the context to be the root of the repo. +# We then have to specify the path from there to the Dockerfile. +x-shared-build-context: &shared-build-context + context: ../.. + dockerfile: containers/spindle-flux-ubuntu/Dockerfile + args: *shared-build-args + +# Name of the node that runs the Flux broker +x-shared-environment: &shared-environment + mainHost: node-1 + <<: *workers + +networks: + flux: + driver: bridge + +# Common parameters for all nodes. +x-shared-node-parameters: &shared-node-parameters + build: *shared-build-context + networks: + - flux + environment: *shared-environment + cap_add: + - SYS_NICE # Required for libnuma + +services: + node-1: + <<: *shared-node-parameters + hostname: node-1 + container_name: node-1 + # Check whether all the workers have registered + # with the broker on the head node. + healthcheck: + test: ["CMD", "./flux_healthcheck.sh"] + start_period: 15s + interval: 5s + timeout: 10s + retries: 5 + + node-2: + <<: *shared-node-parameters + hostname: node-2 + container_name: node-2 + + node-3: + <<: *shared-node-parameters + hostname: node-3 + container_name: node-3 + + node-4: + <<: *shared-node-parameters + hostname: node-4 + container_name: node-4 + diff --git a/containers/spindle-flux-ubuntu/flux/broker.toml b/containers/spindle-flux-ubuntu/flux/broker.toml new file mode 100644 index 0000000..ad94017 --- /dev/null +++ b/containers/spindle-flux-ubuntu/flux/broker.toml @@ -0,0 +1,19 @@ +[exec] +imp = "/usr/libexec/flux/flux-imp" + +[access] +allow-guest-user = true +allow-root-owner = true + +[resource] +path = "/etc/flux/system/R" + +[bootstrap] +curve_cert = "/mnt/curve/curve.cert" +default_port = 8050 +default_bind = "tcp://eth0:%%p" +default_connect = "tcp://%%h:%%p" +hosts = [ + { host="node-[1-4]"}, +] + diff --git a/containers/spindle-flux-ubuntu/flux/imp.toml b/containers/spindle-flux-ubuntu/flux/imp.toml new file mode 100644 index 0000000..156e3b5 --- /dev/null +++ b/containers/spindle-flux-ubuntu/flux/imp.toml @@ -0,0 +1,3 @@ +[exec] +allowed-users = [ "flux", "root" ] +allowed-shells = [ "/usr/libexec/flux/flux-shell" ] diff --git a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh new file mode 100755 index 0000000..9257f85 --- /dev/null +++ b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -euxo pipefail + +mkdir -p /home/${USER}/Spindle-build +cd /home/${USER}/Spindle-build +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +make -j$(nproc) +make install + diff --git a/containers/spindle-flux-ubuntu/scripts/entrypoint.sh b/containers/spindle-flux-ubuntu/scripts/entrypoint.sh new file mode 100755 index 0000000..8d3944e --- /dev/null +++ b/containers/spindle-flux-ubuntu/scripts/entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Starts munged and the flux broker. +# +# For documentation on running Flux in containers, see +# https://flux-framework.readthedocs.io/en/latest/tutorials/containers + +brokerOptions="-Scron.directory=/etc/flux/system/cron.d \ + -Stbon.fanout=256 \ + -Srundir=/run/flux \ + -Sstatedir=${STATE_DIRECTORY:-/var/lib/flux} \ + -Slocal-uri=local:///run/flux/local \ + -Slog-stderr-level=6 \ + -Slog-stderr-mode=local" + +# Get the hostname that will resolve for the Docker bridge network. +address=$(echo $( nslookup "$( hostname -i )" | head -n 1 )) +parts=(${address//=/ }) +hostName=${parts[2]} +thisHost=(${hostName//./ }) +thisHost=${thisHost[0]} +echo $thisHost +export FLUX_FAKE_HOSTNAME=$thisHost + +# Start munged +sudo -u munge /usr/sbin/munged + +if [ ${thisHost} != "${mainHost}" ]; then + # Worker node -- wait for head node before connecting + sleep 15 + FLUX_FAKE_HOSTNAME=$thisHost flux start -o --config /etc/flux/config ${brokerOptions} sleep inf +else + # Head node + FLUX_FAKE_HOSTNAME=$thisHost flux start -o --config /etc/flux/config ${brokerOptions} sleep inf +fi + diff --git a/containers/spindle-flux-ubuntu/scripts/flux_healthcheck.sh b/containers/spindle-flux-ubuntu/scripts/flux_healthcheck.sh new file mode 100755 index 0000000..c8816db --- /dev/null +++ b/containers/spindle-flux-ubuntu/scripts/flux_healthcheck.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +FLUX_FREE_NODES=$(flux resource list -s free -n | awk '{print $2}') + +if [[ ${FLUX_FREE_NODES} -ne ${replicas} ]] ; then + echo "FAILED: Incorrect number of Flux free nodes: expected ${replicas}, got ${FLUX_FREE_NODES}" + exit 1 +fi + +echo "PASSED: Found ${FLUX_FREE_NODES} nodes out of ${replicas}; all nodes registered." + diff --git a/containers/spindle-serial-ubuntu/Dockerfile b/containers/spindle-serial-ubuntu/Dockerfile new file mode 100644 index 0000000..3070596 --- /dev/null +++ b/containers/spindle-serial-ubuntu/Dockerfile @@ -0,0 +1,60 @@ +ARG ubuntu_version=noble +FROM ubuntu:${ubuntu_version} +USER root + +RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ +# install latest pkg utils: + && apt-get -qq install -y --no-install-recommends \ + apt-utils + +RUN DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + locales \ + ca-certificates \ + wget \ + git \ + ssh \ + sudo \ + build-essential \ + pkg-config \ + autotools-dev \ + libtool \ + autoconf \ + automake \ + make \ + gfortran-13 \ + gcc-13 \ + g++-13 \ + munge \ + libmunge-dev \ + libhwloc-dev \ + mpich \ + libmpich-dev + +# Prevent hwloc from trying to use graphics cards +# as this fails when X is not running. +ENV HWLOC_COMPONENTS=-gl + +# Set up munge +RUN mkdir -p /run/munge && \ + chown munge:munge /run/munge && \ + chmod 0755 /run/munge + +ARG USER=spindleuser +ARG UID=1001 +ARG BUILD_ROOT=./containers/spindle-serial-ubuntu +COPY ${BUILD_ROOT}/scripts/add_docker_user.sh /add_docker_user.sh +RUN /add_docker_user.sh + +USER ${USER} +WORKDIR /home/${USER} +RUN mkdir -p /home/${USER}/Spindle +# Copy the Spindle repo into the container +COPY . /home/${USER}/Spindle +COPY ${BUILD_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN ./build_spindle.sh + +COPY ${BUILD_ROOT}/scripts/entrypoint.sh /home/${USER}/entrypoint.sh +ENV PATH /home/${USER}/Spindle-inst/bin:$PATH + +ENTRYPOINT /bin/bash ./entrypoint.sh + diff --git a/containers/spindle-serial-ubuntu/docker-compose.yml b/containers/spindle-serial-ubuntu/docker-compose.yml new file mode 100644 index 0000000..f90f4e8 --- /dev/null +++ b/containers/spindle-serial-ubuntu/docker-compose.yml @@ -0,0 +1,16 @@ +services: + spindlenode: + # The build context is set to ../.. so that we can copy + # the Spindle directory into the created image. + # Because we set the context, we then have to specify + # the path from the context to the Dockerfile. + build: + context: ../.. + dockerfile: containers/spindle-serial-ubuntu/Dockerfile + hostname: spindlenode + container_name: spindlenode + tty: true + # The NUMA tests require CAP_SYS_NICE. + cap_add: + - SYS_NICE + diff --git a/containers/spindle-serial-ubuntu/scripts/add_docker_user.sh b/containers/spindle-serial-ubuntu/scripts/add_docker_user.sh new file mode 100755 index 0000000..9ecfbfd --- /dev/null +++ b/containers/spindle-serial-ubuntu/scripts/add_docker_user.sh @@ -0,0 +1,7 @@ +#!/bin/sh +sudo groupadd -g ${UID} ${USER} +sudo useradd -g ${USER} -u ${UID} -d /home/${USER} -m ${USER} +# Allow user to run as other users so that munge can be started as the munge user +sudo sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers" +sudo adduser ${USER} sudo + diff --git a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh new file mode 100755 index 0000000..37b6491 --- /dev/null +++ b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -euxo pipefail + +mkdir -p /home/${USER}/Spindle-build +cd /home/${USER}/Spindle-build +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +make -j$(nproc) +make install + diff --git a/containers/spindle-serial-ubuntu/scripts/entrypoint.sh b/containers/spindle-serial-ubuntu/scripts/entrypoint.sh new file mode 100755 index 0000000..1d1f236 --- /dev/null +++ b/containers/spindle-serial-ubuntu/scripts/entrypoint.sh @@ -0,0 +1,4 @@ +printf "\nStarting munged\n" + +sudo -u munge /usr/sbin/munged --foreground + diff --git a/containers/spindle-slurm-ubuntu/base/Dockerfile b/containers/spindle-slurm-ubuntu/base/Dockerfile new file mode 100644 index 0000000..d393f9e --- /dev/null +++ b/containers/spindle-slurm-ubuntu/base/Dockerfile @@ -0,0 +1,81 @@ +ARG UBUNTU_VERSION=noble +FROM ubuntu:${UBUNTU_VERSION} +USER root + +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + apt-utils + +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + locales \ + ca-certificates \ + wget \ + git \ + ssh \ + sudo \ + build-essential \ + pkg-config \ + autotools-dev \ + libtool \ + autoconf \ + automake \ + make \ + gfortran-13 \ + gcc-13 \ + g++-13 \ + munge \ + libmunge-dev \ + libhwloc-dev \ + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + python-is-python3 \ + openssh-server \ + openssh-client \ + mariadb-client \ + libmariadb-dev \ + libhttp-parser-dev \ + psmisc \ + libjson-c-dev \ + fd-find \ + silversearcher-ag \ + vim + + +# Prevent hwloc from trying to use graphics cards +# as this fails when X is not running. +ENV HWLOC_COMPONENTS=-gl + +# Set up munge +RUN mkdir -p /run/munge && \ + chown munge:munge /run/munge && \ + chmod 0755 /run/munge + +ARG BUILD_ROOT=. +COPY ${BUILD_ROOT}/scripts/add_docker_user.sh /add_docker_user.sh + +# Slurm daemons run as $SLURM_USER +ARG SLURM_USER=slurm +ARG USER=${SLURM_USER} +ARG UID=1002 +RUN /add_docker_user.sh + +# Applications run as $USER +ARG USER=slurmuser +ARG UID=1001 +RUN /add_docker_user.sh + +ARG SLURM_VERSION=slurm-25-05-3-1 +COPY ${BUILD_ROOT}/scripts/build_slurm.sh /build_slurm.sh +RUN /build_slurm.sh + +ARG MPICH_VERSION=4.2.2 +COPY ${BUILD_ROOT}/scripts/build_mpich.sh /build_mpich.sh +RUN /build_mpich.sh + +COPY ${BUILD_ROOT}/conf/mpich.conf /etc/ld.so.conf.d/mpich.conf +RUN ldconfig + +ENV SLURM_MPI_TYPE pmi2 diff --git a/containers/spindle-slurm-ubuntu/base/conf/mpich.conf b/containers/spindle-slurm-ubuntu/base/conf/mpich.conf new file mode 100644 index 0000000..0bf940b --- /dev/null +++ b/containers/spindle-slurm-ubuntu/base/conf/mpich.conf @@ -0,0 +1 @@ +/usr/lib diff --git a/containers/spindle-slurm-ubuntu/base/scripts/add_docker_user.sh b/containers/spindle-slurm-ubuntu/base/scripts/add_docker_user.sh new file mode 100755 index 0000000..ace8c61 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/base/scripts/add_docker_user.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euxo pipefail + +sudo groupadd -g ${UID} ${USER} +sudo useradd -g ${USER} -u ${UID} -d /home/${USER} -m ${USER} +# Allow user to run as other users so that munge can be started as the munge user +sudo sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers" +sudo adduser ${USER} sudo +sudo usermod -s /bin/bash ${USER} + diff --git a/containers/spindle-slurm-ubuntu/base/scripts/build_mpich.sh b/containers/spindle-slurm-ubuntu/base/scripts/build_mpich.sh new file mode 100755 index 0000000..1fe17cc --- /dev/null +++ b/containers/spindle-slurm-ubuntu/base/scripts/build_mpich.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euxo pipefail + +mkdir mpich +pushd mpich +wget -O - https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz | tar xvz --strip-components 1 +mkdir -p build +pushd build +../configure --prefix=/usr --disable-fortran --with-slurm=/usr/include/slurm +make -j$(nproc) install +popd +popd +rm -rf mpich + diff --git a/containers/spindle-slurm-ubuntu/base/scripts/build_slurm.sh b/containers/spindle-slurm-ubuntu/base/scripts/build_slurm.sh new file mode 100755 index 0000000..9c97a5a --- /dev/null +++ b/containers/spindle-slurm-ubuntu/base/scripts/build_slurm.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euxo pipefail + +export SLURM_SRC="/home/${SLURM_USER}/slurm" +git clone -b ${SLURM_VERSION} --single-branch --depth=1 https://github.com/SchedMD/slurm.git ${SLURM_SRC} +cd ${SLURM_SRC} +./configure --enable-debug --prefix=/usr --sysconfdir=/etc/slurm --with-mysql_config=/usr/bin --libdir=/usr/lib +make -j$(nproc) +make install + diff --git a/src/client/client/intercept.c b/src/client/client/intercept.c index 84c87ee..ea73253 100644 --- a/src/client/client/intercept.c +++ b/src/client/client/intercept.c @@ -52,14 +52,14 @@ struct spindle_binding_t spindle_bindings[] = { { "getenv", (void **) &orig_getenv, NULL, NULL }, { "setenv", (void **) &orig_setenv, NULL, NULL }, { "unsetenv", (void **) &orig_unsetenv, NULL, NULL }, - { "spindle_enable", NULL, "int_spindle_enable", (void *) int_spindle_enable }, - { "spindle_disable", NULL, "int_spindle_disable", (void *) int_spindle_disable }, - { "spindle_is_enabled", NULL, "int_spindle_is_enabled", (void *) int_spindle_is_enabled }, - { "spindle_is_present", NULL, "int_spindle_is_present", (void *) int_spindle_is_present }, - { "spindle_open", NULL, "int_spindle_open", (void *) int_spindle_open }, - { "spindle_stat", NULL, "int_spindle_stat", (void *) int_spindle_stat }, - { "spindle_lstat", NULL, "int_spindle_lstat", (void *) int_spindle_lstat }, - { "spindle_fopen", NULL, "int_spindle_fopen", (void *) int_spindle_fopen }, + { "spindle_enable", (void **) &orig_spindle_enable, "int_spindle_enable", (void *) int_spindle_enable }, + { "spindle_disable", (void **) &orig_spindle_disable, "int_spindle_disable", (void *) int_spindle_disable }, + { "spindle_is_enabled", (void **) &orig_spindle_is_enabled, "int_spindle_is_enabled", (void *) int_spindle_is_enabled }, + { "spindle_is_present", (void **) &orig_spindle_is_present, "int_spindle_is_present", (void *) int_spindle_is_present }, + { "spindle_open", (void **) &orig_spindle_open, "int_spindle_open", (void *) int_spindle_open }, + { "spindle_stat", (void **) &orig_spindle_stat, "int_spindle_stat", (void *) int_spindle_stat }, + { "spindle_lstat", (void **) &orig_spindle_lstat, "int_spindle_lstat", (void *) int_spindle_lstat }, + { "spindle_fopen", (void **) &orig_spindle_fopen, "int_spindle_fopen", (void *) int_spindle_fopen }, { "spindle_test_log_msg", NULL, "int_spindle_test_log_msg", (void *) int_spindle_test_log_msg }, { NULL, NULL, NULL, NULL } }; diff --git a/src/client/client/intercept.h b/src/client/client/intercept.h index 0233374..4ace232 100644 --- a/src/client/client/intercept.h +++ b/src/client/client/intercept.h @@ -53,6 +53,15 @@ extern int (*orig_setenv)(const char *name, const char *value, int overwrite); extern int (*orig_unsetenv)(const char *name); extern char* (*orig_dlerror)(); extern char* (*orig_realpath)(const char *, char *); + +extern int (*orig_spindle_open)(const char *pathname, int flags, ...); +extern FILE* (*orig_spindle_fopen)(const char *path, const char *opts); +extern int (*orig_spindle_stat)(const char *path, struct stat *buf); +extern int (*orig_spindle_lstat)(const char *path, struct stat *buf); +extern int (*orig_spindle_is_present)(); +extern int (*orig_spindle_enable)(); +extern int (*orig_spindle_disable)(); +extern int (*orig_spindle_is_enabled)(); int rtcache_stat(const char *path, struct stat *buf); int rtcache_lstat(const char *path, struct stat *buf); diff --git a/src/client/client/intercept_spindleapi.c b/src/client/client/intercept_spindleapi.c index d5ce1ea..18bd076 100644 --- a/src/client/client/intercept_spindleapi.c +++ b/src/client/client/intercept_spindleapi.c @@ -40,6 +40,15 @@ static TLS int under_spindle_call = 0; #include "sym_alias.h" #endif +int (*orig_spindle_open)(const char *pathname, int flags, ...); +FILE* (*orig_spindle_fopen)(const char *path, const char *opts); +int (*orig_spindle_stat)(const char *path, struct stat *buf); +int (*orig_spindle_lstat)(const char *path, struct stat *buf); +int (*orig_spindle_is_present)(); +int (*orig_spindle_enable)(); +int (*orig_spindle_disable)(); +int (*orig_spindle_is_enabled)(); + int int_spindle_open(const char *pathname, int flags, ...) { va_list argp; diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c index d7978eb..4999a4d 100644 --- a/src/client/client_comlib/client_api.c +++ b/src/client/client_comlib/client_api.c @@ -57,7 +57,7 @@ int send_file_query(int fd, char* path, int dso, char** newpath, int *errcode) { COMM_LOCK; - debug_printf3("sending message of type: file_query len=%d data='%s' ...(%s)\n", + debug_printf3("sending message of type: file_query len=%lu data='%s' ...(%s)\n", message.header.len, message.data, path); client_send_msg(fd, &message); @@ -105,7 +105,7 @@ int send_stat_request(int fd, char *path, int is_lstat, char *newpath) COMM_LOCK; - debug_printf3("sending message of type: %sstat_query len=%d data='%s' ...(%s)\n", + debug_printf3("sending message of type: %sstat_query len=%lu data='%s' ...(%s)\n", is_lstat ? "l" : "", message.header.len, message.data, path); client_send_msg(fd, &message); @@ -149,7 +149,7 @@ int send_existance_test(int fd, char *path, int *exists) message.header.len = strlen(path) + 1; message.data = (void *) buffer; - debug_printf3("Sending message of type: file_exist_query len=%d, data=%s\n", + debug_printf3("Sending message of type: file_exist_query len=%lu, data=%s\n", message.header.len, path); COMM_LOCK; @@ -186,7 +186,7 @@ int send_orig_path_request(int fd, const char *path, char *newpath) message.header.len = strlen(path) + 1; message.data = (void *) buffer; - debug_printf3("Sending message of type: file_orig_path len=%d, data=%s\n", + debug_printf3("Sending message of type: file_orig_path len=%lu, data=%s\n", message.header.len, path); COMM_LOCK; diff --git a/src/client/client_comlib/client_api_biter.c b/src/client/client_comlib/client_api_biter.c index 8dbf166..8b04deb 100644 --- a/src/client/client_comlib/client_api_biter.c +++ b/src/client/client_comlib/client_api_biter.c @@ -72,7 +72,7 @@ int client_send_msg_biter(int connid, ldcs_message_t *msg) return -1; } - debug_printf3("sending message of size len=%d\n", msg->header.len); + debug_printf3("sending message of size len=%lu\n", msg->header.len); result = biterc_write(connid, &msg->header, sizeof(msg->header)); if (result == -1) { diff --git a/src/client/client_comlib/client_api_pipe.c b/src/client/client_comlib/client_api_pipe.c index f97bff5..54d9333 100644 --- a/src/client/client_comlib/client_api_pipe.c +++ b/src/client/client_comlib/client_api_pipe.c @@ -320,7 +320,7 @@ int client_send_msg_pipe(int fd, ldcs_message_t *msg) { assert(fd >= 0 && fd < MAX_FD); - debug_printf3("sending message of size len=%d\n", msg->header.len); + debug_printf3("sending message of size len=%lu\n", msg->header.len); result = write_pipe(fdlist_pipe[fd].out_fd, &msg->header, sizeof(msg->header)); if (result == -1) @@ -360,7 +360,7 @@ static int client_recv_msg_pipe(int fd, ldcs_message_t *msg, ldcs_read_block_t b msg->data = (char *) spindle_malloc(msg->header.len); } - debug_printf3("Reading %d bytes for payload from pipe\n", msg->header.len); + debug_printf3("Reading %lu bytes for payload from pipe\n", msg->header.len); result = read_pipe(fdlist_pipe[fd].in_fd, msg->data, msg->header.len); return result; } diff --git a/src/client/client_comlib/client_api_socket.c b/src/client/client_comlib/client_api_socket.c index c216bc6..5d536d7 100644 --- a/src/client/client_comlib/client_api_socket.c +++ b/src/client/client_comlib/client_api_socket.c @@ -198,7 +198,7 @@ int client_send_msg_socket(int fd, ldcs_message_t * msg) { connfd=ldcs_socket_fdlist[fd].fd; bzero(help,41);if(msg->data) strncpy(help,msg->data,40); - debug_printf3("sending message of type: %s len=%d data=%s ...\n", + debug_printf3("sending message of type: %s len=%lu data=%s ...\n", _message_type_to_str(msg->header.type), msg->header.len,help ); @@ -241,7 +241,7 @@ int client_recv_msg_static_socket(int fd, ldcs_message_t *msg, ldcs_read_block_ } bzero(help,41);if(msg->data) strncpy(help,msg->data,40); - debug_printf3("received message of type: %s len=%d data=%s ...\n", + debug_printf3("received message of type: %s len=%lu data=%s ...\n", _message_type_to_str(msg->header.type), msg->header.len,help ); diff --git a/src/include/ldcs_api.h b/src/include/ldcs_api.h index aaf0e2a..e8ffa43 100644 --- a/src/include/ldcs_api.h +++ b/src/include/ldcs_api.h @@ -112,7 +112,7 @@ typedef enum { struct ldcs_message_header_struct { ldcs_message_ids_t type; - int len; + size_t len; }; typedef struct ldcs_message_header_struct ldcs_message_header_t; diff --git a/src/server/auditserver/ldcs_audit_server_filemngt.c b/src/server/auditserver/ldcs_audit_server_filemngt.c index f7e432b..bb2e929 100644 --- a/src/server/auditserver/ldcs_audit_server_filemngt.c +++ b/src/server/auditserver/ldcs_audit_server_filemngt.c @@ -190,7 +190,7 @@ int filemngt_read_file(char *filename, void *buffer, size_t *size, int strip, int filemngt_encode_packet(char *filename, void *filecontents, size_t filesize, int stripped, char **buffer, size_t *buffer_size) { - int cur_pos = 0; + size_t cur_pos = 0; int filename_len = strlen(filename) + 1; int is_elf = filemngt_is_elf_file(filecontents, filesize); //TODO: Remove filesize from allocation if we're doing a non-contig send. Wastes memory. @@ -227,7 +227,7 @@ int filemngt_encode_packet(char *filename, void *filecontents, size_t filesize, return 0; } -int filemngt_decode_packet(node_peer_t peer, ldcs_message_t *msg, char *filename, size_t *filesize, int *bytes_read, int *is_elf, int *stripped) +int filemngt_decode_packet(node_peer_t peer, ldcs_message_t *msg, char *filename, size_t *filesize, size_t *bytes_read, int *is_elf, int *stripped) { int filename_len = 0; int result; @@ -259,7 +259,7 @@ int filemngt_decode_packet(node_peer_t peer, ldcs_message_t *msg, char *filename *bytes_read = sizeof(filename_len) + sizeof(*filesize) + filename_len; } else { - int pos = 0; + size_t pos = 0; unsigned char *data = (unsigned char *) msg->data; *is_elf = *((int *) (data+pos)); pos += sizeof(int); diff --git a/src/server/auditserver/ldcs_audit_server_filemngt.h b/src/server/auditserver/ldcs_audit_server_filemngt.h index 0187a7e..6f45c48 100644 --- a/src/server/auditserver/ldcs_audit_server_filemngt.h +++ b/src/server/auditserver/ldcs_audit_server_filemngt.h @@ -28,7 +28,7 @@ int ldcs_audit_server_filemngt_init (char* location); int filemngt_read_file(char *filename, void *buffer, size_t *size, int strip, int *err, int *was_stripped); int filemngt_encode_packet(char *filename, void *filecontents, size_t filesize, int stripped, char **buffer, size_t *buffer_size); -int filemngt_decode_packet(node_peer_t peer, ldcs_message_t *msg, char *filename, size_t *buffer_size, int *bytes_read, int *is_elf, int *stripped); +int filemngt_decode_packet(node_peer_t peer, ldcs_message_t *msg, char *filename, size_t *buffer_size, size_t *bytes_read, int *is_elf, int *stripped); typedef enum { clt_unknown, diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c index aded40e..8a42f40 100644 --- a/src/server/auditserver/ldcs_audit_server_handlers.c +++ b/src/server/auditserver/ldcs_audit_server_handlers.c @@ -1563,8 +1563,8 @@ static int handle_file_recv(ldcs_process_data_t *procdata, ldcs_message_t *msg, { char pathname[MAX_PATH_LEN+1], *localname; char *buffer = NULL; - size_t size = 0; - int result, global_error = 0, already_loaded, fd = -1, bytes_read = 0, is_stripped = 0; + size_t size = 0, bytes_read = 0; + int result, global_error = 0, already_loaded, fd = -1, is_stripped = 0; int replicate, is_elf; pathname[MAX_PATH_LEN] = '\0'; diff --git a/src/server/comlib/ldcs_api_biter.c b/src/server/comlib/ldcs_api_biter.c index 78f068f..be00c53 100644 --- a/src/server/comlib/ldcs_api_biter.c +++ b/src/server/comlib/ldcs_api_biter.c @@ -167,7 +167,7 @@ int ldcs_send_msg_biter(int fd, ldcs_message_t *msg) if (spindle_debug_prints) { int rank = biterd_get_rank(session, proc); - debug_printf3("Sending message of size %d to rank %d (session = %d, proc = %d)\n", + debug_printf3("Sending message of size %lu to rank %d (session = %d, proc = %d)\n", msg->header.len, rank, session, proc); } @@ -183,7 +183,7 @@ int ldcs_send_msg_biter(int fd, ldcs_message_t *msg) result = biterd_write(session, proc, msg->data, msg->header.len); if (result == -1) { - err_printf("Error writing message of size %d to session %d, proc %d: %s\n", + err_printf("Error writing message of size %lu to session %d, proc %d: %s\n", msg->header.len, session, proc, biterd_lasterror_str()); return -1; } @@ -271,7 +271,7 @@ int ldcs_recv_msg_static_biter(int connid, ldcs_message_t *msg, ldcs_read_block_ return -1; } - debug_printf3("Message to be read is of size %d (session = %d, proc = %d)\n", + debug_printf3("Message to be read is of size %lu (session = %d, proc = %d)\n", msg->header.len, session, proc); if (msg->header.len == 0) { @@ -281,7 +281,7 @@ int ldcs_recv_msg_static_biter(int connid, ldcs_message_t *msg, ldcs_read_block_ result = biterd_read(session, proc, msg->data, msg->header.len); if (result == -1) { - err_printf("Error reading message of size %d from session %d, proc %d: %s\n", + err_printf("Error reading message of size %lu from session %d, proc %d: %s\n", msg->header.len, session, proc, biterd_lasterror_str()); return -1; } diff --git a/src/server/comlib/ldcs_api_pipe.c b/src/server/comlib/ldcs_api_pipe.c index 8587a4e..c487259 100644 --- a/src/server/comlib/ldcs_api_pipe.c +++ b/src/server/comlib/ldcs_api_pipe.c @@ -274,7 +274,7 @@ int ldcs_send_msg_pipe(int fd, ldcs_message_t * msg) { if ((fd<0) || (fd>fdlist_pipe_size) ) _error("wrong fd"); - debug_printf3("sending message of type: %s len=%d data=%s ...\n", + debug_printf3("sending message of type: %s len=%lu data=%s ...\n", _message_type_to_str(msg->header.type), msg->header.len,msg->data ); @@ -319,7 +319,7 @@ ldcs_message_t * ldcs_recv_msg_pipe(int fd, ldcs_read_block_t block ) { msg->data = NULL; } - debug_printf3("received message of type: %s len=%d data=%s ...\n", + debug_printf3("received message of type: %s len=%lu data=%s ...\n", _message_type_to_str(msg->header.type), msg->header.len, msg->data ); @@ -365,7 +365,7 @@ int ldcs_recv_msg_static_pipe(int fd, ldcs_message_t *msg, ldcs_read_block_t blo *msg->data = '\0'; } - debug_printf3("received message of type: %s len=%d data=%s ...\n", + debug_printf3("received message of type: %s len=%lu data=%s ...\n", _message_type_to_str(msg->header.type), msg->header.len, msg->data ); diff --git a/src/server/comlib/ldcs_api_socket.c b/src/server/comlib/ldcs_api_socket.c index b7b48ef..43cd348 100644 --- a/src/server/comlib/ldcs_api_socket.c +++ b/src/server/comlib/ldcs_api_socket.c @@ -247,7 +247,7 @@ int ldcs_send_msg_socket(int fd, ldcs_message_t * msg) { connfd=ldcs_socket_fdlist[fd].fd; bzero(help,41);if(msg->data) strncpy(help,msg->data,40); - debug_printf3("sending message of type: %s len=%d data=%s ...\n", + debug_printf3("sending message of type: %s len=%lu data=%s ...\n", _message_type_to_str(msg->header.type), msg->header.len,help ); @@ -295,7 +295,7 @@ ldcs_message_t * ldcs_recv_msg_socket(int fd, ldcs_read_block_t block) { } bzero(help,41);if(msg->data) strncpy(help,msg->data,40); - debug_printf3("received message of type: %s len=%d data=%s ...\n", + debug_printf3("received message of type: %s len=%lu data=%s ...\n", _message_type_to_str(msg->header.type), msg->header.len,help ); @@ -330,7 +330,7 @@ int ldcs_recv_msg_static_socket(int fd, ldcs_message_t *msg, ldcs_read_block_t } bzero(help,41);if(msg->data) strncpy(help,msg->data,40); - debug_printf3("received message of type: %s len=%d data=%s ...\n", + debug_printf3("received message of type: %s len=%lu data=%s ...\n", _message_type_to_str(msg->header.type), msg->header.len,help ); diff --git a/testsuite/runTests_template b/testsuite/runTests_template index 43af2f8..e0af401 100644 --- a/testsuite/runTests_template +++ b/testsuite/runTests_template @@ -18,6 +18,8 @@ CHECK_RETCODE CHECK_RETCODE ./run_driver --ldpreload --push CHECK_RETCODE +./run_driver --spindleapi --push +CHECK_RETCODE ./run_driver --dependency --pull CHECK_RETCODE @@ -31,6 +33,8 @@ CHECK_RETCODE CHECK_RETCODE ./run_driver --ldpreload --pull CHECK_RETCODE +./run_driver --spindleapi --pull +CHECK_RETCODE ./run_driver --dependency --numa CHECK_RETCODE @@ -44,6 +48,8 @@ CHECK_RETCODE CHECK_RETCODE ./run_driver --ldpreload --numa CHECK_RETCODE +./run_driver --spindleapi --numa +CHECK_RETCODE if test "x$SPINDLE_BLUEGENE" != "xtrue"; then ./run_driver --dependency --fork @@ -58,6 +64,8 @@ CHECK_RETCODE CHECK_RETCODE ./run_driver --ldpreload --fork CHECK_RETCODE +./run_driver --spindleapi --fork +CHECK_RETCODE fi if test "x$SPINDLE_BLUEGENE" != "xtrue"; then @@ -73,6 +81,8 @@ CHECK_RETCODE CHECK_RETCODE ./run_driver --ldpreload --forkexec CHECK_RETCODE +./run_driver --spindleapi --forkexec +CHECK_RETCODE fi ./run_driver --dependency --chdir @@ -87,6 +97,8 @@ CHECK_RETCODE CHECK_RETCODE ./run_driver --ldpreload --chdir CHECK_RETCODE +./run_driver --spindleapi --chdir +CHECK_RETCODE ./run_driver --dependency --preload CHECK_RETCODE @@ -100,6 +112,8 @@ CHECK_RETCODE CHECK_RETCODE ./run_driver --ldpreload --preload CHECK_RETCODE +./run_driver --spindleapi --preload +CHECK_RETCODE if test "x$TEST_RM" != "xserial"; then export SESSION_ID=`./run_driver --start-session` @@ -115,6 +129,8 @@ CHECK_RETCODE CHECK_RETCODE ./run_driver --ldpreload --session CHECK_RETCODE +./run_driver --spindleapi --session +CHECK_RETCODE ./run_driver --end-session $SESSION_ID CHECK_RETCODE unset SESSION_ID @@ -125,6 +141,7 @@ echo Skipping: ./run_driver --dlreopen --session echo Skipping: ./run_driver --reorder --session echo Skipping: ./run_driver --partial --session echo Skipping: ./run_driver --ldpreload --session +echo Skipping: ./run_driver --spindleapi --session fi ./run_driver --serial ./spindle_exec_test diff --git a/testsuite/test_driver.c b/testsuite/test_driver.c index 8fc41e8..a4b3b46 100644 --- a/testsuite/test_driver.c +++ b/testsuite/test_driver.c @@ -1086,6 +1086,10 @@ void checkTlsSum() { int i; int sum = 0; + + if (open_mode == om_spindleapi) + return; + for (i = 0; libraries[i].libname; i++) { if (!libraries[i].tls_func) continue;