diff --git a/.containerignore b/.containerignore new file mode 100644 index 00000000..09c5d592 --- /dev/null +++ b/.containerignore @@ -0,0 +1,23 @@ +# Podman/Docker ignore file +# Excludes files from being copied into container images during build + +# Exclude the saved images tarball (6+ GB) +spindle-podman-images.tar + +# Exclude build artifacts +workspace-Spindle/build/ +workspace-Spindle/install/ +*.o +*.a +*.so + +# Exclude git metadata +.git/ +.gitignore + +# Exclude test output +out.* +*.log + +# Exclude investigation directory +podman-concurrency-investigation/ diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 00000000..092a75da --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.containerignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index 626ce7fa..394459e4 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,11 @@ run_driver run_driver_rm preload_file_list build + +# Generated by generate_config.sh +containers/spindle-slurm-ubuntu/testing/conf/slurmdbd.conf +containers/spindle-slurm-ubuntu/testing/mariadb.env +containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf +containers/spindle-slurm-ubuntu/testing-srun/mariadb.env + + diff --git a/containers/spindle-flux-ubuntu/Dockerfile.podman b/containers/spindle-flux-ubuntu/Dockerfile.podman new file mode 100644 index 00000000..56862c53 --- /dev/null +++ b/containers/spindle-flux-ubuntu/Dockerfile.podman @@ -0,0 +1,97 @@ +# This is based on the Flux Container Tutorial +# See https://flux-framework.readthedocs.io/en/latest/tutorials/containers +ARG flux_sched_version=noble-v0.48.0-amd64 +FROM fluxrm/flux-sched:${flux_sched_version} AS builder +ARG replicas=4 +ENV workers=${replicas} +USER root +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ + && apt-get -qq install -y --no-install-recommends \ + autotools-dev \ + autoconf \ + automake \ + cmake \ + git \ + python3 \ + openssh-server \ + openssh-client \ + libdb-dev \ + apt-utils \ + dnsutils \ + iputils-ping \ + python3-pip \ + libgcrypt20 \ + libgcrypt20-dev \ + gdb \ + libc6-dbg \ + software-properties-common + +ARG USER=fluxuser +ARG CONFIG_ROOT=containers/spindle-flux-ubuntu + +# Allow fluxuser to run as other users so it can start munged +RUN sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers" + +# Configure flux +ENV STATE_DIR=/var/lib/flux +RUN mkdir -p ${STATE_DIR} /etc/flux/system /etc/flux/system/cron.d /etc/flux/config /run/flux /etc/flux/imp/conf.d +COPY ${CONFIG_ROOT}/flux/imp.toml /etc/flux/imp/conf.d/ +COPY ${CONFIG_ROOT}/flux/broker.toml /etc/flux/config/ +RUN mkdir -p /etc/flux/system/cron.d && \ + mkdir -p /mnt/curve && \ + flux keygen /mnt/curve/curve.cert && \ + flux R encode --hosts="node-[1-${workers}]" > /etc/flux/system/R && \ + chmod -R a+rX /etc/flux && \ + chown -R ${USER}:${USER} /run/flux ${STATE_DIR} /mnt/curve/curve.cert + +# Build Spindle +WORKDIR /home/${USER} +# Copy the whole git repo into the container +COPY . /home/${USER}/Spindle + +# Fix permissions on Spindle source (configure needs +x) +RUN chmod -R u+rwX /home/${USER}/Spindle && \ + chown -R ${USER}:${USER} /home/${USER}/Spindle + +# Copy and prepare build script as root +COPY ${CONFIG_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN chmod +rx /home/${USER}/build_spindle.sh && \ + chown ${USER}:${USER} /home/${USER}/build_spindle.sh + +# Switch to user to run the build +USER ${USER} +RUN bash ./build_spindle.sh + +# Copy scripts as root +USER root +RUN chown -R ${USER}:${USER} /home/fluxuser && \ + chown -R ${USER}:${USER} /run/flux + +COPY ${CONFIG_ROOT}/scripts/flux_healthcheck.sh /home/${USER}/flux_healthcheck.sh +COPY ${CONFIG_ROOT}/scripts/entrypoint.sh.podman /home/${USER}/entrypoint.sh +RUN chmod +rx /home/${USER}/flux_healthcheck.sh /home/${USER}/entrypoint.sh && \ + chown ${USER}:${USER} /home/${USER}/flux_healthcheck.sh /home/${USER}/entrypoint.sh + +USER ${USER} +WORKDIR /home/${USER} +ENV PATH /home/${USER}/Spindle-inst/bin:${PATH} +# Make libfabric work with fork. +ENV RDMAV_FORK_SAFE 1 +# Silence warning from hwloc about unsupported PCI device +# on GitHub-hosted runners. +ENV HWLOC_HIDE_ERRORS 2 +# Suppress UCX warnings about /proc/sys tuning parameters +# These are expected in rootless podman containers +ENV UCX_LOG_LEVEL=error + +ENTRYPOINT /bin/bash ./entrypoint.sh diff --git a/containers/spindle-flux-ubuntu/scripts/entrypoint.sh.podman b/containers/spindle-flux-ubuntu/scripts/entrypoint.sh.podman new file mode 100644 index 00000000..3fb7f9ac --- /dev/null +++ b/containers/spindle-flux-ubuntu/scripts/entrypoint.sh.podman @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Podman-compatible entrypoint for Flux containers +# Starts munged and the flux broker. +# +# For documentation on running Flux in containers, see +# https://flux-framework.readthedocs.io/en/latest/tutorials/containers + +set -x # Debug output + +brokerOptions="-Scron.directory=/etc/flux/system/cron.d \ + -Stbon.fanout=256 \ + -Srundir=/run/flux \ + -Sstatedir=${STATE_DIRECTORY:-/var/lib/flux} \ + -Slocal-uri=local:///run/flux/local \ + -Slog-stderr-level=6 \ + -Slog-stderr-mode=local" + +# Get the hostname that will resolve for the Docker bridge network. +address=$(echo $( nslookup "$( hostname -i )" | head -n 1 )) +parts=(${address//=/ }) +hostName=${parts[2]} +thisHost=(${hostName//./ }) +thisHost=${thisHost[0]} +echo "This host: $thisHost" +echo "Main host: $mainHost" +export FLUX_FAKE_HOSTNAME=$thisHost + +if [ -d /shared ]; then + sudo chown -R "$(id -un):$(id -gn)" /shared + sudo chmod 755 /shared +fi + +# Start munged +echo "Starting munged..." +sudo -u munge /usr/sbin/munged + +# Give munge time to start +sleep 2 + +if [ "${thisHost}" != "${mainHost}" ]; then + # Worker node -- wait for head node before connecting + echo "Worker node: waiting for head node..." + sleep 15 + echo "Starting flux broker (worker)..." + exec flux start -o --config /etc/flux/config ${brokerOptions} sleep inf +else + # Head node + echo "Head node: starting flux broker..." + exec flux start -o --config /etc/flux/config ${brokerOptions} sleep inf +fi diff --git a/containers/spindle-hello-podman/01-basic/Dockerfile b/containers/spindle-hello-podman/01-basic/Dockerfile new file mode 100644 index 00000000..41fc7907 --- /dev/null +++ b/containers/spindle-hello-podman/01-basic/Dockerfile @@ -0,0 +1,28 @@ +FROM ubuntu:noble + +# LC-specific fix for podman setgroups issue +# This is a no-op in Docker, but required for podman on LC systems +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +# Install curl to test both apt-get and SSL connectivity +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Test script that validates: +# 1. Container runs +# 2. Network access works +# 3. SSL certificates work +CMD echo "=== Hello from Podman! ===" && \ + echo "Container is running successfully." && \ + echo "" && \ + echo "Testing network and SSL connectivity..." && \ + curl -I https://www.google.com 2>&1 | head -5 && \ + echo "" && \ + echo "If you see HTTP headers above, SSL is working!" && \ + echo "" && \ + echo "=== 01-basic test complete ===" diff --git a/containers/spindle-hello-podman/02-user-switch/Dockerfile b/containers/spindle-hello-podman/02-user-switch/Dockerfile new file mode 100644 index 00000000..e05c7018 --- /dev/null +++ b/containers/spindle-hello-podman/02-user-switch/Dockerfile @@ -0,0 +1,78 @@ +FROM ubuntu:noble + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +# Install basic tools +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo \ + && rm -rf /var/lib/apt/lists/* + +# Create a non-root user similar to how Spindle containers work +# This pattern is used in all Spindle containers +ARG USER=testuser +ARG UID=1001 +ARG GID=1001 + +RUN groupadd -g ${GID} ${USER} && \ + useradd -m -u ${UID} -g ${GID} -s /bin/bash ${USER} && \ + echo "${USER} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/${USER} && \ + chmod 0440 /etc/sudoers.d/${USER} + +# Create a test directory owned by root to verify permission handling +RUN mkdir -p /root-owned && \ + echo "This is owned by root" > /root-owned/root-file.txt && \ + chmod 644 /root-owned/root-file.txt + +# Create a test directory that will be owned by the user +RUN mkdir -p /user-owned && \ + chown ${UID}:${GID} /user-owned + +# Switch to non-root user +USER ${USER} +WORKDIR /home/${USER} + +# Create a file as the user to verify we're running as that user +RUN echo "This is owned by ${USER}" > /home/${USER}/user-file.txt + +# Test script that validates: +# 1. Running as the correct user +# 2. Can read files owned by root +# 3. Can write to user-owned directories +# 4. Cannot write to root-owned directories +# 5. Can use sudo if needed +CMD echo "=== User Switch Test ===" && \ + echo "" && \ + echo "Current user information:" && \ + id && \ + echo "" && \ + echo "Home directory:" && \ + pwd && \ + ls -la /home/${USER} && \ + echo "" && \ + echo "Testing file permissions:" && \ + echo " Reading root-owned file..." && \ + cat /root-owned/root-file.txt && \ + echo " ✓ Can read root-owned files" && \ + echo "" && \ + echo " Writing to user-owned directory..." && \ + echo "test content" > /user-owned/test.txt && \ + cat /user-owned/test.txt && \ + echo " ✓ Can write to user-owned directories" && \ + echo "" && \ + echo " Testing root-owned directory (should fail)..." && \ + (echo "test" > /root-owned/fail.txt 2>&1 && echo " ✗ UNEXPECTED: Could write to root directory" || echo " ✓ Correctly denied write to root directory") && \ + echo "" && \ + echo " Testing sudo access..." && \ + sudo -n echo " ✓ Sudo works (passwordless for container convenience)" && \ + echo "" && \ + echo "=== 02-user-switch test complete ===" && \ + echo "" && \ + echo "Key patterns validated:" && \ + echo " ✓ Non-root user created (uid=${UID})" && \ + echo " ✓ User has home directory" && \ + echo " ✓ Correct permission handling" && \ + echo " ✓ Sudo available when needed" diff --git a/containers/spindle-hello-podman/03-filesystem/Dockerfile b/containers/spindle-hello-podman/03-filesystem/Dockerfile new file mode 100644 index 00000000..b1fb59c3 --- /dev/null +++ b/containers/spindle-hello-podman/03-filesystem/Dockerfile @@ -0,0 +1,85 @@ +FROM ubuntu:noble + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +# Install tools for filesystem operations +RUN apt-get update && apt-get install -y --no-install-recommends \ + file \ + && rm -rf /var/lib/apt/lists/* + +# Create a non-root user (matching pattern from 02-user-switch) +ARG USER=testuser +ARG UID=1001 +ARG GID=1001 + +# Make USER available at runtime (ARG only available at build time) +ENV USER=${USER} + +RUN groupadd -g ${GID} ${USER} && \ + useradd -m -u ${UID} -g ${GID} -s /bin/bash ${USER} + +# Switch to non-root user +USER ${USER} +WORKDIR /home/${USER} + +# Create mount points for testing different volume scenarios +# These will be populated via volume mounts at runtime +RUN mkdir -p /home/${USER}/host-data && \ + mkdir -p /home/${USER}/output + +# Test script that validates volume mounts and filesystem operations +# Note: The actual volume content will be mounted at runtime +CMD echo "=== Filesystem Mount Test ===" && \ + echo "" && \ + echo "Testing volume mounts and file operations..." && \ + echo "" && \ + echo "1. Checking mounted host directory:" && \ + if [ -d "/home/${USER}/host-data" ]; then \ + echo " Mount point exists: /home/${USER}/host-data" && \ + if [ "$(ls -A /home/${USER}/host-data 2>/dev/null)" ]; then \ + echo " ✓ Host directory mounted with content:" && \ + ls -lh /home/${USER}/host-data | head -10 && \ + echo "" && \ + echo " Reading a file from host..." && \ + if [ -f "/home/${USER}/host-data/test-input.txt" ]; then \ + cat /home/${USER}/host-data/test-input.txt && \ + echo " ✓ Can read files from host mount" ; \ + else \ + echo " ℹ No test-input.txt found (expected for manual testing)" ; \ + fi ; \ + else \ + echo " ⚠ Mount point empty - volume may not be mounted" ; \ + fi ; \ + else \ + echo " ✗ Mount point missing" ; \ + fi && \ + echo "" && \ + echo "2. Testing write to output directory:" && \ + echo "Test data written from container at $(date)" > /home/${USER}/output/container-output.txt && \ + echo " ✓ Created file: container-output.txt" && \ + cat /home/${USER}/output/container-output.txt && \ + echo "" && \ + echo "3. Checking file ownership:" && \ + ls -ln /home/${USER}/output/container-output.txt && \ + echo " ℹ File owned by UID=$(stat -c %u /home/${USER}/output/container-output.txt) GID=$(stat -c %g /home/${USER}/output/container-output.txt)" && \ + echo "" && \ + echo "4. Testing file type detection:" && \ + file /home/${USER}/output/container-output.txt && \ + echo "" && \ + echo "=== 03-filesystem test complete ===" && \ + echo "" && \ + echo "Key patterns validated:" && \ + echo " ✓ Volume mounts work" && \ + echo " ✓ Can read files from host" && \ + echo " ✓ Can write files to mounted volumes" && \ + echo " ✓ File ownership preserved" && \ + echo "" && \ + echo "This demonstrates the pattern used for Spindle:" && \ + echo " - Mount source code from host (read-only)" && \ + echo " - Mount build directory (read-write)" && \ + echo " - Build artifacts persist on host" && \ + echo " - Logs accessible after container exits" diff --git a/containers/spindle-hello-podman/04-networking/Dockerfile b/containers/spindle-hello-podman/04-networking/Dockerfile new file mode 100644 index 00000000..c2be0154 --- /dev/null +++ b/containers/spindle-hello-podman/04-networking/Dockerfile @@ -0,0 +1,41 @@ +FROM ubuntu:noble + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +# Install networking tools +RUN apt-get update && apt-get install -y --no-install-recommends \ + iputils-ping \ + iproute2 \ + curl \ + netcat-openbsd \ + && rm -rf /var/lib/apt/lists/* + +# Create a non-root user +ARG USER=testuser +ARG UID=1001 +ARG GID=1001 + +# Make USER available at runtime +ENV USER=${USER} + +RUN groupadd -g ${GID} ${USER} && \ + useradd -m -u ${UID} -g ${GID} -s /bin/bash ${USER} + +# Switch to non-root user +USER ${USER} +WORKDIR /home/${USER} + +# Default command: run as server listening on port 8080 +# This can be overridden when running as a client +CMD echo "=== Network Container ===" && \ + echo "Container: $(hostname)" && \ + echo "IP Address: $(hostname -I)" && \ + echo "" && \ + echo "Starting simple HTTP server on port 8080..." && \ + while true; do \ + echo "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nHello from $(hostname) at $(date)" | nc -l -p 8080 -q 1; \ + done diff --git a/containers/spindle-hello-podman/05-flux-multi/Dockerfile b/containers/spindle-hello-podman/05-flux-multi/Dockerfile new file mode 100644 index 00000000..8b90d595 --- /dev/null +++ b/containers/spindle-hello-podman/05-flux-multi/Dockerfile @@ -0,0 +1,59 @@ +# Simplified Flux multi-node example +# Based on https://flux-framework.readthedocs.io/en/latest/tutorials/containers +# Uses the official Flux Framework base image + +ARG flux_sched_version=noble-v0.48.0-amd64 +FROM fluxrm/flux-sched:${flux_sched_version} + +USER root + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +# Install basic tools for testing +RUN apt-get update && apt-get install -y --no-install-recommends \ + iputils-ping \ + iproute2 \ + && rm -rf /var/lib/apt/lists/* + +# Configure Flux for multi-node cluster +ARG workers=3 +ENV workers=${workers} +ENV STATE_DIR=/var/lib/flux + +# Set up Flux directories and configuration +RUN mkdir -p ${STATE_DIR} /etc/flux/system /etc/flux/config /run/flux /mnt/curve && \ + flux keygen /mnt/curve/curve.cert && \ + flux R encode --hosts="flux-node-[1-${workers}]" > /etc/flux/system/R + +# Create broker configuration +RUN echo '[bootstrap]' > /etc/flux/config/broker.toml && \ + echo 'default_port = 8050' >> /etc/flux/config/broker.toml && \ + echo 'default_bind = "tcp://eth0:%p"' >> /etc/flux/config/broker.toml && \ + echo 'default_connect = "tcp://%h.local:%p"' >> /etc/flux/config/broker.toml && \ + echo 'hosts = [' >> /etc/flux/config/broker.toml && \ + echo ' { host = "flux-node-1" },' >> /etc/flux/config/broker.toml && \ + echo ' { host = "flux-node-[2-'${workers}']" },' >> /etc/flux/config/broker.toml && \ + echo ']' >> /etc/flux/config/broker.toml + +# Set proper ownership +ARG USER=fluxuser +RUN chown -R ${USER}:${USER} /run/flux ${STATE_DIR} /mnt/curve /etc/flux + +# Allow fluxuser to run munge +RUN sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: /usr/sbin/munged\\n\" >> /etc/sudoers" + +# Copy entrypoint as root, make executable, then set ownership +COPY containers/spindle-hello-podman/05-flux-multi/entrypoint.sh /home/${USER}/entrypoint.sh +RUN chmod +x /home/${USER}/entrypoint.sh && \ + chown ${USER}:${USER} /home/${USER}/entrypoint.sh + +USER ${USER} +WORKDIR /home/${USER} + +ENV HWLOC_HIDE_ERRORS=2 + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/containers/spindle-hello-podman/05-flux-multi/entrypoint.sh b/containers/spindle-hello-podman/05-flux-multi/entrypoint.sh new file mode 100755 index 00000000..bac5a870 --- /dev/null +++ b/containers/spindle-hello-podman/05-flux-multi/entrypoint.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Flux multi-node entrypoint +# Based on Spindle's Flux container setup +# +# All nodes run the same image but behave differently based on hostname. +# flux-node-1 is the head node, others are workers. + +set -e + +echo "=== Flux Container Startup ===" +echo "Hostname: $(hostname)" +echo "IP: $(hostname -I)" + +# Start munge for authentication +echo "Starting munge..." +sudo /usr/sbin/munged +sleep 1 + +# Determine if this is the head node or a worker +MAIN_HOST="flux-node-1" +THIS_HOST=$(hostname) + +brokerOptions="-Stbon.fanout=256 \ + -Srundir=/run/flux \ + -Sstatedir=${STATE_DIR} \ + -Slog-stderr-level=6 \ + -Slog-stderr-mode=local" + +if [ "${THIS_HOST}" != "${MAIN_HOST}" ]; then + # Worker node - wait for head node to be ready + echo "Worker node: connecting to ${MAIN_HOST}..." + sleep 5 + + # Start flux broker and connect to head node + flux start -o --config /etc/flux/config ${brokerOptions} sleep inf +else + # Head node + echo "Head node: starting Flux broker..." + + # Start flux broker + flux start -o --config /etc/flux/config ${brokerOptions} bash -c ' + echo "" + echo "=== Flux Instance Started ===" + echo "" + + # Wait for workers to connect + echo "Waiting for workers to connect..." + sleep 10 + + echo "" + echo "Flux instance status:" + flux resource list + + echo "" + echo "Running test job across cluster..." + flux run -N 3 hostname + + echo "" + echo "=== Flux cluster is operational ===" + echo "You can now run flux commands." + echo "" + + # Keep running + sleep inf + ' +fi diff --git a/containers/spindle-hello-podman/06-slurm-multi/Dockerfile b/containers/spindle-hello-podman/06-slurm-multi/Dockerfile new file mode 100644 index 00000000..2385e36e --- /dev/null +++ b/containers/spindle-hello-podman/06-slurm-multi/Dockerfile @@ -0,0 +1,90 @@ +FROM ubuntu:noble + +USER root + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +# Install build dependencies and runtime tools +# Build Slurm from source instead of using Ubuntu packages +# to avoid dpkg post-install chown failures in rootless podman +RUN apt-get update && \ + DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ + ca-certificates \ + build-essential \ + git \ + autoconf \ + automake \ + libtool \ + pkg-config \ + libmunge-dev \ + libmariadb-dev \ + libhwloc-dev \ + libjson-c-dev \ + libhttp-parser-dev \ + python3-dev \ + munge \ + sudo \ + iputils-ping \ + && rm -rf /var/lib/apt/lists/* + +# Set up munge +RUN mkdir -p /run/munge /etc/munge && \ + chown munge:munge /run/munge && \ + chmod 0755 /run/munge + +# Create munge key (same key on all nodes for simplicity) +RUN dd if=/dev/urandom bs=1 count=1024 > /etc/munge/munge.key && \ + chown munge:munge /etc/munge/munge.key && \ + chmod 400 /etc/munge/munge.key + +# Create slurm user BEFORE building Slurm +RUN groupadd -r -g 900 slurm && \ + useradd -r -u 900 -g 900 -s /bin/false -d /var/lib/slurmd slurm + +# Build Slurm from source +# Based on Spindle's build_slurm.sh +ARG SLURM_VERSION=slurm-25-05-3-1 +RUN git clone -b ${SLURM_VERSION} --single-branch --depth=1 \ + https://github.com/SchedMD/slurm.git /tmp/slurm && \ + cd /tmp/slurm && \ + ./configure \ + --prefix=/usr \ + --sysconfdir=/etc/slurm \ + --with-mysql_config=/usr/bin \ + --libdir=/usr/lib && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf /tmp/slurm + +# Create Slurm directories with correct ownership +# Based on Spindle's setup_slurm.sh +RUN mkdir -p \ + /etc/slurm \ + /var/spool/slurmd \ + /var/spool/slurmctld \ + /var/run/slurmd \ + /var/lib/slurmd \ + /var/log/slurm && \ + chown -R slurm:slurm \ + /etc/slurm \ + /var/spool/slurmd \ + /var/spool/slurmctld \ + /var/run/slurmd \ + /var/lib/slurmd \ + /var/log/slurm + +# Copy Slurm configuration +COPY containers/spindle-hello-podman/06-slurm-multi/slurm.conf /etc/slurm/slurm.conf +COPY containers/spindle-hello-podman/06-slurm-multi/cgroup.conf /etc/slurm/cgroup.conf +RUN chown slurm:slurm /etc/slurm/slurm.conf /etc/slurm/cgroup.conf + +# Copy entrypoint +COPY containers/spindle-hello-podman/06-slurm-multi/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/containers/spindle-hello-podman/06-slurm-multi/cgroup.conf b/containers/spindle-hello-podman/06-slurm-multi/cgroup.conf new file mode 100644 index 00000000..e59e9aee --- /dev/null +++ b/containers/spindle-hello-podman/06-slurm-multi/cgroup.conf @@ -0,0 +1 @@ +CgroupPlugin=cgroup/v1 diff --git a/containers/spindle-hello-podman/06-slurm-multi/entrypoint.sh b/containers/spindle-hello-podman/06-slurm-multi/entrypoint.sh new file mode 100755 index 00000000..77741aa5 --- /dev/null +++ b/containers/spindle-hello-podman/06-slurm-multi/entrypoint.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Slurm multi-node entrypoint +# Simplified version for hello-world demonstration +# Based on Spindle's working entrypoint +# +# Determines role based on SLURM_ROLE environment variable: +# - ctl: runs slurmctld (head node) +# - worker: runs slurmd (compute node) + +set -e + +echo "=== Slurm Container Startup ===" +echo "Hostname: $(hostname)" +echo "Role: ${SLURM_ROLE}" + +# Start munge for authentication +echo "Starting munge..." +sudo -u munge /usr/sbin/munged +sleep 2 + +case "${SLURM_ROLE}" in + ctl) + echo "Starting slurmctld (controller daemon)..." + # Run as slurm user in foreground with high verbosity + sudo -u slurm /usr/sbin/slurmctld -i -Dvvv + ;; + + worker) + echo "Waiting for controller to be ready..." + sleep 5 + + echo "Testing connectivity to controller..." + ping -c 3 slurm-head || echo "WARNING: Cannot ping slurm-head" + + echo "Testing munge authentication..." + munge -n | unmunge || echo "WARNING: Munge test failed" + + echo "Starting slurmd (compute daemon)..." + # Run as root (via sudo) in foreground - slurmd needs root for cgroup management + # Based on Spindle's pattern: sudo bash -c 'exec /usr/sbin/slurmd -Dvvv' + sudo /usr/sbin/slurmd -Dvvv + ;; + + *) + echo "ERROR: SLURM_ROLE must be 'ctl' or 'worker'" + echo "Got: ${SLURM_ROLE}" + exit 1 + ;; +esac diff --git a/containers/spindle-hello-podman/06-slurm-multi/slurm.conf b/containers/spindle-hello-podman/06-slurm-multi/slurm.conf new file mode 100644 index 00000000..1c3f756f --- /dev/null +++ b/containers/spindle-hello-podman/06-slurm-multi/slurm.conf @@ -0,0 +1,46 @@ +# Minimal Slurm configuration for hello-world demo +# Based on Spindle's working configuration +ClusterName=hello-slurm +ControlMachine=slurm-head +ControlAddr=slurm-head +SlurmUser=slurm +SlurmctldPort=6817 +SlurmdPort=6818 + +# Authentication +AuthType=auth/munge + +# Logging +SlurmctldLogFile=/var/log/slurm/slurmctld.log +SlurmdLogFile=/var/log/slurm/slurmd.log +SlurmctldDebug=3 +SlurmdDebug=3 + +# Process tracking +ProctrackType=proctrack/linuxproc +TaskPlugin=task/none + +# Scheduling +SchedulerType=sched/builtin +SelectType=select/linear + +# State preservation +StateSaveLocation=/var/lib/slurmd +SlurmdSpoolDir=/var/spool/slurmd +SlurmctldPidFile=/var/run/slurmd/slurmctld.pid +SlurmdPidFile=/var/run/slurmd/slurmd.pid + +# Accounting (minimal - no database) +AccountingStorageType=accounting_storage/none +JobAcctGatherType=jobacct_gather/none + +# Return to service +ReturnToService=2 + +# MPI +MpiDefault=none + +# Node definitions - explicitly set NodeAddr +NodeName=slurm-node-1 NodeAddr=slurm-node-1 CPUs=1 State=UNKNOWN +NodeName=slurm-node-2 NodeAddr=slurm-node-2 CPUs=1 State=UNKNOWN +PartitionName=debug Nodes=slurm-node-[1-2] Default=YES MaxTime=INFINITE State=UP diff --git a/containers/spindle-hello-podman/README.md b/containers/spindle-hello-podman/README.md new file mode 100644 index 00000000..60999e8f --- /dev/null +++ b/containers/spindle-hello-podman/README.md @@ -0,0 +1,115 @@ +# Spindle Hello World - Podman Tutorial + +This directory contains a series of progressive test containers that demonstrate and validate the patterns needed to run Spindle containers with podman on LC systems. + +## Purpose + +These containers serve as: +1. **Validation** - Test that podman is properly configured +2. **Tutorial** - Show common container patterns (user switching, volumes, networking) +3. **Documentation** - Demonstrate LC-specific workarounds + +## Tests + +### 01-basic (✓ Available) +**What it tests:** +- Container builds with apt-get (tests setgroups fix) +- Container runs successfully +- Network connectivity works +- SSL certificates are properly configured + +**Run from outside the sandbox:** +```bash +cd /path/to/workspace-Spindle/Spindle/podman-port +./scripts/podman/run-hello-01-basic.sh +``` + +### 02-user-switch (✓ Available) +**What it tests:** +- Creating a non-root user (uid=1001, gid=1001) +- Switching to that user with USER directive +- File permission handling (read root files, write user files) +- Sudo access (passwordless for container convenience) + +**Run from outside the sandbox:** +```bash +cd /path/to/workspace-Spindle/Spindle/podman-port +./scripts/podman/run-hello-02-user.sh +``` + +### 03-filesystem (✓ Available) +**What it tests:** +- Volume mounts with -v flag +- Reading files from host directories +- Writing files to mounted volumes +- File persistence after container exit +- SELinux context handling (:Z flag) + +**Run from outside the sandbox:** +```bash +cd /path/to/workspace-Spindle/Spindle/podman-port +./scripts/podman/run-hello-03-filesystem.sh +``` + +This creates temporary test directories, mounts them into the container, +and verifies files can be read/written. Artifacts persist after the +container exits, demonstrating how Spindle build/test output is preserved. + +### 04-networking (✓ Available) +**What it tests:** +- Custom network creation +- Multiple containers on same network +- DNS resolution (hostname lookups) +- Container-to-container HTTP communication +- Service discovery patterns + +**Run from outside the sandbox:** +```bash +cd /path/to/workspace-Spindle/Spindle/podman-port +./scripts/podman/run-hello-04-networking.sh +``` + +This creates a server container and two client containers, all on a custom +network. Clients can resolve the server by hostname and communicate via HTTP. +This demonstrates the pattern used for Slurm/Flux multi-node clusters. + +## LC-Specific Issues Addressed + +### Issue 1: setgroups errors with apt-get +**Problem:** Ubuntu/Debian apt fails with "setgroups 65534 failed" on LC systems + +**Solution:** Add to Dockerfile: +```dockerfile +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi +``` + +This is controlled via build arg (automatically set by scripts). + +### Issue 2: SSL certificate errors +**Problem:** HTTPS fails with "certificate signed by unknown authority" + +**Solution:** Mount LC certificates into container: +- Build: `-v /etc/pki/ca-trust/source/anchors/PAN-cspca.llnl.gov.crt.pem:/usr/local/share/ca-certificates/cspca.crt:ro` +- Run: `-v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/ssl/certs/ca-certificates.crt:ro` + +This is handled automatically by `scripts/podman/common.sh`. + +## Usage + +All scripts are designed to be run from **outside the sandbox** where podman is available: + +```bash +# From your normal home directory or workspace +cd /path/to/workspace-Spindle/Spindle/podman-port +./scripts/podman/run-hello-01-basic.sh +``` + +## Next Steps + +After completing these hello-world tests, proceed to: +- `scripts/podman/run-serial.sh` - Single container Spindle tests +- `scripts/podman/run-flux.sh` - Flux resource manager tests +- `scripts/podman/run-slurm-srun.sh` - Multi-container Slurm cluster tests diff --git a/containers/spindle-serial-ubuntu/Dockerfile.podman b/containers/spindle-serial-ubuntu/Dockerfile.podman new file mode 100644 index 00000000..6151e276 --- /dev/null +++ b/containers/spindle-serial-ubuntu/Dockerfile.podman @@ -0,0 +1,93 @@ +ARG ubuntu_version=noble +FROM ubuntu:${ubuntu_version} +USER root +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ +# install latest pkg utils: + && apt-get -qq install -y --no-install-recommends \ + apt-utils + +RUN DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + locales \ + ca-certificates \ + wget \ + git \ + ssh \ + sudo \ + build-essential \ + pkg-config \ + autotools-dev \ + libtool \ + autoconf \ + automake \ + make \ + gfortran-13 \ + gcc-13 \ + g++-13 \ + gdb \ + libc6-dbg \ + munge \ + libmunge-dev \ + libhwloc-dev \ + mpich \ + libmpich-dev + +# Prevent hwloc from trying to use graphics cards +# as this fails when X is not running. +ENV HWLOC_COMPONENTS=-gl + +# Suppress UCX warnings about /proc/sys tuning parameters +# These are expected in rootless podman containers +ENV UCX_LOG_LEVEL=error + +# Set up munge +RUN mkdir -p /run/munge && \ + chown munge:munge /run/munge && \ + chmod 0755 /run/munge + +ARG USER=spindleuser +ARG UID=1001 +ARG BUILD_ROOT=./containers/spindle-serial-ubuntu +COPY ${BUILD_ROOT}/scripts/add_docker_user.sh /add_docker_user.sh +RUN /add_docker_user.sh + +USER ${USER} +WORKDIR /home/${USER} +RUN mkdir -p /home/${USER}/Spindle +# Copy the Spindle repo into the container +COPY . /home/${USER}/Spindle + +# Fix permissions on copied files (configure, scripts, etc.) as root +USER root +RUN chmod -R u+rwX /home/${USER}/Spindle && \ + chown -R ${USER}:${USER} /home/${USER}/Spindle + +# Copy and prepare build script as root, then chown to user +COPY ${BUILD_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN chmod +rx /home/${USER}/build_spindle.sh && \ + chown ${USER}:${USER} /home/${USER}/build_spindle.sh + +# Switch back to user to run the build +USER ${USER} +RUN bash ./build_spindle.sh + +# Copy entrypoint as root and set permissions +USER root +COPY ${BUILD_ROOT}/scripts/entrypoint.sh.podman /home/${USER}/entrypoint.sh +RUN chmod +rx /home/${USER}/entrypoint.sh && \ + chown ${USER}:${USER} /home/${USER}/entrypoint.sh + +# Final switch to user for runtime +USER ${USER} +ENV PATH /home/${USER}/Spindle-inst/bin:$PATH + +ENTRYPOINT /bin/bash ./entrypoint.sh diff --git a/containers/spindle-serial-ubuntu/scripts/entrypoint.sh.podman b/containers/spindle-serial-ubuntu/scripts/entrypoint.sh.podman new file mode 100644 index 00000000..b4e7a05d --- /dev/null +++ b/containers/spindle-serial-ubuntu/scripts/entrypoint.sh.podman @@ -0,0 +1,10 @@ +#!/bin/bash +# Podman-compatible entrypoint for Spindle serial container +# Start munged in background and keep container alive + +printf "\nStarting munged\n" + +sudo -u munge /usr/sbin/munged + +# Keep container alive +sleep inf diff --git a/containers/spindle-slurm-ubuntu/base/Dockerfile.podman b/containers/spindle-slurm-ubuntu/base/Dockerfile.podman new file mode 100644 index 00000000..0e44ee6e --- /dev/null +++ b/containers/spindle-slurm-ubuntu/base/Dockerfile.podman @@ -0,0 +1,90 @@ +ARG UBUNTU_VERSION=noble +FROM ubuntu:${UBUNTU_VERSION} +USER root +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + apt-utils + +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + locales \ + ca-certificates \ + wget \ + git \ + ssh \ + sudo \ + psmisc \ + build-essential \ + pkg-config \ + autotools-dev \ + libtool \ + autoconf \ + automake \ + make \ + gfortran-13 \ + gcc-13 \ + g++-13 \ + gdb \ + munge \ + libmunge-dev \ + libhwloc-dev \ + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + python-is-python3 \ + openssh-server \ + openssh-client \ + mariadb-client \ + libmariadb-dev \ + libhttp-parser-dev \ + libjson-c-dev + + +# Prevent hwloc from trying to use graphics cards +# as this fails when X is not running. +ENV HWLOC_COMPONENTS=-gl + +# Suppress UCX warnings about /proc/sys tuning parameters +ENV UCX_LOG_LEVEL=error + +# Set up munge +RUN mkdir -p /run/munge && \ + chown munge:munge /run/munge && \ + chmod 0755 /run/munge + +ARG BUILD_ROOT=. +COPY ${BUILD_ROOT}/scripts/add_docker_user.sh /add_docker_user.sh + +# Slurm daemons run as $SLURM_USER +ARG SLURM_USER=slurm +ARG USER=${SLURM_USER} +ARG UID=1002 +RUN /add_docker_user.sh + +# Applications run as $USER +ARG USER=slurmuser +ARG UID=1001 +RUN /add_docker_user.sh + +ARG SLURM_VERSION=slurm-25-05-3-1 +COPY ${BUILD_ROOT}/scripts/build_slurm.sh /build_slurm.sh +RUN chmod +x /build_slurm.sh && /build_slurm.sh + +ARG MPICH_VERSION=4.2.2 +COPY ${BUILD_ROOT}/scripts/build_mpich.sh.podman /build_mpich.sh +RUN chmod +x /build_mpich.sh && /build_mpich.sh + +COPY ${BUILD_ROOT}/conf/mpich.conf /etc/ld.so.conf.d/mpich.conf +RUN ldconfig + +ENV SLURM_MPI_TYPE pmi2 diff --git a/containers/spindle-slurm-ubuntu/base/scripts/build_mpich.sh.podman b/containers/spindle-slurm-ubuntu/base/scripts/build_mpich.sh.podman new file mode 100644 index 00000000..b35a7e48 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/base/scripts/build_mpich.sh.podman @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euxo pipefail + +mkdir mpich +pushd mpich +# Add --no-same-owner for rootless podman compatibility +wget -O - https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz | tar xvz --no-same-owner --strip-components 1 +mkdir -p build +pushd build +../configure --prefix=/usr --disable-fortran --with-slurm=/usr/include/slurm +make -j$(nproc) install +popd +popd +rm -rf mpich diff --git a/containers/spindle-slurm-ubuntu/testing-srun/Dockerfile.podman b/containers/spindle-slurm-ubuntu/testing-srun/Dockerfile.podman new file mode 100644 index 00000000..681eb582 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-srun/Dockerfile.podman @@ -0,0 +1,55 @@ +FROM spindle-slurm-base:latest +ARG replicas=4 +ENV workers=${replicas} +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 + +ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing-srun + +# Slurm daemons run as $SLURM_USER +ARG SLURM_USER=slurm + +# Applications run as $USER +ARG USER=slurmuser +ARG UID=1001 + +# Set up the Slurm install already present in the base image +USER root +COPY ${BUILD_ROOT}/scripts/setup_slurm.sh.podman /setup_slurm.sh +COPY ${BUILD_ROOT}/conf/slurm.conf.template /home/${SLURM_USER}/slurm.conf.template +COPY ${BUILD_ROOT}/conf/slurmdbd.conf.template /home/${SLURM_USER}/slurmdbd.conf.template +COPY ${BUILD_ROOT}/conf/cgroup.conf /home/${SLURM_USER}/cgroup.conf +RUN chmod +x /setup_slurm.sh && /setup_slurm.sh + +USER ${USER} +WORKDIR /home/${USER} + +# Copy the Spindle repo into the container and build it +RUN mkdir -p /home/${USER}/Spindle +COPY . /home/${USER}/Spindle + +# Fix permissions on Spindle source +USER root +RUN chmod -R u+rwX /home/${USER}/Spindle && \ + chown -R ${USER}:${USER} /home/${USER}/Spindle + +# Copy and prepare build script +COPY ${BUILD_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN chmod +rx /home/${USER}/build_spindle.sh && \ + chown ${USER}:${USER} /home/${USER}/build_spindle.sh + +# Build Spindle as user +USER ${USER} +RUN bash ./build_spindle.sh + +# Copy entrypoint as root +USER root +COPY ${BUILD_ROOT}/scripts/entrypoint.sh.podman /home/${USER}/entrypoint.sh +RUN chmod +rx /home/${USER}/entrypoint.sh && \ + chown ${USER}:${USER} /home/${USER}/entrypoint.sh + +USER ${USER} +ENV PATH /home/${USER}/Spindle-inst/bin:$PATH + +ENTRYPOINT /bin/bash ./entrypoint.sh diff --git a/containers/spindle-slurm-ubuntu/testing-srun/conf/slurm.conf.template b/containers/spindle-slurm-ubuntu/testing-srun/conf/slurm.conf.template new file mode 100644 index 00000000..22de2c23 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-srun/conf/slurm.conf.template @@ -0,0 +1,40 @@ +ClusterName=linux +ControlMachine=${SLURM_HEAD_NODE} +ControlAddr=${SLURM_HEAD_NODE} +SlurmUser=slurm +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +StateSaveLocation=/var/lib/slurmd +SlurmdSpoolDir=/var/spool/slurmd +SwitchType=switch/none +MpiDefault=none +SlurmctldPidFile=/var/run/slurmd/slurmctld.pid +SlurmdPidFile=/var/run/slurmd/slurmd.pid +ProctrackType=proctrack/linuxproc +# TaskPlugin=task/affinity disabled - causes "Operation not permitted" in rootless podman +TaskPlugin=task/none +ReturnToService=2 +SlurmctldTimeout=300 +SlurmdTimeout=300 +InactiveLimit=0 +MinJobAge=300 +KillWait=30 +Waittime=0 +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory +SlurmctldDebug=3 +SlurmctldLogFile=/var/log/slurm/slurmctld.log +SlurmdDebug=3 +SlurmdLogFile=/var/log/slurm/slurmd.log +JobCompType=jobcomp/filetxt +JobCompLoc=/var/log/slurm/jobcomp.log +JobAcctGatherType=jobacct_gather/linux +JobAcctGatherFrequency=30 +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost=${SLURM_DB_HOST} +AccountingStoragePort=6819 +# NodeName entries are generated at runtime by entrypoint based on $workers and $SLURM_NODE_PREFIX +PartitionName=debug Nodes=ALL Default=YES MaxTime=INFINITE State=UP + diff --git a/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf b/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf new file mode 100644 index 00000000..01c05970 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf @@ -0,0 +1,11 @@ +AuthType=auth/munge +DbdAddr=slurm-db +DbdHost=slurm-db +SlurmUser=slurm +DebugLevel=4 +LogFile=/var/log/slurm/slurmdbd.log +PidFile=/var/run/slurmdbd/slurmdbd.pid +StorageType=accounting_storage/mysql +StorageHost=slurm-mariadb +StorageUser=slurm +StoragePass=LUJWpLWqKaowyZHEOaQIXQ diff --git a/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf.template b/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf.template index 0e274118..52907024 100644 --- a/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf.template +++ b/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf.template @@ -1,10 +1,10 @@ AuthType=auth/munge -DbdAddr=slurm-db -DbdHost=slurm-db +DbdAddr=${SLURM_DB_HOST} +DbdHost=${SLURM_DB_HOST} SlurmUser=slurm DebugLevel=4 LogFile=/var/log/slurm/slurmdbd.log PidFile=/var/run/slurmdbd/slurmdbd.pid StorageType=accounting_storage/mysql -StorageHost=slurm-mariadb +StorageHost=${SLURM_MARIADB_HOST} StorageUser=slurm diff --git a/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env b/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env new file mode 100644 index 00000000..d61fe3eb --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env @@ -0,0 +1 @@ +MARIADB_PASSWORD: "LUJWpLWqKaowyZHEOaQIXQ" diff --git a/containers/spindle-slurm-ubuntu/testing-srun/scripts/entrypoint.sh.podman b/containers/spindle-slurm-ubuntu/testing-srun/scripts/entrypoint.sh.podman new file mode 100644 index 00000000..c9333e84 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-srun/scripts/entrypoint.sh.podman @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# +# Podman-compatible entrypoint for Slurm srun containers + +set -x # Debug output + +echo "SLURM_ROLE: ${SLURM_ROLE}" + +echo "Starting munged..." +sudo -u munge /usr/sbin/munged +sleep 2 + +# Generate slurm.conf from template with runtime hostname substitution +echo "Generating slurm.conf from template..." +export SLURM_HEAD_NODE=${SLURM_HEAD_NODE:-slurm-head} +export SLURM_DB_HOST=${SLURM_DB_HOST:-slurm-db} +export SLURM_NODE_PREFIX=${SLURM_NODE_PREFIX:-slurm-node} +echo "DEBUG: SLURM_HEAD_NODE=$SLURM_HEAD_NODE" +echo "DEBUG: SLURM_DB_HOST=$SLURM_DB_HOST" +echo "DEBUG: SLURM_NODE_PREFIX=$SLURM_NODE_PREFIX" +echo "DEBUG: Template first 3 lines:" +sudo head -3 /etc/slurm/slurm.conf.template +echo "DEBUG: Running sed to substitute variables..." +sudo sed -e "s/\${SLURM_HEAD_NODE}/$SLURM_HEAD_NODE/g" \ + -e "s/\${SLURM_DB_HOST}/$SLURM_DB_HOST/g" \ + -e "s/\${SLURM_NODE_PREFIX}/$SLURM_NODE_PREFIX/g" \ + /etc/slurm/slurm.conf.template | sudo tee /etc/slurm/slurm.conf > /dev/null + +# Add NodeName entries dynamically based on $workers +workers=${workers:-4} +for i in $(seq 1 $workers); do + echo "NodeName=${SLURM_NODE_PREFIX}-${i} NodeAddr=${SLURM_NODE_PREFIX}-${i} CPUs=3 RealMemory=1000 State=UNKNOWN" | sudo tee -a /etc/slurm/slurm.conf > /dev/null +done +sudo chmod 644 /etc/slurm/slurm.conf + +echo "Generated slurm.conf (first 10 lines):" +sudo head -10 /etc/slurm/slurm.conf +echo "..." +echo "Last 5 lines:" +sudo tail -5 /etc/slurm/slurm.conf +echo "" + +if [ "${SLURM_ROLE}" = "db" ]; then + echo "=== ENTRYPOINT DEBUG (v2) ===" + echo "MARIADB_PASSWORD is set: $([ -n "$MARIADB_PASSWORD" ] && echo YES || echo NO)" + echo "MARIADB_PASSWORD length: ${#MARIADB_PASSWORD}" + echo "Template exists: $([ -f /etc/slurm/slurmdbd.conf.template ] && echo YES || echo NO)" + + # Generate slurmdbd.conf from template with runtime hostname and password substitution + if [ -n "$MARIADB_PASSWORD" ]; then + echo "Generating slurmdbd.conf from template..." + export SLURM_MARIADB_HOST=${SLURM_MARIADB_HOST:-slurm-mariadb} + echo "DEBUG: SLURM_DB_HOST=$SLURM_DB_HOST" + echo "DEBUG: SLURM_MARIADB_HOST=$SLURM_MARIADB_HOST" + echo "DEBUG: Template first 3 lines:" + sudo head -3 /etc/slurm/slurmdbd.conf.template + echo "DEBUG: Running sed to substitute variables..." + sudo sed -e "s/\${SLURM_DB_HOST}/$SLURM_DB_HOST/g" \ + -e "s/\${SLURM_MARIADB_HOST}/$SLURM_MARIADB_HOST/g" \ + /etc/slurm/slurmdbd.conf.template | sudo tee /etc/slurm/slurmdbd.conf > /dev/null + echo "StoragePass=${MARIADB_PASSWORD}" | sudo tee -a /etc/slurm/slurmdbd.conf > /dev/null + sudo chmod 600 /etc/slurm/slurmdbd.conf + sudo chown slurm:slurm /etc/slurm/slurmdbd.conf + echo "Generated slurmdbd.conf (complete file):" + sudo cat /etc/slurm/slurmdbd.conf + else + echo "ERROR: MARIADB_PASSWORD not set!" + echo "Old config exists: $([ -f /etc/slurm/slurmdbd.conf ] && echo YES || echo NO)" + fi + echo "=== END DEBUG ===" + echo "" + + echo "Starting slurmdbd..." + exec sudo -u slurm /usr/sbin/slurmdbd -Dvvv +elif [ "${SLURM_ROLE}" = "ctl" ] ; then + echo "Starting slurmctld..." + exec sudo -u slurm /usr/sbin/slurmctld -i -Dvvv +elif [ "${SLURM_ROLE}" = "worker" ] ; then + echo "Starting slurmd..." + exec sudo /usr/sbin/slurmd -Dvvv +else + echo "Unknown SLURM_ROLE: ${SLURM_ROLE}" + echo "Valid roles: db, ctl, worker" + exit 1 +fi diff --git a/containers/spindle-slurm-ubuntu/testing-srun/scripts/setup_slurm.sh.podman b/containers/spindle-slurm-ubuntu/testing-srun/scripts/setup_slurm.sh.podman new file mode 100644 index 00000000..7d73ccac --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-srun/scripts/setup_slurm.sh.podman @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euxo pipefail + +mkdir -p /etc/slurm /etc/sysconfig/slurm /var/spool/slurmd /var/spool/slurmctld /var/run/slurmd /var/run/slurmdbd /var/lib/slurmd /var/log/slurm +touch /var/lib/slurmd/node_state /var/lib/slurmd/front_end_state /var/lib/slurmd/job_state /var/lib/slurmd/resv_state /var/lib/slurmd/trigger_state /var/lib/slurmd/assoc_mgr_state /var/lib/slurmd/assoc_usage /var/lib/slurmd/qos_usage /var/lib/slurmd/fed_mgr_state +cp /home/${SLURM_USER}/slurm.conf.template /etc/slurm/slurm.conf.template +cp /home/${SLURM_USER}/slurmdbd.conf.template /etc/slurm/slurmdbd.conf.template +cp /home/${SLURM_USER}/cgroup.conf /etc/slurm/cgroup.conf +chown -R slurm:slurm /etc/slurm /etc/sysconfig/slurm /var/spool/slurmd /var/spool/slurmctld /var/run/slurmd /var/run/slurmdbd /var/lib/slurmd /var/log/slurm +# Templates should be readable by slurm (will be used to generate configs at runtime) +chmod 644 /etc/slurm/slurm.conf.template /etc/slurm/slurmdbd.conf.template +# cgroup.conf should be world-readable (needed by slurmuser for salloc/srun) +chmod 644 /etc/slurm/cgroup.conf diff --git a/containers/spindle-slurm-ubuntu/testing/Dockerfile.podman b/containers/spindle-slurm-ubuntu/testing/Dockerfile.podman new file mode 100644 index 00000000..96ab2219 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing/Dockerfile.podman @@ -0,0 +1,68 @@ +FROM spindle-slurm-base:latest +ARG replicas=4 +ENV workers=${replicas} +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 + +ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing + +# Slurm daemons run as $SLURM_USER +ARG SLURM_USER=slurm + +# Applications run as $USER +ARG USER=slurmuser +ARG UID=1001 + +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + gdb \ + libc6-dbg + +# Set up the Slurm install already present in the base image +USER root +COPY ${BUILD_ROOT}/scripts/setup_slurm.sh.podman /setup_slurm.sh +COPY ${BUILD_ROOT}/conf/slurm.conf.podman /home/${SLURM_USER}/slurm.conf +COPY ${BUILD_ROOT}/conf/slurmdbd.conf /home/${SLURM_USER}/slurmdbd.conf +COPY ${BUILD_ROOT}/conf/cgroup.conf /home/${SLURM_USER}/cgroup.conf +RUN chmod +x /setup_slurm.sh && /setup_slurm.sh + +# Slurm without Spank plugin needs passwordless ssh +USER root +COPY ${BUILD_ROOT}/conf/ssh_config /home/${USER}/ +COPY ${BUILD_ROOT}/scripts/setup_ssh.sh /home/${USER}/ +RUN chmod +x /home/${USER}/setup_ssh.sh && \ + chown ${USER}:${USER} /home/${USER}/ssh_config /home/${USER}/setup_ssh.sh + +USER ${USER} +WORKDIR /home/${USER} +RUN bash ./setup_ssh.sh + +# Copy the Spindle repo into the container and build it +RUN mkdir -p /home/${USER}/Spindle +COPY . /home/${USER}/Spindle + +# Fix permissions on Spindle source +USER root +RUN chmod -R u+rwX /home/${USER}/Spindle && \ + chown -R ${USER}:${USER} /home/${USER}/Spindle + +# Copy and prepare build script +COPY ${BUILD_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN chmod +rx /home/${USER}/build_spindle.sh && \ + chown ${USER}:${USER} /home/${USER}/build_spindle.sh + +# Build Spindle as user +USER ${USER} +RUN bash ./build_spindle.sh + +# Copy entrypoint as root +USER root +COPY ${BUILD_ROOT}/scripts/entrypoint.sh.podman /home/${USER}/entrypoint.sh +RUN chmod +rx /home/${USER}/entrypoint.sh && \ + chown ${USER}:${USER} /home/${USER}/entrypoint.sh + +USER ${USER} +ENV PATH /home/${USER}/Spindle-inst/bin:$PATH + +ENTRYPOINT /bin/bash ./entrypoint.sh diff --git a/containers/spindle-slurm-ubuntu/testing/conf/slurm.conf.podman b/containers/spindle-slurm-ubuntu/testing/conf/slurm.conf.podman new file mode 100644 index 00000000..1a308beb --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing/conf/slurm.conf.podman @@ -0,0 +1,43 @@ +ClusterName=linux +ControlMachine=slurm-head +ControlAddr=slurm-head +SlurmUser=slurm +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +StateSaveLocation=/var/lib/slurmd +SlurmdSpoolDir=/var/spool/slurmd +SwitchType=switch/none +MpiDefault=none +SlurmctldPidFile=/var/run/slurmd/slurmctld.pid +SlurmdPidFile=/var/run/slurmd/slurmd.pid +ProctrackType=proctrack/linuxproc +# TaskPlugin=task/affinity disabled - causes "Operation not permitted" in rootless podman +TaskPlugin=task/none +ReturnToService=2 +SlurmctldTimeout=300 +SlurmdTimeout=300 +InactiveLimit=0 +MinJobAge=300 +KillWait=30 +Waittime=0 +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory +SlurmctldDebug=3 +SlurmctldLogFile=/var/log/slurm/slurmctld.log +SlurmdDebug=3 +SlurmdLogFile=/var/log/slurm/slurmd.log +JobCompType=jobcomp/filetxt +JobCompLoc=/var/log/slurm/jobcomp.log +JobAcctGatherType=jobacct_gather/linux +JobAcctGatherFrequency=30 +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost=slurm-db +AccountingStoragePort=6819 +NodeName=slurm-node-1 NodeAddr=slurm-node-1 CPUs=3 RealMemory=1000 State=UNKNOWN +NodeName=slurm-node-2 NodeAddr=slurm-node-2 CPUs=3 RealMemory=1000 State=UNKNOWN +NodeName=slurm-node-3 NodeAddr=slurm-node-3 CPUs=3 RealMemory=1000 State=UNKNOWN +NodeName=slurm-node-4 NodeAddr=slurm-node-4 CPUs=3 RealMemory=1000 State=UNKNOWN +PartitionName=debug Nodes=ALL Default=YES MaxTime=INFINITE State=UP + diff --git a/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh.podman b/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh.podman new file mode 100644 index 00000000..72d2d681 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh.podman @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# Podman-compatible entrypoint for Slurm rshlaunch containers + +set -x # Debug output + +echo "SLURM_ROLE: ${SLURM_ROLE}" + +echo "Configuring SSH..." +# Create privilege separation directory +sudo mkdir -p /run/sshd +sudo chmod 755 /run/sshd + +# Regenerate SSH host keys if they don't exist or are problematic +if [ ! -f /etc/ssh/ssh_host_rsa_key ]; then + sudo ssh-keygen -A +fi +# Fix permissions +sudo chmod 600 /etc/ssh/ssh_host_*_key +sudo chmod 644 /etc/ssh/ssh_host_*_key.pub + +echo "Starting sshd..." +# Disable privilege separation - setgroups fails in rootless podman on LC systems +sudo bash -c 'ulimit -c unlimited; /usr/sbin/sshd -o UsePrivilegeSeparation=no' + +echo "Starting munged..." +sudo -u munge /usr/sbin/munged +sleep 2 + +if [ -d /shared ]; then + sudo chown -R "$(id -un):$(id -gn)" /shared + sudo chmod 755 /shared +fi + +if [ "${SLURM_ROLE}" = "db" ]; then + echo "Starting slurmdbd..." + exec sudo -u slurm /usr/sbin/slurmdbd -Dvvv +elif [ "${SLURM_ROLE}" = "ctl" ] ; then + echo "Starting slurmctld..." + exec sudo -u slurm /usr/sbin/slurmctld -i -Dvvv +elif [ "${SLURM_ROLE}" = "worker" ] ; then + echo "Starting slurmd..." + exec sudo bash -c 'ulimit -c unlimited; exec /usr/sbin/slurmd -Dvvv' +else + echo "Unknown SLURM_ROLE: ${SLURM_ROLE}" + echo "Valid roles: db, ctl, worker" + exit 1 +fi diff --git a/containers/spindle-slurm-ubuntu/testing/scripts/setup_slurm.sh.podman b/containers/spindle-slurm-ubuntu/testing/scripts/setup_slurm.sh.podman new file mode 100644 index 00000000..1adb6e42 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing/scripts/setup_slurm.sh.podman @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euxo pipefail + +mkdir -p /etc/slurm /etc/sysconfig/slurm /var/spool/slurmd /var/spool/slurmctld /var/run/slurmd /var/run/slurmdbd /var/lib/slurmd /var/log/slurm +touch /var/lib/slurmd/node_state /var/lib/slurmd/front_end_state /var/lib/slurmd/job_state /var/lib/slurmd/resv_state /var/lib/slurmd/trigger_state /var/lib/slurmd/assoc_mgr_state /var/lib/slurmd/assoc_usage /var/lib/slurmd/qos_usage /var/lib/slurmd/fed_mgr_state +cp /home/${SLURM_USER}/slurm.conf /etc/slurm/slurm.conf +cp /home/${SLURM_USER}/slurmdbd.conf /etc/slurm/slurmdbd.conf +cp /home/${SLURM_USER}/cgroup.conf /etc/slurm/cgroup.conf +chown -R slurm:slurm /etc/slurm /etc/sysconfig/slurm /var/spool/slurmd /var/spool/slurmctld /var/run/slurmd /var/run/slurmdbd /var/lib/slurmd /var/log/slurm +# slurmdbd.conf should be readable only by slurm (has password) +chmod 600 /etc/slurm/slurmdbd.conf +# slurm.conf and cgroup.conf should be world-readable (needed by slurmuser for salloc/srun) +chmod 644 /etc/slurm/slurm.conf /etc/slurm/cgroup.conf diff --git a/scripts/podman/README.md b/scripts/podman/README.md new file mode 100644 index 00000000..2c87e895 --- /dev/null +++ b/scripts/podman/README.md @@ -0,0 +1,219 @@ +# Spindle Podman Scripts + +Scripts for running Spindle containers locally with podman on LC systems. + +## Prerequisites + +- Podman installed and working +- **LC systems: Run `enable-podman` before using these scripts** +- Subuid/subgid configured for your user + +## Quick Start + +### Hello World Tests (verify environment) + +```bash +./run-hello-01-basic.sh # Basic container execution +./run-hello-02-user.sh # User switching +./run-hello-03-filesystem.sh # Volume mounts +./run-hello-04-networking.sh # Multi-container networking +./run-hello-05-flux.sh # Flux cluster +./run-hello-06-slurm.sh # Slurm cluster +``` + +### Spindle Tests + +```bash +# Build images +./build-spindle-serial.sh +./build-spindle-slurm-base.sh && ./build-spindle-slurm-srun.sh +./build-spindle-flux.sh + +# Run tests +./test-spindle-serial.sh +./test-spindle-slurm-srun.sh +./test-spindle-flux.sh + +# Debug variants (keeps containers running) +./test-spindle-serial-debug.sh + +# Crash tests +./test-spindle-serial-crash.sh +``` + +### Parallel Testing on Compute Nodes + +```bash +# On login node: save images to tarball +./save-images.sh + +# On compute node: load images +./load-images.sh /path/to/spindle-podman-images.tar + +# Run multiple tests in parallel +for i in $(seq 1 10); do + ./test-spindle-slurm-srun-parallel.sh $i > out.$i 2>&1 & +done +wait + +# Clean up +./cleanup.sh +``` + +## Script Reference + +### Infrastructure + +- **common.sh** - Shared functions and LC-specific configurations + - SSL certificate mounts for LC systems + - `podman_build()` and `podman_run()` wrappers with PODMAN_BUILD arg + - Handles setgroups workaround + +- **cleanup.sh** - Remove all or specific test containers/networks + - Usage: `./cleanup.sh` (all) or `./cleanup.sh ` (specific) + - Parallel cleanup for speed + +- **save-images.sh** - Save Spindle images to tarball for compute node deployment + - Creates `spindle-podman-images.tar` in repo root + - Bundles `mariadb.env` for portable Slurm testing + +- **load-images.sh** - Load images from tarball on compute nodes + - Usage: `./load-images.sh ` + - Extracts and loads each image separately + +### Hello World Tests + +- **run-hello-01-basic.sh** - Basic execution (validates environment) +- **run-hello-02-user.sh** - User switching (validates non-root patterns) +- **run-hello-03-filesystem.sh** - Volume mounts (validates host/container filesystem) +- **run-hello-04-networking.sh** - Multi-container networking (validates cluster patterns) +- **run-hello-05-flux.sh** - Flux cluster (validates Flux setup) +- **run-hello-06-slurm.sh** - Slurm cluster (validates Slurm setup) + +### Build Scripts + +- **build-spindle-serial.sh** - Build serial Spindle container +- **build-spindle-slurm-base.sh** - Build Slurm base image (Slurm + MPICH from source, ~6 min) +- **build-spindle-slurm-srun.sh** - Build Slurm srun test layer (requires base) +- **build-spindle-slurm-rshlaunch.sh** - Build Slurm rshlaunch test layer (BLOCKED: SSH issues) +- **build-spindle-flux.sh** - Build Flux Spindle container (~7 min) + +### Test Scripts + +- **test-spindle-serial.sh** - Serial launcher tests +- **test-spindle-serial-debug.sh** - Serial tests with containers kept running for debugging +- **test-spindle-serial-crash.sh** - Serial crash dump tests + +- **test-spindle-slurm-srun.sh** - Slurm srun tests (7 containers: MariaDB, slurmdbd, slurmctld, 4 workers) +- **test-spindle-slurm-srun-parallel.sh** - Parallel-safe variant with unique container names + - Usage: `./test-spindle-slurm-srun-parallel.sh ` + - Enables multiple concurrent test runs without conflicts + +- **test-spindle-slurm-rshlaunch.sh** - Slurm rshlaunch tests (BLOCKED: SSH setgroups issue) + +- **test-spindle-flux.sh** - Flux launcher tests (PARTIALLY WORKING: cluster starts but Spindle tests fail) + +## Deployment Workflow + +### Local Development (Login Node) + +1. Build images: `./build-spindle-*.sh` +2. Test locally: `./test-spindle-*.sh` +3. Iterate and debug + +### Compute Node Deployment + +1. **On login node:** + ```bash + ./save-images.sh + # Creates spindle-podman-images.tar + mariadb.env + ``` + +2. **On compute node:** + ```bash + enable-podman # REQUIRED on LC systems + ./load-images.sh /path/to/spindle-podman-images.tar + ``` + +3. **Run parallel tests:** + ```bash + for i in $(seq 1 50); do + ./test-spindle-slurm-srun-parallel.sh $i > out.$i 2>&1 & + done + wait + + # Check results + grep -l "ALL TESTS PASSED" out.* + grep -l "SOME TESTS FAILED" out.* + ``` + +4. **Cleanup:** + ```bash + ./cleanup.sh + ``` + +## Known Issues + +### On LC Systems: Must Run enable-podman First + +LC systems require running `enable-podman` before using podman. This configures storage and clears any stale state. If you see "Access denied" or image loading issues, run `enable-podman` and retry. + +### Slurm Tests Require mariadb.env + +Slurm tests need the MariaDB password from `mariadb.env`. The `save-images.sh` script copies this to the repo root for portability. If you get "Access denied for user 'slurm'@...", check that `mariadb.env` exists in the repo root. + +### Parallel Test Limits + +Rootless podman has resource limits. On compute nodes, 50-100 parallel tests work well. Beyond that, you may hit: +- File descriptor limits +- Network namespace limits +- Memory pressure + +### Flux Tests Partially Working + +Flux cluster starts correctly, but Spindle tests fail with "spindleRunBE failed!". Needs debugging. See PODMAN.md for details. + +### Slurm rshlaunch Blocked + +SSH privilege separation calls `setgroups()`, which fails in rootless podman on LC systems. Tried `UsePrivilegeSeparation=no` but still fails. May need sshd rebuild or alternative SSH implementation. + +## Troubleshooting + +### "newuidmap failed: Operation not permitted" +Your user doesn't have subuid/subgid mappings. Check: +```bash +grep $(whoami) /etc/subuid /etc/subgid +``` + +If empty, contact your sysadmin to add entries. + +### "certificate signed by unknown authority" +The SSL certificate mounts may be incorrect for your system. Check if these files exist: +```bash +ls -l /etc/pki/ca-trust/source/anchors/PAN-cspca.llnl.gov.crt.pem +ls -l /etc/pki/tls/certs/ca-bundle.trust.crt +``` + +### "setgroups 65534 failed" +The Dockerfile needs the apt sandbox fix. Verify the Dockerfile has: +```dockerfile +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi +``` + +And you're using `podman_build()` from common.sh (not raw `podman build`). + +### Image IDs All the Same After Load + +If `podman images | grep spindle` shows the same image ID for all tags, the tarball wasn't created correctly. Re-run `save-images.sh` which saves each image separately then combines them. + +### Containers Exit Immediately + +Check that you ran `enable-podman` before loading images. Also verify images loaded correctly with `podman images | grep spindle` - each should have a different ID. + +## LC System Documentation + +For more details on podman on LC systems: +https://hpc.llnl.gov/documentation/user-guides/using-containers-lc-hpc-systems/containers-how-build-container diff --git a/scripts/podman/build-spindle-flux.sh b/scripts/podman/build-spindle-flux.sh new file mode 100755 index 00000000..17b6397a --- /dev/null +++ b/scripts/podman/build-spindle-flux.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Build Spindle Flux container for podman +# +# This builds the Spindle Flux multi-node container. +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-flux-ubuntu" +DOCKERFILE="$REPO_ROOT/containers/spindle-flux-ubuntu/Dockerfile.podman" + +echo "==========================================" +echo "Building Spindle Flux Container" +echo "==========================================" +echo "" +echo "This builds the Spindle Flux multi-node test container." +echo "Image: $IMAGE_NAME" +echo "Dockerfile: $DOCKERFILE" +echo "" + +# Build the image +# Pass replicas=4 to match docker-compose configuration +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" "--build-arg replicas=4" + +echo "" +echo "==========================================" +echo "Build complete!" +echo "==========================================" +echo "" +echo "Image: $IMAGE_NAME" +echo "" +echo "Next steps:" +echo " 1. Run regular tests: ./scripts/podman/test-spindle-flux.sh" +echo " 2. Run crash tests: ./scripts/podman/test-spindle-flux-crash.sh" +echo "" diff --git a/scripts/podman/build-spindle-serial.sh b/scripts/podman/build-spindle-serial.sh new file mode 100755 index 00000000..b7ae2428 --- /dev/null +++ b/scripts/podman/build-spindle-serial.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Build Spindle serial container for podman +# +# This builds the actual Spindle serial container (not a hello-world demo). +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-serial-ubuntu" +DOCKERFILE="$REPO_ROOT/containers/spindle-serial-ubuntu/Dockerfile.podman" + +echo "==========================================" +echo "Building Spindle Serial Container" +echo "==========================================" +echo "" +echo "This builds the actual Spindle serial test container." +echo "Image: $IMAGE_NAME" +echo "Dockerfile: $DOCKERFILE" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" +echo "==========================================" +echo "Build complete!" +echo "==========================================" +echo "" +echo "Image: $IMAGE_NAME" +echo "" +echo "Next steps:" +echo " 1. Run regular tests: ./scripts/podman/test-spindle-serial.sh" +echo " 2. Run crash tests: ./scripts/podman/test-spindle-serial-crash.sh" +echo " 3. Or manually: podman run --rm -it $IMAGE_NAME" +echo "" diff --git a/scripts/podman/build-spindle-slurm-base.sh b/scripts/podman/build-spindle-slurm-base.sh new file mode 100755 index 00000000..678bc3d6 --- /dev/null +++ b/scripts/podman/build-spindle-slurm-base.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Build Spindle Slurm base image for podman +# +# This builds the base image with Slurm and MPICH. +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-slurm-base" +DOCKERFILE="$REPO_ROOT/containers/spindle-slurm-ubuntu/base/Dockerfile.podman" +CONTEXT="$REPO_ROOT/containers/spindle-slurm-ubuntu/base" + +echo "==========================================" +echo "Building Spindle Slurm Base Image" +echo "==========================================" +echo "" +echo "This builds the base image with Slurm and MPICH." +echo "This will take 5-10 minutes (building Slurm from source)." +echo "Image: $IMAGE_NAME" +echo "Dockerfile: $DOCKERFILE" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$CONTEXT" + +echo "" +echo "==========================================" +echo "Build complete!" +echo "==========================================" +echo "" +echo "Image: $IMAGE_NAME" +echo "" +echo "Next step: Build the testing image" +echo " ./scripts/podman/build-spindle-slurm-srun.sh" +echo "" diff --git a/scripts/podman/build-spindle-slurm-rshlaunch.sh b/scripts/podman/build-spindle-slurm-rshlaunch.sh new file mode 100755 index 00000000..e319fb1d --- /dev/null +++ b/scripts/podman/build-spindle-slurm-rshlaunch.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Build Spindle Slurm rshlaunch test container for podman +# +# This builds the Slurm testing image with Spindle (rshlaunch launcher). +# Requires: spindle-slurm-base image (build with build-spindle-slurm-base.sh first) +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +BASE_IMAGE="spindle-slurm-base" +IMAGE_NAME="spindle-slurm-rshlaunch" +DOCKERFILE="$REPO_ROOT/containers/spindle-slurm-ubuntu/testing/Dockerfile.podman" + +echo "==========================================" +echo "Building Spindle Slurm Rshlaunch Container" +echo "==========================================" +echo "" + +# Check if base image exists (with or without localhost/ prefix) +if ! podman images --format "{{.Repository}}" | grep -qE "^(localhost/)?${BASE_IMAGE}$"; then + echo "Error: Base image $BASE_IMAGE not found" + echo "" + echo "Available images:" + podman images | grep spindle || echo " (no spindle images found)" + echo "" + echo "Build it first with:" + echo " ./scripts/podman/build-spindle-slurm-base.sh" + echo "" + exit 1 +fi + +echo "Base image: $BASE_IMAGE" +echo "Test image: $IMAGE_NAME" +echo "Dockerfile: $DOCKERFILE" +echo "" + +# Generate MariaDB configuration +echo "Generating MariaDB configuration..." +cd "$REPO_ROOT/containers/spindle-slurm-ubuntu/testing" +./generate_config.sh +echo "✓ Configuration generated" +cd "$REPO_ROOT" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" "--build-arg replicas=4" + +echo "" +echo "==========================================" +echo "Build complete!" +echo "==========================================" +echo "" +echo "Image: $IMAGE_NAME" +echo "" +echo "Next steps:" +echo " 1. Run regular tests: ./scripts/podman/test-spindle-slurm-rshlaunch.sh" +echo "" diff --git a/scripts/podman/build-spindle-slurm-srun.sh b/scripts/podman/build-spindle-slurm-srun.sh new file mode 100755 index 00000000..a2a7e64e --- /dev/null +++ b/scripts/podman/build-spindle-slurm-srun.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Build Spindle Slurm srun test container for podman +# +# This builds the Slurm testing image with Spindle. +# Requires: spindle-slurm-base image (build with build-spindle-slurm-base.sh first) +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +BASE_IMAGE="spindle-slurm-base" +IMAGE_NAME="spindle-slurm-srun" +DOCKERFILE="$REPO_ROOT/containers/spindle-slurm-ubuntu/testing-srun/Dockerfile.podman" + +echo "==========================================" +echo "Building Spindle Slurm Srun Container" +echo "==========================================" +echo "" + +# Check if base image exists (with or without localhost/ prefix) +if ! podman images --format "{{.Repository}}" | grep -qE "^(localhost/)?${BASE_IMAGE}$"; then + echo "Error: Base image $BASE_IMAGE not found" + echo "" + echo "Available images:" + podman images | grep spindle || echo " (no spindle images found)" + echo "" + echo "Build it first with:" + echo " ./scripts/podman/build-spindle-slurm-base.sh" + echo "" + exit 1 +fi + +echo "Base image: $BASE_IMAGE" +echo "Test image: $IMAGE_NAME" +echo "Dockerfile: $DOCKERFILE" +echo "" + +# Generate MariaDB configuration +echo "Generating MariaDB configuration..." +cd "$REPO_ROOT/containers/spindle-slurm-ubuntu/testing-srun" +./generate_config.sh +echo "✓ Configuration generated" +cd "$REPO_ROOT" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" "--build-arg replicas=4" + +echo "" +echo "==========================================" +echo "Build complete!" +echo "==========================================" +echo "" +echo "Image: $IMAGE_NAME" +echo "" +echo "Next steps:" +echo " 1. Run regular tests: ./scripts/podman/test-spindle-slurm-srun.sh" +echo "" diff --git a/scripts/podman/cleanup.sh b/scripts/podman/cleanup.sh new file mode 100755 index 00000000..c917d7c3 --- /dev/null +++ b/scripts/podman/cleanup.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Clean up Spindle test containers and networks +# +# Usage: cleanup.sh [run-id] +# +# With no arguments: removes ALL slurm-srun-* containers and networks +# With run-id: removes only containers/networks for that specific run + +set -e + +RUN_ID="${1}" + +if [ -n "$RUN_ID" ]; then + echo "==========================================" + echo "Cleaning up run $RUN_ID..." + echo "==========================================" + echo "" + + NAME_PREFIX="slurm-srun-${RUN_ID}" + NETWORK_NAME="slurm-srun-${RUN_ID}-net" + + # Stop and remove containers for this run + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + if podman ps -a --format "{{.Names}}" | grep -q "^${container}$"; then + echo "Removing $container..." + podman stop "$container" 2>/dev/null || true + podman rm -f "$container" 2>/dev/null || true + fi + done + + # Remove network + if podman network exists "$NETWORK_NAME" 2>/dev/null; then + echo "Removing network $NETWORK_NAME..." + podman network rm -f "$NETWORK_NAME" 2>/dev/null || true + fi + + echo "✓ Cleanup complete for run $RUN_ID" +else + echo "==========================================" + echo "Cleaning up ALL Spindle test containers" + echo "==========================================" + echo "" + + # Find all slurm-srun containers + CONTAINERS=$(podman ps -a --format "{{.Names}}" | grep "^slurm-srun-" || true) + + if [ -n "$CONTAINERS" ]; then + COUNT=$(echo "$CONTAINERS" | wc -l) + echo "Found $COUNT containers to remove" + echo "" + + # Skip stop, just force-remove (stops and removes in one step) + # Use timeout and lower parallelism to avoid hangs + echo "$CONTAINERS" | while read container; do + echo "Removing $container..." + timeout 10 podman rm -f "$container" 2>/dev/null || echo " (timed out, skipping)" & + done + wait + else + echo "No slurm-srun containers found" + fi + + echo "" + + # Find all slurm-srun networks + NETWORKS=$(podman network ls --format "{{.Name}}" | grep "^slurm-srun-" || true) + + if [ -n "$NETWORKS" ]; then + COUNT=$(echo "$NETWORKS" | wc -l) + echo "Found $COUNT networks to remove" + echo "" + + # Remove networks sequentially with timeout to avoid hangs + echo "$NETWORKS" | while read network; do + echo "Removing $network..." + timeout 5 podman network rm -f "$network" 2>/dev/null || echo " (timed out or in use, skipping)" + done + else + echo "No slurm-srun networks found" + fi + + echo "" + echo "✓ Cleanup complete" +fi diff --git a/scripts/podman/common.sh b/scripts/podman/common.sh new file mode 100755 index 00000000..aac43ca5 --- /dev/null +++ b/scripts/podman/common.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# +# Common functions for running Spindle containers with podman on LC systems +# +# LC systems require special handling for: +# 1. SSL certificates (volume mounts) +# 2. apt setgroups errors (handled via Dockerfile ARG) + +set -e + +# LC-specific SSL certificate mounts +# Build: Mount LLNL cert into ca-certificates directory +LC_CERT_BUILD_MOUNT="-v /etc/pki/ca-trust/source/anchors/PAN-cspca.llnl.gov.crt.pem:/usr/local/share/ca-certificates/cspca.crt:ro" + +# Run: Mount system CA bundle +LC_CERT_RUN_MOUNT="-v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/ssl/certs/ca-certificates.crt:ro" + +# Helper function to build images with LC-specific settings +podman_build() { + local image_name=$1 + local dockerfile=$2 + local context=${3:-.} + local extra_args="$4" # Optional extra build args + + echo "========================================" + echo "Building: $image_name" + echo "Dockerfile: $dockerfile" + echo "Context: $context" + echo "========================================" + + podman build \ + --build-arg PODMAN_BUILD=true \ + $LC_CERT_BUILD_MOUNT \ + $extra_args \ + -t "$image_name" \ + -f "$dockerfile" \ + "$context" + + local rc=$? + if [ $rc -eq 0 ]; then + echo "✓ Build successful: $image_name" + else + echo "✗ Build failed with exit code $rc" + return $rc + fi +} + +# Helper function to run containers with LC-specific settings +podman_run() { + local image_name=$1 + shift + + echo "========================================" + echo "Running: $image_name" + echo "========================================" + + podman run \ + --rm \ + $LC_CERT_RUN_MOUNT \ + "$image_name" \ + "$@" + + local rc=$? + if [ $rc -eq 0 ]; then + echo "✓ Container exited successfully" + else + echo "✗ Container exited with code $rc" + return $rc + fi +} + +# Helper function to clean up images +podman_cleanup() { + local image_name=$1 + echo "Cleaning up image: $image_name" + podman rmi "$image_name" 2>/dev/null || true +} diff --git a/scripts/podman/load-images.sh b/scripts/podman/load-images.sh new file mode 100755 index 00000000..b23748f8 --- /dev/null +++ b/scripts/podman/load-images.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Load Spindle podman images from tarball +# +# Run this on compute nodes to load images saved with save-images.sh + +set -e + +TARBALL="${1}" + +if [ -z "$TARBALL" ]; then + echo "Usage: $0 " + echo "" + echo "Example:" + echo " $0 /g/g24/rountree/v/rzadams/sandbox/spindle-podman-images.tar" + exit 1 +fi + +if [ ! -f "$TARBALL" ]; then + echo "Error: Tarball not found: $TARBALL" + exit 1 +fi + +echo "==========================================" +echo "Loading Spindle Podman Images" +echo "==========================================" +echo "" +echo "Source: $TARBALL" +echo "This may take a minute..." +echo "" + +# Extract and load each image separately +TEMP_DIR=$(mktemp -d) +trap "rm -rf $TEMP_DIR" EXIT + +echo "Extracting tarball..." +tar -xf "$TARBALL" -C "$TEMP_DIR" + +echo "Loading base image..." +podman load -i "$TEMP_DIR/base.tar" + +echo "Loading srun image..." +podman load -i "$TEMP_DIR/srun.tar" + +echo "Loading serial image..." +podman load -i "$TEMP_DIR/serial.tar" + +echo "Loading MariaDB image..." +podman load -i "$TEMP_DIR/mariadb.tar" + +echo "" +echo "==========================================" +echo "✓ Images loaded successfully" +echo "==========================================" +echo "" +echo "Available Spindle images:" +podman images | grep -E '(spindle|mariadb)' || echo " (none - something went wrong)" +echo "" diff --git a/scripts/podman/rebuild-and-test-workflow.sh b/scripts/podman/rebuild-and-test-workflow.sh new file mode 100755 index 00000000..850a00c2 --- /dev/null +++ b/scripts/podman/rebuild-and-test-workflow.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# +# Bulletproof workflow for testing podman changes +# +# This script runs ON THE LOGIN NODE and guides you through the process +# +# Run from anywhere in the repo - it will find the right paths + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +BRANCH=$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") + +echo "==========================================" +echo "Podman Testing Workflow" +echo "==========================================" +echo "" +echo "Branch: $BRANCH" +echo "Repo root: $REPO_ROOT" +echo "" +echo "This script runs on the LOGIN NODE" +echo "It will rebuild, save, and give you commands for the compute node" +echo "" + +cd "$REPO_ROOT" + +TARBALL="$REPO_ROOT/spindle-podman-images.tar" + +# Check if base image exists +if ! podman images | grep -q "spindle-slurm-base"; then + echo "Base image not found in podman." + echo "" + + # Check if tarball exists + if [ -f "$TARBALL" ]; then + echo "Found tarball: $TARBALL" + echo "Loading images from tarball..." + echo "Command: ./scripts/podman/load-images.sh $TARBALL" + echo "" + ./scripts/podman/load-images.sh "$TARBALL" 2>&1 | ts + echo "" + + # Check again if image is now available + if podman images | grep -q "spindle-slurm-base"; then + echo "✓ Base image loaded from tarball" + echo "" + else + echo "ERROR: Base image not in tarball. Need to rebuild." + echo "" + echo "Building spindle-slurm-base from source..." + echo "Command: ./scripts/podman/build-spindle-slurm-base.sh" + echo "(This takes ~6 minutes - compiles Slurm + MPICH from source)" + echo "" + ./scripts/podman/build-spindle-slurm-base.sh 2>&1 | ts + echo "" + echo "✓ Base image built" + echo "" + fi + else + echo "Tarball not found at: $TARBALL" + echo "Building spindle-slurm-base from source..." + echo "Command: ./scripts/podman/build-spindle-slurm-base.sh" + echo "(This takes ~6 minutes - compiles Slurm + MPICH from source)" + echo "" + ./scripts/podman/build-spindle-slurm-base.sh 2>&1 | ts + echo "" + echo "✓ Base image built" + echo "" + fi +else + echo "✓ Base image already loaded" + echo "" +fi + +echo "Step 1: Rebuild slurm-srun image" +echo "Command: ./scripts/podman/build-spindle-slurm-srun.sh" +./scripts/podman/build-spindle-slurm-srun.sh 2>&1 | ts + +echo "" +echo "✓ Build complete" +echo "" + +echo "Step 2: Save images to tarball" +echo "Command: ./scripts/podman/save-images.sh" +./scripts/podman/save-images.sh 2>&1 | ts + +TARBALL_PATH="$REPO_ROOT/spindle-podman-images.tar" +echo "" +echo "✓ Images saved to: $TARBALL_PATH" +echo "" + +echo "==========================================" +echo "NOW SWITCH TO COMPUTE NODE" +echo "==========================================" +echo "" +echo "Run these commands on the compute node:" +echo "" +echo " # Get allocation (if needed)" +echo " salloc -N1 -t 60" +echo "" +echo " # Setup podman" +echo " enable-podman" +echo "" +echo " # Navigate to repo" +echo " cd $REPO_ROOT" +echo "" +echo " # Load images" +echo " ./scripts/podman/load-images.sh ./spindle-podman-images.tar" +echo "" +echo " # Verify canary - should show 'ENTRYPOINT DEBUG (v2)'" +echo " podman run --rm -e SLURM_ROLE=db -e MARIADB_PASSWORD=test123 localhost/spindle-slurm-srun:latest 2>&1 | head -30" +echo "" +echo " # Run test" +echo " ./scripts/podman/test-spindle-slurm-srun.sh 1 2>&1 | tee test-output.log" +echo "" +echo "==========================================" +echo "" +echo "Copy the commands above to your compute node terminal" +echo "" diff --git a/scripts/podman/run-hello-01-basic.sh b/scripts/podman/run-hello-01-basic.sh new file mode 100755 index 00000000..b0876951 --- /dev/null +++ b/scripts/podman/run-hello-01-basic.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Test 01-basic: Validate podman environment +# +# This test verifies: +# - Container can build with apt-get (tests setgroups fix) +# - Container can run +# - Network connectivity works +# - SSL certificates are properly configured +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-hello-01-basic" +DOCKERFILE="$REPO_ROOT/containers/spindle-hello-podman/01-basic/Dockerfile" + +echo "==========================================" +echo "Spindle Hello World - 01-basic" +echo "==========================================" +echo "" +echo "This test validates the podman environment on LC systems." +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" + +# Run the container +podman_run "$IMAGE_NAME" + +echo "" +echo "==========================================" +echo "Test complete!" +echo "" +echo "What was tested:" +echo " ✓ apt-get works (setgroups fix applied)" +echo " ✓ Container runs successfully" +echo " ✓ Network connectivity" +echo " ✓ SSL certificates configured" +echo "" +echo "Next step: run-hello-02-user.sh" +echo "==========================================" + +# Optional: Clean up the image +# Uncomment if you want to remove the image after testing +# podman_cleanup "$IMAGE_NAME" diff --git a/scripts/podman/run-hello-02-user.sh b/scripts/podman/run-hello-02-user.sh new file mode 100755 index 00000000..c53ae455 --- /dev/null +++ b/scripts/podman/run-hello-02-user.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Test 02-user-switch: Validate user switching and permissions +# +# This test verifies: +# - Creating a non-root user in the container +# - Switching to that user (USER directive) +# - File permission handling +# - Sudo access when needed +# +# This pattern is used in all Spindle containers where builds and tests +# run as a non-root user for security and to match typical HPC environments. +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-hello-02-user" +DOCKERFILE="$REPO_ROOT/containers/spindle-hello-podman/02-user-switch/Dockerfile" + +echo "==========================================" +echo "Spindle Hello World - 02-user-switch" +echo "==========================================" +echo "" +echo "This test validates non-root user patterns used in Spindle containers." +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" + +# Run the container +podman_run "$IMAGE_NAME" + +echo "" +echo "==========================================" +echo "Test complete!" +echo "" +echo "What was tested:" +echo " ✓ Non-root user creation" +echo " ✓ User switching with USER directive" +echo " ✓ Home directory setup" +echo " ✓ File permission handling" +echo " ✓ Sudo access" +echo "" +echo "This pattern is used in all Spindle containers to:" +echo " - Match HPC security practices" +echo " - Test file permissions realistically" +echo " - Avoid running builds as root" +echo "" +echo "Next step: run-hello-03-filesystem.sh" +echo "==========================================" + +# Optional: Clean up the image +# Uncomment if you want to remove the image after testing +# podman_cleanup "$IMAGE_NAME" diff --git a/scripts/podman/run-hello-03-filesystem.sh b/scripts/podman/run-hello-03-filesystem.sh new file mode 100755 index 00000000..6e7c0875 --- /dev/null +++ b/scripts/podman/run-hello-03-filesystem.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# +# Test 03-filesystem: Validate volume mounts and filesystem operations +# +# This test verifies: +# - Mounting host directories into containers +# - Reading files from host mounts +# - Writing files to mounted volumes +# - File ownership and permissions with :Z flag +# - Artifacts persisting after container exits +# +# This pattern is critical for Spindle containers: +# - Source code is mounted from host (read-only or read-write) +# - Build artifacts are written to mounted volumes +# - Logs persist on host for debugging +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-hello-03-filesystem" +DOCKERFILE="$REPO_ROOT/containers/spindle-hello-podman/03-filesystem/Dockerfile" + +# Create temporary directories for testing +TEST_DIR=$(mktemp -d) +HOST_DATA_DIR="$TEST_DIR/host-data" +OUTPUT_DIR="$TEST_DIR/output" + +mkdir -p "$HOST_DATA_DIR" +mkdir -p "$OUTPUT_DIR" + +# Create test input file +echo "Hello from the host filesystem!" > "$HOST_DATA_DIR/test-input.txt" +echo "This file was created outside the container." >> "$HOST_DATA_DIR/test-input.txt" +echo "Container should be able to read this." >> "$HOST_DATA_DIR/test-input.txt" + +# No special permissions needed - we'll use --userns=keep-id to map host UID into container + +echo "==========================================" +echo "Spindle Hello World - 03-filesystem" +echo "==========================================" +echo "" +echo "This test validates volume mount patterns used in Spindle containers." +echo "" +echo "Test setup:" +echo " Host data directory: $HOST_DATA_DIR" +echo " Output directory: $OUTPUT_DIR" +echo " Using --userns=keep-id to map host UID into container" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" +echo "Running container with volume mounts..." +echo "" + +# Run the container with volume mounts +# --userns=keep-id - Map host UID into container (avoids permission issues) +# -v host:container:Z - Z flag sets SELinux context for container access +podman run \ + --rm \ + --userns=keep-id \ + $LC_CERT_RUN_MOUNT \ + -v "$HOST_DATA_DIR:/home/testuser/host-data:Z" \ + -v "$OUTPUT_DIR:/home/testuser/output:Z" \ + "$IMAGE_NAME" + +EXIT_CODE=$? + +echo "" +echo "==========================================" +echo "Container exited. Checking results..." +echo "==========================================" +echo "" + +if [ $EXIT_CODE -eq 0 ]; then + echo "✓ Container executed successfully" +else + echo "✗ Container exited with code $EXIT_CODE" +fi + +echo "" +echo "Files created by container in output directory:" +ls -lh "$OUTPUT_DIR" + +echo "" +echo "Content of container-output.txt:" +if [ -f "$OUTPUT_DIR/container-output.txt" ]; then + cat "$OUTPUT_DIR/container-output.txt" + echo "" + echo "✓ Container successfully wrote to mounted volume" +else + echo "✗ Expected output file not found" +fi + +echo "" +echo "==========================================" +echo "Test complete!" +echo "" +echo "What was tested:" +echo " ✓ Volume mounts (-v host:container:Z)" +echo " ✓ User namespace mapping (--userns=keep-id)" +echo " ✓ Reading files from host" +echo " ✓ Writing files to mounted volumes" +echo " ✓ File persistence after container exit" +echo " ✓ SELinux context handling (:Z flag)" +echo "" +echo "This pattern enables Spindle to:" +echo " - Access source code from host" +echo " - Write build artifacts to persistent storage" +echo " - Generate logs accessible after tests" +echo " - Share data between container and host" +echo "" +echo "Cleaning up test directories..." +rm -rf "$TEST_DIR" +echo "✓ Cleanup complete" +echo "" +echo "Next step: run-hello-04-networking.sh" +echo "==========================================" + +# Optional: Clean up the image +# Uncomment if you want to remove the image after testing +# podman_cleanup "$IMAGE_NAME" diff --git a/scripts/podman/run-hello-04-networking.sh b/scripts/podman/run-hello-04-networking.sh new file mode 100755 index 00000000..2a1b44ba --- /dev/null +++ b/scripts/podman/run-hello-04-networking.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# +# Test 04-networking: Validate multi-container networking +# +# This test verifies: +# - Creating custom podman networks +# - Multiple containers on the same network +# - Container-to-container communication by hostname +# - Service discovery (DNS resolution) +# - Network isolation +# +# This pattern is essential for multi-container setups like Slurm clusters +# where head nodes need to communicate with compute nodes. +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-hello-04-networking" +DOCKERFILE="$REPO_ROOT/containers/spindle-hello-podman/04-networking/Dockerfile" +NETWORK_NAME="spindle-test-net" +SERVER_NAME="test-server" +CLIENT1_NAME="test-client1" +CLIENT2_NAME="test-client2" + +echo "==========================================" +echo "Spindle Hello World - 04-networking" +echo "==========================================" +echo "" +echo "This test validates multi-container networking for Slurm/Flux clusters." +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up containers and network..." + podman rm -f "$SERVER_NAME" "$CLIENT1_NAME" "$CLIENT2_NAME" 2>/dev/null || true + podman network rm "$NETWORK_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" +echo "==========================================" +echo "Setting up network infrastructure..." +echo "==========================================" +echo "" + +# Create a custom network +echo "Creating network: $NETWORK_NAME" +podman network create "$NETWORK_NAME" +echo "✓ Network created" + +echo "" +echo "Starting server container..." +podman run \ + --name "$SERVER_NAME" \ + --network "$NETWORK_NAME" \ + --hostname "$SERVER_NAME" \ + -d \ + "$IMAGE_NAME" +echo "✓ Server started: $SERVER_NAME" + +# Give server time to start +sleep 2 + +echo "" +echo "==========================================" +echo "Testing container networking..." +echo "==========================================" +echo "" + +echo "1. Testing DNS resolution (can clients resolve server hostname?)..." +podman run \ + --name "$CLIENT1_NAME" \ + --network "$NETWORK_NAME" \ + --hostname "$CLIENT1_NAME" \ + --rm \ + "$IMAGE_NAME" \ + /bin/bash -c " + echo 'Client: $CLIENT1_NAME' + echo 'Resolving $SERVER_NAME...' + if ping -c 1 -W 2 $SERVER_NAME > /dev/null 2>&1; then + echo '✓ DNS resolution works: $SERVER_NAME is reachable' + echo 'Server IP:' \$(getent hosts $SERVER_NAME | awk '{print \$1}') + else + echo '✗ Cannot resolve $SERVER_NAME' + exit 1 + fi + " + +echo "" +echo "2. Testing HTTP communication between containers..." +podman run \ + --name "$CLIENT2_NAME" \ + --network "$NETWORK_NAME" \ + --hostname "$CLIENT2_NAME" \ + --rm \ + "$IMAGE_NAME" \ + /bin/bash -c " + echo 'Client: $CLIENT2_NAME' + echo 'Fetching from http://$SERVER_NAME:8080...' + response=\$(curl -s --max-time 5 http://$SERVER_NAME:8080) + if [ -n \"\$response\" ]; then + echo 'Response from server:' + echo \"\$response\" + echo '✓ HTTP communication works' + else + echo '✗ No response from server' + exit 1 + fi + " + +echo "" +echo "3. Checking server logs..." +echo "Server received requests from:" +podman logs "$SERVER_NAME" 2>&1 | tail -5 + +echo "" +echo "==========================================" +echo "Test complete!" +echo "" +echo "What was tested:" +echo " ✓ Custom network creation" +echo " ✓ Multiple containers on same network" +echo " ✓ DNS resolution (hostname lookups)" +echo " ✓ Container-to-container HTTP communication" +echo " ✓ Service discovery patterns" +echo "" +echo "This pattern enables:" +echo " - Slurm head node + compute nodes" +echo " - Flux broker + worker nodes" +echo " - Server-client architectures" +echo " - Service discovery by hostname" +echo "" +echo "Next steps:" +echo " - 05-flux-multi: Multi-container Flux cluster" +echo " - 06-slurm-multi: Multi-container Slurm cluster" +echo " - Then: Real Spindle containers" +echo "==========================================" + +# Cleanup happens automatically via trap diff --git a/scripts/podman/run-hello-05-flux.sh b/scripts/podman/run-hello-05-flux.sh new file mode 100755 index 00000000..173f4bf4 --- /dev/null +++ b/scripts/podman/run-hello-05-flux.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# +# Test 05-flux-multi: Validate Flux multi-node cluster +# +# This test verifies: +# - Using official fluxrm/flux-sched base image +# - Multi-node Flux cluster setup +# - Head node and worker node coordination +# - Running distributed jobs with `flux run` +# - Flux resource management +# +# This demonstrates the pattern used for Spindle's Flux tests, +# simplified to just show Flux itself working. +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-hello-05-flux" +DOCKERFILE="$REPO_ROOT/containers/spindle-hello-podman/05-flux-multi/Dockerfile" +NETWORK_NAME="flux-test-net" +NUM_WORKERS=3 + +echo "==========================================" +echo "Spindle Hello World - 05-flux-multi" +echo "==========================================" +echo "" +echo "This test validates a multi-node Flux cluster setup." +echo "Nodes: 1 head + ${NUM_WORKERS} workers = $(($NUM_WORKERS + 1)) total" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up containers and network..." + for i in $(seq 1 $NUM_WORKERS); do + podman rm -f "flux-node-$i" 2>/dev/null || true + done + podman network rm "$NETWORK_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +echo "Building Flux container image..." +echo "(This may take a few minutes - downloading fluxrm/flux-sched base image)" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" +echo "==========================================" +echo "Setting up Flux cluster..." +echo "==========================================" +echo "" + +# Create network +echo "Creating network: $NETWORK_NAME" +podman network create "$NETWORK_NAME" +echo "✓ Network created" + +echo "" +echo "Starting Flux nodes..." + +# Start head node first +echo " Starting flux-node-1 (head node)..." +podman run \ + --name "flux-node-1" \ + --hostname "flux-node-1" \ + --network "$NETWORK_NAME" \ + -d \ + "$IMAGE_NAME" + +sleep 2 + +# Start worker nodes +for i in $(seq 2 $NUM_WORKERS); do + echo " Starting flux-node-$i (worker)..." + podman run \ + --name "flux-node-$i" \ + --hostname "flux-node-$i" \ + --network "$NETWORK_NAME" \ + -d \ + "$IMAGE_NAME" + sleep 1 +done + +echo "" +echo "✓ All nodes started" +echo "" +echo "Waiting for Flux cluster to initialize..." +echo "(Munge authentication + broker connections)" +sleep 15 + +echo "" +echo "==========================================" +echo "Flux Cluster Status" +echo "==========================================" +echo "" + +# Show head node logs +echo "Head node logs:" +echo "----------------------------------------" +podman logs flux-node-1 2>&1 | tail -30 + +echo "" +echo "==========================================" +echo "Test complete!" +echo "" +echo "What was tested:" +echo " ✓ Official fluxrm/flux-sched base image" +echo " ✓ Multi-node Flux cluster (1 head + ${NUM_WORKERS} workers)" +echo " ✓ Munge authentication" +echo " ✓ Flux broker connections" +echo " ✓ Distributed job execution (flux run)" +echo " ✓ Resource management" +echo "" +echo "This demonstrates the Flux pattern used by Spindle:" +echo " - Same image for all nodes" +echo " - Head node vs worker behavior based on hostname" +echo " - Workers connect to head node" +echo " - Jobs can run across all nodes" +echo "" +echo "To interact with the cluster:" +echo " podman exec -it flux-node-1 flux resource list" +echo " podman exec -it flux-node-1 flux run -N ${NUM_WORKERS} hostname" +echo "" +echo "Next: 06-slurm-multi (Slurm cluster example)" +echo "==========================================" + +# Cleanup happens automatically via trap diff --git a/scripts/podman/run-hello-06-slurm.sh b/scripts/podman/run-hello-06-slurm.sh new file mode 100755 index 00000000..ee9e4405 --- /dev/null +++ b/scripts/podman/run-hello-06-slurm.sh @@ -0,0 +1,168 @@ +#!/bin/bash +# +# Test 06-slurm-multi: Validate Slurm multi-node cluster +# +# This test verifies: +# - Installing Slurm from Ubuntu packages +# - Multi-node Slurm cluster setup +# - Controller (slurmctld) and compute (slurmd) daemons +# - Munge authentication +# - Node registration and job submission +# +# This demonstrates a simplified Slurm pattern. The full Spindle +# Slurm setup includes MariaDB and slurmdbd for accounting. +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-hello-06-slurm" +DOCKERFILE="$REPO_ROOT/containers/spindle-hello-podman/06-slurm-multi/Dockerfile" +NETWORK_NAME="slurm-test-net" + +echo "==========================================" +echo "Spindle Hello World - 06-slurm-multi" +echo "==========================================" +echo "" +echo "This test validates a simplified multi-node Slurm cluster." +echo "Cluster: 1 controller + 2 compute nodes" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up containers and network..." + podman rm -f slurm-head slurm-node-1 slurm-node-2 2>/dev/null || true + podman network rm "$NETWORK_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Clean up any existing resources from previous runs +cleanup + +# Set trap to cleanup on exit +trap cleanup EXIT + +echo "Building Slurm container image..." +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" +echo "==========================================" +echo "Setting up Slurm cluster..." +echo "==========================================" +echo "" + +# Create network (or reuse if exists) +if podman network exists "$NETWORK_NAME" 2>/dev/null; then + echo "Network $NETWORK_NAME already exists, reusing" +else + echo "Creating network: $NETWORK_NAME" + podman network create "$NETWORK_NAME" + echo "✓ Network created" +fi + +echo "" +echo "Starting Slurm controller (head node)..." +podman run \ + --name slurm-head \ + --hostname slurm-head \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=ctl \ + -d \ + "$IMAGE_NAME" + +echo "✓ Controller started" +sleep 5 + +echo "" +echo "Starting compute nodes..." + +podman run \ + --name slurm-node-1 \ + --hostname slurm-node-1 \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=worker \ + -d \ + "$IMAGE_NAME" +echo " ✓ slurm-node-1 started" + +podman run \ + --name slurm-node-2 \ + --hostname slurm-node-2 \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=worker \ + -d \ + "$IMAGE_NAME" +echo " ✓ slurm-node-2 started" + +echo "" +echo "Waiting for Slurm cluster to initialize..." +echo "(Munge + node registration)" +sleep 10 + +echo "" +echo "==========================================" +echo "Slurm Cluster Status" +echo "==========================================" +echo "" + +echo "Checking node status with 'sinfo':" +podman exec slurm-head sinfo || echo " (Still initializing...)" + +echo "" +echo "Attempting to run a test job..." +echo "Running: srun -N 2 hostname" +echo "" + +# Try to run a simple job +podman exec slurm-head srun -N 2 hostname || { + echo "" + echo "Job may have failed. Checking controller logs:" + echo "----------------------------------------" + podman logs slurm-head 2>&1 | tail -20 + echo "" + echo "Compute node logs:" + echo "----------------------------------------" + podman logs slurm-node-1 2>&1 | tail -10 +} + +echo "" +echo "==========================================" +echo "Test complete!" +echo "" +echo "What was tested:" +echo " ✓ Slurm installation from Ubuntu packages" +echo " ✓ Multi-node cluster (1 controller + 2 compute)" +echo " ✓ Munge authentication" +echo " ✓ slurmctld (controller daemon)" +echo " ✓ slurmd (compute daemon)" +echo " ✓ Node registration" +echo " ✓ Job submission with srun" +echo "" +echo "This demonstrates a simplified Slurm pattern." +echo "The full Spindle Slurm setup adds:" +echo " - MariaDB for accounting database" +echo " - slurmdbd for accounting" +echo " - More complex configuration" +echo " - MPICH for MPI jobs" +echo "" +echo "To interact with the cluster:" +echo " podman exec slurm-head sinfo" +echo " podman exec slurm-head srun -N 2 hostname" +echo " podman exec slurm-head scontrol show nodes" +echo "" +echo "Ready to port real Spindle containers!" +echo "==========================================" + +# Cleanup happens automatically via trap diff --git a/scripts/podman/save-images.sh b/scripts/podman/save-images.sh new file mode 100755 index 00000000..9e1a956d --- /dev/null +++ b/scripts/podman/save-images.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Save Spindle podman images to tarball for compute node deployment +# +# Run this on the login node where images were built. +# The tarball can then be loaded on compute nodes via load-images.sh + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +OUTPUT_FILE="${1:-$REPO_ROOT/spindle-podman-images.tar}" + +echo "==========================================" +echo "Saving Spindle Podman Images" +echo "==========================================" +echo "" +echo "This saves all Spindle images to a tarball for deployment to compute nodes." +echo "Output: $OUTPUT_FILE" +echo "" + +# Check what images exist +echo "Available Spindle images:" +podman images | grep spindle || { + echo "Error: No Spindle images found. Build them first." + exit 1 +} + +echo "" +echo "Saving images to tarball..." +echo "This may take several minutes..." +echo "" + +# Save images separately to avoid parent-child ID collisions +# When multiple images share a base, podman save can collapse them incorrectly +TEMP_DIR=$(mktemp -d) +trap "rm -rf $TEMP_DIR" EXIT + +echo "Saving spindle-slurm-base..." +podman save localhost/spindle-slurm-base:latest -o "$TEMP_DIR/base.tar" + +echo "Saving spindle-slurm-srun..." +podman save localhost/spindle-slurm-srun:latest -o "$TEMP_DIR/srun.tar" + +echo "Saving spindle-serial-ubuntu..." +podman save localhost/spindle-serial-ubuntu:latest -o "$TEMP_DIR/serial.tar" + +echo "Saving MariaDB..." +podman save docker.io/library/mariadb:12 -o "$TEMP_DIR/mariadb.tar" + +echo "Combining into single tarball..." +tar -cf "$OUTPUT_FILE" -C "$TEMP_DIR" base.tar srun.tar serial.tar mariadb.tar + +# Copy mariadb.env for portable deployment +MARIADB_ENV_SOURCE="$REPO_ROOT/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env" +MARIADB_ENV_DEST="$REPO_ROOT/mariadb.env" +if [ -f "$MARIADB_ENV_SOURCE" ]; then + echo "Copying mariadb.env for portable deployment..." + cp "$MARIADB_ENV_SOURCE" "$MARIADB_ENV_DEST" + echo " ✓ mariadb.env copied to repo root" +fi + +SIZE=$(du -h "$OUTPUT_FILE" | cut -f1) +echo "" +echo "==========================================" +echo "✓ Images saved successfully" +echo "==========================================" +echo "" +echo "File: $OUTPUT_FILE" +echo "Size: $SIZE" +echo "" +echo "To load on compute nodes:" +echo " ./scripts/podman/load-images.sh $OUTPUT_FILE" +echo "" diff --git a/scripts/podman/test-spindle-flux.sh b/scripts/podman/test-spindle-flux.sh new file mode 100755 index 00000000..b447e66c --- /dev/null +++ b/scripts/podman/test-spindle-flux.sh @@ -0,0 +1,167 @@ +#!/bin/bash +# +# Run Spindle Flux tests in podman +# +# This runs the Spindle testsuite in a 4-node Flux cluster. +# Based on the CI workflow and docker-compose configuration. +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-flux-ubuntu" +NETWORK_NAME="flux-test-net" +WORKERS=4 +MAIN_HOST="node-1" + +echo "==========================================" +echo "Spindle Flux Tests" +echo "==========================================" +echo "" +echo "This runs the Spindle testsuite in a 4-node Flux cluster." +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up containers and network..." + for i in $(seq 1 $WORKERS); do + podman rm -f "node-$i" 2>/dev/null || true + done + podman network rm "$NETWORK_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# Initial cleanup +cleanup + +echo "==========================================" +echo "Setting up Flux cluster..." +echo "==========================================" +echo "" + +# Create network (or reuse if exists) +if podman network exists "$NETWORK_NAME" 2>/dev/null; then + echo "Network $NETWORK_NAME already exists, reusing" +else + echo "Creating network: $NETWORK_NAME" + podman network create "$NETWORK_NAME" + echo "✓ Network created" +fi + +echo "" +echo "Starting Flux nodes..." +echo "" + +# Start all 4 nodes +for i in $(seq 1 $WORKERS); do + NODE_NAME="node-$i" + echo "Starting $NODE_NAME..." + podman run \ + --name "$NODE_NAME" \ + --hostname "$NODE_NAME" \ + --network "$NETWORK_NAME" \ + -e mainHost="$MAIN_HOST" \ + -e workers="$WORKERS" \ + --cap-add SYS_NICE \ + -d \ + "$IMAGE_NAME" + echo " ✓ $NODE_NAME started" +done + +echo "" +echo "Waiting for Flux cluster to initialize..." +echo "(This takes ~20-30 seconds for all nodes to register)" +sleep 30 + +echo "" +echo "==========================================" +echo "Checking container status..." +echo "==========================================" +echo "" + +# Check if containers are still running +for i in $(seq 1 $WORKERS); do + NODE_NAME="node-$i" + if podman ps --filter "name=$NODE_NAME" --format "{{.Names}}" | grep -q "$NODE_NAME"; then + echo " ✓ $NODE_NAME is running" + else + echo " ✗ $NODE_NAME has exited!" + echo "" + echo "Last 30 lines of $NODE_NAME logs:" + echo "----------------------------------------" + podman logs "$NODE_NAME" 2>&1 | tail -30 + echo "----------------------------------------" + echo "" + echo "Container exited unexpectedly. Check logs above." + exit 1 + fi +done + +echo "" +echo "==========================================" +echo "Verifying munge authentication..." +echo "==========================================" +echo "" + +podman exec "$MAIN_HOST" bash -c 'munge -n | unmunge' + +echo "" +echo "✓ Munge working" + +echo "" +echo "==========================================" +echo "Verifying Flux cluster health..." +echo "==========================================" +echo "" + +echo "Checking Flux status..." +podman exec "$MAIN_HOST" bash -c 'flux resource list' || echo " (Flux may still be initializing)" + +echo "" +echo "Running flux healthcheck..." +podman exec "$MAIN_HOST" bash -c './flux_healthcheck.sh' || echo " (Some nodes may not be registered yet)" + +echo "" +echo "==========================================" +echo "Running Spindle testsuite..." +echo "==========================================" +echo "" +echo "This will take several minutes." +echo "" + +# Run the testsuite +# Based on CI: docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./runTests --nodes=${workers} --tasks-per-node=3' +if podman exec "$MAIN_HOST" bash -c "cd Spindle-build/testsuite && flux alloc --nodes=${WORKERS} ./runTests --nodes=${WORKERS} --tasks-per-node=3"; then + echo "" + echo "==========================================" + echo "✓ All tests passed!" + echo "==========================================" + echo "" + exit 0 +else + echo "" + echo "==========================================" + echo "✗ Some tests failed" + echo "==========================================" + echo "" + echo "To inspect the cluster:" + echo " podman exec -it $MAIN_HOST bash" + echo " flux resource list" + echo " cd Spindle-build/testsuite" + echo "" + exit 1 +fi + +# Cleanup happens automatically via trap diff --git a/scripts/podman/test-spindle-serial-crash.sh b/scripts/podman/test-spindle-serial-crash.sh new file mode 100755 index 00000000..462adab8 --- /dev/null +++ b/scripts/podman/test-spindle-serial-crash.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# +# Run Spindle serial crash tests in podman +# +# This runs only the crash tests (slower). +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-serial-ubuntu" +CONTAINER_NAME="spindlenode" + +echo "==========================================" +echo "Spindle Serial Crash Tests" +echo "==========================================" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up container..." + podman rm -f "$CONTAINER_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# Initial cleanup +cleanup + +echo "Starting Spindle serial container..." +echo "" + +# Start the container +podman run \ + --name "$CONTAINER_NAME" \ + --hostname "$CONTAINER_NAME" \ + --cap-add SYS_NICE \ + -d \ + -t \ + "$IMAGE_NAME" + +echo "✓ Container started" +sleep 3 + +echo "" +echo "==========================================" +echo "Running crash tests..." +echo "==========================================" +echo "" +echo "This will take several minutes." +echo "" + +# Run crash tests +if podman exec "$CONTAINER_NAME" bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=serial --scratch=/tmp/spindle_crash_test'; then + echo "" + echo "==========================================" + echo "✓ All crash tests passed!" + echo "==========================================" + echo "" + exit 0 +else + echo "" + echo "==========================================" + echo "✗ Some crash tests failed" + echo "==========================================" + echo "" + echo "To inspect the container:" + echo " podman exec -it $CONTAINER_NAME bash" + echo "" + exit 1 +fi + +# Cleanup happens automatically via trap diff --git a/scripts/podman/test-spindle-serial-debug.sh b/scripts/podman/test-spindle-serial-debug.sh new file mode 100755 index 00000000..6bebb7dc --- /dev/null +++ b/scripts/podman/test-spindle-serial-debug.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# +# Run Spindle serial tests with debug logging enabled +# +# This demonstrates: +# 1. Running tests with SPINDLE_DEBUG=3 (verbose logging) +# 2. Extracting logs from the container to the host +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-serial-ubuntu" +CONTAINER_NAME="spindlenode-debug" +LOG_DIR="$REPO_ROOT/spindle-debug-logs" + +echo "==========================================" +echo "Spindle Serial Debug Test" +echo "==========================================" +echo "" +echo "This runs a single test with SPINDLE_DEBUG=3 enabled" +echo "and extracts the debug logs to the host." +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up container..." + podman rm -f "$CONTAINER_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# Initial cleanup +cleanup + +# Create log directory on host +mkdir -p "$LOG_DIR" +echo "Debug logs will be saved to: $LOG_DIR" +echo "" + +echo "Starting Spindle serial container..." +echo "" + +# Start the container +podman run \ + --name "$CONTAINER_NAME" \ + --hostname "$CONTAINER_NAME" \ + --cap-add SYS_NICE \ + -d \ + -t \ + "$IMAGE_NAME" + +echo "✓ Container started" +sleep 3 + +echo "" +echo "==========================================" +echo "Running single test with SPINDLE_DEBUG=3" +echo "==========================================" +echo "" +echo "Test: ./run_driver --dependency --preload" +echo "" + +# Run a single test with SPINDLE_DEBUG=3 +# This should generate verbose logs +podman exec "$CONTAINER_NAME" bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 ./run_driver --dependency --preload' || { + echo "" + echo "Note: Test may have generated logs even if it failed" +} + +echo "" +echo "==========================================" +echo "Extracting logs from container" +echo "==========================================" +echo "" + +# Spindle log files are named: spindle_output.. +echo "Looking for spindle_output.* files in testsuite directory..." +if podman exec "$CONTAINER_NAME" bash -c 'ls -la /home/spindleuser/Spindle-build/testsuite/spindle_output.* 2>/dev/null'; then + echo "" + echo "Found Spindle log files! Extracting..." + + # Get list of log files + LOG_FILES=$(podman exec "$CONTAINER_NAME" bash -c 'cd /home/spindleuser/Spindle-build/testsuite && ls spindle_output.* 2>/dev/null' || echo "") + + if [ -n "$LOG_FILES" ]; then + for logfile in $LOG_FILES; do + echo " Copying $logfile..." + podman cp "$CONTAINER_NAME:/home/spindleuser/Spindle-build/testsuite/$logfile" "$LOG_DIR/" + done + echo "" + echo "✓ Logs extracted to: $LOG_DIR" + echo "" + echo "View logs with:" + for logfile in $LOG_FILES; do + echo " cat $LOG_DIR/$logfile" + done + fi +else + echo "No spindle_output.* files found in testsuite directory" +fi + +echo "" +echo "Checking for other spindle files..." +podman exec "$CONTAINER_NAME" bash -c 'find /home/spindleuser/Spindle-build/testsuite -name "spindle*" -type f 2>/dev/null | head -20' || echo "No other spindle files found" + +echo "" +echo "==========================================" +echo "Summary" +echo "==========================================" +echo "" +echo "Container: $CONTAINER_NAME (still running)" +echo "Log directory: $LOG_DIR" +echo "" +echo "To explore interactively:" +echo " podman exec -it $CONTAINER_NAME bash" +echo " cd /home/spindleuser/Spindle-build/testsuite" +echo " ls spindle_output.*" +echo "" +echo "When done, cleanup with:" +echo " podman rm -f $CONTAINER_NAME" +echo "" + +# Don't cleanup automatically - let user explore +trap - EXIT +echo "Note: Container left running for inspection. Clean up manually when done." +echo "" diff --git a/scripts/podman/test-spindle-serial.sh b/scripts/podman/test-spindle-serial.sh new file mode 100755 index 00000000..c1d52ae2 --- /dev/null +++ b/scripts/podman/test-spindle-serial.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# +# Run Spindle serial regular tests in podman +# +# This runs the main testsuite (not crash tests). +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-serial-ubuntu" +CONTAINER_NAME="spindlenode" + +echo "==========================================" +echo "Spindle Serial Regular Tests" +echo "==========================================" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up container..." + podman rm -f "$CONTAINER_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# Initial cleanup +cleanup + +echo "Starting Spindle serial container..." +echo "" + +# Start the container +podman run \ + --name "$CONTAINER_NAME" \ + --hostname "$CONTAINER_NAME" \ + --cap-add SYS_NICE \ + -d \ + -t \ + "$IMAGE_NAME" + +echo "✓ Container started" +sleep 3 + +echo "" +echo "==========================================" +echo "Verifying munge authentication..." +echo "==========================================" +echo "" + +podman exec "$CONTAINER_NAME" bash -c 'munge -n | unmunge' + +echo "" +echo "✓ Munge working" + +echo "" +echo "==========================================" +echo "Running Spindle testsuite..." +echo "==========================================" +echo "" +echo "This will take several minutes." +echo "" + +# Run the testsuite +if podman exec "$CONTAINER_NAME" bash -c 'cd Spindle-build/testsuite && ./runTests'; then + echo "" + echo "==========================================" + echo "✓ All tests passed!" + echo "==========================================" + echo "" + exit 0 +else + echo "" + echo "==========================================" + echo "✗ Some tests failed" + echo "==========================================" + echo "" + echo "To inspect the container:" + echo " podman exec -it $CONTAINER_NAME bash" + echo "" + exit 1 +fi + +# Cleanup happens automatically via trap diff --git a/scripts/podman/test-spindle-slurm-rshlaunch.sh b/scripts/podman/test-spindle-slurm-rshlaunch.sh new file mode 100755 index 00000000..def418cc --- /dev/null +++ b/scripts/podman/test-spindle-slurm-rshlaunch.sh @@ -0,0 +1,247 @@ +#!/bin/bash +# +# Run Spindle Slurm rshlaunch tests in podman +# +# This runs the Spindle testsuite in a Slurm cluster with rshlaunch launcher. +# Based on the CI workflow and docker-compose configuration. +# +# Cluster: 1 MariaDB + 1 slurmdbd + 1 slurmctld + 4 slurmd workers +# +# CPU Pinning: Cores 24-30 (avoiding system cores 0-23) +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-slurm-rshlaunch" +NETWORK_NAME="slurm-rshlaunch-test-net" +WORKERS=4 + +# CPU pinning disabled - not supported in rootless podman on this system +# See PODMAN.md for details +# CPU_MARIADB=24 +# CPU_DB=25 +# CPU_HEAD=26 +# CPU_NODE_BASE=27 # nodes 1-4 get 27-30 + +echo "==========================================" +echo "Spindle Slurm Rshlaunch Tests" +echo "==========================================" +echo "" +echo "This runs the Spindle testsuite in a Slurm cluster." +echo "Cluster: MariaDB + slurmdbd + slurmctld + 4 workers" +echo "Note: CPU pinning disabled (not supported in rootless podman)" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up containers and network..." + # Stop and remove all containers (force removal even if running) + for container in slurm-rshlaunch-mariadb slurm-rshlaunch-db slurm-rshlaunch-head slurm-rshlaunch-node-{1..4}; do + podman stop "$container" 2>/dev/null || true + podman rm -f "$container" 2>/dev/null || true + done + podman network rm -f "$NETWORK_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +# DISABLED for debugging - cleanup manually with: podman rm -f slurm-rshlaunch-{mariadb,db,head,node-{1..4}}; podman network rm -f slurm-rshlaunch-test-net +# trap cleanup EXIT + +# Initial cleanup +cleanup + +echo "==========================================" +echo "Setting up Slurm cluster..." +echo "==========================================" +echo "" + +# Create network +if podman network exists "$NETWORK_NAME" 2>/dev/null; then + echo "Network $NETWORK_NAME already exists, reusing" +else + echo "Creating network: $NETWORK_NAME" + podman network create "$NETWORK_NAME" + echo "✓ Network created" +fi + +echo "" +echo "Starting MariaDB..." +# Read password from generated mariadb.env +MARIADB_PASSWORD=$(grep MARIADB_PASSWORD "$REPO_ROOT/containers/spindle-slurm-ubuntu/testing/mariadb.env" | cut -d'"' -f2) +if [ -z "$MARIADB_PASSWORD" ]; then + echo "Error: Could not read password from mariadb.env" + exit 1 +fi +podman run \ + --name slurm-rshlaunch-mariadb \ + --hostname slurm-mariadb \ + --network "$NETWORK_NAME" \ + -e MYSQL_RANDOM_ROOT_PASSWORD=yes \ + -e MYSQL_DATABASE=slurm_acct_db \ + -e MYSQL_USER=slurm \ + -e MYSQL_PASSWORD="$MARIADB_PASSWORD" \ + -d \ + mariadb:12 + +echo " ✓ MariaDB started" +echo "Waiting for MariaDB to initialize..." +sleep 15 + +echo "" +echo "Starting slurmdbd (accounting daemon)..." +podman run \ + --name slurm-rshlaunch-db \ + --hostname slurm-db \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=db \ + -e SLURM_HEAD_NODE=slurm-head \ + -e workers="$WORKERS" \ + -d \ + "$IMAGE_NAME" + +echo " ✓ slurmdbd started" +sleep 10 + +echo "" +echo "Starting slurmctld (controller)..." +podman run \ + --name slurm-rshlaunch-head \ + --hostname slurm-head \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=ctl \ + -e SLURM_HEAD_NODE=slurm-head \ + -e workers="$WORKERS" \ + -t \ + -d \ + "$IMAGE_NAME" + +echo " ✓ slurmctld started" +sleep 10 + +echo "" +echo "Starting worker nodes..." +for i in $(seq 1 $WORKERS); do + echo "Starting slurm-node-$i..." + podman run \ + --name "slurm-rshlaunch-node-$i" \ + --hostname "slurm-node-$i" \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=worker \ + -e SLURM_HEAD_NODE=slurm-head \ + -e workers="$WORKERS" \ + -d \ + "$IMAGE_NAME" + echo " ✓ slurm-node-$i started" +done + +echo "" +echo "Waiting for Slurm cluster to initialize..." +echo "(This takes ~30 seconds for all daemons and nodes)" +sleep 30 + +echo "" +echo "==========================================" +echo "Checking container status..." +echo "==========================================" +echo "" + +# Check if containers are still running +ALL_RUNNING=true +for container in slurm-rshlaunch-mariadb slurm-rshlaunch-db slurm-rshlaunch-head slurm-rshlaunch-node-{1..4}; do + if podman ps --filter "name=$container" --format "{{.Names}}" | grep -q "$container"; then + echo " ✓ $container is running" + else + echo " ✗ $container has exited!" + ALL_RUNNING=false + echo "" + echo "Last 30 lines of $container logs:" + echo "----------------------------------------" + podman logs "$container" 2>&1 | tail -30 + echo "----------------------------------------" + fi +done + +if [ "$ALL_RUNNING" = false ]; then + echo "" + echo "Some containers exited. Check logs above." + exit 1 +fi + +echo "" +echo "==========================================" +echo "Verifying munge authentication..." +echo "==========================================" +echo "" + +podman exec slurm-rshlaunch-head bash -c 'munge -n | unmunge' + +echo "" +echo "✓ Munge working" + +echo "" +echo "==========================================" +echo "Verifying Slurm cluster..." +echo "==========================================" +echo "" + +echo "Checking node status with sinfo:" +podman exec slurm-rshlaunch-head sinfo || echo " (Nodes may still be registering)" + +echo "" +echo "Checking cluster status with scontrol:" +podman exec slurm-rshlaunch-head scontrol show nodes || echo " (Still initializing)" + +echo "" +echo "==========================================" +echo "Running Spindle testsuite..." +echo "==========================================" +echo "" +echo "This will take several minutes." +echo "" + +# Run the testsuite +# Based on CI: docker exec slurm-rshlaunch-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} ./runTests ${workers}' +if podman exec slurm-rshlaunch-head bash -c "cd Spindle-build/testsuite && salloc -n${WORKERS} -N${WORKERS} ./runTests ${WORKERS}"; then + echo "" + echo "==========================================" + echo "✓ All tests passed!" + echo "==========================================" + echo "" + exit 0 +else + echo "" + echo "==========================================" + echo "✗ Some tests failed" + echo "==========================================" + echo "" + echo "To inspect the cluster:" + echo " podman exec -it slurm-rshlaunch-head bash" + echo " sinfo" + echo " scontrol show nodes" + echo " cd Spindle-build/testsuite" + echo "" + echo "Manual cleanup when done:" + echo " podman rm -f slurm-rshlaunch-mariadb slurm-rshlaunch-db slurm-rshlaunch-head slurm-rshlaunch-node-{1..4}" + echo " podman network rm -f slurm-rshlaunch-test-net" + echo "" + exit 1 +fi + +echo "" +echo "Manual cleanup when done:" +echo " podman rm -f slurm-rshlaunch-mariadb slurm-rshlaunch-db slurm-rshlaunch-head slurm-rshlaunch-node-{1..4}" +echo " podman network rm -f slurm-rshlaunch-test-net" +echo "" + +# Cleanup disabled for debugging - do manually diff --git a/scripts/podman/test-spindle-slurm-srun-parallel.sh b/scripts/podman/test-spindle-slurm-srun-parallel.sh new file mode 100755 index 00000000..6e05eef2 --- /dev/null +++ b/scripts/podman/test-spindle-slurm-srun-parallel.sh @@ -0,0 +1,261 @@ +#!/bin/bash +# +# Run Spindle Slurm srun test with unique container names for parallel execution +# +# Usage: test-spindle-slurm-srun-parallel.sh +# +# The run-id is appended to all container and network names to avoid conflicts +# when running multiple tests in parallel. + +set -e + +RUN_ID="${1}" + +if [ -z "$RUN_ID" ]; then + echo "Usage: $0 " + echo "" + echo "Example: $0 42" + echo " Creates containers: slurm-srun-42-mariadb, slurm-srun-42-head, etc." + exit 1 +fi + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration - all names include RUN_ID for uniqueness +IMAGE_NAME="spindle-slurm-srun" +NETWORK_NAME="slurm-srun-${RUN_ID}-net" +WORKERS=4 +NAME_PREFIX="slurm-srun-${RUN_ID}" + +echo "==========================================" +echo "Spindle Slurm Srun Tests (Run ID: $RUN_ID)" +echo "==========================================" +echo "" +echo "Container prefix: $NAME_PREFIX" +echo "Network: $NETWORK_NAME" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up run $RUN_ID..." + # Stop all containers in parallel for faster cleanup + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + (podman stop "$container" 2>/dev/null || true) & + done + wait + # Remove all containers in parallel + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + (podman rm -f "$container" 2>/dev/null || true) & + done + wait + podman network rm -f "$NETWORK_NAME" 2>/dev/null || true + echo "✓ Cleanup complete for run $RUN_ID" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# Initial cleanup +cleanup + +echo "==========================================" +echo "Setting up Slurm cluster..." +echo "==========================================" +echo "" + +# Create network +if podman network exists "$NETWORK_NAME" 2>/dev/null; then + echo "Network $NETWORK_NAME already exists, reusing" +else + echo "Creating network: $NETWORK_NAME" + podman network create "$NETWORK_NAME" + echo "✓ Network created" +fi + +echo "" +echo "Starting MariaDB..." +# Read password from mariadb.env +# First check repo root (portable deployment), then fall back to source location +MARIADB_ENV="" +if [ -f "$REPO_ROOT/mariadb.env" ]; then + MARIADB_ENV="$REPO_ROOT/mariadb.env" +elif [ -f "$REPO_ROOT/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env" ]; then + MARIADB_ENV="$REPO_ROOT/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env" +else + echo "Error: Could not find mariadb.env" + echo " Looked in:" + echo " $REPO_ROOT/mariadb.env" + echo " $REPO_ROOT/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env" + exit 1 +fi + +MARIADB_PASSWORD=$(grep MARIADB_PASSWORD "$MARIADB_ENV" | cut -d'"' -f2) +if [ -z "$MARIADB_PASSWORD" ]; then + echo "Error: Could not read password from $MARIADB_ENV" + exit 1 +fi +echo " Using password from: $MARIADB_ENV" +podman run \ + --name "${NAME_PREFIX}-mariadb" \ + --hostname slurm-mariadb \ + --network "$NETWORK_NAME" \ + -e MYSQL_RANDOM_ROOT_PASSWORD=yes \ + -e MYSQL_DATABASE=slurm_acct_db \ + -e MYSQL_USER=slurm \ + -e MYSQL_PASSWORD="$MARIADB_PASSWORD" \ + -d \ + mariadb:12 + +echo " ✓ MariaDB started" +echo "Waiting for MariaDB to initialize..." +sleep 15 + +echo "" +echo "Starting slurmdbd (accounting daemon)..." +podman run \ + --name "${NAME_PREFIX}-db" \ + --hostname slurm-db \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=db \ + -e SLURM_HEAD_NODE=slurm-head \ + -e workers="$WORKERS" \ + -d \ + "$IMAGE_NAME" + +echo " ✓ slurmdbd started" +sleep 10 + +echo "" +echo "Starting slurmctld (controller)..." +podman run \ + --name "${NAME_PREFIX}-head" \ + --hostname slurm-head \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=ctl \ + -e SLURM_HEAD_NODE=slurm-head \ + -e workers="$WORKERS" \ + -t \ + -d \ + "$IMAGE_NAME" + +echo " ✓ slurmctld started" +sleep 10 + +echo "" +echo "Starting worker nodes..." +for i in $(seq 1 $WORKERS); do + echo "Starting slurm-node-$i..." + podman run \ + --name "${NAME_PREFIX}-node-$i" \ + --hostname "slurm-node-$i" \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=worker \ + -e SLURM_HEAD_NODE=slurm-head \ + -e workers="$WORKERS" \ + -d \ + "$IMAGE_NAME" + echo " ✓ slurm-node-$i started" +done + +echo "" +echo "Waiting for Slurm cluster to initialize..." +echo "(This takes ~60 seconds for all daemons and nodes)" +sleep 60 + +echo "" +echo "==========================================" +echo "Checking container status..." +echo "==========================================" +echo "" + +# Check if containers are still running +ALL_RUNNING=true +for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + if podman ps --filter "name=$container" --format "{{.Names}}" | grep -q "$container"; then + echo " ✓ $container is running" + else + echo " ✗ $container has exited!" + ALL_RUNNING=false + echo "" + echo "Last 30 lines of $container logs:" + echo "----------------------------------------" + podman logs "$container" 2>&1 | tail -30 + echo "----------------------------------------" + fi +done + +if [ "$ALL_RUNNING" = false ]; then + echo "" + echo "Some containers exited. Check logs above." + exit 1 +fi + +echo "" +echo "==========================================" +echo "Verifying munge authentication..." +echo "==========================================" +echo "" + +podman exec "${NAME_PREFIX}-head" bash -c 'munge -n | unmunge' + +echo "" +echo "✓ Munge working" + +echo "" +echo "==========================================" +echo "Verifying Slurm cluster..." +echo "==========================================" +echo "" + +echo "Checking node status with sinfo:" +if podman exec "${NAME_PREFIX}-head" sinfo; then + echo " ✓ Slurm cluster ready" +else + echo "" + echo "✗ Slurm cluster verification FAILED" + echo "" + echo "Containers are still running for debugging." + echo "Press ENTER to cleanup and exit, or Ctrl-C to keep them running." + echo "" + echo "Useful debug commands:" + echo " podman logs ${NAME_PREFIX}-head" + echo " podman logs ${NAME_PREFIX}-db" + echo " podman exec ${NAME_PREFIX}-head sinfo" + echo " podman exec ${NAME_PREFIX}-head scontrol show nodes" + echo "" + read -r + exit 1 +fi + +echo "" +echo "==========================================" +echo "Running Spindle testsuite..." +echo "==========================================" +echo "" +echo "This will take several minutes." +echo "" + +# Run the testsuite +if podman exec "${NAME_PREFIX}-head" bash -c "cd Spindle-build/testsuite && salloc -n${WORKERS} -N${WORKERS} ./runTests ${WORKERS}"; then + echo "" + echo "==========================================" + echo "✓ All tests passed! (Run $RUN_ID)" + echo "==========================================" + echo "" + exit 0 +else + echo "" + echo "==========================================" + echo "✗ Some tests failed (Run $RUN_ID)" + echo "==========================================" + echo "" + exit 1 +fi + +# Cleanup happens automatically via trap diff --git a/scripts/podman/test-spindle-slurm-srun.sh b/scripts/podman/test-spindle-slurm-srun.sh new file mode 100755 index 00000000..e0fc5ebf --- /dev/null +++ b/scripts/podman/test-spindle-slurm-srun.sh @@ -0,0 +1,360 @@ +#!/bin/bash +# +# Run N parallel Spindle Slurm srun tests in podman +# +# Usage: test-spindle-slurm-srun.sh +# +# Creates N independent Slurm clusters and runs tests in parallel. +# Each instance outputs to both stdout and out. with timestamps. +# +# Run this from outside the sandbox where podman is available. + +set -e + +NUM_INSTANCES="${1}" + +if [ -z "$NUM_INSTANCES" ]; then + echo "Usage: $0 " + echo "" + echo "Example: $0 10" + echo " Creates 10 independent Slurm clusters and runs tests in parallel" + exit 1 +fi + +if ! [[ "$NUM_INSTANCES" =~ ^[0-9]+$ ]] || [ "$NUM_INSTANCES" -lt 1 ]; then + echo "Error: num-instances must be a positive integer" + exit 1 +fi + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-slurm-srun" +WORKERS=4 + +echo "==========================================" +echo "Spindle Slurm Srun Parallel Tests" +echo "==========================================" +echo "" +echo "Instances: $NUM_INSTANCES" +echo "Each cluster: MariaDB + slurmdbd + slurmctld + 4 workers" +echo "Output: stdout + out. files (timestamped)" +echo "" + +# Function to run a single test instance +run_instance() { + local INSTANCE_ID=$1 + local NAME_PREFIX="slurm-srun-${INSTANCE_ID}" + + # Instance-specific hostnames for shared network + local MARIADB_HOST="${NAME_PREFIX}-mariadb" + local DB_HOST="${NAME_PREFIX}-db" + local HEAD_NODE="${NAME_PREFIX}-head" + local NODE_PREFIX="${NAME_PREFIX}-node" + + # All output from this function goes through ts and tee + { + echo "[Instance $INSTANCE_ID] Starting test" + echo "" + + echo "[Instance $INSTANCE_ID] Setting up Slurm cluster..." + echo "" + + # Generate random password for this cluster + MARIADB_PASSWORD=$(openssl rand -base64 16) + echo "[Instance $INSTANCE_ID] Generated MariaDB password" + echo "" + + # Start MariaDB + echo "[Instance $INSTANCE_ID] Starting MariaDB..." + podman run \ + --name "${NAME_PREFIX}-mariadb" \ + --hostname "$MARIADB_HOST" \ + --network "$SHARED_NETWORK" \ + -e MYSQL_RANDOM_ROOT_PASSWORD=yes \ + -e MYSQL_DATABASE=slurm_acct_db \ + -e MYSQL_USER=slurm \ + -e MYSQL_PASSWORD="$MARIADB_PASSWORD" \ + -d \ + mariadb:12 >/dev/null + echo "[Instance $INSTANCE_ID] MariaDB started" + echo "[Instance $INSTANCE_ID] Waiting for MariaDB to initialize (15s)..." + sleep 15 + echo "" + + # Start worker nodes FIRST so they're available for DNS resolution + # when slurmctld starts and tries to resolve node addresses + echo "[Instance $INSTANCE_ID] Starting worker nodes..." + for i in $(seq 1 $WORKERS); do + echo "[Instance $INSTANCE_ID] Starting ${NODE_PREFIX}-$i..." + podman run \ + --name "${NAME_PREFIX}-node-$i" \ + --hostname "${NODE_PREFIX}-$i" \ + --network "$SHARED_NETWORK" \ + -e SLURM_ROLE=worker \ + -e SLURM_HEAD_NODE="$HEAD_NODE" \ + -e SLURM_DB_HOST="$DB_HOST" \ + -e SLURM_NODE_PREFIX="$NODE_PREFIX" \ + -e workers="$WORKERS" \ + -d \ + "$IMAGE_NAME" >/dev/null + echo "[Instance $INSTANCE_ID] ${NODE_PREFIX}-$i started" + done + echo "[Instance $INSTANCE_ID] Waiting for workers to initialize (5s)..." + sleep 5 + echo "" + + # Start slurmdbd + echo "[Instance $INSTANCE_ID] Starting slurmdbd..." + podman run \ + --name "${NAME_PREFIX}-db" \ + --hostname "$DB_HOST" \ + --network "$SHARED_NETWORK" \ + -e SLURM_ROLE=db \ + -e SLURM_HEAD_NODE="$HEAD_NODE" \ + -e SLURM_DB_HOST="$DB_HOST" \ + -e SLURM_MARIADB_HOST="$MARIADB_HOST" \ + -e SLURM_NODE_PREFIX="$NODE_PREFIX" \ + -e workers="$WORKERS" \ + -e MARIADB_PASSWORD="$MARIADB_PASSWORD" \ + -d \ + "$IMAGE_NAME" >/dev/null + echo "[Instance $INSTANCE_ID] slurmdbd started" + sleep 10 + echo "" + + # Start slurmctld LAST so all other containers are reachable via DNS + echo "[Instance $INSTANCE_ID] Starting slurmctld..." + podman run \ + --name "${NAME_PREFIX}-head" \ + --hostname "$HEAD_NODE" \ + --network "$SHARED_NETWORK" \ + -e SLURM_ROLE=ctl \ + -e SLURM_HEAD_NODE="$HEAD_NODE" \ + -e SLURM_DB_HOST="$DB_HOST" \ + -e SLURM_NODE_PREFIX="$NODE_PREFIX" \ + -e workers="$WORKERS" \ + -t \ + -d \ + "$IMAGE_NAME" >/dev/null + echo "[Instance $INSTANCE_ID] slurmctld started" + sleep 10 + echo "" + + echo "[Instance $INSTANCE_ID] Waiting for Slurm cluster to initialize (60s)..." + sleep 60 + echo "" + + # Verify cluster + echo "[Instance $INSTANCE_ID] Verifying cluster..." + if podman exec "${NAME_PREFIX}-head" sinfo >/dev/null 2>&1; then + echo "[Instance $INSTANCE_ID] Cluster ready" + else + echo "[Instance $INSTANCE_ID] WARNING: sinfo failed" + echo "" + echo "[Instance $INSTANCE_ID] ========== DIAGNOSTICS ==========" + + # Check container status + echo "[Instance $INSTANCE_ID] Container status:" + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head; do + if podman ps --filter "name=$container" --format "{{.Names}}" | grep -q "$container"; then + echo "[Instance $INSTANCE_ID] ✓ $container is running" + else + echo "[Instance $INSTANCE_ID] ✗ $container has exited!" + fi + done + echo "" + + # Show MariaDB logs + echo "[Instance $INSTANCE_ID] MariaDB logs (last 20 lines):" + podman logs "${NAME_PREFIX}-mariadb" 2>&1 | tail -20 | sed "s/^/[Instance $INSTANCE_ID] /" + echo "" + + # Show slurmdbd logs + echo "[Instance $INSTANCE_ID] slurmdbd logs (last 30 lines):" + podman logs "${NAME_PREFIX}-db" 2>&1 | tail -30 | sed "s/^/[Instance $INSTANCE_ID] /" + echo "" + + # Show slurmctld logs + echo "[Instance $INSTANCE_ID] slurmctld logs (last 30 lines):" + podman logs "${NAME_PREFIX}-head" 2>&1 | tail -30 | sed "s/^/[Instance $INSTANCE_ID] /" + echo "" + + # Test MariaDB connectivity + echo "[Instance $INSTANCE_ID] Testing MariaDB connectivity:" + if podman exec "${NAME_PREFIX}-mariadb" mysqladmin ping 2>&1 | grep -q "mysqld is alive"; then + echo "[Instance $INSTANCE_ID] ✓ MariaDB is responding" + else + echo "[Instance $INSTANCE_ID] ✗ MariaDB not responding" + fi + echo "" + + # Check if slurmdbd can resolve MariaDB hostname + echo "[Instance $INSTANCE_ID] DNS check from slurmdbd:" + podman exec "${NAME_PREFIX}-db" getent hosts slurm-mariadb 2>&1 | sed "s/^/[Instance $INSTANCE_ID] /" || echo "[Instance $INSTANCE_ID] ✗ Cannot resolve slurm-mariadb" + echo "" + + echo "[Instance $INSTANCE_ID] ========== END DIAGNOSTICS ==========" + echo "" + echo "[Instance $INSTANCE_ID] Continuing with tests anyway..." + fi + echo "" + + # Run tests + echo "[Instance $INSTANCE_ID] Running Spindle testsuite..." + if podman exec "${NAME_PREFIX}-head" bash -c "cd Spindle-build/testsuite && export SPINDLE_DEBUG=3 && salloc -n${WORKERS} -N${WORKERS} ./runTests ${WORKERS}"; then + echo "" + echo "[Instance $INSTANCE_ID] ==========================================" + echo "[Instance $INSTANCE_ID] ALL TESTS PASSED" + echo "[Instance $INSTANCE_ID] ==========================================" + RESULT=0 + else + echo "" + echo "[Instance $INSTANCE_ID] ==========================================" + echo "[Instance $INSTANCE_ID] SOME TESTS FAILED" + echo "[Instance $INSTANCE_ID] ==========================================" + RESULT=1 + fi + echo "" + echo "[Instance $INSTANCE_ID] Test complete (cleanup will happen in serial phase)" + echo "" + + exit $RESULT + } 2>&1 | ts | tee "out.${INSTANCE_ID}" +} + +# Shared network for all instances (avoids 30s timeout per instance) +SHARED_NETWORK="slurm-srun-shared" + +# Serial phase: Verify prerequisites and cleanup +echo "==========================================" +echo "Serial Phase: Prerequisites & Cleanup" +echo "==========================================" +echo "" + +echo "Checking for required images..." +if ! podman images | grep -q "spindle-slurm-srun"; then + echo "ERROR: spindle-slurm-srun image not found" + echo "Build it first with: ./build-spindle-slurm-srun.sh" + exit 1 +fi + +if ! podman images | grep -q "mariadb.*12"; then + echo "ERROR: mariadb:12 image not found" + echo "Pull it first with: podman pull mariadb:12" + echo "Or load from tarball with: ./load-images.sh " + exit 1 +fi +echo "✓ All required images present" +echo "" + +echo "Cleaning up any existing test containers and network..." +for i in $(seq 1 $NUM_INSTANCES); do + NAME_PREFIX="slurm-srun-${i}" + + # Remove containers for this instance + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + podman rm -f "$container" 2>/dev/null || true + done +done + +# Remove shared network +podman network rm -f "$SHARED_NETWORK" 2>/dev/null || true +echo "✓ Cleanup complete" +echo "" + +echo "Creating shared network: $SHARED_NETWORK" +podman network create "$SHARED_NETWORK" >/dev/null +echo "✓ Shared network created (this may take ~30s due to systemd session bus timeout)" +echo "" + +# Parallel phase: Launch all instances +echo "==========================================" +echo "Parallel Phase: Launching $NUM_INSTANCES instances" +echo "==========================================" +echo "" + +PIDS=() +for i in $(seq 1 $NUM_INSTANCES); do + echo "Launching instance $i..." + run_instance $i & + PIDS+=($!) +done + +echo "" +echo "All instances launched. Waiting for completion..." +echo "(Output to stdout and out. files)" +echo "" + +# Wait for all instances and collect results +FAILED=0 +for i in $(seq 1 $NUM_INSTANCES); do + if ! wait ${PIDS[$((i-1))]}; then + FAILED=$((FAILED + 1)) + fi +done + +echo "" +echo "==========================================" +echo "Serial Phase: Cleanup" +echo "==========================================" +echo "" + +echo "Cleaning up $NUM_INSTANCES test clusters..." +for i in $(seq 1 $NUM_INSTANCES); do + NAME_PREFIX="slurm-srun-${i}" + + echo "Cleaning up instance $i..." + + # Stop containers + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + podman stop "$container" 2>/dev/null || true + done + + # Remove containers + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + podman rm -f "$container" 2>/dev/null || true + done +done +echo "✓ Cleanup complete" +echo "" + +# Remove shared network +echo "Removing shared network..." +podman network rm -f "$SHARED_NETWORK" 2>/dev/null || true +echo "✓ Network removed" +echo "" + +# Serial phase: Summary +echo "==========================================" +echo "Serial Phase: Summary" +echo "==========================================" +echo "" +echo "Total instances: $NUM_INSTANCES" +echo "Passed: $((NUM_INSTANCES - FAILED))" +echo "Failed: $FAILED" +echo "" + +if [ $FAILED -eq 0 ]; then + echo "✓ All instances passed!" + exit 0 +else + echo "✗ Some instances failed" + echo "" + echo "Check individual logs:" + for i in $(seq 1 $NUM_INSTANCES); do + if grep -q "SOME TESTS FAILED" "out.$i" 2>/dev/null; then + echo " out.$i - FAILED" + elif grep -q "ALL TESTS PASSED" "out.$i" 2>/dev/null; then + echo " out.$i - PASSED" + else + echo " out.$i - UNKNOWN" + fi + done + exit 1 +fi diff --git a/src/client/beboot/spindle_bootstrap.c b/src/client/beboot/spindle_bootstrap.c index e769fe4c..21aff5f1 100644 --- a/src/client/beboot/spindle_bootstrap.c +++ b/src/client/beboot/spindle_bootstrap.c @@ -91,16 +91,39 @@ extern char *realize(char *path); static int establish_connection() { + fprintf(stderr, "[BOOTSTRAP] establish_connection: commpath=%s number=%lu\n", commpath, (unsigned long)number); + fflush(stderr); debug_printf2("Opening connection to server\n"); + + fprintf(stderr, "[BOOTSTRAP] About to call client_open_connection\n"); + fflush(stderr); ldcsid = client_open_connection(commpath, number); - if (ldcsid == -1) + if (ldcsid == -1) { + fprintf(stderr, "[BOOTSTRAP] ERROR: client_open_connection returned -1\n"); + fflush(stderr); return -1; + } + fprintf(stderr, "[BOOTSTRAP] client_open_connection succeeded, ldcsid=%d\n", ldcsid); + fflush(stderr); + fprintf(stderr, "[BOOTSTRAP] About to send_pid\n"); + fflush(stderr); send_pid(ldcsid); - send_rankinfo_query(ldcsid, &rankinfo[0], &rankinfo[1], &rankinfo[2], &rankinfo[3]); - if (opts & OPT_NUMA) + + fprintf(stderr, "[BOOTSTRAP] About to send_rankinfo_query\n"); + fflush(stderr); + send_rankinfo_query(ldcsid, &rankinfo[0], &rankinfo[1], &rankinfo[2], &rankinfo[3]); + fprintf(stderr, "[BOOTSTRAP] Received rankinfo: [%d,%d,%d,%d]\n", rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3]); + fflush(stderr); + + if (opts & OPT_NUMA) { + fprintf(stderr, "[BOOTSTRAP] About to send_cpu\n"); + fflush(stderr); send_cpu(ldcsid, get_cur_cpu()); + } + fprintf(stderr, "[BOOTSTRAP] establish_connection complete\n"); + fflush(stderr); return 0; } @@ -334,47 +357,92 @@ int main(int argc, char *argv[]) int result; char **j, *spindle_env; + fprintf(stderr, "[BOOTSTRAP] Starting spindle_bootstrap pid=%d argc=%d\n", getpid(), argc); + fflush(stderr); + + fprintf(stderr, "[BOOTSTRAP] About to init logging\n"); + fflush(stderr); LOGGING_INIT_PREEXEC("Client"); + fprintf(stderr, "[BOOTSTRAP] Logging initialized\n"); + fflush(stderr); debug_printf("Launched Spindle Bootstrapper\n"); + fprintf(stderr, "[BOOTSTRAP] About to parse cmdline\n"); + fflush(stderr); result = parse_cmdline(argc, argv); if (result == -1) { + fprintf(stderr, "[BOOTSTRAP] ERROR: parse_cmdline failed\n"); + fflush(stderr); fprintf(stderr, "spindle_boostrap cannot be invoked directly\n"); return -1; } + fprintf(stderr, "[BOOTSTRAP] Parsed cmdline successfully\n"); + fflush(stderr); + debug_printf("Bootstrap parsed cmdline successfully\n"); + fprintf(stderr, "[BOOTSTRAP] Checking SPINDLE env var\n"); + fflush(stderr); spindle_env = getenv("SPINDLE"); if (spindle_env) { + fprintf(stderr, "[BOOTSTRAP] SPINDLE=%s\n", spindle_env); + fflush(stderr); if (strcasecmp(spindle_env, "false") == 0 || strcmp(spindle_env, "0") == 0) { + fprintf(stderr, "[BOOTSTRAP] Turning off spindle, executing directly\n"); + fflush(stderr); debug_printf("Turning off spindle from bootstrapper because SPINDLE is %s\n", spindle_env); execvp(cmdline[0], cmdline); return handle_exec_failure(cmdline, errno); } } + fprintf(stderr, "[BOOTSTRAP] About to parse commpath location\n"); + fflush(stderr); char *orig_commpath = parse_location(symbolic_commpath, number); if (!orig_commpath) { + fprintf(stderr, "[BOOTSTRAP] ERROR: parse_location failed\n"); + fflush(stderr); return -1; } + fprintf(stderr, "[BOOTSTRAP] About to realize commpath: %s\n", orig_commpath); + fflush(stderr); commpath = realize(orig_commpath); + fprintf(stderr, "[BOOTSTRAP] Realized commpath: %s\n", commpath); + fflush(stderr); + debug_printf("Bootstrap using commpath: %s\n", commpath); if (daemon_args) { + fprintf(stderr, "[BOOTSTRAP] About to launch daemon\n"); + fflush(stderr); launch_daemon(commpath); + fprintf(stderr, "[BOOTSTRAP] Daemon launch returned\n"); + fflush(stderr); } - + + fprintf(stderr, "[BOOTSTRAP] About to establish connection\n"); + fflush(stderr); result = establish_connection(); if (result == -1) { + fprintf(stderr, "[BOOTSTRAP] ERROR: establish_connection failed\n"); + fflush(stderr); err_printf("spindle_bootstrap failed to connect to daemons\n"); return -1; } + fprintf(stderr, "[BOOTSTRAP] Connection established successfully\n"); + fflush(stderr); + debug_printf("Bootstrap connection established\n"); + fprintf(stderr, "[BOOTSTRAP] Checking if exec is excluded: %s\n", cmdline[0]); + fflush(stderr); if (isExecExcluded(cmdline[0])) { + fprintf(stderr, "[BOOTSTRAP] Exec excluded, running directly\n"); + fflush(stderr); debug_printf("Turning off spindle because we're running an excluded binary: %s\n", cmdline[0]); execvp(cmdline[0], cmdline); return handle_exec_failure(cmdline, errno); } - + fprintf(stderr, "[BOOTSTRAP] Checking shmcache opts=0x%x cachesize=%u\n", opts, cachesize); + fflush(stderr); if ((opts & OPT_SHMCACHE) && cachesize) { unsigned int shm_cache_limit; cachesize *= 1024; @@ -383,14 +451,26 @@ int main(int argc, char *argv[]) #else shm_cache_limit = cachesize; #endif + fprintf(stderr, "[BOOTSTRAP] About to init shmcache\n"); + fflush(stderr); shmcache_init(commpath, number, cachesize, shm_cache_limit); use_cache = 1; - } - + fprintf(stderr, "[BOOTSTRAP] Shmcache initialized\n"); + fflush(stderr); + } + + fprintf(stderr, "[BOOTSTRAP] About to get_executable\n"); + fflush(stderr); get_executable(); + fprintf(stderr, "[BOOTSTRAP] About to get_clientlib\n"); + fflush(stderr); get_clientlib(); + fprintf(stderr, "[BOOTSTRAP] About to adjust_script\n"); + fflush(stderr); adjust_script(); - + fprintf(stderr, "[BOOTSTRAP] Completed pre-exec setup\n"); + fflush(stderr); + /** * Exec setup **/ @@ -406,11 +486,18 @@ int main(int argc, char *argv[]) } bare_printf("\n"); + fprintf(stderr, "[BOOTSTRAP] About to setup environment\n"); + fflush(stderr); /** * Exec the user's application. **/ setup_environment(); + fprintf(stderr, "[BOOTSTRAP] About to execvp: %s\n", executable ? executable : ""); + fflush(stderr); execvp(executable, cmdline); + + fprintf(stderr, "[BOOTSTRAP] ERROR: execvp failed errno=%d\n", errno); + fflush(stderr); /** diff --git a/src/logging/spindle_logc.c b/src/logging/spindle_logc.c index 4e82300f..a9f90f7c 100644 --- a/src/logging/spindle_logc.c +++ b/src/logging/spindle_logc.c @@ -33,6 +33,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include +#include #if !defined(LIBEXEC) #error Expected to have LIBEXEC defined @@ -59,6 +60,40 @@ int run_tests; #define SPAWN_TIMEOUT 300 #define CONNECT_TIMEOUT 100 +// Timestamp support +static double start_time_monotonic = 0.0; + +static double get_monotonic_time() { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec + ts.tv_nsec / 1e9; +} + +void format_timestamp_dual(char *buf, size_t bufsize) { + struct timespec ts; + struct tm tm_info; + double now_monotonic; + double elapsed; + + // Get absolute wall-clock time + clock_gettime(CLOCK_REALTIME, &ts); + localtime_r(&ts.tv_sec, &tm_info); + + // Get relative time + now_monotonic = get_monotonic_time(); + elapsed = now_monotonic - start_time_monotonic; + + // Format: [HH:MM:SS.usec +elapsed] + snprintf(buf, bufsize, "%02d:%02d:%02d.%06ld +%8.6f", + tm_info.tm_hour, tm_info.tm_min, tm_info.tm_sec, + ts.tv_nsec / 1000, elapsed); +} + +void init_timestamp() { + if (start_time_monotonic == 0.0) + start_time_monotonic = get_monotonic_time(); +} + extern int spindle_mkdir(char *orig_path); int fileExists(char *name) @@ -328,7 +363,10 @@ void init_spindle_debugging(char *name, int survive_exec) if (debug_fd != -1) spindle_debug_output_f = fdopen(debug_fd, "w"); if (test_fd != -1) - spindle_test_output_f = fdopen(test_fd, "w"); + spindle_test_output_f = fdopen(test_fd, "w"); + + /* Initialize timestamp */ + init_timestamp(); } void spindle_dump_on_error() diff --git a/src/logging/spindle_logc.h b/src/logging/spindle_logc.h index f25ff44e..f76d38a1 100644 --- a/src/logging/spindle_logc.h +++ b/src/logging/spindle_logc.h @@ -34,8 +34,10 @@ extern void spindle_dump_on_error(); #define debug_printf(format, ...) \ do { \ if (spindle_debug_prints && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, "[%s.%d@%s:%u] %s - " format, \ - spindle_debug_name, getpid(), \ + char timestamp_buf[64]; \ + format_timestamp_dual(timestamp_buf, sizeof(timestamp_buf)); \ + fprintf(spindle_debug_output_f, "[%s] [%s.%d@%s:%u] %s - " format, \ + timestamp_buf, spindle_debug_name, getpid(), \ BASE_FILE, __LINE__, __func__, ## __VA_ARGS__); \ fflush(spindle_debug_output_f); \ } \ @@ -44,8 +46,10 @@ extern void spindle_dump_on_error(); #define debug_printf2(format, ...) \ do { \ if (spindle_debug_prints > 1 && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, "[%s.%d@%s:%u] %s - " format, \ - spindle_debug_name, getpid(), \ + char timestamp_buf[64]; \ + format_timestamp_dual(timestamp_buf, sizeof(timestamp_buf)); \ + fprintf(spindle_debug_output_f, "[%s] [%s.%d@%s:%u] %s - " format, \ + timestamp_buf, spindle_debug_name, getpid(), \ BASE_FILE, __LINE__, __func__, ## __VA_ARGS__); \ fflush(spindle_debug_output_f); \ } \ @@ -54,8 +58,10 @@ extern void spindle_dump_on_error(); #define debug_printf3(format, ...) \ do { \ if (spindle_debug_prints > 2 && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, "[%s.%d@%s:%u] %s - " format, \ - spindle_debug_name, getpid(), \ + char timestamp_buf[64]; \ + format_timestamp_dual(timestamp_buf, sizeof(timestamp_buf)); \ + fprintf(spindle_debug_output_f, "[%s] [%s.%d@%s:%u] %s - " format, \ + timestamp_buf, spindle_debug_name, getpid(), \ BASE_FILE, __LINE__, __func__, ## __VA_ARGS__); \ fflush(spindle_debug_output_f); \ } \ @@ -64,7 +70,10 @@ extern void spindle_dump_on_error(); #define bare_printf(format, ...) \ do { \ if (spindle_debug_prints && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, format, ## __VA_ARGS__); \ + char timestamp_buf[64]; \ + format_timestamp_dual(timestamp_buf, sizeof(timestamp_buf)); \ + fprintf(spindle_debug_output_f, "[%s] " format, \ + timestamp_buf, ## __VA_ARGS__); \ fflush(spindle_debug_output_f); \ } \ } while (0) @@ -72,7 +81,10 @@ extern void spindle_dump_on_error(); #define bare_printf2(format, ...) \ do { \ if (spindle_debug_prints > 1 && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, format, ## __VA_ARGS__); \ + char timestamp_buf[64]; \ + format_timestamp_dual(timestamp_buf, sizeof(timestamp_buf)); \ + fprintf(spindle_debug_output_f, "[%s] " format, \ + timestamp_buf, ## __VA_ARGS__); \ fflush(spindle_debug_output_f); \ } \ } while (0) @@ -80,7 +92,10 @@ extern void spindle_dump_on_error(); #define bare_printf3(format, ...) \ do { \ if (spindle_debug_prints > 2 && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, format, ## __VA_ARGS__); \ + char timestamp_buf[64]; \ + format_timestamp_dual(timestamp_buf, sizeof(timestamp_buf)); \ + fprintf(spindle_debug_output_f, "[%s] " format, \ + timestamp_buf, ## __VA_ARGS__); \ fflush(spindle_debug_output_f); \ } \ } while (0) @@ -88,8 +103,10 @@ extern void spindle_dump_on_error(); #define err_printf(format, ...) \ do { \ if (spindle_debug_prints && spindle_debug_output_f) { \ - fprintf(spindle_debug_output_f, "[%s.%d@%s:%u] - ERROR: " \ - format, spindle_debug_name, getpid(), \ + char timestamp_buf[64]; \ + format_timestamp_dual(timestamp_buf, sizeof(timestamp_buf)); \ + fprintf(spindle_debug_output_f, "[%s] [%s.%d@%s:%u] - ERROR: " \ + format, timestamp_buf, spindle_debug_name, getpid(), \ BASE_FILE, __LINE__, ## __VA_ARGS__); \ spindle_dump_on_error(); \ fflush(spindle_debug_output_f); \ @@ -109,5 +126,7 @@ void init_spindle_debugging(char *name, int survive_exec); void fini_spindle_debugging(); void reset_spindle_debugging(); int is_debug_fd(int fd); +void format_timestamp_dual(char *buf, size_t bufsize); +void init_timestamp(); #endif