Podman port

.containerignore

@@ -0,0 +1,23 @@
+# Podman/Docker ignore file
+# Excludes files from being copied into container images during build
+
+# Exclude the saved images tarball (6+ GB)
+spindle-podman-images.tar
+
+# Exclude build artifacts
+workspace-Spindle/build/
+workspace-Spindle/install/
+*.o
+*.a
+*.so
+
+# Exclude git metadata
+.git/
+.gitignore
+
+# Exclude test output
+out.*
+*.log
+
+# Exclude investigation directory
+podman-concurrency-investigation/

.dockerignore

@@ -0,0 +1 @@
+.containerignore

.gitignore

@@ -35,3 +35,11 @@ run_driver
 run_driver_rm
 preload_file_list
 build
+
+# Generated by generate_config.sh
+containers/spindle-slurm-ubuntu/testing/conf/slurmdbd.conf
+containers/spindle-slurm-ubuntu/testing/mariadb.env
+containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf
+containers/spindle-slurm-ubuntu/testing-srun/mariadb.env
+
+

containers/spindle-flux-ubuntu/Dockerfile.podman

@@ -0,0 +1,97 @@
+# This is based on the Flux Container Tutorial
+# See https://flux-framework.readthedocs.io/en/latest/tutorials/containers
+ARG flux_sched_version=noble-v0.48.0-amd64
+FROM fluxrm/flux-sched:${flux_sched_version} AS builder
+ARG replicas=4
+ENV workers=${replicas}
+USER root
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
+ENV SPINDLE_TEST_CONTAINER=1
+
+# LC-specific fix for podman setgroups issue
+ARG PODMAN_BUILD=false
+RUN if [ "$PODMAN_BUILD" = "true" ]; then \
+      echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \
+    fi
+
+RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
+    && apt-get -qq install -y --no-install-recommends \
+        autotools-dev \
+        autoconf \
+        automake \
+        cmake \
+        git \
+        python3 \
+        openssh-server \
+        openssh-client \
+        libdb-dev \
+        apt-utils \
+        dnsutils \
+        iputils-ping \
+        python3-pip \
+        libgcrypt20 \
+        libgcrypt20-dev \
+        gdb \
+        libc6-dbg \
+        software-properties-common
+
+ARG USER=fluxuser
+ARG CONFIG_ROOT=containers/spindle-flux-ubuntu
+
+# Allow fluxuser to run as other users so it can start munged
+RUN sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers"
+
+# Configure flux
+ENV STATE_DIR=/var/lib/flux
+RUN mkdir -p ${STATE_DIR} /etc/flux/system /etc/flux/system/cron.d /etc/flux/config /run/flux /etc/flux/imp/conf.d
+COPY ${CONFIG_ROOT}/flux/imp.toml /etc/flux/imp/conf.d/
+COPY ${CONFIG_ROOT}/flux/broker.toml /etc/flux/config/
+RUN mkdir -p /etc/flux/system/cron.d && \
+    mkdir -p /mnt/curve && \
+    flux keygen /mnt/curve/curve.cert && \
+    flux R encode --hosts="node-[1-${workers}]" > /etc/flux/system/R && \
+    chmod -R a+rX /etc/flux && \
+    chown -R ${USER}:${USER} /run/flux ${STATE_DIR} /mnt/curve/curve.cert
+
+# Build Spindle
+WORKDIR /home/${USER}
+# Copy the whole git repo into the container
+COPY . /home/${USER}/Spindle
+
+# Fix permissions on Spindle source (configure needs +x)
+RUN chmod -R u+rwX /home/${USER}/Spindle && \
+    chown -R ${USER}:${USER} /home/${USER}/Spindle
+
+# Copy and prepare build script as root
+COPY ${CONFIG_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh
+RUN chmod +rx /home/${USER}/build_spindle.sh && \
+    chown ${USER}:${USER} /home/${USER}/build_spindle.sh
+
+# Switch to user to run the build
+USER ${USER}
+RUN bash ./build_spindle.sh
+
+# Copy scripts as root
+USER root
+RUN chown -R ${USER}:${USER} /home/fluxuser && \
+    chown -R ${USER}:${USER} /run/flux
+
+COPY ${CONFIG_ROOT}/scripts/flux_healthcheck.sh /home/${USER}/flux_healthcheck.sh
+COPY ${CONFIG_ROOT}/scripts/entrypoint.sh.podman /home/${USER}/entrypoint.sh
+RUN chmod +rx /home/${USER}/flux_healthcheck.sh /home/${USER}/entrypoint.sh && \
+    chown ${USER}:${USER} /home/${USER}/flux_healthcheck.sh /home/${USER}/entrypoint.sh
+
+USER ${USER}
+WORKDIR /home/${USER}
+ENV PATH /home/${USER}/Spindle-inst/bin:${PATH}
+# Make libfabric work with fork.
+ENV RDMAV_FORK_SAFE 1
+# Silence warning from hwloc about unsupported PCI device
+# on GitHub-hosted runners.
+ENV HWLOC_HIDE_ERRORS 2
+# Suppress UCX warnings about /proc/sys tuning parameters
+# These are expected in rootless podman containers
+ENV UCX_LOG_LEVEL=error
+
+ENTRYPOINT /bin/bash ./entrypoint.sh

containers/spindle-flux-ubuntu/scripts/entrypoint.sh.podman

@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Podman-compatible entrypoint for Flux containers
+# Starts munged and the flux broker.
+#
+# For documentation on running Flux in containers, see
+# https://flux-framework.readthedocs.io/en/latest/tutorials/containers
+
+set -x  # Debug output
+
+brokerOptions="-Scron.directory=/etc/flux/system/cron.d \
+  -Stbon.fanout=256 \
+  -Srundir=/run/flux \
+  -Sstatedir=${STATE_DIRECTORY:-/var/lib/flux} \
+  -Slocal-uri=local:///run/flux/local \
+  -Slog-stderr-level=6 \
+  -Slog-stderr-mode=local"
+
+# Get the hostname that will resolve for the Docker bridge network.
+address=$(echo $( nslookup "$( hostname -i )" | head -n 1 ))
+parts=(${address//=/ })
+hostName=${parts[2]}
+thisHost=(${hostName//./ })
+thisHost=${thisHost[0]}
+echo "This host: $thisHost"
+echo "Main host: $mainHost"
+export FLUX_FAKE_HOSTNAME=$thisHost
+
+if [ -d /shared ]; then
+    sudo chown -R "$(id -un):$(id -gn)" /shared
+    sudo chmod 755 /shared
+fi
+
+# Start munged
+echo "Starting munged..."
+sudo -u munge /usr/sbin/munged
+
+# Give munge time to start
+sleep 2
+
+if [ "${thisHost}" != "${mainHost}" ]; then
+    # Worker node -- wait for head node before connecting
+    echo "Worker node: waiting for head node..."
+    sleep 15
+    echo "Starting flux broker (worker)..."
+    exec flux start -o --config /etc/flux/config ${brokerOptions} sleep inf
+else
+    # Head node
+    echo "Head node: starting flux broker..."
+    exec flux start -o --config /etc/flux/config ${brokerOptions} sleep inf
+fi

containers/spindle-hello-podman/01-basic/Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:noble
+
+# LC-specific fix for podman setgroups issue
+# This is a no-op in Docker, but required for podman on LC systems
+ARG PODMAN_BUILD=false
+RUN if [ "$PODMAN_BUILD" = "true" ]; then \
+      echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \
+    fi
+
+# Install curl to test both apt-get and SSL connectivity
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+# Test script that validates:
+# 1. Container runs
+# 2. Network access works
+# 3. SSL certificates work
+CMD echo "=== Hello from Podman! ===" && \
+    echo "Container is running successfully." && \
+    echo "" && \
+    echo "Testing network and SSL connectivity..." && \
+    curl -I https://www.google.com 2>&1 | head -5 && \
+    echo "" && \
+    echo "If you see HTTP headers above, SSL is working!" && \
+    echo "" && \
+    echo "=== 01-basic test complete ==="

containers/spindle-hello-podman/02-user-switch/Dockerfile

@@ -0,0 +1,78 @@
+FROM ubuntu:noble
+
+# LC-specific fix for podman setgroups issue
+ARG PODMAN_BUILD=false
+RUN if [ "$PODMAN_BUILD" = "true" ]; then \
+      echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \
+    fi
+
+# Install basic tools
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    sudo \
+ && rm -rf /var/lib/apt/lists/*
+
+# Create a non-root user similar to how Spindle containers work
+# This pattern is used in all Spindle containers
+ARG USER=testuser
+ARG UID=1001
+ARG GID=1001
+
+RUN groupadd -g ${GID} ${USER} && \
+    useradd -m -u ${UID} -g ${GID} -s /bin/bash ${USER} && \
+    echo "${USER} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/${USER} && \
+    chmod 0440 /etc/sudoers.d/${USER}
+
+# Create a test directory owned by root to verify permission handling
+RUN mkdir -p /root-owned && \
+    echo "This is owned by root" > /root-owned/root-file.txt && \
+    chmod 644 /root-owned/root-file.txt
+
+# Create a test directory that will be owned by the user
+RUN mkdir -p /user-owned && \
+    chown ${UID}:${GID} /user-owned
+
+# Switch to non-root user
+USER ${USER}
+WORKDIR /home/${USER}
+
+# Create a file as the user to verify we're running as that user
+RUN echo "This is owned by ${USER}" > /home/${USER}/user-file.txt
+
+# Test script that validates:
+# 1. Running as the correct user
+# 2. Can read files owned by root
+# 3. Can write to user-owned directories
+# 4. Cannot write to root-owned directories
+# 5. Can use sudo if needed
+CMD echo "=== User Switch Test ===" && \
+    echo "" && \
+    echo "Current user information:" && \
+    id && \
+    echo "" && \
+    echo "Home directory:" && \
+    pwd && \
+    ls -la /home/${USER} && \
+    echo "" && \
+    echo "Testing file permissions:" && \
+    echo "  Reading root-owned file..." && \
+    cat /root-owned/root-file.txt && \
+    echo "  ✓ Can read root-owned files" && \
+    echo "" && \
+    echo "  Writing to user-owned directory..." && \
+    echo "test content" > /user-owned/test.txt && \
+    cat /user-owned/test.txt && \
+    echo "  ✓ Can write to user-owned directories" && \
+    echo "" && \
+    echo "  Testing root-owned directory (should fail)..." && \
+    (echo "test" > /root-owned/fail.txt 2>&1 && echo "  ✗ UNEXPECTED: Could write to root directory" || echo "  ✓ Correctly denied write to root directory") && \
+    echo "" && \
+    echo "  Testing sudo access..." && \
+    sudo -n echo "  ✓ Sudo works (passwordless for container convenience)" && \
+    echo "" && \
+    echo "=== 02-user-switch test complete ===" && \
+    echo "" && \
+    echo "Key patterns validated:" && \
+    echo "  ✓ Non-root user created (uid=${UID})" && \
+    echo "  ✓ User has home directory" && \
+    echo "  ✓ Correct permission handling" && \
+    echo "  ✓ Sudo available when needed"

containers/spindle-hello-podman/03-filesystem/Dockerfile

@@ -0,0 +1,85 @@
+FROM ubuntu:noble
+
+# LC-specific fix for podman setgroups issue
+ARG PODMAN_BUILD=false
+RUN if [ "$PODMAN_BUILD" = "true" ]; then \
+      echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \
+    fi
+
+# Install tools for filesystem operations
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    file \
+ && rm -rf /var/lib/apt/lists/*
+
+# Create a non-root user (matching pattern from 02-user-switch)
+ARG USER=testuser
+ARG UID=1001
+ARG GID=1001
+
+# Make USER available at runtime (ARG only available at build time)
+ENV USER=${USER}
+
+RUN groupadd -g ${GID} ${USER} && \
+    useradd -m -u ${UID} -g ${GID} -s /bin/bash ${USER}
+
+# Switch to non-root user
+USER ${USER}
+WORKDIR /home/${USER}
+
+# Create mount points for testing different volume scenarios
+# These will be populated via volume mounts at runtime
+RUN mkdir -p /home/${USER}/host-data && \
+    mkdir -p /home/${USER}/output
+
+# Test script that validates volume mounts and filesystem operations
+# Note: The actual volume content will be mounted at runtime
+CMD echo "=== Filesystem Mount Test ===" && \
+    echo "" && \
+    echo "Testing volume mounts and file operations..." && \
+    echo "" && \
+    echo "1. Checking mounted host directory:" && \
+    if [ -d "/home/${USER}/host-data" ]; then \
+        echo "   Mount point exists: /home/${USER}/host-data" && \
+        if [ "$(ls -A /home/${USER}/host-data 2>/dev/null)" ]; then \
+            echo "   ✓ Host directory mounted with content:" && \
+            ls -lh /home/${USER}/host-data | head -10 && \
+            echo "" && \
+            echo "   Reading a file from host..." && \
+            if [ -f "/home/${USER}/host-data/test-input.txt" ]; then \
+                cat /home/${USER}/host-data/test-input.txt && \
+                echo "   ✓ Can read files from host mount" ; \
+            else \
+                echo "   ℹ No test-input.txt found (expected for manual testing)" ; \
+            fi ; \
+        else \
+            echo "   ⚠ Mount point empty - volume may not be mounted" ; \
+        fi ; \
+    else \
+        echo "   ✗ Mount point missing" ; \
+    fi && \
+    echo "" && \
+    echo "2. Testing write to output directory:" && \
+    echo "Test data written from container at $(date)" > /home/${USER}/output/container-output.txt && \
+    echo "   ✓ Created file: container-output.txt" && \
+    cat /home/${USER}/output/container-output.txt && \
+    echo "" && \
+    echo "3. Checking file ownership:" && \
+    ls -ln /home/${USER}/output/container-output.txt && \
+    echo "   ℹ File owned by UID=$(stat -c %u /home/${USER}/output/container-output.txt) GID=$(stat -c %g /home/${USER}/output/container-output.txt)" && \
+    echo "" && \
+    echo "4. Testing file type detection:" && \
+    file /home/${USER}/output/container-output.txt && \
+    echo "" && \
+    echo "=== 03-filesystem test complete ===" && \
+    echo "" && \
+    echo "Key patterns validated:" && \
+    echo "  ✓ Volume mounts work" && \
+    echo "  ✓ Can read files from host" && \
+    echo "  ✓ Can write files to mounted volumes" && \
+    echo "  ✓ File ownership preserved" && \
+    echo "" && \
+    echo "This demonstrates the pattern used for Spindle:" && \
+    echo "  - Mount source code from host (read-only)" && \
+    echo "  - Mount build directory (read-write)" && \
+    echo "  - Build artifacts persist on host" && \
+    echo "  - Logs accessible after container exits"