From e25642cdca7448a374fd2ec53c04a1664ca9b5d9 Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Thu, 20 Aug 2026 15:36:08 -0700 Subject: [PATCH 1/2] Send exit message with MSG_NOSIGNAL; ensure log daemon exits between tests --- src/cobo/cobo_comm.c | 58 +++++++++++++++++++++++++++++++ src/cobo/cobo_comm.h | 2 ++ src/fe/comlib/cobo_fe_comm.c | 4 ++- testsuite/Makefile.am | 9 +++-- testsuite/Makefile.in | 53 +++++++++++++++------------- testsuite/logd_kill_test_template | 36 +++++++++++++++++++ testsuite/runTests_template | 2 ++ testsuite/run_driver_template | 44 +++++++++++++++++++++++ 8 files changed, 181 insertions(+), 27 deletions(-) create mode 100644 testsuite/logd_kill_test_template diff --git a/src/cobo/cobo_comm.c b/src/cobo/cobo_comm.c index 8f59e0cb..7f2084b2 100644 --- a/src/cobo/cobo_comm.c +++ b/src/cobo/cobo_comm.c @@ -241,3 +241,61 @@ int write_msg(int fd, ldcs_message_t *msg) return 0; } + +/* ll_send_nosignal is a variation on ll_write which uses + * send(..., MSG_NOSIGNAL) to avoid SIGPIPE when writing to + * a broken socket. + */ +int ll_send_nosignal(int fd, void *buf, size_t count) +{ + int error; + ssize_t result; + size_t pos = 0; + unsigned char *cbuf = (unsigned char *) buf; + + debug_printf3("Have %lu bytes at %p to write to network\n", count, buf); + + while (pos < count) { + result = send(fd, cbuf + pos, count - pos, MSG_NOSIGNAL); + debug_printf3("Wrote %d bytes at %p to network: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x...\n", (int) result, cbuf + pos, + result > 0 ? ((int) cbuf[pos+0]) : 0, + result > 1 ? ((int) cbuf[pos+1]) : 0, + result > 2 ? ((int) cbuf[pos+2]) : 0, + result > 3 ? ((int) cbuf[pos+3]) : 0, + result > 4 ? ((int) cbuf[pos+4]) : 0, + result > 5 ? ((int) cbuf[pos+5]) : 0, + result > 6 ? ((int) cbuf[pos+6]) : 0, + result > 7 ? ((int) cbuf[pos+7]) : 0); + if (result == -1 && (errno == EINTR || errno == EAGAIN)) + continue; + if (result <= 0) { + error = errno; + err_printf("Error writing to cobo FD %d: %s\n", fd, strerror(error)); + return -1; + } + pos += result; + } + debug_printf3("Sent %lu bytes to fd %d\n", count, fd); + return 0; +} + +/* write_msg_nosignal is a variation on write_msg + * which uses ll_send_nosignal instead of ll_send + * to avoid SIGPIPE when writing to a broken socket. + */ +int write_msg_nosignal(int fd, ldcs_message_t *msg) +{ + int result = ll_send_nosignal(fd, msg, sizeof(*msg)); + if (result == -1) { + return -1; + } + + if (msg->header.len && msg->data) { + result = ll_send_nosignal(fd, msg->data, msg->header.len); + if (result == -1) { + return -1; + } + } + + return 0; +} diff --git a/src/cobo/cobo_comm.h b/src/cobo/cobo_comm.h index 38a72511..f6612ad9 100644 --- a/src/cobo/cobo_comm.h +++ b/src/cobo/cobo_comm.h @@ -24,7 +24,9 @@ Place, Suite 330, Boston, MA 02111-1307 USA int ldcs_cobo_read_fd(int fd, void* buf, int size); int ldcs_cobo_write_fd(int fd, void* buf, int size); int ll_write(int fd, void *buf, size_t count); +int ll_send_nosignal(int fd, void *buf, size_t count); int ll_read(int fd, void *buf, size_t count); int write_msg(int fd, ldcs_message_t *msg); +int write_msg_nosignal(int fd, ldcs_message_t *msg); #endif /* _COBO_COMM_H */ diff --git a/src/fe/comlib/cobo_fe_comm.c b/src/fe/comlib/cobo_fe_comm.c index 297fc1d3..0df792e8 100644 --- a/src/fe/comlib/cobo_fe_comm.c +++ b/src/fe/comlib/cobo_fe_comm.c @@ -178,7 +178,9 @@ int ldcs_audit_server_fe_md_close ( void *data ) { out_msg.data = NULL; cobo_server_get_root_socket(&root_fd); - write_msg(root_fd, &out_msg); + /* We use write_msg_nosignal to avoid SIGPIPE if the socket is broken. + * We're exiting here, so ignore if the server already exited. */ + write_msg_nosignal(root_fd, &out_msg); return cobo_server_close(); } diff --git a/testsuite/Makefile.am b/testsuite/Makefile.am index 1ce88385..675b62d5 100644 --- a/testsuite/Makefile.am +++ b/testsuite/Makefile.am @@ -1,7 +1,7 @@ noinst_PROGRAMS = libgenerator ABS_TEST_DIR = $(abspath $(top_builddir)/testsuite) -BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so libtest14.so libtest15.so libtest16.so libtest17.so libtest18.so libtest19.so libtest20.so libtest50.so libtest100.so libtest500.so libtest1000.so libtest2000.so libtest4000.so libtest6000.so libtest8000.so libtest10000.so libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libsymlink.so libdepC.so libdepB.so libdepA.so libcxxexceptB.so libcxxexceptA.so origin_dir/liboriginlib.so origin_dir/origin_subdir/liborigintarget.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc preload_file_list test_driver test_driver_libs retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py spindle_exec_test spindle_deactivated.sh liblocal.so symbind_test interpreter_test interpreter_test_dir/interpreter_test_perl alias/aliastest.py exec_shell.sh exec_shell.tcsh exec_shell_expected_output exec_shell exec_shell_env.sh +BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so libtest14.so libtest15.so libtest16.so libtest17.so libtest18.so libtest19.so libtest20.so libtest50.so libtest100.so libtest500.so libtest1000.so libtest2000.so libtest4000.so libtest6000.so libtest8000.so libtest10000.so libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libsymlink.so libdepC.so libdepB.so libdepA.so libcxxexceptB.so libcxxexceptA.so origin_dir/liboriginlib.so origin_dir/origin_subdir/liborigintarget.so libtestoutput.so libfuncdict.so runTests run_driver logd_kill_test run_driver_rm spindle.rc preload_file_list test_driver test_driver_libs retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py spindle_exec_test spindle_deactivated.sh liblocal.so symbind_test interpreter_test interpreter_test_dir/interpreter_test_perl alias/aliastest.py exec_shell.sh exec_shell.tcsh exec_shell_expected_output exec_shell exec_shell_env.sh BUILT_SOURCES += crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh @@ -401,6 +401,11 @@ run_driver: $(srcdir)/run_driver_template $(top_builddir)/Makefile $(AM_V_GEN)$(SED) -e s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g\;s,BLUEGENE_TEST,$(IS_BLUEGENE),g < $(srcdir)/run_driver_template > $(top_builddir)/testsuite/run_driver @chmod 700 $(top_builddir)/testsuite/run_driver +logd_kill_test: $(srcdir)/logd_kill_test_template $(top_builddir)/Makefile + @rm -f ./logd_kill_test + $(AM_V_GEN)$(SED) -e s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g < $(srcdir)/logd_kill_test_template > $(top_builddir)/testsuite/logd_kill_test + @chmod 700 $(top_builddir)/testsuite/logd_kill_test + run_driver_rm: $(srcdir)/run_driver_$(TESTRM) $(top_builddir)/Makefile $(AM_V_GEN)cp $(srcdir)/run_driver_$(TESTRM) $(top_builddir)/testsuite/run_driver_rm @chmod 700 $(top_builddir)/testsuite/run_driver_rm @@ -470,4 +475,4 @@ exec_shell_expected_output: $(srcdir)/exec_shell_expected_output_template exec_shell: $(srcdir)/exec_shell.c $(AM_V_CCLD)$(CC) $(CFLAGS) -o $@ $< -CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so libtest100.c libtest100.so libtest500.c libtest500.so libtest1000.c libtest1000.so libtest2000.c libtest2000.so libtest4000.c libtest4000.so libtest6000.c libtest6000.so libtest8000.c libtest8000.so libtest10000.c libtest10000.so libsymlink.so libdepA.so libdepB.so libdepC.so libcxxexceptA.so libcxxexceptB.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc test_driver test_driver_libs preload_file_list retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so libsymbind_d.so libsymbind_e.so libsymbind_f.so libsymbind_g.so interpreter_test interpreter_test_dir/interpreter_test_perl alias exec_shell.sh exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output exec_shell crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh +CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so libtest100.c libtest100.so libtest500.c libtest500.so libtest1000.c libtest1000.so libtest2000.c libtest2000.so libtest4000.c libtest4000.so libtest6000.c libtest6000.so libtest8000.c libtest8000.so libtest10000.c libtest10000.so libsymlink.so libdepA.so libdepB.so libdepC.so libcxxexceptA.so libcxxexceptB.so libtestoutput.so libfuncdict.so runTests run_driver logd_kill_test run_driver_rm spindle.rc test_driver test_driver_libs preload_file_list retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so libsymbind_d.so libsymbind_e.so libsymbind_f.so libsymbind_g.so interpreter_test interpreter_test_dir/interpreter_test_perl alias exec_shell.sh exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output exec_shell crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh diff --git a/testsuite/Makefile.in b/testsuite/Makefile.in index f3402c0a..c3c41927 100644 --- a/testsuite/Makefile.in +++ b/testsuite/Makefile.in @@ -351,17 +351,17 @@ BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so \ libdepB.so libdepA.so libcxxexceptB.so libcxxexceptA.so \ origin_dir/liboriginlib.so \ origin_dir/origin_subdir/liborigintarget.so libtestoutput.so \ - libfuncdict.so runTests run_driver run_driver_rm spindle.rc \ - preload_file_list test_driver test_driver_libs retzero_rx \ - retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py \ - hello_rx.py hello_.py hello_l.py badlink.py spindle_exec_test \ - spindle_deactivated.sh liblocal.so symbind_test \ - interpreter_test interpreter_test_dir/interpreter_test_perl \ - alias/aliastest.py exec_shell.sh exec_shell.tcsh \ - exec_shell_expected_output exec_shell exec_shell_env.sh \ - crash_test crash_test_fixedaddr crash_test_pie \ - libcrashfuncs.so libcrashctor.so libcrashfixed.so \ - run_crash_tests.sh + libfuncdict.so runTests run_driver logd_kill_test \ + run_driver_rm spindle.rc preload_file_list test_driver \ + test_driver_libs retzero_rx retzero_r retzero_x retzero_ \ + badinterp hello_r.py hello_x.py hello_rx.py hello_.py \ + hello_l.py badlink.py spindle_exec_test spindle_deactivated.sh \ + liblocal.so symbind_test interpreter_test \ + interpreter_test_dir/interpreter_test_perl alias/aliastest.py \ + exec_shell.sh exec_shell.tcsh exec_shell_expected_output \ + exec_shell exec_shell_env.sh crash_test crash_test_fixedaddr \ + crash_test_pie libcrashfuncs.so libcrashctor.so \ + libcrashfixed.so run_crash_tests.sh @BGQ_BLD_FALSE@DYNAMIC_FLAG = @BGQ_BLD_TRUE@DYNAMIC_FLAG = -dynamic @BGQ_BLD_FALSE@IS_BLUEGENE = false @@ -387,19 +387,19 @@ CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c \ libtest8000.c libtest8000.so libtest10000.c libtest10000.so \ libsymlink.so libdepA.so libdepB.so libdepC.so \ libcxxexceptA.so libcxxexceptB.so libtestoutput.so \ - libfuncdict.so runTests run_driver run_driver_rm spindle.rc \ - test_driver test_driver_libs preload_file_list retzero_rx \ - retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py \ - hello_rx.py hello_.py hello_l.py badlink.py libtls1.c \ - libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c \ - libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c \ - libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c \ - libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so \ - libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so \ - libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so \ - libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so \ - libtls18.so libtls19.so libtls20.so symbind_test \ - libsymbind_a.so libsymbind_b.so libsymbind_c.so \ + libfuncdict.so runTests run_driver logd_kill_test \ + run_driver_rm spindle.rc test_driver test_driver_libs \ + preload_file_list retzero_rx retzero_r retzero_x retzero_ \ + badinterp hello_r.py hello_x.py hello_rx.py hello_.py \ + hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c \ + libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c \ + libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c \ + libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c \ + libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so \ + libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so \ + libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so \ + libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so \ + symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so \ libsymbind_d.so libsymbind_e.so libsymbind_f.so \ libsymbind_g.so interpreter_test \ interpreter_test_dir/interpreter_test_perl alias exec_shell.sh \ @@ -1085,6 +1085,11 @@ run_driver: $(srcdir)/run_driver_template $(top_builddir)/Makefile $(AM_V_GEN)$(SED) -e s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g\;s,BLUEGENE_TEST,$(IS_BLUEGENE),g < $(srcdir)/run_driver_template > $(top_builddir)/testsuite/run_driver @chmod 700 $(top_builddir)/testsuite/run_driver +logd_kill_test: $(srcdir)/logd_kill_test_template $(top_builddir)/Makefile + @rm -f ./logd_kill_test + $(AM_V_GEN)$(SED) -e s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g < $(srcdir)/logd_kill_test_template > $(top_builddir)/testsuite/logd_kill_test + @chmod 700 $(top_builddir)/testsuite/logd_kill_test + run_driver_rm: $(srcdir)/run_driver_$(TESTRM) $(top_builddir)/Makefile $(AM_V_GEN)cp $(srcdir)/run_driver_$(TESTRM) $(top_builddir)/testsuite/run_driver_rm @chmod 700 $(top_builddir)/testsuite/run_driver_rm diff --git a/testsuite/logd_kill_test_template b/testsuite/logd_kill_test_template new file mode 100644 index 00000000..21c59793 --- /dev/null +++ b/testsuite/logd_kill_test_template @@ -0,0 +1,36 @@ +#!/bin/bash + +# This test forces a log daemon to run and verifies that +# run_driver will kill it + +cd TEST_RUN_DIR +export SPINDLE=SPINDLE_EXEC + +if [ "x$SESSION_ID" != "x" ] || [ "x$SPANK_SPINDLE_USE_SESSION" != "x" ] ; then + echo "Skipping logd kill test: session active" + exit 0 +fi + +echo Running: ./logd_kill_test + +LOGD_TMP="${TMPDIR:-${TEMPDIR:-/tmp}}" +LOGD_BIN="$(dirname "$SPINDLE")/../libexec/spindle/spindlef_logd" +fail() { + echo "FAILED logd kill test: $1" + kill -9 $LOGD_PID 2>/dev/null + rm -f "$LOGD_TMP/spindle_log" "$LOGD_TMP/spindle_test" \ + "$LOGD_TMP/spindle_log_lock" "$LOGD_TMP/spindle_log_reset" + exit 1 +} +[ -x "$LOGD_BIN" ] || fail "no spindle_logd at $LOGD_BIN" +"$LOGD_BIN" "$LOGD_TMP" -test spindle_test & +LOGD_PID=$! +SECONDS=0 +while [ ! -f "$LOGD_TMP/spindle_log_lock" ] || [ ! -S "$LOGD_TMP/spindle_test" ] ; do + [ $SECONDS -ge 5 ] && fail "spindle_logd did not start" + sleep 0.1 +done +SPINDLE_LOGD_SHUTDOWN_TIMEOUT=2 ./run_driver --wait-for-logd +wait $LOGD_PID 2>/dev/null +echo "PASSED" +exit 0 diff --git a/testsuite/runTests_template b/testsuite/runTests_template index 2e04b924..76e08495 100644 --- a/testsuite/runTests_template +++ b/testsuite/runTests_template @@ -16,6 +16,8 @@ fi if test "x$SKIP_NONSESSION" != "xtrue"; then +./logd_kill_test +CHECK_RETCODE ./run_driver --dependency --push CHECK_RETCODE ./run_driver --dlopen --push diff --git a/testsuite/run_driver_template b/testsuite/run_driver_template index fddf2d3e..d64b968f 100644 --- a/testsuite/run_driver_template +++ b/testsuite/run_driver_template @@ -10,6 +10,50 @@ export SPINDLE_TEST=1 export SPINDLE=SPINDLE_EXEC export PATH=$PATH:. +# This is a command to run on a node to check whether the log daemon +# has already exited, and, after a delay, forcibly kill it. +# This ensures that each test connects to a fresh daemon. +LOGD_WAIT_CMD=' +SECONDS=0 +TMP="${TMPDIR:-${TEMPDIR:-/tmp}}" +LOCK="$TMP/spindle_log_lock" +TIMEOUT="${SPINDLE_LOGD_SHUTDOWN_TIMEOUT:-15}" +while PID=$(cat "$LOCK" 2>/dev/null) && kill -0 "$PID" 2>/dev/null ; do + if [ $SECONDS -ge $TIMEOUT ]; then + echo "WARNING: $(hostname): spindle_logd (pid $PID) still running after $TIMEOUT sec; killing it" >&2 + kill -9 "$PID" 2>/dev/null + break + fi + sleep 0.1 +done +rm -f "$TMP/spindle_log" "$TMP/spindle_test" "$TMP/spindle_log_lock" "$TMP/spindle_log_reset" +' +# Runs LOGD_WAIT_CMD on all nodes in job +wait_for_logd_exit() { + bash -c "$LOGD_WAIT_CMD" + if [ "x$TEST_RM" == "xslurm" -o "x$TEST_RM" == "xslurm-plugin" ] && [ "x$SLURM_NNODES" != "x" ] ; then + srun --overlap -N $SLURM_NNODES -n $SLURM_NNODES bash -c "$LOGD_WAIT_CMD" + elif [ "x$TEST_RM" == "xflux" ] ; then + flux exec -r all bash -c "$LOGD_WAIT_CMD" + fi +} + +SESSION_ACTIVE=false +if [ "x$SESSION_ID" != "x" ] || [ "x$SPANK_SPINDLE_USE_SESSION" != "x" ] ; then + SESSION_ACTIVE=true +fi + +# If we're not running in a session, wait for the log daemon to exit +# before proceeding to the actual test. +if [ $SESSION_ACTIVE == false ] && [ "x$1" != "x--end-session" ] ; then + wait_for_logd_exit +fi + +# If --wait-for-logd was passed, run ONLY the log daemon check. +if [ $1 == --wait-for-logd ] ; then + exit 0 +fi + if [ $1 == --start-session ] ; then # With SPANK plugin, sessions are started by argument to salloc/sbatch instead if [ "x$TEST_RM" == "xslurm-plugin" ] ; then From 88fa5a4487ea229e163cbdaa4c4fe5436cbe9f8d Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Mon, 24 Aug 2026 11:13:33 -0700 Subject: [PATCH 2/2] Remove logd_kill_test; relocate logd check --- testsuite/Makefile.am | 9 ++---- testsuite/Makefile.in | 53 ++++++++++++++----------------- testsuite/logd_kill_test_template | 36 --------------------- testsuite/runTests_template | 2 -- testsuite/run_driver_template | 34 +++++++------------- 5 files changed, 38 insertions(+), 96 deletions(-) delete mode 100644 testsuite/logd_kill_test_template diff --git a/testsuite/Makefile.am b/testsuite/Makefile.am index 675b62d5..1ce88385 100644 --- a/testsuite/Makefile.am +++ b/testsuite/Makefile.am @@ -1,7 +1,7 @@ noinst_PROGRAMS = libgenerator ABS_TEST_DIR = $(abspath $(top_builddir)/testsuite) -BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so libtest14.so libtest15.so libtest16.so libtest17.so libtest18.so libtest19.so libtest20.so libtest50.so libtest100.so libtest500.so libtest1000.so libtest2000.so libtest4000.so libtest6000.so libtest8000.so libtest10000.so libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libsymlink.so libdepC.so libdepB.so libdepA.so libcxxexceptB.so libcxxexceptA.so origin_dir/liboriginlib.so origin_dir/origin_subdir/liborigintarget.so libtestoutput.so libfuncdict.so runTests run_driver logd_kill_test run_driver_rm spindle.rc preload_file_list test_driver test_driver_libs retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py spindle_exec_test spindle_deactivated.sh liblocal.so symbind_test interpreter_test interpreter_test_dir/interpreter_test_perl alias/aliastest.py exec_shell.sh exec_shell.tcsh exec_shell_expected_output exec_shell exec_shell_env.sh +BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so libtest14.so libtest15.so libtest16.so libtest17.so libtest18.so libtest19.so libtest20.so libtest50.so libtest100.so libtest500.so libtest1000.so libtest2000.so libtest4000.so libtest6000.so libtest8000.so libtest10000.so libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libsymlink.so libdepC.so libdepB.so libdepA.so libcxxexceptB.so libcxxexceptA.so origin_dir/liboriginlib.so origin_dir/origin_subdir/liborigintarget.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc preload_file_list test_driver test_driver_libs retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py spindle_exec_test spindle_deactivated.sh liblocal.so symbind_test interpreter_test interpreter_test_dir/interpreter_test_perl alias/aliastest.py exec_shell.sh exec_shell.tcsh exec_shell_expected_output exec_shell exec_shell_env.sh BUILT_SOURCES += crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh @@ -401,11 +401,6 @@ run_driver: $(srcdir)/run_driver_template $(top_builddir)/Makefile $(AM_V_GEN)$(SED) -e s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g\;s,BLUEGENE_TEST,$(IS_BLUEGENE),g < $(srcdir)/run_driver_template > $(top_builddir)/testsuite/run_driver @chmod 700 $(top_builddir)/testsuite/run_driver -logd_kill_test: $(srcdir)/logd_kill_test_template $(top_builddir)/Makefile - @rm -f ./logd_kill_test - $(AM_V_GEN)$(SED) -e s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g < $(srcdir)/logd_kill_test_template > $(top_builddir)/testsuite/logd_kill_test - @chmod 700 $(top_builddir)/testsuite/logd_kill_test - run_driver_rm: $(srcdir)/run_driver_$(TESTRM) $(top_builddir)/Makefile $(AM_V_GEN)cp $(srcdir)/run_driver_$(TESTRM) $(top_builddir)/testsuite/run_driver_rm @chmod 700 $(top_builddir)/testsuite/run_driver_rm @@ -475,4 +470,4 @@ exec_shell_expected_output: $(srcdir)/exec_shell_expected_output_template exec_shell: $(srcdir)/exec_shell.c $(AM_V_CCLD)$(CC) $(CFLAGS) -o $@ $< -CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so libtest100.c libtest100.so libtest500.c libtest500.so libtest1000.c libtest1000.so libtest2000.c libtest2000.so libtest4000.c libtest4000.so libtest6000.c libtest6000.so libtest8000.c libtest8000.so libtest10000.c libtest10000.so libsymlink.so libdepA.so libdepB.so libdepC.so libcxxexceptA.so libcxxexceptB.so libtestoutput.so libfuncdict.so runTests run_driver logd_kill_test run_driver_rm spindle.rc test_driver test_driver_libs preload_file_list retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so libsymbind_d.so libsymbind_e.so libsymbind_f.so libsymbind_g.so interpreter_test interpreter_test_dir/interpreter_test_perl alias exec_shell.sh exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output exec_shell crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh +CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c libtest14.c libtest15.c libtest16.c libtest17.c libtest18.c libtest19.c libtest20.c libtest10.so libtest50.c libtest50.so libtest100.c libtest100.so libtest500.c libtest500.so libtest1000.c libtest1000.so libtest2000.c libtest2000.so libtest4000.c libtest4000.so libtest6000.c libtest6000.so libtest8000.c libtest8000.so libtest10000.c libtest10000.so libsymlink.so libdepA.so libdepB.so libdepC.so libcxxexceptA.so libcxxexceptB.so libtestoutput.so libfuncdict.so runTests run_driver run_driver_rm spindle.rc test_driver test_driver_libs preload_file_list retzero_rx retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py hello_rx.py hello_.py hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so libsymbind_d.so libsymbind_e.so libsymbind_f.so libsymbind_g.so interpreter_test interpreter_test_dir/interpreter_test_perl alias exec_shell.sh exec_shell.tcsh exec_shell_env.sh exec_shell_expected_output exec_shell crash_test crash_test_fixedaddr crash_test_pie libcrashfuncs.so libcrashctor.so libcrashfixed.so run_crash_tests.sh diff --git a/testsuite/Makefile.in b/testsuite/Makefile.in index c3c41927..f3402c0a 100644 --- a/testsuite/Makefile.in +++ b/testsuite/Makefile.in @@ -351,17 +351,17 @@ BUILT_SOURCES = libtest10.so libtest11.so libtest12.so libtest13.so \ libdepB.so libdepA.so libcxxexceptB.so libcxxexceptA.so \ origin_dir/liboriginlib.so \ origin_dir/origin_subdir/liborigintarget.so libtestoutput.so \ - libfuncdict.so runTests run_driver logd_kill_test \ - run_driver_rm spindle.rc preload_file_list test_driver \ - test_driver_libs retzero_rx retzero_r retzero_x retzero_ \ - badinterp hello_r.py hello_x.py hello_rx.py hello_.py \ - hello_l.py badlink.py spindle_exec_test spindle_deactivated.sh \ - liblocal.so symbind_test interpreter_test \ - interpreter_test_dir/interpreter_test_perl alias/aliastest.py \ - exec_shell.sh exec_shell.tcsh exec_shell_expected_output \ - exec_shell exec_shell_env.sh crash_test crash_test_fixedaddr \ - crash_test_pie libcrashfuncs.so libcrashctor.so \ - libcrashfixed.so run_crash_tests.sh + libfuncdict.so runTests run_driver run_driver_rm spindle.rc \ + preload_file_list test_driver test_driver_libs retzero_rx \ + retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py \ + hello_rx.py hello_.py hello_l.py badlink.py spindle_exec_test \ + spindle_deactivated.sh liblocal.so symbind_test \ + interpreter_test interpreter_test_dir/interpreter_test_perl \ + alias/aliastest.py exec_shell.sh exec_shell.tcsh \ + exec_shell_expected_output exec_shell exec_shell_env.sh \ + crash_test crash_test_fixedaddr crash_test_pie \ + libcrashfuncs.so libcrashctor.so libcrashfixed.so \ + run_crash_tests.sh @BGQ_BLD_FALSE@DYNAMIC_FLAG = @BGQ_BLD_TRUE@DYNAMIC_FLAG = -dynamic @BGQ_BLD_FALSE@IS_BLUEGENE = false @@ -387,19 +387,19 @@ CLEANFILES = libtest10.c libtest11.c libtest12.c libtest13.c \ libtest8000.c libtest8000.so libtest10000.c libtest10000.so \ libsymlink.so libdepA.so libdepB.so libdepC.so \ libcxxexceptA.so libcxxexceptB.so libtestoutput.so \ - libfuncdict.so runTests run_driver logd_kill_test \ - run_driver_rm spindle.rc test_driver test_driver_libs \ - preload_file_list retzero_rx retzero_r retzero_x retzero_ \ - badinterp hello_r.py hello_x.py hello_rx.py hello_.py \ - hello_l.py badlink.py libtls1.c libtls2.c libtls3.c libtls4.c \ - libtls5.c libtls6.c libtls7.c libtls8.c libtls9.c libtls10.c \ - libtls11.c libtls12.c libtls13.c libtls14.c libtls15.c \ - libtls16.c libtls17.c libtls18.c libtls19.c libtls20.c \ - libtls1.so libtls2.so libtls3.so libtls4.so libtls5.so \ - libtls6.so libtls7.so libtls8.so libtls9.so libtls10.so \ - libtls11.so libtls12.so libtls13.so libtls14.so libtls15.so \ - libtls16.so libtls17.so libtls18.so libtls19.so libtls20.so \ - symbind_test libsymbind_a.so libsymbind_b.so libsymbind_c.so \ + libfuncdict.so runTests run_driver run_driver_rm spindle.rc \ + test_driver test_driver_libs preload_file_list retzero_rx \ + retzero_r retzero_x retzero_ badinterp hello_r.py hello_x.py \ + hello_rx.py hello_.py hello_l.py badlink.py libtls1.c \ + libtls2.c libtls3.c libtls4.c libtls5.c libtls6.c libtls7.c \ + libtls8.c libtls9.c libtls10.c libtls11.c libtls12.c \ + libtls13.c libtls14.c libtls15.c libtls16.c libtls17.c \ + libtls18.c libtls19.c libtls20.c libtls1.so libtls2.so \ + libtls3.so libtls4.so libtls5.so libtls6.so libtls7.so \ + libtls8.so libtls9.so libtls10.so libtls11.so libtls12.so \ + libtls13.so libtls14.so libtls15.so libtls16.so libtls17.so \ + libtls18.so libtls19.so libtls20.so symbind_test \ + libsymbind_a.so libsymbind_b.so libsymbind_c.so \ libsymbind_d.so libsymbind_e.so libsymbind_f.so \ libsymbind_g.so interpreter_test \ interpreter_test_dir/interpreter_test_perl alias exec_shell.sh \ @@ -1085,11 +1085,6 @@ run_driver: $(srcdir)/run_driver_template $(top_builddir)/Makefile $(AM_V_GEN)$(SED) -e s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g\;s,BLUEGENE_TEST,$(IS_BLUEGENE),g < $(srcdir)/run_driver_template > $(top_builddir)/testsuite/run_driver @chmod 700 $(top_builddir)/testsuite/run_driver -logd_kill_test: $(srcdir)/logd_kill_test_template $(top_builddir)/Makefile - @rm -f ./logd_kill_test - $(AM_V_GEN)$(SED) -e s,SPINDLE_EXEC,$(bindir)/spindle,g\;s,TEST_RUN_DIR,$(ABS_TEST_DIR),g < $(srcdir)/logd_kill_test_template > $(top_builddir)/testsuite/logd_kill_test - @chmod 700 $(top_builddir)/testsuite/logd_kill_test - run_driver_rm: $(srcdir)/run_driver_$(TESTRM) $(top_builddir)/Makefile $(AM_V_GEN)cp $(srcdir)/run_driver_$(TESTRM) $(top_builddir)/testsuite/run_driver_rm @chmod 700 $(top_builddir)/testsuite/run_driver_rm diff --git a/testsuite/logd_kill_test_template b/testsuite/logd_kill_test_template deleted file mode 100644 index 21c59793..00000000 --- a/testsuite/logd_kill_test_template +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# This test forces a log daemon to run and verifies that -# run_driver will kill it - -cd TEST_RUN_DIR -export SPINDLE=SPINDLE_EXEC - -if [ "x$SESSION_ID" != "x" ] || [ "x$SPANK_SPINDLE_USE_SESSION" != "x" ] ; then - echo "Skipping logd kill test: session active" - exit 0 -fi - -echo Running: ./logd_kill_test - -LOGD_TMP="${TMPDIR:-${TEMPDIR:-/tmp}}" -LOGD_BIN="$(dirname "$SPINDLE")/../libexec/spindle/spindlef_logd" -fail() { - echo "FAILED logd kill test: $1" - kill -9 $LOGD_PID 2>/dev/null - rm -f "$LOGD_TMP/spindle_log" "$LOGD_TMP/spindle_test" \ - "$LOGD_TMP/spindle_log_lock" "$LOGD_TMP/spindle_log_reset" - exit 1 -} -[ -x "$LOGD_BIN" ] || fail "no spindle_logd at $LOGD_BIN" -"$LOGD_BIN" "$LOGD_TMP" -test spindle_test & -LOGD_PID=$! -SECONDS=0 -while [ ! -f "$LOGD_TMP/spindle_log_lock" ] || [ ! -S "$LOGD_TMP/spindle_test" ] ; do - [ $SECONDS -ge 5 ] && fail "spindle_logd did not start" - sleep 0.1 -done -SPINDLE_LOGD_SHUTDOWN_TIMEOUT=2 ./run_driver --wait-for-logd -wait $LOGD_PID 2>/dev/null -echo "PASSED" -exit 0 diff --git a/testsuite/runTests_template b/testsuite/runTests_template index 76e08495..2e04b924 100644 --- a/testsuite/runTests_template +++ b/testsuite/runTests_template @@ -16,8 +16,6 @@ fi if test "x$SKIP_NONSESSION" != "xtrue"; then -./logd_kill_test -CHECK_RETCODE ./run_driver --dependency --push CHECK_RETCODE ./run_driver --dlopen --push diff --git a/testsuite/run_driver_template b/testsuite/run_driver_template index d64b968f..a486766b 100644 --- a/testsuite/run_driver_template +++ b/testsuite/run_driver_template @@ -10,10 +10,18 @@ export SPINDLE_TEST=1 export SPINDLE=SPINDLE_EXEC export PATH=$PATH:. -# This is a command to run on a node to check whether the log daemon -# has already exited, and, after a delay, forcibly kill it. -# This ensures that each test connects to a fresh daemon. -LOGD_WAIT_CMD=' +SESSION_ACTIVE=false +if [ "x$SESSION_ID" != "x" ] || [ "x$SPANK_SPINDLE_USE_SESSION" != "x" ] ; then + SESSION_ACTIVE=true +fi + +# If we're not running in a session, wait, on every node in the job, for +# the log daemon to exit before proceeding to the actual test; after a +# delay, forcibly kill it. This ensures that each test connects to a +# fresh daemon. +if [ $SESSION_ACTIVE == false ] && [ "x$1" != "x--end-session" ] ; then + # This is the command run on each node. + LOGD_WAIT_CMD=' SECONDS=0 TMP="${TMPDIR:-${TEMPDIR:-/tmp}}" LOCK="$TMP/spindle_log_lock" @@ -28,30 +36,12 @@ while PID=$(cat "$LOCK" 2>/dev/null) && kill -0 "$PID" 2>/dev/null ; do done rm -f "$TMP/spindle_log" "$TMP/spindle_test" "$TMP/spindle_log_lock" "$TMP/spindle_log_reset" ' -# Runs LOGD_WAIT_CMD on all nodes in job -wait_for_logd_exit() { bash -c "$LOGD_WAIT_CMD" if [ "x$TEST_RM" == "xslurm" -o "x$TEST_RM" == "xslurm-plugin" ] && [ "x$SLURM_NNODES" != "x" ] ; then srun --overlap -N $SLURM_NNODES -n $SLURM_NNODES bash -c "$LOGD_WAIT_CMD" elif [ "x$TEST_RM" == "xflux" ] ; then flux exec -r all bash -c "$LOGD_WAIT_CMD" fi -} - -SESSION_ACTIVE=false -if [ "x$SESSION_ID" != "x" ] || [ "x$SPANK_SPINDLE_USE_SESSION" != "x" ] ; then - SESSION_ACTIVE=true -fi - -# If we're not running in a session, wait for the log daemon to exit -# before proceeding to the actual test. -if [ $SESSION_ACTIVE == false ] && [ "x$1" != "x--end-session" ] ; then - wait_for_logd_exit -fi - -# If --wait-for-logd was passed, run ONLY the log daemon check. -if [ $1 == --wait-for-logd ] ; then - exit 0 fi if [ $1 == --start-session ] ; then