From 9fdc1fdb69df0eaf9aba84f183a2a16e62ccb66f Mon Sep 17 00:00:00 2001 From: Connor Briggs Date: Wed, 3 Dec 2025 09:44:10 -0500 Subject: [PATCH 1/8] Update sapelo_programs.py Update software versions and fix an issue with the ORCA format string where `{}` is treated like it needs an argument. --- submitscripts/slurm/sapelo_programs.py | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/submitscripts/slurm/sapelo_programs.py b/submitscripts/slurm/sapelo_programs.py index 2b8dd76..3208905 100644 --- a/submitscripts/slurm/sapelo_programs.py +++ b/submitscripts/slurm/sapelo_programs.py @@ -1,8 +1,8 @@ # use sapelo2 work area (called scratch) # no need to copy set psi_scratch variable -fermi = """module load Julia/1.8.2-linux-x86_64 -module load intel/2023a +fermi = """module load Julia/1.11.6-gfbf-2023b +module load intel/2023b julia {input_name} @@ -29,16 +29,13 @@ # mpi only # set scratch dir to home area but run from submit_dir -molpro_mpi = """module load intel/2023a - +molpro_mpi = """ # to change scratch dir to use local machine scratch export SCRATCH_DIR=/scratch/$USER/tmp/$SLURM_JOB_ID mkdir -p $SCRATCH_DIR export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR" # This binds the directory into the container so that output can be written. -mpirun -n $NSLOTS apptainer exec /work/jttlab/containers/molpro_mpipr.sif \ -molpro.exe input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR - +singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR rm $SCRATCH_DIR -r """ @@ -46,22 +43,21 @@ # mpi only # copy everything to lscratch to run and set scratch to lscratch -molpro_mpi_lscratch = """module load intel/2023a - +molpro_mpi_lscratch = """ # to change scratch dir to use local machine scratch export SCRATCH_DIR=/lscratch/$USER/tmp/$SLURM_JOB_ID mkdir -p $SCRATCH_DIR export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR" # This binds the directory into the container so that output can be written. -mpirun -n $NSLOTS apptainer exec /work/jttlab/containers/molpro_mpipr.sif \ -molpro.exe input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR + +singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR rm $SCRATCH_DIR -r """ orca_common = """#Set MPI Variables -module load ORCA/5.0.4-gompi-2022a +module load ORCA/6.1.0-OpenMPI-4.1.8-GCC-13.3.0-avx2 export OMP_NUM_THREADS=1 # Set other variables @@ -79,12 +75,12 @@ echo " Running calculation..." cd $scratch_dir -orca {input_name} >& $SLURM_SUBMIT_DIR/{output_name} || exit 1 +/apps/eb/ORCA/6.1.0-OpenMPI-4.1.8-GCC-13.3.0-avx2/bin/orca {input_name} >& $SLURM_SUBMIT_DIR/{output_name} || exit 1 echo " Saving data and cleaning up..." # delete any temporary files that my be hanging around. rm -f *.tmp* -find . -type f -size +50M -exec rm -f {} \; +find . -type f -size +50M -exec rm -f {{}} \; tar --exclude='*tmp*' --transform "s,^,Job_Data_$SLURM_JOB_ID/," -vzcf $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar.gz * echo " Job complete on `hostname`." @@ -148,7 +144,7 @@ rm $scratch_dir -r """ -cfour_serial = """module=cfour/2.1-intel-2021b-serial +cfour_serial = """module=cfour/2.1-intel-2023a-serial export OMP_NUM_THREADS=$NSLOTS scratch_dir=/scratch/$USER/tmp/$SLURM_JOB_ID @@ -156,7 +152,7 @@ """ + cfour_common -cfour_serial_lscratch = """module=cfour/2.1-intel-2021b-serial +cfour_serial_lscratch = """module=cfour/2.1-intel-2023a-serial export OMP_NUM_THREADS=$NSLOTS scratch_dir=/lscratch/$USER/tmp/$SLURM_JOB_ID @@ -164,7 +160,9 @@ """ + cfour_common -cfour_mpi = """module=cfour/2.1-intel-2021b-mpi +cfour_mpi = """ +echo "Sapelo no longer has MPI CFOUR. Use the container version instead." +module=cfour/2.1-intel-2023a-mpi scratch_dir=/scratch/$USER/tmp/$SLURM_JOB_ID mkdir -p $scratch_dir @@ -172,7 +170,9 @@ echo -e "\t$NSLOTS" > $scratch_dir/ncpu """ + cfour_common -cfour_mpi_lscratch = """module=cfour/2.1-intel-2021b-mpi +cfour_mpi_lscratch = """ +echo "Sapelo no longer has MPI CFOUR. Use the container version instead." +module=cfour/2.1-intel-2023a-mpi scratch_dir=/lscratch/$USER/tmp/$SLURM_JOB_ID mkdir -p $scratch_dir From f0cce2232c747bae3feef57178848b97d459d320 Mon Sep 17 00:00:00 2001 From: Connor Briggs Date: Fri, 19 Dec 2025 15:18:59 -0500 Subject: [PATCH 2/8] Update sapelo_programs.py --- submitscripts/slurm/sapelo_programs.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/submitscripts/slurm/sapelo_programs.py b/submitscripts/slurm/sapelo_programs.py index 3208905..a850b0a 100644 --- a/submitscripts/slurm/sapelo_programs.py +++ b/submitscripts/slurm/sapelo_programs.py @@ -35,7 +35,8 @@ mkdir -p $SCRATCH_DIR export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR" # This binds the directory into the container so that output can be written. -singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR +singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR \ +-m $(($SLURM_MEM_PER_NODE/10/$NSLOTS))M # Divide by 10 so that it leaves room for extra variables. Word size is normally 8. rm $SCRATCH_DIR -r """ @@ -49,8 +50,8 @@ mkdir -p $SCRATCH_DIR export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR" # This binds the directory into the container so that output can be written. - -singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR +singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR \ +-m $(($SLURM_MEM_PER_NODE/10/$NSLOTS))M # Divide by 10 so that it leaves room for extra variables. Word size is normally 8. rm $SCRATCH_DIR -r From 38a57392a04b39d9a62f07e2241622daf9f36ddd Mon Sep 17 00:00:00 2001 From: Connor Briggs Date: Fri, 19 Dec 2025 16:33:44 -0500 Subject: [PATCH 3/8] Update sapelo_programs.py --- submitscripts/slurm/sapelo_programs.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/submitscripts/slurm/sapelo_programs.py b/submitscripts/slurm/sapelo_programs.py index a850b0a..7d1e41e 100644 --- a/submitscripts/slurm/sapelo_programs.py +++ b/submitscripts/slurm/sapelo_programs.py @@ -35,8 +35,7 @@ mkdir -p $SCRATCH_DIR export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR" # This binds the directory into the container so that output can be written. -singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR \ --m $(($SLURM_MEM_PER_NODE/10/$NSLOTS))M # Divide by 10 so that it leaves room for extra variables. Word size is normally 8. +singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR rm $SCRATCH_DIR -r """ @@ -50,8 +49,7 @@ mkdir -p $SCRATCH_DIR export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR" # This binds the directory into the container so that output can be written. -singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR \ --m $(($SLURM_MEM_PER_NODE/10/$NSLOTS))M # Divide by 10 so that it leaves room for extra variables. Word size is normally 8. +singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR rm $SCRATCH_DIR -r From 4d6643b1b574563b76d873721c1371259123fbb9 Mon Sep 17 00:00:00 2001 From: Alexander Heide Date: Tue, 12 May 2026 15:01:45 -0400 Subject: [PATCH 4/8] Adds molpro_24 as separate option. --- options.py | 23 +++++++++++++++++- submitscripts/slurm/sapelo_programs.py | 33 ++++++++++++++++++-------- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/options.py b/options.py index 7cd3c4c..c20cc44 100644 --- a/options.py +++ b/options.py @@ -58,6 +58,14 @@ class Options(object): The name should at least be sufficiently long to load the module with. i.e. cfour@2.0+mpi is sufficient for vulcan load psi4 is also sufficient isntead of psi4@master. + The naming convention is similar for Sapelo, however, most of the submit scripts are hardwired to specific versions. + Some submit scripts use the container in `/work/jttlab/containers,` some use the modules on Sapelo. + Again, the string should generally include the name and a version number. In general, for containers, + the name of the container should lead to the correct version being used. + + optavc only uses the gapr containers for molpro. Edit the submit scripts to use soc. + For cfour requesting mpi will force use of the container but serial will use the module + template_file_path : string default : 'template.dat' @@ -459,6 +467,10 @@ def program(self): @program.setter def program(self, val=""): prog = val.lower().split("@") + if len(prog) == 1: + # no @ in version string. try to split on -. This is needed for molpro if + # user provides the container name as the program. + prog = val.lower().split("-", 1) self._program = prog[0] if prog[0] == 'fermi': @@ -470,9 +482,18 @@ def program(self, val=""): elif prog[0] in ['orca', 'molpro']: self.parallel = 'mpi' + if prog[0] == 'molpro' and '24' in prog[-1]: + # molpro will be set above already change to molpro_24 + self._program = f"{prog[0]}_24" + if not self.parallel: - if '+mpi' in prog[-1]: + if '+mpi' in prog[-1] or '-mpi' in prog[-1]: # two common strings include +mpi or just name-mpi self.parallel = 'mpi' + + if prog[0] == 'molpro' and '24' in prog[-1]: + # molpro will be set above already change to molpro_24 + self._program = f"{prog[0]}_24" + elif 'serial' in prog[-1]: self.parallel = 'serial' elif '~mpi' in prog[-1]: diff --git a/submitscripts/slurm/sapelo_programs.py b/submitscripts/slurm/sapelo_programs.py index 7d1e41e..26fd416 100644 --- a/submitscripts/slurm/sapelo_programs.py +++ b/submitscripts/slurm/sapelo_programs.py @@ -29,32 +29,43 @@ # mpi only # set scratch dir to home area but run from submit_dir -molpro_mpi = """ +molpro_scratch_prefix = """ # to change scratch dir to use local machine scratch export SCRATCH_DIR=/scratch/$USER/tmp/$SLURM_JOB_ID mkdir -p $SCRATCH_DIR export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR" # This binds the directory into the container so that output can be written. - -singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR -rm $SCRATCH_DIR -r - """ -# mpi only -# copy everything to lscratch to run and set scratch to lscratch - -molpro_mpi_lscratch = """ +molpro_lscratch_prefix = """ # to change scratch dir to use local machine scratch export SCRATCH_DIR=/lscratch/$USER/tmp/$SLURM_JOB_ID mkdir -p $SCRATCH_DIR export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR" # This binds the directory into the container so that output can be written. +""" -singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR +molpro = """module load intel/2022a +mpirun -n $NSLOTS apptainer exec /work/jttlab/containers/molpro_mpipr.sif \ +molpro.exe input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR rm $SCRATCH_DIR -r """ +molpro_24 = """ +singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat \ +--output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR +rm $SCRATCH_DIR -r + +""" + +# mpi only +# copy everything to lscratch to run and set scratch to lscratch + +molpro_24_mpi = molpro_scratch_prefix + molpro_24 +molpro_24_mpi_lscratch = molpro_lscratch_prefix + molpro_24 +molpro_mpi = molpro_scratch_prefix + molpro +molpro_mpi_lscratch = molpro_lscratch_prefix + molpro + orca_common = """#Set MPI Variables module load ORCA/6.1.0-OpenMPI-4.1.8-GCC-13.3.0-avx2 export OMP_NUM_THREADS=1 @@ -196,11 +207,13 @@ "lscratch": { "orca": orca_lscratch, "molpro": molpro_mpi_lscratch, + "molpro_24": molpro_24_mpi_lscratch, "cfour": cfour_mpi_lscratch }, "scratch": { "orca": orca, "molpro": molpro_mpi, + "molpro_24": molpro_24_mpi, "cfour": cfour_mpi } } From fb9ac675de4c7725fe604160bf467652b6a77434 Mon Sep 17 00:00:00 2001 From: Alexander Heide Date: Mon, 18 May 2026 16:31:05 -0400 Subject: [PATCH 5/8] Fixes submit script error. Use container for cfour. Add simple program test --- submitscripts/slurm/sapelo_programs.py | 37 +++++++++++++++++--------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/submitscripts/slurm/sapelo_programs.py b/submitscripts/slurm/sapelo_programs.py index 26fd416..e7fa772 100644 --- a/submitscripts/slurm/sapelo_programs.py +++ b/submitscripts/slurm/sapelo_programs.py @@ -106,7 +106,7 @@ mkdir -p $scratch_dir """ + orca_common -cfour_common = """ +cfour_prefix = """ # make sure MRCC is around just in case export PATH=$PATH:/work/jttlab/mrcc/2020/ prefix=/apps/eb/$module/ @@ -136,9 +136,9 @@ echo " Running calculation..." cd $scratch_dir -xcfour >& $SLURM_SUBMIT_DIR/{output_name} -xja2fja +""" +cfour_suffix = """ echo " Saving data and cleaning up..." if [ -e ZMATnew ]; then cp -f ZMATnew $SLURM_SUBMIT_DIR/ZMATnew ; fi if [ -e GRD ]; then cp -f GRD $SLURM_SUBMIT_DIR/GRD ; fi @@ -154,13 +154,28 @@ rm $scratch_dir -r """ +xcfour_module = cfour_prefix + """ +xcfour >& $SLURM_SUBMIT_DIR/{output_name} +xja2fja +""" + cfour_suffix + +xcfour_container = cfour_prefix + """ +# Silence all the IEEE signaling messages +export NO_STOP_MESSAGE=yes +# request devices (inifiniband) use openib BTL interface for openmpi 4 +export OMPI_MCA_btl_openib_allow_ib=true + +apptainer exec /work/jttlab/containers/cfour-2.1-foss-ompi.sif xcfour >& $SLURM_SUBMIT_DIR/{output_name} +apptainer exec /work/jttlab/containers/cfour-2.1-foss-ompi.sif xja2fja +""" + cfour_suffix + cfour_serial = """module=cfour/2.1-intel-2023a-serial export OMP_NUM_THREADS=$NSLOTS scratch_dir=/scratch/$USER/tmp/$SLURM_JOB_ID mkdir -p $scratch_dir -""" + cfour_common +""" + xcfour_module cfour_serial_lscratch = """module=cfour/2.1-intel-2023a-serial export OMP_NUM_THREADS=$NSLOTS @@ -168,27 +183,23 @@ scratch_dir=/lscratch/$USER/tmp/$SLURM_JOB_ID mkdir -p $scratch_dir -""" + cfour_common +""" + xcfour_module -cfour_mpi = """ -echo "Sapelo no longer has MPI CFOUR. Use the container version instead." -module=cfour/2.1-intel-2023a-mpi +cfour_mpi = """module=OpenMPI/4.1.1-GCC-11.2.0 # no cfour mpi by gacrc scratch_dir=/scratch/$USER/tmp/$SLURM_JOB_ID mkdir -p $scratch_dir echo -e "\t$NSLOTS" > ./ncpu # CFour appears to just claim any and all cpus echo -e "\t$NSLOTS" > $scratch_dir/ncpu -""" + cfour_common +""" + xcfour_container -cfour_mpi_lscratch = """ -echo "Sapelo no longer has MPI CFOUR. Use the container version instead." -module=cfour/2.1-intel-2023a-mpi +cfour_mpi_lscratch = """module=OpenMPI/4.1.1-GCC-11.2.0 # no cfour mpi by gacrc scratch_dir=/lscratch/$USER/tmp/$SLURM_JOB_ID mkdir -p $scratch_dir echo -e "\t$NSLOTS" > ./ncpu # CFour appears to just claim any and all cpus echo -e "\t$NSLOTS" > $scratch_dir/ncpu -""" + cfour_common +""" + xcfour_container progdict = { "serial": { From a811231053d29c101f697375e5862245c679defd Mon Sep 17 00:00:00 2001 From: Alexander Heide Date: Mon, 18 May 2026 16:47:39 -0400 Subject: [PATCH 6/8] Add stored submit scripts for testing against --- tests/sub_templates/cfour@2.1-mpi_template.sh | 77 +++++++++++++++++++ tests/sub_templates/cfour@2.1_template.sh | 69 +++++++++++++++++ tests/sub_templates/molpro@2021_template.sh | 29 +++++++ tests/sub_templates/molpro@2024_template.sh | 28 +++++++ tests/sub_templates/orca_template.sh | 51 ++++++++++++ tests/sub_templates/psi4_template.sh | 20 +++++ 6 files changed, 274 insertions(+) create mode 100644 tests/sub_templates/cfour@2.1-mpi_template.sh create mode 100644 tests/sub_templates/cfour@2.1_template.sh create mode 100644 tests/sub_templates/molpro@2021_template.sh create mode 100644 tests/sub_templates/molpro@2024_template.sh create mode 100644 tests/sub_templates/orca_template.sh create mode 100644 tests/sub_templates/psi4_template.sh diff --git a/tests/sub_templates/cfour@2.1-mpi_template.sh b/tests/sub_templates/cfour@2.1-mpi_template.sh new file mode 100644 index 0000000..6e717c0 --- /dev/null +++ b/tests/sub_templates/cfour@2.1-mpi_template.sh @@ -0,0 +1,77 @@ +#!/bin/bash +#SBATCH --job-name=STEP # Job name +#SBATCH --partition=batch # Partition (queue) name +#SBATCH --constraint=EPYC|Intel +#SBATCH --nodes=1 # Number of nodes +#SBATCH --ntasks=4 # Number of MPI ranks +#SBATCH --ntasks-per-node=4 # How many tasks on each node +#SBATCH --cpus-per-task=1 # Number of cores per MPI rank +#SBATCH --mem=32GB # Total Memory +#SBATCH --time=10:00:00 +#SBATCH --output="%x.%j".out # Standard output log +#SBATCH --error="%x.%j".err # Standard error log + +cd $SLURM_SUBMIT_DIR +export NSLOTS=4 +export THREADS=1 + +module=OpenMPI/4.1.1-GCC-11.2.0 # no cfour mpi by gacrc +scratch_dir=/scratch/$USER/tmp/$SLURM_JOB_ID +mkdir -p $scratch_dir + +echo -e " $NSLOTS" > ./ncpu # CFour appears to just claim any and all cpus +echo -e " $NSLOTS" > $scratch_dir/ncpu + +# make sure MRCC is around just in case +export PATH=$PATH:/work/jttlab/mrcc/2020/ +prefix=/apps/eb/$module/ +module load $module + +# Copy job data +if [[ -e input.dat && ! -e ZMAT ]]; then + cp input.dat $scratch_dir/ZMAT +else + cp $SLURM_SUBMIT_DIR/ZMAT $scratch_dir +fi + +cp $prefix/basis/GENBAS $scratch_dir +cp $prefix/basis/ECPDATA $scratch_dir +if [ -e JAINDX ]; then cp JAINDX $scratch_dir ; fi +if [ -e JOBARC ]; then cp JOBARC $scratch_dir ; fi +if [ -e FCMINT ]; then cp FCMINT $scratch_dir ; fi +if [ -e GENBAS ]; then cp GENBAS $scratch_dir ; fi +if [ -e ECPDATA ]; then cp ECPDATA $scratch_dir ; fi +if [ -e OPTARC ]; then cp OPTARC $scratch_dir ; fi +if [ -e ISOTOPES ]; then cp ISOTOPES $scratch_dir ; fi +if [ -e ISOMASS ]; then cp ISOMASS $scratch_dir ; fi +if [ -e initden.dat ]; then cp initden.dat $scratch_dir ; fi +if [ -e OLDMOS ]; then cp OLDMOS $scratch_dir ; fi + +echo " Running cfour on `hostname`" +echo " Running calculation..." + +cd $scratch_dir + +# Silence all the IEEE signaling messages +export NO_STOP_MESSAGE=yes +# request devices (inifiniband) use openib BTL interface for openmpi 4 +export OMPI_MCA_btl_openib_allow_ib=true + +apptainer exec /work/jttlab/containers/cfour-2.1-foss-ompi.sif xcfour >& $SLURM_SUBMIT_DIR/output.dat +apptainer exec /work/jttlab/containers/cfour-2.1-foss-ompi.sif xja2fja + +echo " Saving data and cleaning up..." +if [ -e ZMATnew ]; then cp -f ZMATnew $SLURM_SUBMIT_DIR/ZMATnew ; fi +if [ -e GRD ]; then cp -f GRD $SLURM_SUBMIT_DIR/GRD ; fi +if [ -e FCMFINAL ]; then cp -f FCMFINAL $SLURM_SUBMIT_DIR/FCMFINAL ; fi + +# Create a job data archive file +tar --transform "s,^,Job_Data_$SLURM_JOB_ID/," -vcf $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar OPTARC FCMINT FCMFINAL ZMATnew JMOL.plot JOBARC JAINDX FJOBARC DIPDER HESSIAN MOLDEN NEWMOS den.dat +if [ -e zmat001 ]; then tar --transform "s,^,Job_Data_$SLURM_JOB_ID/," -vrf $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar zmat* ; fi +gzip $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar + +echo " Job complete on `hostname`." + +rm $scratch_dir -r + +#ignored line -- do not remove diff --git a/tests/sub_templates/cfour@2.1_template.sh b/tests/sub_templates/cfour@2.1_template.sh new file mode 100644 index 0000000..1a9a470 --- /dev/null +++ b/tests/sub_templates/cfour@2.1_template.sh @@ -0,0 +1,69 @@ +#!/bin/bash +#SBATCH --partition=batch +#SBATCH --constraint="EPYC|Intel" +#SBATCH --job-name=STEP +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=4 +#SBATCH --time=10:00:00 +#SBATCH --mem=32GB +#SBATCH --output="%x.%j".out # Standard output log +#SBATCH --error="%x.%j".err # Standard error log + +cd $SLURM_SUBMIT_DIR +export NSLOTS=4 + +module=cfour/2.1-intel-2023a-serial +export OMP_NUM_THREADS=$NSLOTS + +scratch_dir=/scratch/$USER/tmp/$SLURM_JOB_ID +mkdir -p $scratch_dir + + +# make sure MRCC is around just in case +export PATH=$PATH:/work/jttlab/mrcc/2020/ +prefix=/apps/eb/$module/ +module load $module + +# Copy job data +if [[ -e input.dat && ! -e ZMAT ]]; then + cp input.dat $scratch_dir/ZMAT +else + cp $SLURM_SUBMIT_DIR/ZMAT $scratch_dir +fi + +cp $prefix/basis/GENBAS $scratch_dir +cp $prefix/basis/ECPDATA $scratch_dir +if [ -e JAINDX ]; then cp JAINDX $scratch_dir ; fi +if [ -e JOBARC ]; then cp JOBARC $scratch_dir ; fi +if [ -e FCMINT ]; then cp FCMINT $scratch_dir ; fi +if [ -e GENBAS ]; then cp GENBAS $scratch_dir ; fi +if [ -e ECPDATA ]; then cp ECPDATA $scratch_dir ; fi +if [ -e OPTARC ]; then cp OPTARC $scratch_dir ; fi +if [ -e ISOTOPES ]; then cp ISOTOPES $scratch_dir ; fi +if [ -e ISOMASS ]; then cp ISOMASS $scratch_dir ; fi +if [ -e initden.dat ]; then cp initden.dat $scratch_dir ; fi +if [ -e OLDMOS ]; then cp OLDMOS $scratch_dir ; fi + +echo " Running cfour on `hostname`" +echo " Running calculation..." + +cd $scratch_dir + +xcfour >& $SLURM_SUBMIT_DIR/output.dat +xja2fja + +echo " Saving data and cleaning up..." +if [ -e ZMATnew ]; then cp -f ZMATnew $SLURM_SUBMIT_DIR/ZMATnew ; fi +if [ -e GRD ]; then cp -f GRD $SLURM_SUBMIT_DIR/GRD ; fi +if [ -e FCMFINAL ]; then cp -f FCMFINAL $SLURM_SUBMIT_DIR/FCMFINAL ; fi + +# Create a job data archive file +tar --transform "s,^,Job_Data_$SLURM_JOB_ID/," -vcf $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar OPTARC FCMINT FCMFINAL ZMATnew JMOL.plot JOBARC JAINDX FJOBARC DIPDER HESSIAN MOLDEN NEWMOS den.dat +if [ -e zmat001 ]; then tar --transform "s,^,Job_Data_$SLURM_JOB_ID/," -vrf $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar zmat* ; fi +gzip $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar + +echo " Job complete on `hostname`." + +rm $scratch_dir -r + +# ignored line -- do not remove diff --git a/tests/sub_templates/molpro@2021_template.sh b/tests/sub_templates/molpro@2021_template.sh new file mode 100644 index 0000000..562c481 --- /dev/null +++ b/tests/sub_templates/molpro@2021_template.sh @@ -0,0 +1,29 @@ +#!/bin/bash +#SBATCH --job-name=STEP # Job name +#SBATCH --partition=batch # Partition (queue) name +#SBATCH --constraint=EPYC|Intel +#SBATCH --nodes=1 # Number of nodes +#SBATCH --ntasks=4 # Number of MPI ranks +#SBATCH --ntasks-per-node=4 # How many tasks on each node +#SBATCH --cpus-per-task=1 # Number of cores per MPI rank +#SBATCH --mem=32GB # Total Memory +#SBATCH --time=10:00:00 +#SBATCH --output="%x.%j".out # Standard output log +#SBATCH --error="%x.%j".err # Standard error log + +cd $SLURM_SUBMIT_DIR +export NSLOTS=4 +export THREADS=1 + + +# to change scratch dir to use local machine scratch +export SCRATCH_DIR=/scratch/$USER/tmp/$SLURM_JOB_ID +mkdir -p $SCRATCH_DIR +export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR" # This binds the directory into the container so that output can be written. +module load intel/2022a +mpirun -n $NSLOTS apptainer exec /work/jttlab/containers/molpro_mpipr.sif molpro.exe input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR + +rm $SCRATCH_DIR -r + + +#ignored line -- do not remove diff --git a/tests/sub_templates/molpro@2024_template.sh b/tests/sub_templates/molpro@2024_template.sh new file mode 100644 index 0000000..0bd86a1 --- /dev/null +++ b/tests/sub_templates/molpro@2024_template.sh @@ -0,0 +1,28 @@ +#!/bin/bash +#SBATCH --job-name=STEP # Job name +#SBATCH --partition=batch # Partition (queue) name +#SBATCH --constraint=EPYC|Intel +#SBATCH --nodes=1 # Number of nodes +#SBATCH --ntasks=4 # Number of MPI ranks +#SBATCH --ntasks-per-node=4 # How many tasks on each node +#SBATCH --cpus-per-task=1 # Number of cores per MPI rank +#SBATCH --mem=32GB # Total Memory +#SBATCH --time=10:00:00 +#SBATCH --output="%x.%j".out # Standard output log +#SBATCH --error="%x.%j".err # Standard error log + +cd $SLURM_SUBMIT_DIR +export NSLOTS=4 +export THREADS=1 + + +# to change scratch dir to use local machine scratch +export SCRATCH_DIR=/scratch/$USER/tmp/$SLURM_JOB_ID +mkdir -p $SCRATCH_DIR +export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR" # This binds the directory into the container so that output can be written. + +singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR +rm $SCRATCH_DIR -r + + +#ignored line -- do not remove diff --git a/tests/sub_templates/orca_template.sh b/tests/sub_templates/orca_template.sh new file mode 100644 index 0000000..f45baa4 --- /dev/null +++ b/tests/sub_templates/orca_template.sh @@ -0,0 +1,51 @@ +#!/bin/bash +#SBATCH --job-name=STEP # Job name +#SBATCH --partition=batch # Partition (queue) name +#SBATCH --constraint=EPYC|Intel +#SBATCH --nodes=1 # Number of nodes +#SBATCH --ntasks=4 # Number of MPI ranks +#SBATCH --ntasks-per-node=4 # How many tasks on each node +#SBATCH --cpus-per-task=1 # Number of cores per MPI rank +#SBATCH --mem=32GB # Total Memory +#SBATCH --time=10:00:00 +#SBATCH --output="%x.%j".out # Standard output log +#SBATCH --error="%x.%j".err # Standard error log + +cd $SLURM_SUBMIT_DIR +export NSLOTS=4 +export THREADS=1 + +scratch_dir=/scratch/$USER/tmp/$SLURM_JOB_ID +mkdir -p $scratch_dir +#Set MPI Variables +module load ORCA/6.1.0-OpenMPI-4.1.8-GCC-13.3.0-avx2 +export OMP_NUM_THREADS=1 + +# Set other variables +base=`basename input.dat .dat` + +# Copy Job/Executable Data +cp $SLURM_SUBMIT_DIR/input.dat $scratch_dir/input.dat +if [ -e $base.xyz ]; then cp $base.xyz $scratch_dir/guess.xyz ; fi +if [ -e $base.gbw ]; then cp $base.gbw $scratch_dir/guess.gbw ; fi +if [ -e $base.hess ]; then cp $base.hess $scratch_dir/guess.hess ; fi +if [ -e product.xyz ]; then cp product.xyz $scratch_dir/product.xyz ; fi +if [ -e ts_guess.xyz ]; then cp ts_guess.xyz $scratch_dir/ts_guess.xyz ; fi + +echo " Running orca on `hostname`" +echo " Running calculation..." + +cd $scratch_dir +/apps/eb/ORCA/6.1.0-OpenMPI-4.1.8-GCC-13.3.0-avx2/bin/orca input.dat >& $SLURM_SUBMIT_DIR/output.dat || exit 1 + +echo " Saving data and cleaning up..." +# delete any temporary files that my be hanging around. +rm -f *.tmp* +find . -type f -size +50M -exec rm -f {} \; +tar --exclude='*tmp*' --transform "s,^,Job_Data_$SLURM_JOB_ID/," -vzcf $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar.gz * + +echo " Job complete on `hostname`." + +rm $scratch_dir -r + +#ignored line -- do not remove diff --git a/tests/sub_templates/psi4_template.sh b/tests/sub_templates/psi4_template.sh new file mode 100644 index 0000000..d266bec --- /dev/null +++ b/tests/sub_templates/psi4_template.sh @@ -0,0 +1,20 @@ +#!/bin/bash +#SBATCH --partition=batch +#SBATCH --constraint="EPYC|Intel" +#SBATCH --job-name=STEP +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=4 +#SBATCH --time=10:00:00 +#SBATCH --mem=32GB +#SBATCH --output="%x.%j".out # Standard output log +#SBATCH --error="%x.%j".err # Standard error log + +cd $SLURM_SUBMIT_DIR +export NSLOTS=4 + +export PSI_SCRATCH=/scratch/$USER/tmp/$SLURM_JOB_ID +mkdir -p $PSI_SCRATCH +psi4 -n $NSLOTS +rm $PSI_SCRATCH -r + +# ignored line -- do not remove From 9afcce47fc84b2a918c688f167a64493e2857dff Mon Sep 17 00:00:00 2001 From: Alexander Heide Date: Mon, 18 May 2026 17:37:41 -0400 Subject: [PATCH 7/8] Fixes molpro container name --- submitscripts/slurm/sapelo_programs.py | 4 ++-- tests/test_submitscripts.py | 32 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 tests/test_submitscripts.py diff --git a/submitscripts/slurm/sapelo_programs.py b/submitscripts/slurm/sapelo_programs.py index e7fa772..1f73ab1 100644 --- a/submitscripts/slurm/sapelo_programs.py +++ b/submitscripts/slurm/sapelo_programs.py @@ -44,7 +44,7 @@ """ molpro = """module load intel/2022a -mpirun -n $NSLOTS apptainer exec /work/jttlab/containers/molpro_mpipr.sif \ +mpirun -n $NSLOTS apptainer exec /work/jttlab/containers/molpro-2021-gapr.sif \ molpro.exe input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR rm $SCRATCH_DIR -r @@ -52,7 +52,7 @@ """ molpro_24 = """ -singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat \ +singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr-v3.sif -n $NSLOTS input.dat \ --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR rm $SCRATCH_DIR -r diff --git a/tests/test_submitscripts.py b/tests/test_submitscripts.py new file mode 100644 index 0000000..91b9ab0 --- /dev/null +++ b/tests/test_submitscripts.py @@ -0,0 +1,32 @@ +import pytest +import optavc + +# This is my simple, dirty way to make sure we're actually generating submit scripts for all programs +# correctly and choosing between for instance molpro 24 and molpro 21. cfour serial and mpi +# note how simple the sprogram specs I'm using are. A better test would be to run an energy with each +# submit script + +@pytest.mark.parametrize("program", [("molpro@2024"), + ("molpro@2021"), + ("cfour@2.1-mpi"), + ("cfour@2.1"), + ("psi4"), + ("orca")]) +def test_programs(program): + + options_kwargs = { + 'input_name' : "input.dat", + 'output_name' : "output.dat", + "program": program, + 'cluster': 'Sapelo', + } + + options = optavc.options.Options(**options_kwargs) + options.job_array_range = (1, 1) # This is a poor design choice + cluster = optavc.cluster.Cluster("SAPELO") + sub_script = cluster.make_sub_script(options) + + with open(f"./sub_templates/{program}_template.sh", "r") as f: + ref_template = str(f.read()) + assert sub_script == ref_template + From 781d7bea5c2a9e7901ac096feaaa6d0ceb1c9c89 Mon Sep 17 00:00:00 2001 From: Alexander Heide Date: Tue, 19 May 2026 15:44:43 -0400 Subject: [PATCH 8/8] Uses symlinks for both molpro verisons. Actually push test updates --- .gitignore | 2 ++ submitscripts/slurm/sapelo_programs.py | 2 +- tests/sub_templates/molpro@2021_template.sh | 2 +- tests/sub_templates/molpro@2024_template.sh | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 17fe3f9..5bdf3db 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,5 @@ dmypy.json /vulcan_examples/ examples tests +!tests/sub_templates +!tests/templates diff --git a/submitscripts/slurm/sapelo_programs.py b/submitscripts/slurm/sapelo_programs.py index 1f73ab1..6d24d8a 100644 --- a/submitscripts/slurm/sapelo_programs.py +++ b/submitscripts/slurm/sapelo_programs.py @@ -52,7 +52,7 @@ """ molpro_24 = """ -singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr-v3.sif -n $NSLOTS input.dat \ +singularity run /work/jttlab/containers/molpro-2024-gapr.sif -n $NSLOTS input.dat \ --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR rm $SCRATCH_DIR -r diff --git a/tests/sub_templates/molpro@2021_template.sh b/tests/sub_templates/molpro@2021_template.sh index 562c481..0a755a1 100644 --- a/tests/sub_templates/molpro@2021_template.sh +++ b/tests/sub_templates/molpro@2021_template.sh @@ -21,7 +21,7 @@ export SCRATCH_DIR=/scratch/$USER/tmp/$SLURM_JOB_ID mkdir -p $SCRATCH_DIR export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR" # This binds the directory into the container so that output can be written. module load intel/2022a -mpirun -n $NSLOTS apptainer exec /work/jttlab/containers/molpro_mpipr.sif molpro.exe input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR +mpirun -n $NSLOTS apptainer exec /work/jttlab/containers/molpro-2021-gapr.sif molpro.exe input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR rm $SCRATCH_DIR -r diff --git a/tests/sub_templates/molpro@2024_template.sh b/tests/sub_templates/molpro@2024_template.sh index 0bd86a1..296ed87 100644 --- a/tests/sub_templates/molpro@2024_template.sh +++ b/tests/sub_templates/molpro@2024_template.sh @@ -21,7 +21,7 @@ export SCRATCH_DIR=/scratch/$USER/tmp/$SLURM_JOB_ID mkdir -p $SCRATCH_DIR export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR" # This binds the directory into the container so that output can be written. -singularity run /work/jttlab/containers/molpro-2024.1.1-mpi-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR +singularity run /work/jttlab/containers/molpro-2024-gapr.sif -n $NSLOTS input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR rm $SCRATCH_DIR -r