Update sapelo_programs.py

.gitignore

@@ -127,3 +127,5 @@ dmypy.json
 /vulcan_examples/
 examples
 tests
+!tests/sub_templates
+!tests/templates

options.py

@@ -58,6 +58,14 @@ class Options(object):
         The name should at least be sufficiently long to load the module with. i.e. cfour@2.0+mpi is sufficient for
         vulcan load psi4 is also sufficient isntead of psi4@master.
 
+        The naming convention is similar for Sapelo, however, most of the submit scripts are hardwired to specific versions.
+        Some submit scripts use the container in `/work/jttlab/containers,` some use the modules on Sapelo.
+        Again, the string should generally include the name and a version number. In general, for containers,
+        the name of the container should lead to the correct version being used.
+
+        optavc only uses the gapr containers for molpro. Edit the submit scripts to use soc.
+        For cfour requesting mpi will force use of the container but serial will use the module
+
     template_file_path : string
         default : 'template.dat'
 
@@ -459,6 +467,10 @@ def program(self):
     @program.setter
     def program(self, val=""):
         prog = val.lower().split("@")
+        if len(prog) == 1:
+            # no @ in version string. try to split on -. This is needed for molpro if
+            # user provides the container name as the program.
+            prog = val.lower().split("-", 1)
         self._program = prog[0]
 
         if prog[0] == 'fermi':
@@ -470,9 +482,18 @@ def program(self, val=""):
             elif prog[0] in ['orca', 'molpro']:
                 self.parallel = 'mpi'
 
+                if prog[0] == 'molpro' and '24' in prog[-1]:
+                    # molpro will be set above already change to molpro_24
+                    self._program = f"{prog[0]}_24"
+
         if not self.parallel:
-            if '+mpi' in prog[-1]:
+            if '+mpi' in prog[-1] or '-mpi' in prog[-1]:  # two common strings include +mpi or just name-mpi
                 self.parallel = 'mpi'
+
+                if prog[0] == 'molpro' and '24' in prog[-1]:
+                    # molpro will be set above already change to molpro_24
+                    self._program = f"{prog[0]}_24"
+
             elif 'serial' in prog[-1]:
                 self.parallel = 'serial'
             elif '~mpi' in prog[-1]:

submitscripts/slurm/sapelo_programs.py

@@ -1,8 +1,8 @@
 # use sapelo2 work area (called scratch)
 # no need to copy set psi_scratch variable
 
-fermi = """module load Julia/1.8.2-linux-x86_64
-module load intel/2023a
+fermi = """module load Julia/1.11.6-gfbf-2023b
+module load intel/2023b
 
 julia {input_name} 
 
@@ -29,39 +29,45 @@
 # mpi only
 # set scratch dir to home area but run from submit_dir
 
-molpro_mpi = """module load intel/2023a
-
+molpro_scratch_prefix = """
 # to change scratch dir to use local machine scratch
 export SCRATCH_DIR=/scratch/$USER/tmp/$SLURM_JOB_ID
 mkdir -p $SCRATCH_DIR
 export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR"  # This binds the directory into the container so that output can be written.
-
-mpirun -n $NSLOTS apptainer exec /work/jttlab/containers/molpro_mpipr.sif \
-molpro.exe input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR
-
-rm $SCRATCH_DIR -r
-
 """
 
-# mpi only
-# copy everything to lscratch to run and set scratch to lscratch
-
-molpro_mpi_lscratch = """module load intel/2023a
-
+molpro_lscratch_prefix = """
 # to change scratch dir to use local machine scratch
 export SCRATCH_DIR=/lscratch/$USER/tmp/$SLURM_JOB_ID
 mkdir -p $SCRATCH_DIR
 export APPTAINER_BIND="$SLURM_SUBMIT_DIR,$SCRATCH_DIR"  # This binds the directory into the container so that output can be written.
+"""
 
-mpirun -n $NSLOTS apptainer exec /work/jttlab/containers/molpro_mpipr.sif \
+molpro = """module load intel/2022a
+mpirun -n $NSLOTS apptainer exec /work/jttlab/containers/molpro-2021-gapr.sif \
 molpro.exe input.dat --output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR
 
 rm $SCRATCH_DIR -r
 
 """
 
+molpro_24 = """
+singularity run /work/jttlab/containers/molpro-2024-gapr.sif -n $NSLOTS input.dat \
+--output $SLURM_SUBMIT_DIR/output.dat --nouse-logfile --directory $SCRATCH_DIR
+rm $SCRATCH_DIR -r
+
+"""
+
+# mpi only
+# copy everything to lscratch to run and set scratch to lscratch
+
+molpro_24_mpi = molpro_scratch_prefix + molpro_24
+molpro_24_mpi_lscratch = molpro_lscratch_prefix + molpro_24
+molpro_mpi = molpro_scratch_prefix + molpro
+molpro_mpi_lscratch = molpro_lscratch_prefix + molpro
+
 orca_common = """#Set MPI Variables
-module load ORCA/5.0.4-gompi-2022a
+module load ORCA/6.1.0-OpenMPI-4.1.8-GCC-13.3.0-avx2
 export OMP_NUM_THREADS=1
 
 # Set other variables
@@ -79,12 +85,12 @@
 echo " Running calculation..."
 
 cd $scratch_dir
-orca {input_name} >& $SLURM_SUBMIT_DIR/{output_name} || exit 1
+/apps/eb/ORCA/6.1.0-OpenMPI-4.1.8-GCC-13.3.0-avx2/bin/orca {input_name} >& $SLURM_SUBMIT_DIR/{output_name} || exit 1
 
 echo " Saving data and cleaning up..."
 # delete any temporary files that my be hanging around.
 rm -f *.tmp*
-find . -type f -size +50M -exec rm -f {} \;
+find . -type f -size +50M -exec rm -f {{}} \;
 tar --exclude='*tmp*' --transform "s,^,Job_Data_$SLURM_JOB_ID/," -vzcf $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar.gz *
 
 echo " Job complete on `hostname`."
@@ -100,7 +106,7 @@
 mkdir -p $scratch_dir
 """ + orca_common
 
-cfour_common = """
+cfour_prefix = """
 # make sure MRCC is around just in case
 export PATH=$PATH:/work/jttlab/mrcc/2020/
 prefix=/apps/eb/$module/
@@ -130,9 +136,9 @@
 echo " Running calculation..."
 
 cd $scratch_dir
-xcfour >& $SLURM_SUBMIT_DIR/{output_name}
-xja2fja
+"""
 
+cfour_suffix = """
 echo " Saving data and cleaning up..."
 if [ -e ZMATnew ]; then cp -f ZMATnew $SLURM_SUBMIT_DIR/ZMATnew ; fi
 if [ -e GRD ]; then cp -f GRD $SLURM_SUBMIT_DIR/GRD ; fi
@@ -148,37 +154,52 @@
 rm $scratch_dir -r
 """
 
-cfour_serial = """module=cfour/2.1-intel-2021b-serial
+xcfour_module = cfour_prefix + """
+xcfour >& $SLURM_SUBMIT_DIR/{output_name}
+xja2fja
+""" + cfour_suffix
+
+xcfour_container = cfour_prefix + """
+# Silence all the IEEE signaling messages
+export NO_STOP_MESSAGE=yes
+# request devices (inifiniband) use openib BTL interface for openmpi 4
+export OMPI_MCA_btl_openib_allow_ib=true
+
+apptainer exec /work/jttlab/containers/cfour-2.1-foss-ompi.sif xcfour >& $SLURM_SUBMIT_DIR/{output_name}
+apptainer exec /work/jttlab/containers/cfour-2.1-foss-ompi.sif xja2fja
+""" + cfour_suffix
+
+cfour_serial = """module=cfour/2.1-intel-2023a-serial
 export OMP_NUM_THREADS=$NSLOTS
 
 scratch_dir=/scratch/$USER/tmp/$SLURM_JOB_ID
 mkdir -p $scratch_dir
 
-""" + cfour_common
+""" + xcfour_module
 
-cfour_serial_lscratch = """module=cfour/2.1-intel-2021b-serial
+cfour_serial_lscratch = """module=cfour/2.1-intel-2023a-serial
 export OMP_NUM_THREADS=$NSLOTS
 
 scratch_dir=/lscratch/$USER/tmp/$SLURM_JOB_ID
 mkdir -p $scratch_dir
 
-""" + cfour_common
+""" + xcfour_module
 
-cfour_mpi = """module=cfour/2.1-intel-2021b-mpi
+cfour_mpi = """module=OpenMPI/4.1.1-GCC-11.2.0 # no cfour mpi by gacrc
 scratch_dir=/scratch/$USER/tmp/$SLURM_JOB_ID
 mkdir -p $scratch_dir
 
 echo -e "\t$NSLOTS" > ./ncpu   # CFour appears to just claim any and all cpus
 echo -e "\t$NSLOTS" > $scratch_dir/ncpu
-""" + cfour_common
+""" + xcfour_container
 
-cfour_mpi_lscratch = """module=cfour/2.1-intel-2021b-mpi
+cfour_mpi_lscratch = """module=OpenMPI/4.1.1-GCC-11.2.0 # no cfour mpi by gacrc
 scratch_dir=/lscratch/$USER/tmp/$SLURM_JOB_ID
 mkdir -p $scratch_dir
 
 echo -e "\t$NSLOTS" > ./ncpu   # CFour appears to just claim any and all cpus
 echo -e "\t$NSLOTS" > $scratch_dir/ncpu
-""" + cfour_common
+""" + xcfour_container
 
 progdict = {
     "serial": {
@@ -197,11 +218,13 @@
         "lscratch": {
             "orca": orca_lscratch,
             "molpro": molpro_mpi_lscratch,
+            "molpro_24": molpro_24_mpi_lscratch,
             "cfour": cfour_mpi_lscratch
             },
         "scratch": {
             "orca": orca,
             "molpro": molpro_mpi,
+            "molpro_24": molpro_24_mpi,
             "cfour": cfour_mpi
             }
         }

tests/sub_templates/cfour@2.1-mpi_template.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+#SBATCH --job-name=STEP             # Job name
+#SBATCH --partition=batch               # Partition (queue) name
+#SBATCH --constraint=EPYC|Intel
+#SBATCH --nodes=1                     # Number of nodes
+#SBATCH --ntasks=4             # Number of MPI ranks
+#SBATCH --ntasks-per-node=4    # How many tasks on each node
+#SBATCH --cpus-per-task=1     # Number of cores per MPI rank 
+#SBATCH --mem=32GB                # Total Memory
+#SBATCH --time=10:00:00
+#SBATCH --output="%x.%j".out     # Standard output log
+#SBATCH --error="%x.%j".err      # Standard error log
+
+cd $SLURM_SUBMIT_DIR
+export NSLOTS=4
+export THREADS=1
+
+module=OpenMPI/4.1.1-GCC-11.2.0 # no cfour mpi by gacrc
+scratch_dir=/scratch/$USER/tmp/$SLURM_JOB_ID
+mkdir -p $scratch_dir
+
+echo -e "	$NSLOTS" > ./ncpu   # CFour appears to just claim any and all cpus
+echo -e "	$NSLOTS" > $scratch_dir/ncpu
+
+# make sure MRCC is around just in case
+export PATH=$PATH:/work/jttlab/mrcc/2020/
+prefix=/apps/eb/$module/
+module load $module
+
+# Copy job data
+if [[ -e input.dat && ! -e ZMAT ]]; then
+  cp input.dat $scratch_dir/ZMAT
+else
+  cp $SLURM_SUBMIT_DIR/ZMAT $scratch_dir
+fi
+
+cp $prefix/basis/GENBAS $scratch_dir
+cp $prefix/basis/ECPDATA $scratch_dir
+if [ -e JAINDX ]; then cp JAINDX $scratch_dir ; fi
+if [ -e JOBARC ]; then cp JOBARC $scratch_dir ; fi
+if [ -e FCMINT ]; then cp FCMINT $scratch_dir ; fi
+if [ -e GENBAS ]; then cp GENBAS $scratch_dir ; fi
+if [ -e ECPDATA ]; then cp ECPDATA $scratch_dir ; fi
+if [ -e OPTARC ]; then cp OPTARC $scratch_dir ; fi
+if [ -e ISOTOPES ]; then cp ISOTOPES $scratch_dir ; fi
+if [ -e ISOMASS ]; then cp ISOMASS $scratch_dir ; fi
+if [ -e initden.dat ]; then cp initden.dat $scratch_dir ; fi
+if [ -e OLDMOS ]; then cp OLDMOS $scratch_dir ; fi
+
+echo " Running cfour on `hostname`"
+echo " Running calculation..."
+
+cd $scratch_dir
+
+# Silence all the IEEE signaling messages
+export NO_STOP_MESSAGE=yes
+# request devices (inifiniband) use openib BTL interface for openmpi 4
+export OMPI_MCA_btl_openib_allow_ib=true
+
+apptainer exec /work/jttlab/containers/cfour-2.1-foss-ompi.sif xcfour >& $SLURM_SUBMIT_DIR/output.dat
+apptainer exec /work/jttlab/containers/cfour-2.1-foss-ompi.sif xja2fja
+
+echo " Saving data and cleaning up..."
+if [ -e ZMATnew ]; then cp -f ZMATnew $SLURM_SUBMIT_DIR/ZMATnew ; fi
+if [ -e GRD ]; then cp -f GRD $SLURM_SUBMIT_DIR/GRD ; fi
+if [ -e FCMFINAL ]; then cp -f FCMFINAL $SLURM_SUBMIT_DIR/FCMFINAL ; fi
+
+# Create a job data archive file
+tar --transform "s,^,Job_Data_$SLURM_JOB_ID/," -vcf $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar OPTARC FCMINT FCMFINAL ZMATnew JMOL.plot JOBARC JAINDX FJOBARC DIPDER HESSIAN MOLDEN NEWMOS den.dat
+if [ -e zmat001 ]; then tar --transform "s,^,Job_Data_$SLURM_JOB_ID/," -vrf $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar zmat* ; fi
+gzip $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar
+
+echo " Job complete on `hostname`."
+
+rm $scratch_dir -r
+
+#ignored line -- do not remove

tests/sub_templates/cfour@2.1_template.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --constraint="EPYC|Intel"
+#SBATCH --job-name=STEP
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --time=10:00:00
+#SBATCH --mem=32GB
+#SBATCH --output="%x.%j".out    # Standard output log
+#SBATCH --error="%x.%j".err     # Standard error log
+
+cd $SLURM_SUBMIT_DIR
+export NSLOTS=4
+
+module=cfour/2.1-intel-2023a-serial
+export OMP_NUM_THREADS=$NSLOTS
+
+scratch_dir=/scratch/$USER/tmp/$SLURM_JOB_ID
+mkdir -p $scratch_dir
+
+
+# make sure MRCC is around just in case
+export PATH=$PATH:/work/jttlab/mrcc/2020/
+prefix=/apps/eb/$module/
+module load $module
+
+# Copy job data
+if [[ -e input.dat && ! -e ZMAT ]]; then
+  cp input.dat $scratch_dir/ZMAT
+else
+  cp $SLURM_SUBMIT_DIR/ZMAT $scratch_dir
+fi
+
+cp $prefix/basis/GENBAS $scratch_dir
+cp $prefix/basis/ECPDATA $scratch_dir
+if [ -e JAINDX ]; then cp JAINDX $scratch_dir ; fi
+if [ -e JOBARC ]; then cp JOBARC $scratch_dir ; fi
+if [ -e FCMINT ]; then cp FCMINT $scratch_dir ; fi
+if [ -e GENBAS ]; then cp GENBAS $scratch_dir ; fi
+if [ -e ECPDATA ]; then cp ECPDATA $scratch_dir ; fi
+if [ -e OPTARC ]; then cp OPTARC $scratch_dir ; fi
+if [ -e ISOTOPES ]; then cp ISOTOPES $scratch_dir ; fi
+if [ -e ISOMASS ]; then cp ISOMASS $scratch_dir ; fi
+if [ -e initden.dat ]; then cp initden.dat $scratch_dir ; fi
+if [ -e OLDMOS ]; then cp OLDMOS $scratch_dir ; fi
+
+echo " Running cfour on `hostname`"
+echo " Running calculation..."
+
+cd $scratch_dir
+
+xcfour >& $SLURM_SUBMIT_DIR/output.dat
+xja2fja
+
+echo " Saving data and cleaning up..."
+if [ -e ZMATnew ]; then cp -f ZMATnew $SLURM_SUBMIT_DIR/ZMATnew ; fi
+if [ -e GRD ]; then cp -f GRD $SLURM_SUBMIT_DIR/GRD ; fi
+if [ -e FCMFINAL ]; then cp -f FCMFINAL $SLURM_SUBMIT_DIR/FCMFINAL ; fi
+
+# Create a job data archive file
+tar --transform "s,^,Job_Data_$SLURM_JOB_ID/," -vcf $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar OPTARC FCMINT FCMFINAL ZMATnew JMOL.plot JOBARC JAINDX FJOBARC DIPDER HESSIAN MOLDEN NEWMOS den.dat
+if [ -e zmat001 ]; then tar --transform "s,^,Job_Data_$SLURM_JOB_ID/," -vrf $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar zmat* ; fi
+gzip $SLURM_SUBMIT_DIR/Job_Data_$SLURM_JOB_ID.tar
+
+echo " Job complete on `hostname`."
+
+rm $scratch_dir -r
+
+# ignored line -- do not remove