improve pbs run

bin/endorse-mlmc

@@ -15,6 +15,12 @@ function safe_mkdir() {
   fi
 }
 
+function input_copy {
+    # links seems to not work from containers
+    #ln -s "$@"
+    cp "$@"
+}
+
 function error() {
     echo -e "${bred}ERROR:$reset ${red} $1!$reset"
     exit 1
@@ -38,9 +44,10 @@ function build_docker_image() {
 
 function setup_dev_env() {
     rm -rf $ENDORSE_SRC_ROOT/venv
-    call_container $ENDORSE_SRC_ROOT/package/setup_venv.sh
+    __call_container $ENDORSE_SRC_ROOT/package/setup_venv.sh
 }
 
+
 function setup_container() {
     if [ -z "$CONTAINER_TOOL" ]; then
         if command -v docker &> /dev/null
@@ -56,7 +63,7 @@ function setup_container() {
     fi
 }
 
-function docker_path_home_translate() {
+function __docker_path_home_translate() {
     # translate  path under $HOME to /mnt$HOME
     # set RETURN return corresponding docker path
     local PATH_HOST=$1
@@ -70,41 +77,51 @@ function docker_path_home_translate() {
     fi
 }
 
-function call_container() {
+
+
+function __call_container() {
+    # Call command in a container for:
+    # WORKDIR - where the command will be executed
+    # ENDORSE_IMAGE - container image
+    # ENDORSE_SRC_ROOT - 
+    # CONTAINER_TOOL
+    # ENDORSE_WORKSPACE
+
+
     # Setup container tool, docker prefered
-    setup_container   # setup CONTAINER_TOOL and ENDORSE_IMAGE vars, prepare environment
     cd ${WORKDIR}
     # TODO: move container tool resolution into swrap
     if [ "$CONTAINER_TOOL" == "singularity" ]
     then
         export PYTHONPATH="$ENDORSE_SRC_ROOT/submodules/swrap/src/swrap"
-        python3 -m sexec -i $ENDORSE_IMAGE -e "$ENDORSE_SRC_ROOT/venv" endorse_mlmc "$@"
+        python3 -m sexec -i $ENDORSE_IMAGE -e "$ENDORSE_SRC_ROOT/venv" "$@"
     else
         # docker
-        envarg="-euid=$uid -egid=$gid -etheme=$theme -ewho=$uname -ehome=/mnt/$HOME -v/$HOME:/mnt/$HOME"
-        docker_path_home_translate $ENDORSE_SRC_ROOT
+        local envarg="-euid=$uid -egid=$gid -etheme=$theme -ewho=$uname -ehome=/mnt/$HOME -v/$HOME:/mnt/$HOME"
+        __docker_path_home_translate $ENDORSE_SRC_ROOT
         local docker_src_root=$RETURN
-        docker_path_home_translate $ENDORSE_WORKSPACE
+        __docker_path_home_translate $ENDORSE_WORKSPACE
         local docker_workspace=$RETURN
-        docker_path_home_translate $WORKDIR
+        __docker_path_home_translate $WORKDIR
         local docker_workdir=$RETURN
 
         args=("${@}")
         for i in ${!args[@]}
         do
-            docker_path_home_translate "${args[$i]}"
+            __docker_path_home_translate "${args[$i]}"
             args[$i]="$RETURN"
         done
 
-        path_env="-eENDORSE_WORKSPACE=$docker_workspace -eWORKDIR=$docker_workdir -ePYTHONPATH=$docker_src_root/submodules/swrap/src/swrap"
-        $CONTAINER_TOOL run $path_env $envarg $bind $ENDORSE_IMAGE "${args[@]}"
+        local path_env="-eENDORSE_WORKSPACE=$docker_workspace -eWORKDIR=$docker_workdir -ePYTHONPATH=$docker_src_root/submodules/swrap/src/swrap"
+        $CONTAINER_TOOL run $path_env $envarg $ENDORSE_IMAGE "${args[@]}"
     fi
 }
 
 
 
 
 
+
 # Set global paths
 # Endorse workspace
 ENDORSE_WORKSPACE="${ENDORSE_WORKSPACE:-${HOME}/endorse_space}"
@@ -220,8 +237,8 @@ while [ "${1#-}" != "$1" ]      # arg starts with '-'
       EXAMPLE="$1"
       # TODO: treat various examples
       test_data="$ENDORSE_SRC_ROOT/tests/test_data"
-      ln -s "$test_data"/*.yaml  "${WORKDIR}"
-      ln -s "$test_data"/large_model_local.msh2  "${WORKDIR}"
+      input_copy "$test_data"/*.yaml  "${WORKDIR}"
+      input_copy "$test_data"/large_model_local.msh2  "${WORKDIR}"
       exit 0
     ;;
     *)
@@ -237,8 +254,9 @@ if [ -z "$1" ]; then print_usage; error "Missing command"; fi
 if [ -z "${QUEUE}" ]
 then
     # without PBS    
-    cont_call endorse_mlmc "$@"
+    EXECUTOR=bash 
 else
+    # with PBS
     stdoe="${WORKDIR}/"
     if [ -n "${CLUSTER}" ]
     then
@@ -252,9 +270,81 @@ else
         queue="-q ${QUEUE}"
     fi
     select="-l select=1:ncpus=8:mem=8gb${cluster_restrict}"
-    qsub ${queue} -S /bin/bash -N endorse-main -j oe -o ${stdoe} ${select} ${walltime} \
-    -v ENDORSE_WORKSPACE,WORKDIR,ENDORSE_SRC_ROOT,call_swrap - <<PBS_END
-    cont_call endorse_mlmc "$@"
-PBS_END
+    EXECUTOR="qsub ${queue} -S /bin/bash -N endorse-main -j oe -o ${stdoe} ${select} ${walltime}  -" 
 fi
 
+
+# Export these two functions in order to pass them to the QSUB script
+# Mostly temporary solution, should be part of swrap
+#export -f __docker_path_home_translate
+#export -f __call_container
+
+setup_container   # setup CONTAINER_TOOL and ENDORSE_IMAGE vars, prepare environment
+$EXECUTOR <<SCRIPT_END
+set -x
+
+
+function __docker_path_home_translate() {
+    # translate  path under $HOME to /mnt$HOME
+    # set RETURN return corresponding docker path
+    local PATH_HOST=$1
+    local rel_host="${PATH_HOST##$HOME}"
+    if [ "$rel_host" == "$PATH_HOST" ]
+    then
+        # preserve path out of home
+        RETURN="$PATH_HOST"
+    else
+        RETURN="/mnt$HOME/$rel_host"
+    fi
+}
+
+function __call_container() {
+    set -x
+    echo \$1
+    echo \$2
+    echo \$3
+    # Call command in a container for:
+    # WORKDIR - where the command will be executed
+    # ENDORSE_IMAGE - container image
+    # ENDORSE_SRC_ROOT - 
+    # CONTAINER_TOOL
+    # ENDORSE_WORKSPACE
+
+
+    # Setup container tool, docker prefered
+    cd ${WORKDIR}
+    # TODO: move container tool resolution into swrap
+    if [ "$CONTAINER_TOOL" == "singularity" ]
+    then
+        export PYTHONPATH="$ENDORSE_SRC_ROOT/submodules/swrap/src/swrap"
+        python3 -m sexec -i $ENDORSE_IMAGE -e "$ENDORSE_SRC_ROOT/venv" "\$@"
+    else
+        # docker
+        local envarg="-euid=$uid -egid=$gid -etheme=$theme -ewho=$uname -ehome=/mnt/$HOME -v/$HOME:/mnt/$HOME"
+        __docker_path_home_translate $ENDORSE_SRC_ROOT
+        local docker_src_root=$RETURN
+        __docker_path_home_translate $ENDORSE_WORKSPACE
+        local docker_workspace=$RETURN
+        __docker_path_home_translate $WORKDIR
+        local docker_workdir=$RETURN
+
+        args=("\${@}")
+        for i in ${!args[@]}
+        do
+            __docker_path_home_translate "${args[$i]}"
+            args[$i]="$RETURN"
+        done
+
+        local path_env="-eENDORSE_WORKSPACE=$docker_workspace -eWORKDIR=$docker_workdir -ePYTHONPATH=$docker_src_root/submodules/swrap/src/swrap"
+        $CONTAINER_TOOL run $path_env $envarg $ENDORSE_IMAGE "\${args[@]}"
+    fi
+}
+
+    CONTAINER_TOOL=$CONTAINER_TOOL
+    ENDORSE_IMAGE=$ENDORSE_IMAGE
+    WORKDIR="$WORKDIR"
+    ENDORSE_SRC_ROOT="$ENDORSE_SRC_ROOT"
+    ENDORSE_WORKSPACE="$ENDORSE_WORKSPACE"    
+    __call_container endorse_mlmc $@
+SCRIPT_END
+

bin/recreate_submodules

@@ -1,6 +1,7 @@
 #!/bin/bash
 # recreate submodules from .gitmodule
 
+set -x
 set -e
 
 git config -f .gitmodules --get-regexp '^submodule\..*\.path$' |

requirements.txt

@@ -1,3 +1,4 @@
+attrs
 pyvista
 pyyaml
 pyyaml-include
@@ -11,13 +12,8 @@ importlib-resources
 
 #numpy==1.21.4
 PyQt5==5.15.6
-#vtk==9.1.0
+#vtk==9.1.0 # should be part of the image
 #bgem # commit 4428b2286b5d1711b3c2fcd9f3651a004d91704f
 pandas==1.3.4
 openpyxl==3.0.9
-#pygimli==1.2.1
 psutil==5.8.0
-attrs==21.2.0
-#obspy==1.2.2
-#pyqtgraph==0.12.3
-#pymeshlab==2021.10

src/endorse/scripts/endorse_mlmc.py

@@ -9,7 +9,10 @@
 import time
 import logging
 
-logging.basicConfig(level=logging.INFO, filename='endorse_mlmc.log')
+logging.basicConfig(
+    level=logging.INFO, 
+    filename='endorse_mlmc.log', 
+    format='%(asctime)s %(message)s')
 logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
 
 from concurrent.futures import ProcessPoolExecutor
@@ -184,6 +187,7 @@ def run_fixed(cfg, n_samples, debug, n_proc):
         sleep=sampling_params.sample_sleep,
         timeout=sampling_params.sample_timeout)    
     logging.info(f"[{work_dir}] init N running: {running}")
+    logging.info(f"[{work_dir}] set init N = {n_samples}")
 
     sampler.set_initial_n_samples(n_samples)
     sampler.schedule_samples()
@@ -666,7 +670,7 @@ def run_case(base_dir:str, case : SimCase, model_dim, np, debug):
             #cfg_file = "cfg_variant.yaml"
             #with open(cfg_file, "w") as f:
             #        yaml.dump(common.dotdict.serialize(cfg_var), f)
-            n_samples = cfg.mlmc.n_samples
+            n_samples = cfg_var.mlmc.n_samples
             cfg_var._model_dim = model_dim
             run_fixed(cfg_var, n_samples, debug, n_proc=np)
 

tests/test_data/_cases.yaml

@@ -34,3 +34,9 @@ edz_4:
 noedz_4:
     transport_fullscale/bulk_field_params/cond_max: 3e-13
     transport_fullscale/fractures/fixed_seed: 632
+test_fast:
+    mlmc/n_samples: 3
+    transport_fullscale/mesh/fracture_mesh_step: 20
+    transport_fullscale/mesh/boreholes_mesh_step: 1
+    transport_fullscale/mesh/edz_mesh_step: 3
+    transport_fullscale/fractures/n_frac_limit: 1

tests/test_data/config.yaml

@@ -54,7 +54,7 @@ machine_config:
     #    - flow123d
 
 mlmc:
-    n_samples: 100
+    n_samples: 10
     n_levels: 1
 
 # TODO: replace hardwired indicator list. 

tests/test_data/download.sh

@@ -19,7 +19,7 @@ update() {
 
 # download big test data
 #update large_model.msh https://flow.nti.tul.cz/endorse_large_data/large_model.tar.gz
-update large_model_local.msh2 http://flow.nti.tul.cz/endorse_large_data/large_model_local.tar.gz 
+update large_model_local.msh2 https://flow.nti.tul.cz/endorse_large_data/large_model_local.tar.gz 
 
 #update flow_fields.pvd https://flow.nti.tul.cz/endorse_large_data/output_transport_2022_5.tar.gz
 #update solute_fields.pvd https://flow.nti.tul.cz/endorse_large_data/output_transport_2022_5.tar.gz