diff --git a/bundle/CMakeLists.txt b/bundle/CMakeLists.txt index f00ba70cb..463ae4c5c 100644 --- a/bundle/CMakeLists.txt +++ b/bundle/CMakeLists.txt @@ -125,6 +125,17 @@ if(BUILD_GDASBUNDLE) # GDASApp ecbuild_bundle( PROJECT gdas SOURCE "../") +# Install jcb (JEDI Configuration Builder) Python package + message(STATUS "Installing jcb (JEDI Configuration Builder)") + execute_process( + COMMAND ${Python3_EXECUTABLE} -m pip install --prefix ${CMAKE_INSTALL_PREFIX} . + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../sorc/jcb + RESULT_VARIABLE jcb_install_result + ) + if(NOT jcb_install_result EQUAL 0) + message(FATAL_ERROR "Failed to install jcb Python package") + endif() + # ioda, ufo, fv3-jedi, and saber test data #--------------------------------- if(CLONE_JCSDADATA) diff --git a/modulefiles/EVA/hercules.lua b/modulefiles/EVA/hercules.lua new file mode 100644 index 000000000..398156f00 --- /dev/null +++ b/modulefiles/EVA/hercules.lua @@ -0,0 +1,18 @@ +-- NOAA HPC Orion Modulefile for EVA +help([[ +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +prepend_path("MODULEPATH", '/work2/noaa/da/python/opt/modulefiles/stack') + +load("hpc/1.2.0") +load("miniconda3/4.6.14") +load("eva/1.0.0") + +whatis("Name: ".. pkgName) +whatis("Version: " .. pkgVersion) +whatis("Category: EVA") +whatis("Description: Load all libraries needed for EVA") diff --git a/modulefiles/EVA/ursa.lua b/modulefiles/EVA/ursa.lua new file mode 100644 index 000000000..768fb2fae --- /dev/null +++ b/modulefiles/EVA/ursa.lua @@ -0,0 +1,48 @@ +help([[ +Load python virtual environment for obs-monitor +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() + +conflict(pkgName) + +-- Load dependencies +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.2.1/install/modulefiles/Core") + +load("stack-oneapi/2024.2.1") +load("stack-intel-oneapi-mpi/2021.13") +load("intel-oneapi-mkl/2024.2.1") +load("stack-python/3.11.7") +load("rocoto/1.3.7") + +-- Python packages +load("py-jinja2/3.1.4") +load("py-netcdf4/1.7.1.post2") +load("py-pybind11/2.13.5") +load("py-pycodestyle/2.11.0") +load("py-pyyaml/6.0.2") +load("py-scipy/1.14.1") +load("py-xarray/2024.7.0") +load("py-pip/23.1.2") +load("py-matplotlib/3.7.4") +load("py-cartopy/0.24.1") +load("proj/9.4.1") +load("py-wxflow/0.2.0") + +-- Set the venv path +local venv_root = "/scratch3/NCEPDEV/da/Edward.Safford/noscrub/python/envs/obs-mon" +local bin_path = pathJoin(venv_root, "bin") +local lib_path = pathJoin(venv_root, "lib/python3.11/site-packages") -- adjust Python version if needed + +-- Modify environment variables to emulate virtualenv activation +prepend_path("PATH", bin_path) +prepend_path("PYTHONPATH", lib_path) + +-- Optional: If you want to make this environment more self-contained +setenv("VIRTUAL_ENV", venv_root) + +whatis("Name: " .. pkgName) +whatis("Version: " .. tostring(pkgVersion)) +whatis("Category: Obs-monitor") +whatis("Description: Load all libraries needed for obs-monitor") \ No newline at end of file diff --git a/test/test_jcb_installation.sh b/test/test_jcb_installation.sh new file mode 100755 index 000000000..e8be58e58 --- /dev/null +++ b/test/test_jcb_installation.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Test script to verify that jcb is properly installed during build +# This script should be run after a successful CMake build + +set -e + +echo "Testing jcb installation in build directory..." + +# Get the build directory - assuming it's passed as first argument or use default +BUILD_DIR="${1:-${CMAKE_INSTALL_PREFIX:-./install}}" + +if [ ! -d "$BUILD_DIR" ]; then + echo "Error: Build directory '$BUILD_DIR' does not exist" + exit 1 +fi + +echo "Using build directory: $BUILD_DIR" + +# Find the Python site-packages directory +PYTHON_SITE_PACKAGES=$(find "$BUILD_DIR" -path "*/lib/python*/dist-packages" -o -path "*/lib/python*/site-packages" | head -1) +if [ -z "$PYTHON_SITE_PACKAGES" ]; then + echo "Error: Could not find Python site-packages directory in $BUILD_DIR" + exit 1 +fi + +# Find the bin directory +BIN_DIR=$(find "$BUILD_DIR" -name "bin" -type d | head -1) +if [ -z "$BIN_DIR" ]; then + echo "Error: Could not find bin directory in $BUILD_DIR" + exit 1 +fi + +echo "Python packages directory: $PYTHON_SITE_PACKAGES" +echo "Binary directory: $BIN_DIR" + +# Check if jcb package is installed +if [ ! -d "$PYTHON_SITE_PACKAGES/jcb" ]; then + echo "Error: jcb Python package not found in $PYTHON_SITE_PACKAGES" + exit 1 +fi + +# Check if jcb command is available +if [ ! -f "$BIN_DIR/jcb" ]; then + echo "Error: jcb command not found in $BIN_DIR" + exit 1 +fi + +# Test if jcb command works +export PYTHONPATH="$PYTHON_SITE_PACKAGES:$PYTHONPATH" +export PATH="$BIN_DIR:$PATH" + +echo "Testing jcb command..." +if ! jcb --version; then + echo "Error: jcb command failed to run" + exit 1 +fi + +echo "✓ jcb is properly installed and functional" +echo "✓ jcb command is available at: $BIN_DIR/jcb" +echo "✓ jcb package is installed at: $PYTHON_SITE_PACKAGES/jcb" + +exit 0 \ No newline at end of file diff --git a/ush/ufoeval/run_ufo_hofx_test.sh b/ush/ufoeval/run_ufo_hofx_test.sh index 2bd4799b2..e2384aa2b 100755 --- a/ush/ufoeval/run_ufo_hofx_test.sh +++ b/ush/ufoeval/run_ufo_hofx_test.sh @@ -67,9 +67,9 @@ obtype=$1 #--------------- User modified options below ----------------- -machine=${machine:-orion} +machine=${machine:-ursa} -if [ $machine = orion ]; then +if [[ "$machine" = "orion" || "$machine" = "hercules" ]]; then if [ $run_filtering == NO ]; then workdir=/work2/noaa/da/$LOGNAME/ufoeval/$cycle/${obtype}_noqc echo "Run without data filtering" @@ -79,15 +79,15 @@ if [ $machine = orion ]; then GDASApp=${GDASApp:-/work2/noaa/da/$LOGNAME/git/GDASApp/} # Change this to your own branch JCBinstall=${JCBinstall:-/work2/noaa/da/cmartin/CI/GDASApp/opt} JCBpylib=$JCBinstall/lib/python3.7/site-packages -elif [ $machine = hera ]; then +elif [[ "$machine" = "ursa" || "$machine" = "hera" ]]; then if [ $run_filtering == NO ]; then - workdir=/scratch1/NCEPDEV/stmp2/$LOGNAME/ufoeval/$cycle/${obtype}_noqc + workdir=/scratch3/NCEPDEV/stmp/$LOGNAME/ufoeval/$cycle/${obtype}_noqc else - workdir=/scratch1/NCEPDEV/stmp2/$LOGNAME/ufoeval/$cycle/${obtype} + workdir=/scratch3/NCEPDEV/stmp/$LOGNAME/ufoeval/$cycle/${obtype} fi - GDASApp=${GDASApp:-/scratch1/NCEPDEV/da/$LOGNAME/git/GDASApp/} # Change this to your own branch - JCBinstall=${JCBinstall:-/scratch1/NCEPDEV/da/Cory.R.Martin/CI/GDASApp/opt} - JCBpylib=$JCBinstall/lib/python3.10/site-packages + GDASApp=${GDASApp:-/scratch3/NCEPDEV/da/$LOGNAME/git/GDASApp/} # Change this to your own branch + JCBinstall=${JCBinstall:-/scratch3/NCEPDEV/da/Andrew.Collard/jcb} + JCBpylib=$JCBinstall/lib/python3.9/site-packages else echo "Machine " $machine "not found" exit 1 @@ -103,7 +103,7 @@ exename=test_ObsFilters.x #-------------- Do not modify below this line ---------------- # paths that should only be changed by an expert user -dataprocdate=20240815 # Production date of test data +dataprocdate=20250731 # Production date of test data obtype_short=${obtype:0:4} if [ $obtype_short = "cris" ] || [ $obtype_short = "iasi" ] || [ $obtype_short = "hirs" ] || [ $obtype_short = "sevi" ] || \ @@ -114,12 +114,12 @@ else radiance="NO" fi -if [ $machine = orion ]; then +if [[ "$machine" = "orion" || "$machine" = "hercules" ]]; then export Datapath='/work2/noaa/da/acollard/UFO_eval/data/gsi_geovals_l127/nofgat_feb2024/'$dataprocdate FixDir=/work2/noaa/da/cmartin/GDASApp/fix -elif [ $machine = hera ]; then - export Datapath='/scratch1/NCEPDEV/da/Emily.Liu/UFO_eval/data/gsi_geovals_l127/nofgat_aug2021/'$dataprocdate - FixDir=/scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix +elif [[ "$machine" = "ursa" || "$machine" = "hera" ]]; then + export Datapath='/scratch3/NCEPDEV/da/Andrew.Collard/UFO_eval/data/gsi_geovals_l127/nofgat_Feb2024/'$dataprocdate + FixDir=/scratch3/NCEPDEV/da/Andrew.Collard/GDASApp/fix else echo "Machine " $machine "not found" exit @@ -155,7 +155,7 @@ export GPREFIX=gdas.t${gcyc}z # Load Modules for GDASApp module use $GDASApp/modulefiles -module load GDAS/$machine +module load GDAS/$machine.intel export PYTHONPATH=$GDASApp/ush:$PYTHONPATH # Create and set up the working directory