From af39eb1f53b86894b3b8ea15ad46689647f99cab Mon Sep 17 00:00:00 2001 From: Synkevych Roman Date: Tue, 20 Feb 2024 12:03:39 +0200 Subject: [PATCH 1/5] fix: used imax as an end date instead of last measure data --- calculation/parser.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/calculation/parser.py b/calculation/parser.py index 4884410..c8c0d59 100644 --- a/calculation/parser.py +++ b/calculation/parser.py @@ -30,9 +30,8 @@ def parse_xml_to_object(file=OPTIONS_FILE): xml_root = xml_tree.getroot() start_date_time_str = xml_root.find('imin').text start_date_time = datetime.strptime(start_date_time_str, '%Y-%m-%d %H:%M:%S') - # TODO: check if end date is needed - # end_date_time_str = xml_root.find('imax').text - # end_date_time = datetime.strptime(end_date_time_str, '%Y-%m-%d %H:%M:%S') + end_date_time_str = xml_root.find('imax').text + end_date_time = datetime.strptime(end_date_time_str, '%Y-%m-%d %H:%M:%S') nx = (xml_root.find('nx').text) ny = (xml_root.find('ny').text) min_height = xml_root.find('minheight').text if xml_root.find( @@ -40,7 +39,7 @@ def parse_xml_to_object(file=OPTIONS_FILE): return { 'start_date_time': start_date_time, - # 'end_date_time': end_date_time, + 'end_date_time': end_date_time, 'out_longitude': xml_root.find('se_lon').text, 'out_latitude': xml_root.find('se_lat').text, 'num_x_grid': nx, @@ -111,7 +110,7 @@ def parse_csv_to_object(file=MEASUREMENTS_FILE): def parse_command_file(user_params): start_date_time = user_params['start_date_time'] - end_date_time = releases_params[-1]['end_date_time'] + timedelta(hours=1) + end_date_time = user_params['end_date_time'] + timedelta(hours=1) command_body = f"""&COMMAND LDIRECT= -1, ! Simulation direction in time ; 1 (forward) or -1 (backward) IBDATE= {start_date_time.strftime('%Y%m%d')}, ! Start date of the simulation ; YYYYMMDD: YYYY=year, MM=month, DD=day @@ -268,8 +267,8 @@ def parse_releases_file(releases_params): def process_releases(releases_params, user_params, start_calc_time): series_dirpath = f"/series/{user_params['series_id']}" - end_release_date = releases_params[-1]['end_date_time'] + timedelta(hours=1) - end_date_time_str = end_release_date.strftime('%Y%m%d%H%M%S') + end_date = user_params['end_date_time'] + timedelta(hours=1) + end_date_time_str = end_date.strftime('%Y%m%d%H%M%S') output_filename_prefix = f"grid_time_{end_date_time_str}" release_count = len(releases_params) @@ -283,7 +282,7 @@ def process_releases(releases_params, user_params, start_calc_time): parse_pathnames_file() # First date from user last is the last release date + 3 hours - download_data(user_params['start_date_time'], end_release_date) + download_data(user_params['start_date_time'], end_date) for i, param in enumerate(releases_params, start=1): id = param['id'] From 63e63a007692c9584615b495a1a63e9a9996689c Mon Sep 17 00:00:00 2001 From: Synkevych Roman Date: Tue, 20 Feb 2024 12:18:37 +0200 Subject: [PATCH 2/5] add openstack scripts --- openstack/delete_instance.sh | 41 +++++++++++++ openstack/launch_instace.sh | 109 +++++++++++++++++++++++++++++++++ openstack/setup_instance.sh | 63 +++++++++++++++++++ openstack/start_calculation.sh | 30 +++++++++ openstack/test_quotas.sh | 40 ++++++++++++ 5 files changed, 283 insertions(+) create mode 100644 openstack/delete_instance.sh create mode 100644 openstack/launch_instace.sh create mode 100644 openstack/setup_instance.sh create mode 100644 openstack/start_calculation.sh create mode 100644 openstack/test_quotas.sh diff --git a/openstack/delete_instance.sh b/openstack/delete_instance.sh new file mode 100644 index 0000000..849173b --- /dev/null +++ b/openstack/delete_instance.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +if [ $# -ne 1 ]; then echo "Usage: $0 VM_name"; exit 1; fi + +# exit on the first error +set -e + +# should be moved to launch_instance +. .WRF-UNG # load openstack environment variables + +VM_NAME=$1 +TIME=$(date "+%d.%m.%y-%H:%M:%S") + +echo "Preparing to remove the virtual machine $VM_NAME ..." + +private_key=".ssh/${VM_NAME}.key" +if [ -f $private_key ]; then + rm $private_key + echo "Private key file '$private_key' has been removed." + echo "$TIME Private key for '$private_key' deleted." >> vm_launching.log +else + echo "Private key file '$private_key' not found." +fi + +# remove keypair if it exists +if openstack keypair show $VM_NAME > /dev/null 2>&1; then + openstack keypair delete $VM_NAME + echo "Keypair ${VM_NAME} has been deleted." + echo "$TIME Keypair $VM_NAME deleted." >> vm_launching.log +else + echo "Keypair ${VM_NAME} does not exist" +fi + +# check if it the instance exists +if openstack server show $VM_NAME > /dev/null 2>&1; then + openstack server stop $VM_NAME && openstack server delete $VM_NAME + echo "Instance ${VM_NAME} has been deleted." + echo -e "$TIME Instance $VM_NAME deleted.\n" >> vm_launching.log +else + echo "Instance ${VM_NAME} does not exist." +fi diff --git a/openstack/launch_instace.sh b/openstack/launch_instace.sh new file mode 100644 index 0000000..d17d4fa --- /dev/null +++ b/openstack/launch_instace.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +HASH=`date --utc +%Y%m%d%H%M`; FLAVOR="d1.xlarge"; VMNAME="flexpart_${FLAVOR/./_}_${HASH}"; +TIME=$(date "+%d.%m.%Y-%H:%M:%S"); TIMER=30; KEY_PATH=.ssh/"${VMNAME}.key" +calculation_dir=$(pwd) +done_file="$calculation_dir/done.txt" + +echo "calculation dir: $calculation_dir" +cd ~/ || exit +echo "home dir: $(pwd)" + +# Ensure calculation folder exists +if [[ ! -d "$calculation_dir" ]]; then + echo "Error: Calculation folder '$calculation_dir' does not exist." >&2 + exit 1 +fi + +# execute the test_quotas.sh script and provide the flavor name as an argument +if ! ./test_quotas.sh $FLAVOR; then + echo "Error: Quotas are exceeded, canceling ..." + echo -e "$TIME Quotas are exceeded, canceling ...\n" >> vm_launching.log + exit 1 +fi + +# create a series dir if not exist +xml_file="$calculation_dir/input/options.xml" +if [ ! -f "$xml_file" ]; then + echo "Error: Input file options.xml does not exist: $xml_file" + exit 1 +fi +series_id=$(grep -oP '\K[0-9]+' "$xml_file" | sed 's/^0*//') +series_path="/home/flexpart/series/$series_id" + +mkdir -p "$series_path" +echo "Series path: $series_path" +sed -i "7s@.*@SERIES_PATH=$series_path@" start_calculation.sh +# provide the calculation directory name to the VM +echo "Calculation path: $calculation_dir" +sed -i "6s@.*@DIR_NAME=$calculation_dir@" start_calculation.sh + +openstack keypair create $VMNAME >> $KEY_PATH; chmod 600 .ssh/"${VMNAME}.key" + +nova boot --flavor $FLAVOR\ + --image 77a8fe4c-c0ea-4ec1-9723-11c8876325e7\ + --key-name $VMNAME\ + --security-groups d134acb2-e6bc-4c82-a294-9617fdf7bf07\ + --user-data start_calculation.sh\ + $VMNAME + +echo -e "$TIME start creating VM $VMNAME, status - $STATUS\n" >> vm_launching.log +total_iterations=$(wc -l < "$xml_file") + +for i in `seq 1 3`; do + echo -ne "$i attempt to start VM \033[0K\r" + sleep $TIMER & wait + + STATUS=`openstack server list | grep $VMNAME | awk '{ print $6 }'` + IP=`openstack server list | grep $VMNAME | awk '{ split($8, v, "="); print v[2]}'` + SYSTEM=`openstack server list | grep $VMNAME | awk '{ print $10 $11 }'` + + if [ "x$STATUS" = "xACTIVE" ]; then + printf "VM $VMNAME is $STATUS, IP address $IP, system $SYSTEM\n" + echo "$TIME VM $VMNAME is $STATUS, IP address $IP, system $SYSTEM" >> vm_launching.log + printf "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n" + echo -e "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n" >> vm_launching.log + + # Main loop to check for done.txt creation + while [[ ! -f "$done_file" ]]; do + echo "Waiting for 'done.txt' file in '$calculation_dir'..." >&2 + sleep "$TIMER" + done + echo "File 'done.txt' detected! Removing the VM $VMNAME" >&2 + # TODO: add argument -test=true as a value from output to make it easy while testing + ./delete_instance.sh $VMNAME + exit 0 + fi +done + +if test -z "$STATUS"; then + echo "Launching $VMNAME failed" + echo -e "$TIME Launching VM $VMNAME failed\n" >> vm_launching.log +else + printf "Launching $VMNAME failed with status: $STATUS" + echo -e "$TIME Launching VM $VMNAME failed\n" >> vm_launching.log +fi + +private_key=".ssh/${VMNAME}.key" +if [ -f $private_key ]; then + rm $private_key + echo "Private key file '$private_key' has been removed." +else + echo "Private key file '$private_key' not found." +fi + +# remove keypair if it exists +if openstack keypair show $VMNAME > /dev/null 2>&1; then + openstack keypair delete $VMNAME + echo "Keypair ${VMNAME} has been deleted." +else + echo "Keypair ${VMNAME} does not exist" +fi + +# check if it the instance exists +if openstack server show $VMNAME > /dev/null 2>&1; then + openstack server stop $VMNAME && openstack server delete $VMNAME + echo "Instance ${VMNAME} has been deleted." +else + echo "Instance ${VMNAME} does not exist, canceling." +fi diff --git a/openstack/setup_instance.sh b/openstack/setup_instance.sh new file mode 100644 index 0000000..b58bdb1 --- /dev/null +++ b/openstack/setup_instance.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +HASH=`date --utc +%Y%m%d%H%M`; FLAVOR="m1.2xlarge"; VMNAME="flexpart_${FLAVOR/./_}_${HASH}"; +TIME=$(date "+%d.%m.%Y-%H:%M:%S"); TIMER=30; KEY_PATH=.ssh/"${VMNAME}.key" + +openstack keypair create $VMNAME >> $KEY_PATH; chmod 600 .ssh/"${VMNAME}.key" + +nova boot --flavor $FLAVOR\ + --image cb3c7000-ed38-4e9d-a726-4d648dcbc9c9\ + --key-name $VMNAME\ + --security-groups d134acb2-e6bc-4c82-a294-9617fdf7bf07\ + $VMNAME + +echo -e "$TIME start creating VM $VMNAME, status - $STATUS\n" >> vm_launching.log + +for i in `seq 1 3`; do + echo -ne "$i attempt to start VM \033[0K\r" + sleep $TIMER & wait + + STATUS=`openstack server list | grep $VMNAME | awk '{ print $6 }'` + IP=`openstack server list | grep $VMNAME | awk '{ split($8, v, "="); print v[2]}'` + SYSTEM=`openstack server list | grep $VMNAME | awk '{ print $10 $11 }'` + + if [ "x$STATUS" = "xACTIVE" ]; then + printf "VM $VMNAME is $STATUS, IP address $IP, system $SYSTEM\n" + echo "$TIME VM $VMNAME is $STATUS, IP address $IP, system $SYSTEM" >> vm_launching.log + printf "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n" + echo -e "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n" >> vm_launching.log + exit 0 + fi +done + +if test -z "$STATUS"; then + echo "Launching $VMNAME failed" + echo -e "$TIME Launching VM $VMNAME failed\n" >> vm_launching.log +else + printf "Launching $VMNAME failed with status: $STATUS" + echo -e "$TIME Launching VM $VMNAME failed\n" >> vm_launching.log +fi + +private_key=".ssh/${VMNAME}.key" +if [ -f $private_key ]; then + rm $private_key + echo "Private key file '$private_key' has been removed." +else + echo "Private key file '$private_key' not found." +fi + +# remove keypair if it exists +if openstack keypair show $VMNAME > /dev/null 2>&1; then + openstack keypair delete $VMNAME + echo "Keypair ${VMNAME} has been deleted." +else + echo "Keypair ${VMNAME} does not exist" +fi + +# check if it the instance exists +if openstack server show $VMNAME > /dev/null 2>&1; then + openstack server stop $VMNAME && openstack server delete $VMNAME + echo "Instance ${VMNAME} has been deleted." +else + echo "Instance ${VMNAME} does not exist." +fi diff --git a/openstack/start_calculation.sh b/openstack/start_calculation.sh new file mode 100644 index 0000000..6f1a68b --- /dev/null +++ b/openstack/start_calculation.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# exit on the first error +set -e + +DIR_NAME=calculation_path +SERIES_PATH=series_path +NFS_SERVER=$(hostname -I) +uid=$(id username) +export LD_LIBRARY_PATH=/home/ubuntu/flexpart_lib/lib:$LD_LIBRARY_PATH + +# mount DIR_NAME folder to /data +sudo mount $NFS_SERVER:$DIR_NAME /data +sudo mount $NFS_SERVER:$SERIES_PATH /series +# could be removed in production +sudo mount $NFS_SERVER:/home/flexpart/series/grid_data /grid_data + +echo "FLEXPART on $(hostname) uses $(nproc) cores, $(free -h | awk '/^Mem:/ {print $2}') RAM for calculation ${DIR_NAME}" >> /data/calculations_server.log + +sudo chown -R ubuntu /data/ /series /grid_data + +su -c "cd /home/ubuntu/calculation && python3.6 parser.py" ubuntu + +echo "FLEXPART on $(hostname) uses $(nproc) cores, $(free -h | awk '/^Mem:/ {print $2}') RAM for calculation ${DIR_NAME}" >> /data/calculations_server.log + +# Change ownership recursively to the specified user +sudo chown -R "$uid:$uid" /data /series /grid_data +sudo umount /data /series /grid_data + +exit 0 diff --git a/openstack/test_quotas.sh b/openstack/test_quotas.sh new file mode 100644 index 0000000..28f21a8 --- /dev/null +++ b/openstack/test_quotas.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +if [ $# -ne 1 ]; then echo "Usage: $0 flavor_name"; exit 1; fi + +set -e + +flavor_info=`openstack flavor list --all | grep $1` + +flavor_cores=`echo $flavor_info | awk '{ print $12 }'` +flavor_ram=`echo $flavor_info | awk '{ print $6 }'` + +limits=`nova limits 2>/dev/null | grep 'Cores\|Instances\|RAM'` + +all_cores=`echo $limits | awk '{ print $6 }'` +used_cores=`echo $limits | awk '{ print $4 }'` + +all_instances=`echo $limits | awk '{ print $13 }'` +used_instances=`echo $limits | awk '{ print $11 }'` + +all_ram=`echo $limits | awk '{ print $20 }'` +used_ram=`echo $limits | awk '{ print $18 }'` +TIME=$(date "+%d.%m.%Y-%H:%M:%S") + +if (( $used_cores + $flavor_cores > $all_cores )); then + echo "Core limit was reached! Your need to fire $(expr $all_cores - $used_cores + $flavor_cores ) cores or change the flavor. Canceling ..." + echo -e "$TIME Core limit was reached! Your need to fire $(expr $used_cores + $flavor_cores - $all_cores) cores." >> vm_launching.log; + exit 1 +fi + +if (( $used_instances + 1 >= $all_instances )); then + echo "Instance limit was reached. You need to delete one of the instances. Canceling ..."; + echo -e "$TIME Instance limit was reached." >> vm_launching.log + exit 1 +fi + +if (( $used_ram + $flavor_ram > $all_ram )); then + echo "RAM limit was reached. You need to delete one of the instances or change the flavor. Canceling ..."; + echo -e "$TIME RAM limit was reached." >> vm_launching.log + exit 1 +fi From 171d8c379a95eb158586b02884feabc8674b8007 Mon Sep 17 00:00:00 2001 From: Synkevych Roman Date: Tue, 20 Feb 2024 12:18:56 +0200 Subject: [PATCH 3/5] fix: simflex for openstack --- simflex_v1/src/read_tintegr_srs.for | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/simflex_v1/src/read_tintegr_srs.for b/simflex_v1/src/read_tintegr_srs.for index 261c52a..a82f6d9 100644 --- a/simflex_v1/src/read_tintegr_srs.for +++ b/simflex_v1/src/read_tintegr_srs.for @@ -109,9 +109,7 @@ srsred=REAL(loutstep)*AllSRS(i)%srstsum(:,:,1) call fname_srs(fsrspattern,id_obs(i),fsrsname,200) curlen=len_trim(fsrsname) - series_dirpath = '/series/'// - & trim(adjustl(int2str(series_id)))// - & '/'//trim(nuclide_name)//'/' + series_dirpath = '/series/'//trim(nuclide_name)//'/' open(1022,FILE=series_dirpath//fsrsname(1:curlen), & FORM='UNFORMATTED') write(1022)nlon,nlat From 4eb1a1a8d8f16a912f4ef50d40b3b87025a4e620 Mon Sep 17 00:00:00 2001 From: Synkevych Roman Date: Mon, 4 Mar 2024 15:09:38 +0200 Subject: [PATCH 4/5] refactor code, add error check --- openstack/delete_instance.sh | 41 ------ openstack/launch_instace.sh | 220 +++++++++++++++++++++------------ openstack/start_calculation.sh | 2 +- openstack/test_quotas.sh | 40 ------ 4 files changed, 143 insertions(+), 160 deletions(-) delete mode 100644 openstack/delete_instance.sh delete mode 100644 openstack/test_quotas.sh diff --git a/openstack/delete_instance.sh b/openstack/delete_instance.sh deleted file mode 100644 index 849173b..0000000 --- a/openstack/delete_instance.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -if [ $# -ne 1 ]; then echo "Usage: $0 VM_name"; exit 1; fi - -# exit on the first error -set -e - -# should be moved to launch_instance -. .WRF-UNG # load openstack environment variables - -VM_NAME=$1 -TIME=$(date "+%d.%m.%y-%H:%M:%S") - -echo "Preparing to remove the virtual machine $VM_NAME ..." - -private_key=".ssh/${VM_NAME}.key" -if [ -f $private_key ]; then - rm $private_key - echo "Private key file '$private_key' has been removed." - echo "$TIME Private key for '$private_key' deleted." >> vm_launching.log -else - echo "Private key file '$private_key' not found." -fi - -# remove keypair if it exists -if openstack keypair show $VM_NAME > /dev/null 2>&1; then - openstack keypair delete $VM_NAME - echo "Keypair ${VM_NAME} has been deleted." - echo "$TIME Keypair $VM_NAME deleted." >> vm_launching.log -else - echo "Keypair ${VM_NAME} does not exist" -fi - -# check if it the instance exists -if openstack server show $VM_NAME > /dev/null 2>&1; then - openstack server stop $VM_NAME && openstack server delete $VM_NAME - echo "Instance ${VM_NAME} has been deleted." - echo -e "$TIME Instance $VM_NAME deleted.\n" >> vm_launching.log -else - echo "Instance ${VM_NAME} does not exist." -fi diff --git a/openstack/launch_instace.sh b/openstack/launch_instace.sh index d17d4fa..090b3b9 100644 --- a/openstack/launch_instace.sh +++ b/openstack/launch_instace.sh @@ -1,109 +1,173 @@ #!/bin/bash HASH=`date --utc +%Y%m%d%H%M`; FLAVOR="d1.xlarge"; VMNAME="flexpart_${FLAVOR/./_}_${HASH}"; -TIME=$(date "+%d.%m.%Y-%H:%M:%S"); TIMER=30; KEY_PATH=.ssh/"${VMNAME}.key" +TIME=$(date "+%d.%m.%Y-%H:%M:%S"); TIMER=30; KEY_PATH="/home/flexpart/.ssh/${VMNAME}.key" calculation_dir=$(pwd) done_file="$calculation_dir/done.txt" -echo "calculation dir: $calculation_dir" -cd ~/ || exit -echo "home dir: $(pwd)" +log_message() { + local message="$1" + local calc_log="$calculation_dir/calculations_server.log" + local vm_log="/home/flexpart/vm_launching.log" + + echo "$message" + echo -e "$TIME $message" | tee -a "$calc_log" >> "$vm_log" +} + +remove_vm() { + log_message "Preparing to remove the virtual machine $VMNAME ..." + + if [ -f $KEY_PATH ]; then + rm $KEY_PATH + log_message "Private key file '$KEY_PATH' has been removed." + else + log_message "Private key file '$KEY_PATH' not found." + fi + + # remove keypair if it exists + if openstack keypair show $VMNAME > /dev/null 2>&1; then + openstack keypair delete $VMNAME + log_message "Keypair ${VMNAME} has been deleted." + else + log_message "Keypair ${VMNAME} does not exist." + fi + + # check if it the instance exists + if openstack server show $VMNAME > /dev/null 2>&1; then + openstack server stop $VMNAME && openstack server delete $VMNAME + log_message "Instance ${VMNAME} has been deleted." + else + log_message "Instance ${VMNAME} does not exist, canceling." + fi +} + +# Define cleanup function for other errors +cleanup() { + vm_created=$1 + + if [[ $# -eq 1 && $1 =~ ^[0-9]+$ ]]; then + log_message "Error: Command exited with status $1" + # Remove VM if it exists + elif [ "$vm_created" = true ]; then + remove_vm + fi + # Create a done.txt to indicate finishing the calculation + touch "$done_file" + exit 1 +} + +test_quotas() { + # Get flavor information + flavor_info=$(openstack flavor show $FLAVOR) + + if [[ -z "$flavor_info" ]]; then + log_message "Error: Flavor information is empty, cant start instance." + cleanup false + fi + + # Extract flavor cores and RAM + flavor_cores=$(echo "$flavor_info" | awk '/vcpus/ { print $4 }') + flavor_ram=$(echo "$flavor_info" | awk '/ram/ { print $4 }') + + # Get Nova limits + limits=$(nova limits 2>/dev/null | awk '/Cores|Instances|RAM/') + # Extract all cores, used cores, all instances, used instances, all RAM, and used RAM + read all_cores used_cores all_instances used_instances all_ram used_ram <<< $(echo $limits | awk '{ print $6, $4, $13, $11, $20, $18 }') + if (( $used_cores + $flavor_cores > $all_cores )); then + log_message "Error: Core limit was reached! Your need to fire $(expr $all_cores - $used_cores + $flavor_cores ) cores or change the flavor. Canceling ..." + cleanup false + fi + + if (( $used_instances + 1 >= $all_instances )); then + log_message "Error: Instance limit was reached. You need to delete one of the instances. Canceling ..." + cleanup false + fi + + if (( $used_ram + $flavor_ram > $all_ram )); then + log_message "Error: RAM limit was reached. You need to delete one of the instances or change the flavor. Canceling ..."; + cleanup false + fi +} + +log_message "Process ID: $$" + +# Trap errors and call cleanup function, it will delete VM and create done.txt +trap 'cleanup true' ERR + +. /home/flexpart/.WRF-UNG # load openstack environment variables # Ensure calculation folder exists if [[ ! -d "$calculation_dir" ]]; then - echo "Error: Calculation folder '$calculation_dir' does not exist." >&2 - exit 1 + log_message "Error: Calculation folder '$calculation_dir' does not exist." + cleanup false fi # execute the test_quotas.sh script and provide the flavor name as an argument -if ! ./test_quotas.sh $FLAVOR; then - echo "Error: Quotas are exceeded, canceling ..." - echo -e "$TIME Quotas are exceeded, canceling ...\n" >> vm_launching.log - exit 1 +if ! test_quotas; then + log_message "Error: Quotas are exceeded, canceling ..." + cleanup false fi # create a series dir if not exist xml_file="$calculation_dir/input/options.xml" if [ ! -f "$xml_file" ]; then - echo "Error: Input file options.xml does not exist: $xml_file" - exit 1 + log_message "Error: File: $xml_file does not exist" + cleanup false fi + series_id=$(grep -oP '\K[0-9]+' "$xml_file" | sed 's/^0*//') series_path="/home/flexpart/series/$series_id" - mkdir -p "$series_path" -echo "Series path: $series_path" -sed -i "7s@.*@SERIES_PATH=$series_path@" start_calculation.sh + # provide the calculation directory name to the VM -echo "Calculation path: $calculation_dir" sed -i "6s@.*@DIR_NAME=$calculation_dir@" start_calculation.sh +log_message "Calculation path: $calculation_dir" +sed -i "7s@.*@SERIES_PATH=$series_path@" start_calculation.sh +log_message "Series path: $series_path" openstack keypair create $VMNAME >> $KEY_PATH; chmod 600 .ssh/"${VMNAME}.key" -nova boot --flavor $FLAVOR\ - --image 77a8fe4c-c0ea-4ec1-9723-11c8876325e7\ +instance_id=$(nova boot --flavor $FLAVOR\ + --image f7eed42e-266d-4576-8ac6-b6dbbfa53233\ --key-name $VMNAME\ --security-groups d134acb2-e6bc-4c82-a294-9617fdf7bf07\ - --user-data start_calculation.sh\ - $VMNAME - -echo -e "$TIME start creating VM $VMNAME, status - $STATUS\n" >> vm_launching.log -total_iterations=$(wc -l < "$xml_file") - -for i in `seq 1 3`; do - echo -ne "$i attempt to start VM \033[0K\r" - sleep $TIMER & wait - - STATUS=`openstack server list | grep $VMNAME | awk '{ print $6 }'` - IP=`openstack server list | grep $VMNAME | awk '{ split($8, v, "="); print v[2]}'` - SYSTEM=`openstack server list | grep $VMNAME | awk '{ print $10 $11 }'` - - if [ "x$STATUS" = "xACTIVE" ]; then - printf "VM $VMNAME is $STATUS, IP address $IP, system $SYSTEM\n" - echo "$TIME VM $VMNAME is $STATUS, IP address $IP, system $SYSTEM" >> vm_launching.log - printf "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n" - echo -e "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n" >> vm_launching.log - - # Main loop to check for done.txt creation - while [[ ! -f "$done_file" ]]; do - echo "Waiting for 'done.txt' file in '$calculation_dir'..." >&2 - sleep "$TIMER" - done - echo "File 'done.txt' detected! Removing the VM $VMNAME" >&2 - # TODO: add argument -test=true as a value from output to make it easy while testing - ./delete_instance.sh $VMNAME - exit 0 - fi -done - -if test -z "$STATUS"; then - echo "Launching $VMNAME failed" - echo -e "$TIME Launching VM $VMNAME failed\n" >> vm_launching.log -else - printf "Launching $VMNAME failed with status: $STATUS" - echo -e "$TIME Launching VM $VMNAME failed\n" >> vm_launching.log -fi - -private_key=".ssh/${VMNAME}.key" -if [ -f $private_key ]; then - rm $private_key - echo "Private key file '$private_key' has been removed." -else - echo "Private key file '$private_key' not found." + --user-data start_calculation.sh\ + $VMNAME\ + 2>/dev/null | awk '/ id / {print $4}') + +# Check if instance creation was successful +if [ -z "$instance_id" ]; then + log_message "Error: Failed to create instance." + cleanup true fi -# remove keypair if it exists -if openstack keypair show $VMNAME > /dev/null 2>&1; then - openstack keypair delete $VMNAME - echo "Keypair ${VMNAME} has been deleted." -else - echo "Keypair ${VMNAME} does not exist" -fi +log_message "$TIME start creating VM $VMNAME, status - $STATUS." + +while true; do + STATUS=$(openstack server show --format value -c status $instance_id) + + if [ "$STATUS" == "ACTIVE" ]; then + IP=`openstack server show --format value -c addresses $instance_id | awk '{ split($1, v, "="); print v[2]}'` + + log_message "$TIME VM $VMNAME is $STATUS, IP address $IP" + log_message "To connect use: ssh -i $KEY_PATH ubuntu@$IP\n" + + # Main loop to check for done.txt creation + log_message "Calculation for '$calculation_dir' and series '$series_path' on VM $VMNAME started." + log_message "Waiting for 'done.txt' file in '$calculation_dir'..." + while [[ ! -f "$done_file" ]]; do + sleep "$TIMER" + done + log_message "File 'done.txt' detected! Removing the VM $VMNAME" + # TODO: add argument -test=true as a value from output to make it easy while testing + remove_vm + exit 0 + break + elif [ "$status" == "ERROR" ]; then + log_message "Error: Instance $VMNAME creation failed." + cleanup true + fi + sleep 5 +done -# check if it the instance exists -if openstack server show $VMNAME > /dev/null 2>&1; then - openstack server stop $VMNAME && openstack server delete $VMNAME - echo "Instance ${VMNAME} has been deleted." -else - echo "Instance ${VMNAME} does not exist, canceling." -fi +trap - ERR diff --git a/openstack/start_calculation.sh b/openstack/start_calculation.sh index 6f1a68b..ca6f422 100644 --- a/openstack/start_calculation.sh +++ b/openstack/start_calculation.sh @@ -12,7 +12,7 @@ export LD_LIBRARY_PATH=/home/ubuntu/flexpart_lib/lib:$LD_LIBRARY_PATH # mount DIR_NAME folder to /data sudo mount $NFS_SERVER:$DIR_NAME /data sudo mount $NFS_SERVER:$SERIES_PATH /series -# could be removed in production +# could be removed in production sudo mount $NFS_SERVER:/home/flexpart/series/grid_data /grid_data echo "FLEXPART on $(hostname) uses $(nproc) cores, $(free -h | awk '/^Mem:/ {print $2}') RAM for calculation ${DIR_NAME}" >> /data/calculations_server.log diff --git a/openstack/test_quotas.sh b/openstack/test_quotas.sh deleted file mode 100644 index 28f21a8..0000000 --- a/openstack/test_quotas.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -if [ $# -ne 1 ]; then echo "Usage: $0 flavor_name"; exit 1; fi - -set -e - -flavor_info=`openstack flavor list --all | grep $1` - -flavor_cores=`echo $flavor_info | awk '{ print $12 }'` -flavor_ram=`echo $flavor_info | awk '{ print $6 }'` - -limits=`nova limits 2>/dev/null | grep 'Cores\|Instances\|RAM'` - -all_cores=`echo $limits | awk '{ print $6 }'` -used_cores=`echo $limits | awk '{ print $4 }'` - -all_instances=`echo $limits | awk '{ print $13 }'` -used_instances=`echo $limits | awk '{ print $11 }'` - -all_ram=`echo $limits | awk '{ print $20 }'` -used_ram=`echo $limits | awk '{ print $18 }'` -TIME=$(date "+%d.%m.%Y-%H:%M:%S") - -if (( $used_cores + $flavor_cores > $all_cores )); then - echo "Core limit was reached! Your need to fire $(expr $all_cores - $used_cores + $flavor_cores ) cores or change the flavor. Canceling ..." - echo -e "$TIME Core limit was reached! Your need to fire $(expr $used_cores + $flavor_cores - $all_cores) cores." >> vm_launching.log; - exit 1 -fi - -if (( $used_instances + 1 >= $all_instances )); then - echo "Instance limit was reached. You need to delete one of the instances. Canceling ..."; - echo -e "$TIME Instance limit was reached." >> vm_launching.log - exit 1 -fi - -if (( $used_ram + $flavor_ram > $all_ram )); then - echo "RAM limit was reached. You need to delete one of the instances or change the flavor. Canceling ..."; - echo -e "$TIME RAM limit was reached." >> vm_launching.log - exit 1 -fi From 0f0a3bc217758866837c4677ff74e9b6ed81cd3b Mon Sep 17 00:00:00 2001 From: Synkevych Roman Date: Mon, 4 Mar 2024 23:10:59 +0200 Subject: [PATCH 5/5] refactor code and messages --- openstack/launch_instace.sh | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/openstack/launch_instace.sh b/openstack/launch_instace.sh index 090b3b9..1322ce1 100644 --- a/openstack/launch_instace.sh +++ b/openstack/launch_instace.sh @@ -43,15 +43,17 @@ remove_vm() { # Define cleanup function for other errors cleanup() { - vm_created=$1 + local vm_created=$1 + local message=$2 if [[ $# -eq 1 && $1 =~ ^[0-9]+$ ]]; then - log_message "Error: Command exited with status $1" + message="Command exited with status $1" # Remove VM if it exists elif [ "$vm_created" = true ]; then remove_vm fi # Create a done.txt to indicate finishing the calculation + log_message "launch_error: $message\n" touch "$done_file" exit 1 } @@ -61,8 +63,7 @@ test_quotas() { flavor_info=$(openstack flavor show $FLAVOR) if [[ -z "$flavor_info" ]]; then - log_message "Error: Flavor information is empty, cant start instance." - cleanup false + cleanup false "Flavor information is empty, can't start instance." fi # Extract flavor cores and RAM @@ -74,18 +75,15 @@ test_quotas() { # Extract all cores, used cores, all instances, used instances, all RAM, and used RAM read all_cores used_cores all_instances used_instances all_ram used_ram <<< $(echo $limits | awk '{ print $6, $4, $13, $11, $20, $18 }') if (( $used_cores + $flavor_cores > $all_cores )); then - log_message "Error: Core limit was reached! Your need to fire $(expr $all_cores - $used_cores + $flavor_cores ) cores or change the flavor. Canceling ..." - cleanup false + cleanup false "Core limit was reached! Fire $(expr $used_cores + $flavor_cores - $all_cores ) cores or change the flavor. Canceling ..." fi if (( $used_instances + 1 >= $all_instances )); then - log_message "Error: Instance limit was reached. You need to delete one of the instances. Canceling ..." - cleanup false + cleanup false "Instance limit was reached. Delete one of the instances. Canceling ..." fi if (( $used_ram + $flavor_ram > $all_ram )); then - log_message "Error: RAM limit was reached. You need to delete one of the instances or change the flavor. Canceling ..."; - cleanup false + cleanup false "RAM limit was reached. Delete one of the instances or change the flavor. Canceling ..." fi } @@ -98,21 +96,18 @@ trap 'cleanup true' ERR # Ensure calculation folder exists if [[ ! -d "$calculation_dir" ]]; then - log_message "Error: Calculation folder '$calculation_dir' does not exist." - cleanup false + cleanup false "Calculation folder '$calculation_dir' does not exist." fi # execute the test_quotas.sh script and provide the flavor name as an argument if ! test_quotas; then - log_message "Error: Quotas are exceeded, canceling ..." - cleanup false + cleanup false "Quotas are exceeded, Canceling ..." fi # create a series dir if not exist xml_file="$calculation_dir/input/options.xml" if [ ! -f "$xml_file" ]; then - log_message "Error: File: $xml_file does not exist" - cleanup false + cleanup false "File $xml_file does not exist." fi series_id=$(grep -oP '\K[0-9]+' "$xml_file" | sed 's/^0*//') @@ -131,14 +126,13 @@ instance_id=$(nova boot --flavor $FLAVOR\ --image f7eed42e-266d-4576-8ac6-b6dbbfa53233\ --key-name $VMNAME\ --security-groups d134acb2-e6bc-4c82-a294-9617fdf7bf07\ - --user-data start_calculation.sh\ + --user-data /usr/local/bin/start_calculation.sh\ $VMNAME\ 2>/dev/null | awk '/ id / {print $4}') # Check if instance creation was successful if [ -z "$instance_id" ]; then - log_message "Error: Failed to create instance." - cleanup true + cleanup true "Failed to create instance." fi log_message "$TIME start creating VM $VMNAME, status - $STATUS." @@ -164,8 +158,7 @@ while true; do exit 0 break elif [ "$status" == "ERROR" ]; then - log_message "Error: Instance $VMNAME creation failed." - cleanup true + cleanup true "Instance $VMNAME creation failed." fi sleep 5 done