From c459ae286d2ee08f8361cc2be755335e135554f8 Mon Sep 17 00:00:00 2001 From: Foster Date: Thu, 16 Apr 2020 13:33:52 +0100 Subject: [PATCH 01/10] testing commit --- disk-recycling/clean-disk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disk-recycling/clean-disk.sh b/disk-recycling/clean-disk.sh index 31e9879..caccad2 100644 --- a/disk-recycling/clean-disk.sh +++ b/disk-recycling/clean-disk.sh @@ -1,5 +1,5 @@ #!/bin/bash - +#test comment for git #This script will unmount and wipe the partitions from a drive. if [ $# -ne 1 ]; then From 753f678c815235112e576d99605555094b1a671f Mon Sep 17 00:00:00 2001 From: Foster Date: Thu, 23 Apr 2020 21:32:51 +0100 Subject: [PATCH 02/10] Adding disk-tooling directory and SAS smartdata parse script. --- disk-tooling/smartdata_sas | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 disk-tooling/smartdata_sas diff --git a/disk-tooling/smartdata_sas b/disk-tooling/smartdata_sas new file mode 100644 index 0000000..140fb35 --- /dev/null +++ b/disk-tooling/smartdata_sas @@ -0,0 +1,6 @@ +#!/bin/bash +#Saves smartdata to a temporary file, then uses multiple searches to parse it. + +smartctl -a /dev/$1 > /tmp/smartdata_temp + +cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}' From f7aeabe880ea49b79101fcf0d47715411d3aafbb Mon Sep 17 00:00:00 2001 From: Foster Date: Thu, 23 Apr 2020 21:39:41 +0100 Subject: [PATCH 03/10] Replacing smartdata_sas with smartdata_sas.sh --- disk-tooling/smartdata_sas.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 disk-tooling/smartdata_sas.sh diff --git a/disk-tooling/smartdata_sas.sh b/disk-tooling/smartdata_sas.sh new file mode 100644 index 0000000..140fb35 --- /dev/null +++ b/disk-tooling/smartdata_sas.sh @@ -0,0 +1,6 @@ +#!/bin/bash +#Saves smartdata to a temporary file, then uses multiple searches to parse it. + +smartctl -a /dev/$1 > /tmp/smartdata_temp + +cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}' From e4085f57d5fe501cbe4ed92bef0d717d1f4df0bd Mon Sep 17 00:00:00 2001 From: Foster Date: Thu, 7 May 2020 11:53:13 +0100 Subject: [PATCH 04/10] Adding report_blue_scsi script. --- disk-tooling/smartdata_sas | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 disk-tooling/smartdata_sas diff --git a/disk-tooling/smartdata_sas b/disk-tooling/smartdata_sas deleted file mode 100644 index 140fb35..0000000 --- a/disk-tooling/smartdata_sas +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -#Saves smartdata to a temporary file, then uses multiple searches to parse it. - -smartctl -a /dev/$1 > /tmp/smartdata_temp - -cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}' From 7c433d7b0846fae5841a126bdc5a0b87a34172f7 Mon Sep 17 00:00:00 2001 From: Foster Date: Tue, 12 May 2020 11:06:41 +0100 Subject: [PATCH 05/10] Adding report_blue_scsi --- disk-tooling/report_blue_scsi.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 disk-tooling/report_blue_scsi.sh diff --git a/disk-tooling/report_blue_scsi.sh b/disk-tooling/report_blue_scsi.sh new file mode 100644 index 0000000..f953afa --- /dev/null +++ b/disk-tooling/report_blue_scsi.sh @@ -0,0 +1,9 @@ +#!/bin/bash +#Prints out parsed smartdata for bluestore SCSI drives. +host=$(ceph osd find $1 | jq -r '.crush_location.host') +dev=$(ceph osd metadata $1 | jq -r '.bluestore_bdev_partition_path' | cut -d '/' -f3) + +echo $host + +ssh $host smartctl -a -d scsi /dev/$dev > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; + From fda52af02fa9656c793ad82d0bd8ca94d200c24f Mon Sep 17 00:00:00 2001 From: Foster Date: Thu, 14 May 2020 10:14:22 +0100 Subject: [PATCH 06/10] Adding report_osd_alltypes, which prints out a report for all different types of drive and OSD. --- disk-tooling/report_osd_alltypes.sh | 85 +++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 disk-tooling/report_osd_alltypes.sh diff --git a/disk-tooling/report_osd_alltypes.sh b/disk-tooling/report_osd_alltypes.sh new file mode 100644 index 0000000..27fba9a --- /dev/null +++ b/disk-tooling/report_osd_alltypes.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +if [[ $(($1)) != $1 ]]; then + echo "Error: Input needs to be an OSD number" + exit 1 +fi + +host=$(ceph osd find $1 | jq -r '.crush_location.host') +obj=$(ceph osd metadata $1 | jq -r '.osd_objectstore') + +multipath_check() { +if ssh $1 multipath -ll ; then + isMultipath=true +else + isMultipath=false +fi +} + +multipath_check $host &> /dev/null + +echo "Hostname is ${host}" + +if [ "$obj" = 'filestore' ]; then +echo "OSD is filestore" + +dev=$(ceph osd metadata $1 | jq -r '.backend_filestore_partition_path' | cut -d '/' -f3) +type=$(ssh $host udevadm info --query=property /dev/$dev | awk -F= -v key="ID_BUS" '$1==key {print $2}') + + if [ "$type" = 'scsi' ]; then + + ssh $host smartctl -a -d scsi /dev/$dev > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; + + elif [ "$type" = 'ata' ]; then + + ssh $host smartctl -a /dev/$dev | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; + + else + echo "Error: unknown type of disk (Not ATA or SCSI)" + + fi + +elif [ "$obj" = 'bluestore' ]; then +echo "OSD is bluestore" + +vg=$(ssh $host ls -l /var/lib/ceph/osd/ceph-$1 | grep /dev | cut -d '/' -f3) + +if [[ "$isMultipath" == true ]]; then +echo "Host is multipath" + +part=$(ssh $host pvs --select vg_name=$vg | grep /dev/mapper | cut -d '/' -f4 | cut -d ' ' -f1) +type=$(ssh $host udevadm info --query=property /dev/mapper/$part | awk -F= -v key="ID_BUS" '$1==key {print $2}') + + if [ "$type" = 'ata' ]; then + + ssh $host smartctl -a /dev/mapper/$part | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; + + else + + ssh $host smartctl -a -d scsi /dev/mapper/$part > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; + + fi + +elif [[ "$isMultipath" == false ]]; then +echo "Host is not multipath" + + part=$(ssh $host pvs --select vg_name=$vg | grep /dev | cut -d '/' -f3 | cut -d ' ' -f1) + type=$(ssh $host udevadm info --query=property /dev/$part | awk -F= -v key="ID_BUS" '$1==key {print $2}') + + if [ "$type" = 'ata' ]; then + + ssh $host smartctl -a /dev/$part | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; + + else + + ssh $host smartctl -a -d scsi /dev/$part > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; + + fi +else +echo "Error: Could not detect if host is multipath or not" + +fi +else +echo "Error: OSD is neither filestore nor bluestore" + +fi From 460d8c20cc164e166b4c78186730455464b5609a Mon Sep 17 00:00:00 2001 From: Foster Date: Thu, 21 May 2020 11:10:01 +0100 Subject: [PATCH 07/10] Updated report_osd_alltypes.sh with exception handling --- disk-tooling/report_osd_alltypes.sh | 31 +++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/disk-tooling/report_osd_alltypes.sh b/disk-tooling/report_osd_alltypes.sh index 27fba9a..6396092 100644 --- a/disk-tooling/report_osd_alltypes.sh +++ b/disk-tooling/report_osd_alltypes.sh @@ -6,8 +6,17 @@ if [[ $(($1)) != $1 ]]; then fi host=$(ceph osd find $1 | jq -r '.crush_location.host') +if [ -z "$host" ]; then +echo "Error: Could not find host location" +exit 1 +fi obj=$(ceph osd metadata $1 | jq -r '.osd_objectstore') +if [ -z "$obj" ]; then +echo "Error: Could not find OSD objectstore information" +exit 1 +fi +#Identifies multipath host by running command conditionally multipath_check() { if ssh $1 multipath -ll ; then isMultipath=true @@ -15,7 +24,7 @@ else isMultipath=false fi } - +#Sends output to /dev/null to prevent it from displaying in console multipath_check $host &> /dev/null echo "Hostname is ${host}" @@ -23,20 +32,26 @@ echo "Hostname is ${host}" if [ "$obj" = 'filestore' ]; then echo "OSD is filestore" -dev=$(ceph osd metadata $1 | jq -r '.backend_filestore_partition_path' | cut -d '/' -f3) -type=$(ssh $host udevadm info --query=property /dev/$dev | awk -F= -v key="ID_BUS" '$1==key {print $2}') +if [[ "$isMultipath" == true ]]; then +echo "Host is multipath" +else +echo "Host is not multipath" +fi + +path=$(ceph osd metadata $1 | jq -r '.backend_filestore_partition_path' | cut -d '/' -f3) +disktype=$(ssh $host udevadm info --query=property /dev/$path | awk -F= -v key="ID_BUS" '$1==key {print $2}') - if [ "$type" = 'scsi' ]; then + if [ "$disktype" = 'scsi' ]; then - ssh $host smartctl -a -d scsi /dev/$dev > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; + ssh $host smartctl -a -d scsi /dev/$path > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; - elif [ "$type" = 'ata' ]; then + elif [ "$disktype" = 'ata' ]; then - ssh $host smartctl -a /dev/$dev | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; + ssh $host smartctl -a /dev/$path | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; else echo "Error: unknown type of disk (Not ATA or SCSI)" - + exit 1 fi elif [ "$obj" = 'bluestore' ]; then From a22398134965211b68d689f018cbc1b901a6d536 Mon Sep 17 00:00:00 2001 From: Foster Date: Thu, 25 Jun 2020 14:38:19 +0100 Subject: [PATCH 08/10] Updated report_osd_alltypes, adding osdsearch and ticket-drive. --- disk-tooling/osdsearch.sh | 33 +++++ disk-tooling/report_osd_alltypes.sh | 191 ++++++++++++++++------------ disk-tooling/ticket-drive.sh | 33 +++++ 3 files changed, 178 insertions(+), 79 deletions(-) create mode 100644 disk-tooling/osdsearch.sh create mode 100644 disk-tooling/ticket-drive.sh diff --git a/disk-tooling/osdsearch.sh b/disk-tooling/osdsearch.sh new file mode 100644 index 0000000..cf416e5 --- /dev/null +++ b/disk-tooling/osdsearch.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +host=$2 +multipath_check() { +if timeout 10 ssh $1 multipath -ll ; then + isMultipath=true +else + isMultipath=false +fi +} + +multipath_check $host &> /dev/null + +path="" +parsedpath="" + +if [[ "$isMultipath" == true ]]; then + + for i in $(ssh $host "ls /dev/mapper/*" | grep -v [0-9] | grep -v control ); do if ( ssh $host smartctl -a -d scsi $i | grep $1 ); then path="$i"; break; fi; done + + vg=$(ssh $host pvs | grep $path | awk '{print $2}' ) + osd=$(ssh $host ls -l /var/lib/ceph/osd/ceph-*/block 2>/dev/null | grep "$vg" 2>/dev/null | cut -d'/' -f6 | cut -d'-' -f2) + parsedpath=$(ssh $host echo "$path" | cut -d '/' -f4) + +else + for i in $(ssh $host "ls /dev/sd*" | grep -v [0-9] ); do if ( ssh $host smartctl -a $i | grep $1 ); then path="$i"; break; fi; done + + osd=$(ssh $host df | grep "${path}1"| cut -d'-' -f2) + parsedpath=$(ssh $host echo "${path%1}" | cut -d '/' -f3) +fi + +echo "$parsedpath" +echo "$osd" diff --git a/disk-tooling/report_osd_alltypes.sh b/disk-tooling/report_osd_alltypes.sh index 6396092..d021e14 100644 --- a/disk-tooling/report_osd_alltypes.sh +++ b/disk-tooling/report_osd_alltypes.sh @@ -1,100 +1,133 @@ #!/bin/bash -if [[ $(($1)) != $1 ]]; then - echo "Error: Input needs to be an OSD number" - exit 1 +#Input sanitization, uses a regular expression which will fail if input contains non-integer characters +if ! [[ "$1" =~ ^[0-9]+$ ]]; then + echo "Error: Input needs to be a valid OSD number" + exit 1 fi -host=$(ceph osd find $1 | jq -r '.crush_location.host') +#Locates the host +host=$(timeout 10 ceph osd find $1 | jq -r '.crush_location.host') if [ -z "$host" ]; then -echo "Error: Could not find host location" -exit 1 + echo "Error: Could not find host location" + exit 1 fi -obj=$(ceph osd metadata $1 | jq -r '.osd_objectstore') +#Finds objectstore type +obj=$(timeout 10 ceph osd metadata $1 | jq -r '.osd_objectstore') if [ -z "$obj" ]; then -echo "Error: Could not find OSD objectstore information" -exit 1 + echo "Error: Could not find OSD objectstore information" + exit 1 +fi +#Checks if SSH successfully connects, exits script if it doesn't +if ! timeout 10 ssh $host exit; then + echo "Error: SSH connection failed" + exit 1 +fi +#Identifies the hardware generation of host +gen=$(timeout 10 ssh $host /usr/sbin/ccm --format json /hardware | jq -r '.model') +if [ -z "$gen" ]; then + echo "Error: Could not identify hardware generation" + exit 1 fi -#Identifies multipath host by running command conditionally +#Multipath identification +#multipath -ll will fail if used on a non-multipath host, so the command is run conditionally +#isMultipath variable is set to true or false depending on if the command succeeded or not multipath_check() { -if ssh $1 multipath -ll ; then - isMultipath=true +if timeout 10 ssh $1 multipath -ll ; then + isMultipath=true else - isMultipath=false + isMultipath=false fi } -#Sends output to /dev/null to prevent it from displaying in console +#multipath -ll output is sent to /dev/null to prevent it from displaying in console window +#The command output isn't relevant, just the command's success or failure multipath_check $host &> /dev/null -echo "Hostname is ${host}" +echo "Hostname: ${host}" +echo "Hardware Generation: ${gen}" if [ "$obj" = 'filestore' ]; then -echo "OSD is filestore" - -if [[ "$isMultipath" == true ]]; then -echo "Host is multipath" -else -echo "Host is not multipath" -fi - -path=$(ceph osd metadata $1 | jq -r '.backend_filestore_partition_path' | cut -d '/' -f3) -disktype=$(ssh $host udevadm info --query=property /dev/$path | awk -F= -v key="ID_BUS" '$1==key {print $2}') - - if [ "$disktype" = 'scsi' ]; then - - ssh $host smartctl -a -d scsi /dev/$path > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; - - elif [ "$disktype" = 'ata' ]; then - - ssh $host smartctl -a /dev/$path | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; - - else - echo "Error: unknown type of disk (Not ATA or SCSI)" - exit 1 - fi + echo "OSD: Filestore" + + if [[ "$isMultipath" == true ]]; then + echo "Multipath: Yes " + else + echo "Multipath: No" + fi + #Locates the partition path i.e. sda, then uses udev info to identify if the disk type is SATA or SAS + path=$(timeout 10 ceph osd metadata $1 | jq -r '.backend_filestore_partition_path' | cut -d '/' -f3) + disktype=$(timeout 10 ssh $host udevadm info --query=property /dev/$path | awk -F= -v key="ID_BUS" '$1==key {print $2}') + + if [ "$disktype" = 'scsi' ]; then + #Filestore SAS + if ! timeout 10 ssh $host smartctl -a -d scsi /dev/$path > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; then + echo "Error: smartctl failed to find path" + exit 1 + fi + + elif [ "$disktype" = 'ata' ]; then + #Filestore SATA + if ! timeout 10 ssh $host smartctl -a /dev/$path | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; then + echo "Error smartctl failed to find path" + exit 1 + fi + + else + #Exception check + echo "Error: unknown type of disk (Not ATA or SCSI)" + exit 1 + fi elif [ "$obj" = 'bluestore' ]; then -echo "OSD is bluestore" - -vg=$(ssh $host ls -l /var/lib/ceph/osd/ceph-$1 | grep /dev | cut -d '/' -f3) - -if [[ "$isMultipath" == true ]]; then -echo "Host is multipath" - -part=$(ssh $host pvs --select vg_name=$vg | grep /dev/mapper | cut -d '/' -f4 | cut -d ' ' -f1) -type=$(ssh $host udevadm info --query=property /dev/mapper/$part | awk -F= -v key="ID_BUS" '$1==key {print $2}') - - if [ "$type" = 'ata' ]; then - - ssh $host smartctl -a /dev/mapper/$part | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; - - else - - ssh $host smartctl -a -d scsi /dev/mapper/$part > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; - - fi - -elif [[ "$isMultipath" == false ]]; then -echo "Host is not multipath" - - part=$(ssh $host pvs --select vg_name=$vg | grep /dev | cut -d '/' -f3 | cut -d ' ' -f1) - type=$(ssh $host udevadm info --query=property /dev/$part | awk -F= -v key="ID_BUS" '$1==key {print $2}') - - if [ "$type" = 'ata' ]; then - - ssh $host smartctl -a /dev/$part | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; - - else - - ssh $host smartctl -a -d scsi /dev/$part > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; - - fi + echo "OSD: Bluestore" + + vg=$(timeout 10 ssh $host ls -l /var/lib/ceph/osd/ceph-$1 | grep /dev | cut -d '/' -f3) + + if [[ "$isMultipath" == true ]]; then + echo "Multipath: Yes" + #Uses the volume group to find the /dev/mapper partition path, then udev to identify SATA or SAS + part=$(timeout 10 ssh $host pvs --select vg_name=$vg | grep /dev/mapper | cut -d '/' -f4 | cut -d ' ' -f1) + disktype=$(timeout 10 ssh $host udevadm info --query=property /dev/mapper/$part | awk -F= -v key="ID_BUS" '$1==key {print $2}') + + if [ "$disktype" = 'ata' ]; then + #Bluestore multipath SATA + timeout 10 ssh $host smartctl -a /dev/mapper/$part | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; + if $? > 0; then + echo "Error: smartctl failed to find mpath" + exit 1 + fi + + else + #Bluestore multipath SAS + timeout 10 ssh $host smartctl -a -d scsi /dev/mapper/$part > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; + + fi + + else + echo "Multipath: No" + #Non-multipath uses te same method to identify SATA/SAS, but the parsing is different + part=$(timeout 10 ssh $host pvs --select vg_name=$vg | grep /dev | cut -d '/' -f3 | cut -d ' ' -f1) + disktype=$(timeout 10 ssh $host udevadm info --query=property /dev/$part | awk -F= -v key="ID_BUS" '$1==key {print $2}') + + if [ "$disktype" = 'ata' ]; then + #Bluestore non-multipath SATA + if ! timeout 10 ssh $host smartctl -a /dev/$part | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; then + echo "Error: smartctl failed to find path" + exit 1 + fi + + else + #Bluestore non-multipath SAS + if ! timeout 10 ssh $host smartctl -a -d scsi /dev/$part > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; then + echo "Error: smartctl failed to find path" + exit 1 + fi + + fi + fi else -echo "Error: Could not detect if host is multipath or not" - -fi -else -echo "Error: OSD is neither filestore nor bluestore" - + #Exception check + echo "Error: OSD is neither filestore nor bluestore" + exit 1 fi diff --git a/disk-tooling/ticket-drive.sh b/disk-tooling/ticket-drive.sh new file mode 100644 index 0000000..a90ba0b --- /dev/null +++ b/disk-tooling/ticket-drive.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +host=$2 +echo "Creating ticket..." +source ~/jamesf/osdsearch $1 $2 &> /dev/null +source ~/jamesf/report_osd_alltypes $osd > /tmp/osdreport +echo "" >> /tmp/osdreport +~/tom/check_osd.sh $2 $parsedpath >> /tmp/osdreport + +echo "Status: new" > /tmp/driveticket +echo "Subject: Disk Replacement $1 on $2" >> /tmp/driveticket +echo "" >> /tmp/driveticket +echo -e "This drive has been ticketed for replacement using the ticket-drive script.\n" >> /tmp/driveticket +echo "Path: $parsedpath" >> /tmp/driveticket +echo "$(cat /tmp/osdreport)" >> /tmp/driveticket +cat /tmp/driveticket + +read -p "Submit this ticket? (Y/N) " -n 1 -r +if [[ $REPLY =~ ^[Yy]$ ]] +then + if ! sendmail ceph-disk@helpdesk.gridpp.rl.ac.uk < /tmp/driveticket + then + echo -e "\nTicket submission failed. (Ceph-Disk)" + fi + if ! sendmail hardware@helpdesk.gridpp.rl.ac.uk < /tmp/driveticket + then + echo -e "\nTicket submission failed. (Fabric-Hardware)" + fi + echo -e "\nTicket submitted to Ceph-Disk and Fabric-Hardware successfully!" +else + echo -e "\nTicket was not submitted." + exit 1 +fi From 0ac2b4aaf502cb2713f0fe193db3d72d4740195a Mon Sep 17 00:00:00 2001 From: Foster Date: Thu, 9 Jul 2020 11:25:39 +0100 Subject: [PATCH 09/10] Updating osdsearch and ticket-drive with comments and final exception handling changes. --- disk-tooling/osdsearch.sh | 75 +++++++++++++++++++++++++++++------- disk-tooling/ticket-drive.sh | 19 +++++++-- 2 files changed, 78 insertions(+), 16 deletions(-) diff --git a/disk-tooling/osdsearch.sh b/disk-tooling/osdsearch.sh index cf416e5..8a5f1be 100644 --- a/disk-tooling/osdsearch.sh +++ b/disk-tooling/osdsearch.sh @@ -1,6 +1,14 @@ #!/bin/bash +#Checks if host can be connected to host=$2 +if ! timeout 10 ssh $host exit; then + echo "Error: SSH connection failed" + exit 1 +fi +#Multipath identification +#multipath -ll will fail if used on a non-multipath host, so the command is run conditionally +#isMultipath variable is set to true or false depending on if the command succeeded or not multipath_check() { if timeout 10 ssh $1 multipath -ll ; then isMultipath=true @@ -8,26 +16,67 @@ else isMultipath=false fi } - +#multipath -ll output is sent to /dev/null to prevent it from displaying in console window +#The command output isn't relevant, just the command's success or failure multipath_check $host &> /dev/null path="" parsedpath="" +echo "Searching for OSD..." if [[ "$isMultipath" == true ]]; then - - for i in $(ssh $host "ls /dev/mapper/*" | grep -v [0-9] | grep -v control ); do if ( ssh $host smartctl -a -d scsi $i | grep $1 ); then path="$i"; break; fi; done - - vg=$(ssh $host pvs | grep $path | awk '{print $2}' ) - osd=$(ssh $host ls -l /var/lib/ceph/osd/ceph-*/block 2>/dev/null | grep "$vg" 2>/dev/null | cut -d'/' -f6 | cut -d'-' -f2) - parsedpath=$(ssh $host echo "$path" | cut -d '/' -f4) +#If the host is multipath, search for /dev/mapper + for i in $(timeout 10 ssh $host "ls /dev/mapper/*" | grep -v [0-9] | grep -v control ); do if (timeout 10 ssh $host smartctl -a -d scsi $i | grep $1 ); then path="$i"; break; fi; done +#Use the path to find the VG + vg=$(timeout 10 ssh $host pvs | grep $path | awk '{print $2}' ) + if [ -z "$vg" ]; then + echo "Error: Volume Group not found" + exit 1 + fi +#Use the VG to search for the OSD number + osd=$(timeout 10 ssh $host ls -l /var/lib/ceph/osd/ceph-*/block 2>/dev/null | grep "$vg" 2>/dev/null | cut -d'/' -f6 | cut -d'-' -f2) +#OSD sanitization + if [ -z "$osd" ]; then + echo "Error: OSD not found" + exit 1 + fi +#Parsing the pathname to only have the last part, i.e mpatha + parsedpath=$(timeout 10 ssh $host echo "$path" | cut -d '/' -f4) + if [ -z "$parsedpath" ]; then + echo "Error: Could not parse path correctly" + exit 1 + fi else - for i in $(ssh $host "ls /dev/sd*" | grep -v [0-9] ); do if ( ssh $host smartctl -a $i | grep $1 ); then path="$i"; break; fi; done - - osd=$(ssh $host df | grep "${path}1"| cut -d'-' -f2) - parsedpath=$(ssh $host echo "${path%1}" | cut -d '/' -f3) +#If the host is not multipath, search for /dev/sd +#The pathname can then be used to search directly for the OSD + for i in $(timeout 10 ssh $host "ls /dev/sd*" | grep -v [0-9] ); do if (timeout 10 ssh $host smartctl -a $i | grep $1 ); then path="$i"; break; fi; done + osd=$(timeout 10 ssh $host df | grep "${path}1"| cut -d'-' -f2) +#If this fails, the OSD is a bluestore + if [ -z "$osd" ]; then +#Search for the VG using the pathname instead, then use the VG to find the OSD + vg=$(timeout 10 ssh $host pvs | grep $path | awk '{print $2}' ) + if [ -z "$vg" ]; then + echo "Error: Volume Group not found" + exit 1 + fi + osd=$(timeout 10 ssh $host ls -l /var/lib/ceph/osd/ceph-*/block 2>/dev/null | grep "$vg" 2>/dev/null | cut -d'/' -f6 | cut -d'-' -f2) + if [ -z "$osd" ]; then + echo "Error: OSD not found" + exit 1 + fi + fi + if ! [[ "$osd" =~ ^[0-9]+$ ]]; then + echo "Error: OSD not found" + exit 1 + fi +#Parsing the pathname to only have the last part, i.e sdaa + parsedpath=$(timeout 10 ssh $host echo "${path%1}" | cut -d '/' -f3) + if [ -z "$parsedpath" ]; then + echo "Error: Path parsing failed (OSD not found)" + exit 1 + fi fi -echo "$parsedpath" -echo "$osd" +echo "${parsedpath} is the path" +echo "${osd} is the OSD number" diff --git a/disk-tooling/ticket-drive.sh b/disk-tooling/ticket-drive.sh index a90ba0b..c1a911d 100644 --- a/disk-tooling/ticket-drive.sh +++ b/disk-tooling/ticket-drive.sh @@ -1,12 +1,22 @@ #!/bin/bash +#Checks if the host can be connected to host=$2 +if ! timeout 10 ssh $host exit; then + echo "Error: SSH connection failed" + exit 1 +fi +#Uses osdsearch to find the osd and parsed pathname +source ~/jamesf/osdsearch $1 $2 echo "Creating ticket..." -source ~/jamesf/osdsearch $1 $2 &> /dev/null +#Uses report_osd_alltypes to print a report of the osd source ~/jamesf/report_osd_alltypes $osd > /tmp/osdreport echo "" >> /tmp/osdreport +#Uses check_osd to verify if osd is ready to be removed ~/tom/check_osd.sh $2 $parsedpath >> /tmp/osdreport +#Creates the ticket in specific format +echo "" echo "Status: new" > /tmp/driveticket echo "Subject: Disk Replacement $1 on $2" >> /tmp/driveticket echo "" >> /tmp/driveticket @@ -15,16 +25,19 @@ echo "Path: $parsedpath" >> /tmp/driveticket echo "$(cat /tmp/osdreport)" >> /tmp/driveticket cat /tmp/driveticket +#Submits ticket via sendmail read -p "Submit this ticket? (Y/N) " -n 1 -r if [[ $REPLY =~ ^[Yy]$ ]] then - if ! sendmail ceph-disk@helpdesk.gridpp.rl.ac.uk < /tmp/driveticket + if ! timeout 10 sendmail ceph-disk@helpdesk.gridpp.rl.ac.uk < /tmp/driveticket then echo -e "\nTicket submission failed. (Ceph-Disk)" + exit fi - if ! sendmail hardware@helpdesk.gridpp.rl.ac.uk < /tmp/driveticket + if ! timeout 10 sendmail hardware@helpdesk.gridpp.rl.ac.uk < /tmp/driveticket then echo -e "\nTicket submission failed. (Fabric-Hardware)" + exit fi echo -e "\nTicket submitted to Ceph-Disk and Fabric-Hardware successfully!" else From f5886f0410876d7df48b2f47d749e54d9b252365 Mon Sep 17 00:00:00 2001 From: Foster Date: Thu, 6 Aug 2020 13:23:51 +0100 Subject: [PATCH 10/10] Adding most up-to-date versions of various tooling scripts. --- disk-tooling/linktickets.sh | 50 +++++++++++++++++++++++++++++ disk-tooling/osdsearch.sh | 20 +++++++----- disk-tooling/report_osd_alltypes.sh | 21 ++++++------ disk-tooling/ticket-drive.sh | 16 ++++++++- disk-tooling/toggleLED.sh | 42 ++++++++++++++++++++++++ 5 files changed, 131 insertions(+), 18 deletions(-) create mode 100644 disk-tooling/linktickets.sh create mode 100644 disk-tooling/toggleLED.sh diff --git a/disk-tooling/linktickets.sh b/disk-tooling/linktickets.sh new file mode 100644 index 0000000..0e4092a --- /dev/null +++ b/disk-tooling/linktickets.sh @@ -0,0 +1,50 @@ +#!/bin/bash +serial=$1 +#RT credentials +read -p "Enter RT Username: " USERNAME +read -s -p "Enter RT Password: " PASSWORD ; echo "" +#Uses RT credentials to get a list of tickets in Ceph-Disk and Fabric-Hardware queues, obtaining the ticket ID and the subject. +#Saves these queues to a tmp file, sorted by newest ticket at the top. +if ! wget --keep-session-cookies --save-cookies /tmp/cookies.txt --post-data="user=$USERNAME&pass=$PASSWORD" --no-check-certificate -O /tmp/cephdiskqueue https://helpdesk.gridpp.rl.ac.uk/REST/1.0/search/ticket?query="Queue='Ceph-Disk'&orderby=-Created" &> /dev/null ; then + echo "Failed to connect to RT" + exit 1 +fi +if ! wget --keep-session-cookies --save-cookies /tmp/cookies.txt --post-data="user=$USERNAME&pass=$PASSWORD" --no-check-certificate -O /tmp/hardwarequeue https://helpdesk.gridpp.rl.ac.uk/REST/1.0/search/ticket?query="Queue='Fabric-Hardware'&orderby=-Created" &> /dev/null ; then + echo "Failed to connect to RT" + exit 1 +fi +#Exits if credentials were incorrect. +if grep -q "Credentials required" /tmp/cephdiskqueue; then + echo "RT Username or Password are incorrect." + exit 1 +fi +if grep -q "Credentials required" /tmp/hardwarequeue; then + echo "RT Username or Password are incorrect." + exit 1 +fi +#The ticket list is then searched by subject to find the appropriate ticket. When a match is found, the search ends. +#If a match isn't found, it ends the script. +#The ID of the ticket is then parsed from the list. +cephTicketID="$(grep "$serial" /tmp/cephdiskqueue | awk -F: '{print $1}' | head -1)" +if [ -z "$cephTicketID" ]; then + echo "Ceph-Disk Ticket ID not found." + exit 1 +fi +fabricTicketID="$(grep "$serial" /tmp/hardwarequeue | awk -F: '{print $1}' | head -1)" +if [ -z "$fabricTicketID" ]; then + echo "Fabric-Hardware Ticket ID not found." + exit 1 +fi + +echo "Ceph-Disk Ticket ID: ${cephTicketID}" +echo "Fabric-Hardware Ticket ID: ${fabricTicketID}" + +#Makes the Ceph-Disk ticket depend on the Fabric-Hardware ticket by posting the ticketlink file. +echo "DependsOn: $fabricTicketID" > /tmp/ticketlink +if ! curl -k --data-urlencode content@/tmp/ticketlink "https://helpdesk.gridpp.rl.ac.uk/REST/1.0/ticket/$cephTicketID/links?user=$USERNAME&pass=$PASSWORD" ; then + echo "Failed to curl ticketlink file to RT" + exit 1 +fi + +#Unsets the password variable for security. +unset PASSWORD diff --git a/disk-tooling/osdsearch.sh b/disk-tooling/osdsearch.sh index 8a5f1be..8fb6143 100644 --- a/disk-tooling/osdsearch.sh +++ b/disk-tooling/osdsearch.sh @@ -7,18 +7,18 @@ if ! timeout 10 ssh $host exit; then exit 1 fi #Multipath identification -#multipath -ll will fail if used on a non-multipath host, so the command is run conditionally -#isMultipath variable is set to true or false depending on if the command succeeded or not -multipath_check() { -if timeout 10 ssh $1 multipath -ll ; then +#Compares the output of pvs on the host to see if it contains mapper. +#If it does, the host is multipath. If not, it is not. +if ! ssh $host pvs &> /dev/null ; then + echo "Error: Could not run pvs for multipath check" + exit 1 +fi +multicheck=$(timeout 10 ssh $host pvs 2>/dev/null) +if echo "$multicheck" | grep -q "mapper" ; then isMultipath=true else isMultipath=false fi -} -#multipath -ll output is sent to /dev/null to prevent it from displaying in console window -#The command output isn't relevant, just the command's success or failure -multipath_check $host &> /dev/null path="" parsedpath="" @@ -40,6 +40,10 @@ if [[ "$isMultipath" == true ]]; then echo "Error: OSD not found" exit 1 fi + if ! [[ "$osd" =~ ^[0-9]+$ ]]; then + echo "Error: OSD not found" + exit 1 + fi #Parsing the pathname to only have the last part, i.e mpatha parsedpath=$(timeout 10 ssh $host echo "$path" | cut -d '/' -f4) if [ -z "$parsedpath" ]; then diff --git a/disk-tooling/report_osd_alltypes.sh b/disk-tooling/report_osd_alltypes.sh index d021e14..60d0bec 100644 --- a/disk-tooling/report_osd_alltypes.sh +++ b/disk-tooling/report_osd_alltypes.sh @@ -31,18 +31,18 @@ if [ -z "$gen" ]; then fi #Multipath identification -#multipath -ll will fail if used on a non-multipath host, so the command is run conditionally -#isMultipath variable is set to true or false depending on if the command succeeded or not -multipath_check() { -if timeout 10 ssh $1 multipath -ll ; then +#Compares the output of pvs on the host to see if it contains mapper. +#If it does, the host is multipath. If not, it is not. +if ! ssh $host pvs &> /dev/null ; then + echo "Error: Could not run pvs for multipath check" + exit 1 +fi +multicheck=$(timeout 10 ssh $host pvs 2>/dev/null) +if echo "$multicheck" | grep -q "mapper" ; then isMultipath=true else isMultipath=false fi -} -#multipath -ll output is sent to /dev/null to prevent it from displaying in console window -#The command output isn't relevant, just the command's success or failure -multipath_check $host &> /dev/null echo "Hostname: ${host}" echo "Hardware Generation: ${gen}" @@ -57,6 +57,7 @@ if [ "$obj" = 'filestore' ]; then fi #Locates the partition path i.e. sda, then uses udev info to identify if the disk type is SATA or SAS path=$(timeout 10 ceph osd metadata $1 | jq -r '.backend_filestore_partition_path' | cut -d '/' -f3) + echo "Path: /dev/$path" disktype=$(timeout 10 ssh $host udevadm info --query=property /dev/$path | awk -F= -v key="ID_BUS" '$1==key {print $2}') if [ "$disktype" = 'scsi' ]; then @@ -88,6 +89,7 @@ elif [ "$obj" = 'bluestore' ]; then echo "Multipath: Yes" #Uses the volume group to find the /dev/mapper partition path, then udev to identify SATA or SAS part=$(timeout 10 ssh $host pvs --select vg_name=$vg | grep /dev/mapper | cut -d '/' -f4 | cut -d ' ' -f1) + echo "Path: /dev/mapper/$part" disktype=$(timeout 10 ssh $host udevadm info --query=property /dev/mapper/$part | awk -F= -v key="ID_BUS" '$1==key {print $2}') if [ "$disktype" = 'ata' ]; then @@ -106,8 +108,9 @@ elif [ "$obj" = 'bluestore' ]; then else echo "Multipath: No" - #Non-multipath uses te same method to identify SATA/SAS, but the parsing is different + #Non-multipath uses the same method to identify SATA/SAS, but the parsing is different part=$(timeout 10 ssh $host pvs --select vg_name=$vg | grep /dev | cut -d '/' -f3 | cut -d ' ' -f1) + echo "Path: /dev/$part" disktype=$(timeout 10 ssh $host udevadm info --query=property /dev/$part | awk -F= -v key="ID_BUS" '$1==key {print $2}') if [ "$disktype" = 'ata' ]; then diff --git a/disk-tooling/ticket-drive.sh b/disk-tooling/ticket-drive.sh index c1a911d..cc0cbd2 100644 --- a/disk-tooling/ticket-drive.sh +++ b/disk-tooling/ticket-drive.sh @@ -21,7 +21,6 @@ echo "Status: new" > /tmp/driveticket echo "Subject: Disk Replacement $1 on $2" >> /tmp/driveticket echo "" >> /tmp/driveticket echo -e "This drive has been ticketed for replacement using the ticket-drive script.\n" >> /tmp/driveticket -echo "Path: $parsedpath" >> /tmp/driveticket echo "$(cat /tmp/osdreport)" >> /tmp/driveticket cat /tmp/driveticket @@ -44,3 +43,18 @@ else echo -e "\nTicket was not submitted." exit 1 fi + +#Uses linktickets to find the ticket IDs and link the two tickets together, then print a URL to them. +echo "Would you like to link the tickets together and receive a URL to them? (Y/N)" +read -p "Note: This will require RT credentials: " -n 1 -r +if [[ $REPLY =~ ^[Yy]$ ]] +then + subject="Disk Replacement $1 on $2" + echo -e "\n" + source ~/jamesf/linktickets $subject + echo "Ceph-Disk ticket URL: https://helpdesk.gridpp.rl.ac.uk/Ticket/Display.html?id=${cephTicketID}" + echo "Fabric-Hardware ticket URL: https://helpdesk.gridpp.rl.ac.uk/Ticket/Display.html?id=${fabricTicketID}" +else + echo -e "\n" + exit 1 +fi diff --git a/disk-tooling/toggleLED.sh b/disk-tooling/toggleLED.sh new file mode 100644 index 0000000..0421c1a --- /dev/null +++ b/disk-tooling/toggleLED.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#The script is run like: ./toggleLED $SERIALNUMBER $HOSTNAME +#Checks if host can be connected to +host=$2 +if ! timeout 10 ssh $host exit; then + echo "Error: SSH connection failed" + exit 1 +fi + +path="" +parsedpath="" +echo "Looking for the path..." +#Searches host for a path linked to a serial number +for i in $(timeout 10 ssh $host "ls /dev/sd*" | grep -v [0-9] ); do if (timeout 10 ssh $host smartctl -a $i | grep $1 ); then path="$i"; break; fi; done &> /dev/null +#Parses the path name to only contain the final part, i.e. sdaa +parsedpath=$(timeout 10 ssh $host echo "${path%1}" | cut -d '/' -f3) +if [ -z "$parsedpath" ]; then + echo "Error: Path parsing failed (Path not found)" + exit 1 +fi +echo "${parsedpath} is the path" +#Checks to see if locate is set to 0 or 1, off or on respectively. +#Then, toggles the LED to be the opposite of what it currently was, i.e. turns off if it was on. +ledStatus=$(timeout 10 ssh $host "cat /sys/class/enclosure/*/*/device/block/$parsedpath/../../enclosure*/locate") +if [[ $ledStatus -eq "0" ]]; then + echo "The drive's LED is currently turned off. Turning LED on..." + if ! timeout 10 ssh $host "echo 1 > /sys/class/enclosure/*/*/device/block/$parsedpath/../../enclosure*/locate" ; then + echo "Error: Failed to make drive LED blink" + exit 1 + fi + echo "Drive LED is now blinking. Re-enter the command again to toggle the LED off." +elif [[ $ledStatus -eq "1" ]]; then + echo "The drive's LED is currently turned on. Turning LED off..." + if ! timeout 10 ssh $host "echo 0 > /sys/class/enclosure/*/*/device/block/$parsedpath/../../enclosure*/locate" ; then + echo "Error: Failed to make drive LED stop blinking" + exit 1 + fi + echo "Drive LED has stopped blinking. Re-enter the command again to toggle the LED on." +else + echo "Error: Couldn't cat the path's enclosure/locate" + exit 1 +fi