diff --git a/disk-recycling/clean-disk.sh b/disk-recycling/clean-disk.sh index 31e9879..caccad2 100644 --- a/disk-recycling/clean-disk.sh +++ b/disk-recycling/clean-disk.sh @@ -1,5 +1,5 @@ #!/bin/bash - +#test comment for git #This script will unmount and wipe the partitions from a drive. if [ $# -ne 1 ]; then diff --git a/disk-tooling/linktickets.sh b/disk-tooling/linktickets.sh new file mode 100644 index 0000000..0e4092a --- /dev/null +++ b/disk-tooling/linktickets.sh @@ -0,0 +1,50 @@ +#!/bin/bash +serial=$1 +#RT credentials +read -p "Enter RT Username: " USERNAME +read -s -p "Enter RT Password: " PASSWORD ; echo "" +#Uses RT credentials to get a list of tickets in Ceph-Disk and Fabric-Hardware queues, obtaining the ticket ID and the subject. +#Saves these queues to a tmp file, sorted by newest ticket at the top. +if ! wget --keep-session-cookies --save-cookies /tmp/cookies.txt --post-data="user=$USERNAME&pass=$PASSWORD" --no-check-certificate -O /tmp/cephdiskqueue https://helpdesk.gridpp.rl.ac.uk/REST/1.0/search/ticket?query="Queue='Ceph-Disk'&orderby=-Created" &> /dev/null ; then + echo "Failed to connect to RT" + exit 1 +fi +if ! wget --keep-session-cookies --save-cookies /tmp/cookies.txt --post-data="user=$USERNAME&pass=$PASSWORD" --no-check-certificate -O /tmp/hardwarequeue https://helpdesk.gridpp.rl.ac.uk/REST/1.0/search/ticket?query="Queue='Fabric-Hardware'&orderby=-Created" &> /dev/null ; then + echo "Failed to connect to RT" + exit 1 +fi +#Exits if credentials were incorrect. +if grep -q "Credentials required" /tmp/cephdiskqueue; then + echo "RT Username or Password are incorrect." + exit 1 +fi +if grep -q "Credentials required" /tmp/hardwarequeue; then + echo "RT Username or Password are incorrect." + exit 1 +fi +#The ticket list is then searched by subject to find the appropriate ticket. When a match is found, the search ends. +#If a match isn't found, it ends the script. +#The ID of the ticket is then parsed from the list. +cephTicketID="$(grep "$serial" /tmp/cephdiskqueue | awk -F: '{print $1}' | head -1)" +if [ -z "$cephTicketID" ]; then + echo "Ceph-Disk Ticket ID not found." + exit 1 +fi +fabricTicketID="$(grep "$serial" /tmp/hardwarequeue | awk -F: '{print $1}' | head -1)" +if [ -z "$fabricTicketID" ]; then + echo "Fabric-Hardware Ticket ID not found." + exit 1 +fi + +echo "Ceph-Disk Ticket ID: ${cephTicketID}" +echo "Fabric-Hardware Ticket ID: ${fabricTicketID}" + +#Makes the Ceph-Disk ticket depend on the Fabric-Hardware ticket by posting the ticketlink file. +echo "DependsOn: $fabricTicketID" > /tmp/ticketlink +if ! curl -k --data-urlencode content@/tmp/ticketlink "https://helpdesk.gridpp.rl.ac.uk/REST/1.0/ticket/$cephTicketID/links?user=$USERNAME&pass=$PASSWORD" ; then + echo "Failed to curl ticketlink file to RT" + exit 1 +fi + +#Unsets the password variable for security. +unset PASSWORD diff --git a/disk-tooling/osdsearch.sh b/disk-tooling/osdsearch.sh new file mode 100644 index 0000000..8fb6143 --- /dev/null +++ b/disk-tooling/osdsearch.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +#Checks if host can be connected to +host=$2 +if ! timeout 10 ssh $host exit; then + echo "Error: SSH connection failed" + exit 1 +fi +#Multipath identification +#Compares the output of pvs on the host to see if it contains mapper. +#If it does, the host is multipath. If not, it is not. +if ! ssh $host pvs &> /dev/null ; then + echo "Error: Could not run pvs for multipath check" + exit 1 +fi +multicheck=$(timeout 10 ssh $host pvs 2>/dev/null) +if echo "$multicheck" | grep -q "mapper" ; then + isMultipath=true +else + isMultipath=false +fi + +path="" +parsedpath="" + +echo "Searching for OSD..." +if [[ "$isMultipath" == true ]]; then +#If the host is multipath, search for /dev/mapper + for i in $(timeout 10 ssh $host "ls /dev/mapper/*" | grep -v [0-9] | grep -v control ); do if (timeout 10 ssh $host smartctl -a -d scsi $i | grep $1 ); then path="$i"; break; fi; done +#Use the path to find the VG + vg=$(timeout 10 ssh $host pvs | grep $path | awk '{print $2}' ) + if [ -z "$vg" ]; then + echo "Error: Volume Group not found" + exit 1 + fi +#Use the VG to search for the OSD number + osd=$(timeout 10 ssh $host ls -l /var/lib/ceph/osd/ceph-*/block 2>/dev/null | grep "$vg" 2>/dev/null | cut -d'/' -f6 | cut -d'-' -f2) +#OSD sanitization + if [ -z "$osd" ]; then + echo "Error: OSD not found" + exit 1 + fi + if ! [[ "$osd" =~ ^[0-9]+$ ]]; then + echo "Error: OSD not found" + exit 1 + fi +#Parsing the pathname to only have the last part, i.e mpatha + parsedpath=$(timeout 10 ssh $host echo "$path" | cut -d '/' -f4) + if [ -z "$parsedpath" ]; then + echo "Error: Could not parse path correctly" + exit 1 + fi + +else +#If the host is not multipath, search for /dev/sd +#The pathname can then be used to search directly for the OSD + for i in $(timeout 10 ssh $host "ls /dev/sd*" | grep -v [0-9] ); do if (timeout 10 ssh $host smartctl -a $i | grep $1 ); then path="$i"; break; fi; done + osd=$(timeout 10 ssh $host df | grep "${path}1"| cut -d'-' -f2) +#If this fails, the OSD is a bluestore + if [ -z "$osd" ]; then +#Search for the VG using the pathname instead, then use the VG to find the OSD + vg=$(timeout 10 ssh $host pvs | grep $path | awk '{print $2}' ) + if [ -z "$vg" ]; then + echo "Error: Volume Group not found" + exit 1 + fi + osd=$(timeout 10 ssh $host ls -l /var/lib/ceph/osd/ceph-*/block 2>/dev/null | grep "$vg" 2>/dev/null | cut -d'/' -f6 | cut -d'-' -f2) + if [ -z "$osd" ]; then + echo "Error: OSD not found" + exit 1 + fi + fi + if ! [[ "$osd" =~ ^[0-9]+$ ]]; then + echo "Error: OSD not found" + exit 1 + fi +#Parsing the pathname to only have the last part, i.e sdaa + parsedpath=$(timeout 10 ssh $host echo "${path%1}" | cut -d '/' -f3) + if [ -z "$parsedpath" ]; then + echo "Error: Path parsing failed (OSD not found)" + exit 1 + fi +fi + +echo "${parsedpath} is the path" +echo "${osd} is the OSD number" diff --git a/disk-tooling/report_blue_scsi.sh b/disk-tooling/report_blue_scsi.sh new file mode 100644 index 0000000..f953afa --- /dev/null +++ b/disk-tooling/report_blue_scsi.sh @@ -0,0 +1,9 @@ +#!/bin/bash +#Prints out parsed smartdata for bluestore SCSI drives. +host=$(ceph osd find $1 | jq -r '.crush_location.host') +dev=$(ceph osd metadata $1 | jq -r '.bluestore_bdev_partition_path' | cut -d '/' -f3) + +echo $host + +ssh $host smartctl -a -d scsi /dev/$dev > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; + diff --git a/disk-tooling/report_osd_alltypes.sh b/disk-tooling/report_osd_alltypes.sh new file mode 100644 index 0000000..60d0bec --- /dev/null +++ b/disk-tooling/report_osd_alltypes.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +#Input sanitization, uses a regular expression which will fail if input contains non-integer characters +if ! [[ "$1" =~ ^[0-9]+$ ]]; then + echo "Error: Input needs to be a valid OSD number" + exit 1 +fi + +#Locates the host +host=$(timeout 10 ceph osd find $1 | jq -r '.crush_location.host') +if [ -z "$host" ]; then + echo "Error: Could not find host location" + exit 1 +fi +#Finds objectstore type +obj=$(timeout 10 ceph osd metadata $1 | jq -r '.osd_objectstore') +if [ -z "$obj" ]; then + echo "Error: Could not find OSD objectstore information" + exit 1 +fi +#Checks if SSH successfully connects, exits script if it doesn't +if ! timeout 10 ssh $host exit; then + echo "Error: SSH connection failed" + exit 1 +fi +#Identifies the hardware generation of host +gen=$(timeout 10 ssh $host /usr/sbin/ccm --format json /hardware | jq -r '.model') +if [ -z "$gen" ]; then + echo "Error: Could not identify hardware generation" + exit 1 +fi + +#Multipath identification +#Compares the output of pvs on the host to see if it contains mapper. +#If it does, the host is multipath. If not, it is not. +if ! ssh $host pvs &> /dev/null ; then + echo "Error: Could not run pvs for multipath check" + exit 1 +fi +multicheck=$(timeout 10 ssh $host pvs 2>/dev/null) +if echo "$multicheck" | grep -q "mapper" ; then + isMultipath=true +else + isMultipath=false +fi + +echo "Hostname: ${host}" +echo "Hardware Generation: ${gen}" + +if [ "$obj" = 'filestore' ]; then + echo "OSD: Filestore" + + if [[ "$isMultipath" == true ]]; then + echo "Multipath: Yes " + else + echo "Multipath: No" + fi + #Locates the partition path i.e. sda, then uses udev info to identify if the disk type is SATA or SAS + path=$(timeout 10 ceph osd metadata $1 | jq -r '.backend_filestore_partition_path' | cut -d '/' -f3) + echo "Path: /dev/$path" + disktype=$(timeout 10 ssh $host udevadm info --query=property /dev/$path | awk -F= -v key="ID_BUS" '$1==key {print $2}') + + if [ "$disktype" = 'scsi' ]; then + #Filestore SAS + if ! timeout 10 ssh $host smartctl -a -d scsi /dev/$path > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; then + echo "Error: smartctl failed to find path" + exit 1 + fi + + elif [ "$disktype" = 'ata' ]; then + #Filestore SATA + if ! timeout 10 ssh $host smartctl -a /dev/$path | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; then + echo "Error smartctl failed to find path" + exit 1 + fi + + else + #Exception check + echo "Error: unknown type of disk (Not ATA or SCSI)" + exit 1 + fi + +elif [ "$obj" = 'bluestore' ]; then + echo "OSD: Bluestore" + + vg=$(timeout 10 ssh $host ls -l /var/lib/ceph/osd/ceph-$1 | grep /dev | cut -d '/' -f3) + + if [[ "$isMultipath" == true ]]; then + echo "Multipath: Yes" + #Uses the volume group to find the /dev/mapper partition path, then udev to identify SATA or SAS + part=$(timeout 10 ssh $host pvs --select vg_name=$vg | grep /dev/mapper | cut -d '/' -f4 | cut -d ' ' -f1) + echo "Path: /dev/mapper/$part" + disktype=$(timeout 10 ssh $host udevadm info --query=property /dev/mapper/$part | awk -F= -v key="ID_BUS" '$1==key {print $2}') + + if [ "$disktype" = 'ata' ]; then + #Bluestore multipath SATA + timeout 10 ssh $host smartctl -a /dev/mapper/$part | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; + if $? > 0; then + echo "Error: smartctl failed to find mpath" + exit 1 + fi + + else + #Bluestore multipath SAS + timeout 10 ssh $host smartctl -a -d scsi /dev/mapper/$part > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; + + fi + + else + echo "Multipath: No" + #Non-multipath uses the same method to identify SATA/SAS, but the parsing is different + part=$(timeout 10 ssh $host pvs --select vg_name=$vg | grep /dev | cut -d '/' -f3 | cut -d ' ' -f1) + echo "Path: /dev/$part" + disktype=$(timeout 10 ssh $host udevadm info --query=property /dev/$part | awk -F= -v key="ID_BUS" '$1==key {print $2}') + + if [ "$disktype" = 'ata' ]; then + #Bluestore non-multipath SATA + if ! timeout 10 ssh $host smartctl -a /dev/$part | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; then + echo "Error: smartctl failed to find path" + exit 1 + fi + + else + #Bluestore non-multipath SAS + if ! timeout 10 ssh $host smartctl -a -d scsi /dev/$part > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; then + echo "Error: smartctl failed to find path" + exit 1 + fi + + fi + fi +else + #Exception check + echo "Error: OSD is neither filestore nor bluestore" + exit 1 +fi diff --git a/disk-tooling/smartdata_sas.sh b/disk-tooling/smartdata_sas.sh new file mode 100644 index 0000000..140fb35 --- /dev/null +++ b/disk-tooling/smartdata_sas.sh @@ -0,0 +1,6 @@ +#!/bin/bash +#Saves smartdata to a temporary file, then uses multiple searches to parse it. + +smartctl -a /dev/$1 > /tmp/smartdata_temp + +cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}' diff --git a/disk-tooling/ticket-drive.sh b/disk-tooling/ticket-drive.sh new file mode 100644 index 0000000..cc0cbd2 --- /dev/null +++ b/disk-tooling/ticket-drive.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +#Checks if the host can be connected to +host=$2 +if ! timeout 10 ssh $host exit; then + echo "Error: SSH connection failed" + exit 1 +fi +#Uses osdsearch to find the osd and parsed pathname +source ~/jamesf/osdsearch $1 $2 +echo "Creating ticket..." +#Uses report_osd_alltypes to print a report of the osd +source ~/jamesf/report_osd_alltypes $osd > /tmp/osdreport +echo "" >> /tmp/osdreport +#Uses check_osd to verify if osd is ready to be removed +~/tom/check_osd.sh $2 $parsedpath >> /tmp/osdreport + +#Creates the ticket in specific format +echo "" +echo "Status: new" > /tmp/driveticket +echo "Subject: Disk Replacement $1 on $2" >> /tmp/driveticket +echo "" >> /tmp/driveticket +echo -e "This drive has been ticketed for replacement using the ticket-drive script.\n" >> /tmp/driveticket +echo "$(cat /tmp/osdreport)" >> /tmp/driveticket +cat /tmp/driveticket + +#Submits ticket via sendmail +read -p "Submit this ticket? (Y/N) " -n 1 -r +if [[ $REPLY =~ ^[Yy]$ ]] +then + if ! timeout 10 sendmail ceph-disk@helpdesk.gridpp.rl.ac.uk < /tmp/driveticket + then + echo -e "\nTicket submission failed. (Ceph-Disk)" + exit + fi + if ! timeout 10 sendmail hardware@helpdesk.gridpp.rl.ac.uk < /tmp/driveticket + then + echo -e "\nTicket submission failed. (Fabric-Hardware)" + exit + fi + echo -e "\nTicket submitted to Ceph-Disk and Fabric-Hardware successfully!" +else + echo -e "\nTicket was not submitted." + exit 1 +fi + +#Uses linktickets to find the ticket IDs and link the two tickets together, then print a URL to them. +echo "Would you like to link the tickets together and receive a URL to them? (Y/N)" +read -p "Note: This will require RT credentials: " -n 1 -r +if [[ $REPLY =~ ^[Yy]$ ]] +then + subject="Disk Replacement $1 on $2" + echo -e "\n" + source ~/jamesf/linktickets $subject + echo "Ceph-Disk ticket URL: https://helpdesk.gridpp.rl.ac.uk/Ticket/Display.html?id=${cephTicketID}" + echo "Fabric-Hardware ticket URL: https://helpdesk.gridpp.rl.ac.uk/Ticket/Display.html?id=${fabricTicketID}" +else + echo -e "\n" + exit 1 +fi diff --git a/disk-tooling/toggleLED.sh b/disk-tooling/toggleLED.sh new file mode 100644 index 0000000..0421c1a --- /dev/null +++ b/disk-tooling/toggleLED.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#The script is run like: ./toggleLED $SERIALNUMBER $HOSTNAME +#Checks if host can be connected to +host=$2 +if ! timeout 10 ssh $host exit; then + echo "Error: SSH connection failed" + exit 1 +fi + +path="" +parsedpath="" +echo "Looking for the path..." +#Searches host for a path linked to a serial number +for i in $(timeout 10 ssh $host "ls /dev/sd*" | grep -v [0-9] ); do if (timeout 10 ssh $host smartctl -a $i | grep $1 ); then path="$i"; break; fi; done &> /dev/null +#Parses the path name to only contain the final part, i.e. sdaa +parsedpath=$(timeout 10 ssh $host echo "${path%1}" | cut -d '/' -f3) +if [ -z "$parsedpath" ]; then + echo "Error: Path parsing failed (Path not found)" + exit 1 +fi +echo "${parsedpath} is the path" +#Checks to see if locate is set to 0 or 1, off or on respectively. +#Then, toggles the LED to be the opposite of what it currently was, i.e. turns off if it was on. +ledStatus=$(timeout 10 ssh $host "cat /sys/class/enclosure/*/*/device/block/$parsedpath/../../enclosure*/locate") +if [[ $ledStatus -eq "0" ]]; then + echo "The drive's LED is currently turned off. Turning LED on..." + if ! timeout 10 ssh $host "echo 1 > /sys/class/enclosure/*/*/device/block/$parsedpath/../../enclosure*/locate" ; then + echo "Error: Failed to make drive LED blink" + exit 1 + fi + echo "Drive LED is now blinking. Re-enter the command again to toggle the LED off." +elif [[ $ledStatus -eq "1" ]]; then + echo "The drive's LED is currently turned on. Turning LED off..." + if ! timeout 10 ssh $host "echo 0 > /sys/class/enclosure/*/*/device/block/$parsedpath/../../enclosure*/locate" ; then + echo "Error: Failed to make drive LED stop blinking" + exit 1 + fi + echo "Drive LED has stopped blinking. Re-enter the command again to toggle the LED on." +else + echo "Error: Couldn't cat the path's enclosure/locate" + exit 1 +fi