James ral ceph tools scripts

disk-recycling/clean-disk.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-
+#test comment for git
 #This script will unmount and wipe the partitions from a drive.
 
 if [ $# -ne 1 ]; then

disk-tooling/linktickets.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+serial=$1
+#RT credentials
+read -p "Enter RT Username: " USERNAME
+read -s -p "Enter RT Password: " PASSWORD ; echo ""
+#Uses RT credentials to get a list of tickets in Ceph-Disk and Fabric-Hardware queues, obtaining the ticket ID and the subject.
+#Saves these queues to a tmp file, sorted by newest ticket at the top.
+if ! wget --keep-session-cookies --save-cookies /tmp/cookies.txt --post-data="user=$USERNAME&pass=$PASSWORD" --no-check-certificate -O /tmp/cephdiskqueue  https://helpdesk.gridpp.rl.ac.uk/REST/1.0/search/ticket?query="Queue='Ceph-Disk'&orderby=-Created" &> /dev/null ; then
+    echo "Failed to connect to RT"
+    exit 1
+fi
+if ! wget --keep-session-cookies --save-cookies /tmp/cookies.txt --post-data="user=$USERNAME&pass=$PASSWORD" --no-check-certificate -O /tmp/hardwarequeue  https://helpdesk.gridpp.rl.ac.uk/REST/1.0/search/ticket?query="Queue='Fabric-Hardware'&orderby=-Created" &> /dev/null ; then
+    echo "Failed to connect to RT"
+    exit 1
+fi
+#Exits if credentials were incorrect.
+if grep -q "Credentials required" /tmp/cephdiskqueue; then
+    echo "RT Username or Password are incorrect."
+    exit 1
+fi
+if grep -q "Credentials required" /tmp/hardwarequeue; then
+    echo "RT Username or Password are incorrect."
+    exit 1
+fi
+#The ticket list is then searched by subject to find the appropriate ticket. When a match is found, the search ends.
+#If a match isn't found, it ends the script.
+#The ID of the ticket is then parsed from the list.
+cephTicketID="$(grep "$serial" /tmp/cephdiskqueue | awk -F: '{print $1}' | head -1)"
+if [ -z "$cephTicketID" ]; then
+    echo "Ceph-Disk Ticket ID not found."
+    exit 1
+fi
+fabricTicketID="$(grep "$serial" /tmp/hardwarequeue | awk -F: '{print $1}' | head -1)"
+if [ -z "$fabricTicketID" ]; then
+    echo "Fabric-Hardware Ticket ID not found."
+    exit 1
+fi
+
+echo "Ceph-Disk Ticket ID: ${cephTicketID}"
+echo "Fabric-Hardware Ticket ID: ${fabricTicketID}"
+
+#Makes the Ceph-Disk ticket depend on the Fabric-Hardware ticket by posting the ticketlink file.
+echo "DependsOn: $fabricTicketID" > /tmp/ticketlink
+if ! curl -k --data-urlencode content@/tmp/ticketlink "https://helpdesk.gridpp.rl.ac.uk/REST/1.0/ticket/$cephTicketID/links?user=$USERNAME&pass=$PASSWORD" ; then
+    echo "Failed to curl ticketlink file to RT"
+    exit 1
+fi
+
+#Unsets the password variable for security.
+unset PASSWORD

disk-tooling/osdsearch.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+#Checks if host can be connected to
+host=$2
+if ! timeout 10 ssh $host exit; then
+    echo "Error: SSH connection failed"
+    exit 1
+fi
+#Multipath identification
+#Compares the output of pvs on the host to see if it contains mapper.
+#If it does, the host is multipath. If not, it is not.
+if ! ssh $host pvs &> /dev/null ; then
+    echo "Error: Could not run pvs for multipath check"
+    exit 1
+fi
+multicheck=$(timeout 10 ssh $host pvs 2>/dev/null)
+if echo "$multicheck" | grep -q "mapper" ; then
+    isMultipath=true
+else
+    isMultipath=false
+fi
+
+path=""
+parsedpath=""
+
+echo "Searching for OSD..."
+if [[ "$isMultipath" == true ]]; then
+#If the host is multipath, search for /dev/mapper
+    for i in $(timeout 10 ssh $host "ls /dev/mapper/*" | grep -v [0-9] | grep -v control ); do if (timeout 10 ssh $host smartctl -a -d scsi $i | grep $1 ); then path="$i"; break; fi; done
+#Use the path to find the VG
+    vg=$(timeout 10 ssh $host pvs | grep $path | awk '{print $2}' )
+    if [ -z "$vg" ]; then
+        echo "Error: Volume Group not found"
+        exit 1
+    fi
+#Use the VG to search for the OSD number
+    osd=$(timeout 10 ssh $host ls -l /var/lib/ceph/osd/ceph-*/block 2>/dev/null | grep "$vg" 2>/dev/null | cut -d'/' -f6 | cut -d'-' -f2)
+#OSD sanitization
+    if [ -z "$osd" ]; then
+        echo "Error: OSD not found"
+        exit 1
+    fi
+    if ! [[ "$osd" =~ ^[0-9]+$ ]]; then
+        echo "Error: OSD not found"
+        exit 1
+    fi
+#Parsing the pathname to only have the last part, i.e mpatha
+    parsedpath=$(timeout 10 ssh $host echo "$path" | cut -d '/' -f4)
+    if [ -z "$parsedpath" ]; then
+        echo "Error: Could not parse path correctly"
+        exit 1
+    fi
+
+else
+#If the host is not multipath, search for /dev/sd
+#The pathname can then be used to search directly for the OSD
+    for i in $(timeout 10 ssh $host "ls /dev/sd*" | grep -v [0-9] ); do if (timeout 10 ssh $host smartctl -a $i | grep $1 ); then path="$i"; break; fi; done
+    osd=$(timeout 10 ssh $host df | grep "${path}1"| cut -d'-' -f2)
+#If this fails, the OSD is a bluestore
+    if [ -z "$osd" ]; then
+#Search for the VG using the pathname instead, then use the VG to find the OSD
+        vg=$(timeout 10 ssh $host pvs | grep $path | awk '{print $2}' )
+        if [ -z "$vg" ]; then
+            echo "Error: Volume Group not found"
+            exit 1
+        fi
+        osd=$(timeout 10 ssh $host ls -l /var/lib/ceph/osd/ceph-*/block 2>/dev/null | grep "$vg" 2>/dev/null | cut -d'/' -f6 | cut -d'-' -f2)
+        if [ -z "$osd" ]; then
+            echo "Error: OSD not found"
+            exit 1
+        fi
+    fi
+    if ! [[ "$osd" =~ ^[0-9]+$ ]]; then
+        echo "Error: OSD not found"
+        exit 1
+    fi
+#Parsing the pathname to only have the last part, i.e sdaa
+    parsedpath=$(timeout 10 ssh $host echo "${path%1}" | cut -d '/' -f3)
+    if [ -z "$parsedpath" ]; then
+        echo "Error: Path parsing failed (OSD not found)"
+        exit 1
+    fi
+fi
+
+echo "${parsedpath} is the path"
+echo "${osd} is the OSD number"

disk-tooling/report_blue_scsi.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+#Prints out parsed smartdata for bluestore SCSI drives.
+host=$(ceph osd find $1 | jq -r '.crush_location.host')
+dev=$(ceph osd metadata $1 | jq -r '.bluestore_bdev_partition_path' | cut -d '/' -f3)
+
+echo $host
+
+ssh $host smartctl -a -d scsi /dev/$dev > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}';
+

disk-tooling/report_osd_alltypes.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+
+#Input sanitization, uses a regular expression which will fail if input contains non-integer characters
+if ! [[ "$1" =~ ^[0-9]+$ ]]; then
+    echo "Error: Input needs to be a valid OSD number"
+    exit 1
+fi
+
+#Locates the host
+host=$(timeout 10 ceph osd find $1 | jq -r '.crush_location.host')
+if [ -z "$host" ]; then
+    echo "Error: Could not find host location"
+    exit 1
+fi
+#Finds objectstore type
+obj=$(timeout 10 ceph osd metadata $1 | jq -r '.osd_objectstore')
+if [ -z "$obj" ]; then
+    echo "Error: Could not find OSD objectstore information"
+    exit 1
+fi
+#Checks if SSH successfully connects, exits script if it doesn't
+if ! timeout 10 ssh $host exit; then
+    echo "Error: SSH connection failed"
+    exit 1
+fi
+#Identifies the hardware generation of host
+gen=$(timeout 10 ssh $host /usr/sbin/ccm --format json /hardware | jq -r '.model')
+if [ -z "$gen" ]; then
+    echo "Error: Could not identify hardware generation"
+    exit 1
+fi
+
+#Multipath identification
+#Compares the output of pvs on the host to see if it contains mapper.
+#If it does, the host is multipath. If not, it is not.
+if ! ssh $host pvs &> /dev/null ; then
+    echo "Error: Could not run pvs for multipath check"
+    exit 1
+fi
+multicheck=$(timeout 10 ssh $host pvs 2>/dev/null)
+if echo "$multicheck" | grep -q "mapper" ; then
+    isMultipath=true
+else
+    isMultipath=false
+fi
+
+echo "Hostname: ${host}"
+echo "Hardware Generation: ${gen}"
+
+if [ "$obj" = 'filestore' ]; then
+    echo "OSD: Filestore"
+
+    if [[ "$isMultipath" == true ]]; then
+        echo "Multipath: Yes "
+    else
+        echo "Multipath: No"
+    fi
+    #Locates the partition path i.e. sda, then uses udev info to identify if the disk type is SATA or SAS
+    path=$(timeout 10 ceph osd metadata $1 | jq -r '.backend_filestore_partition_path' | cut -d '/' -f3)
+    echo "Path: /dev/$path"
+    disktype=$(timeout 10 ssh $host udevadm info --query=property /dev/$path | awk -F= -v key="ID_BUS" '$1==key {print $2}')
+
+    if [ "$disktype" = 'scsi' ]; then
+        #Filestore SAS
+        if ! timeout 10 ssh $host smartctl -a -d scsi /dev/$path > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; then
+            echo "Error: smartctl failed to find path"
+            exit 1
+        fi
+
+    elif [ "$disktype" = 'ata' ]; then
+        #Filestore SATA
+        if ! timeout 10 ssh $host smartctl -a /dev/$path | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; then
+            echo "Error smartctl failed to find path"
+            exit 1
+        fi
+
+    else
+        #Exception check
+        echo "Error: unknown type of disk (Not ATA or SCSI)"
+        exit 1
+    fi
+
+elif [ "$obj" = 'bluestore' ]; then
+    echo "OSD: Bluestore"
+
+    vg=$(timeout 10 ssh $host ls -l /var/lib/ceph/osd/ceph-$1 | grep /dev | cut -d '/' -f3)
+
+    if [[ "$isMultipath" == true ]]; then
+        echo "Multipath: Yes"
+        #Uses the volume group to find the /dev/mapper partition path, then udev to identify SATA or SAS
+        part=$(timeout 10 ssh $host pvs --select vg_name=$vg | grep /dev/mapper | cut -d '/' -f4 | cut -d ' ' -f1)
+        echo "Path: /dev/mapper/$part"
+        disktype=$(timeout 10 ssh $host udevadm info --query=property /dev/mapper/$part | awk -F= -v key="ID_BUS" '$1==key {print $2}')
+
+        if [ "$disktype" = 'ata' ]; then
+            #Bluestore multipath SATA
+            timeout 10 ssh $host smartctl -a /dev/mapper/$part | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)';
+            if $? > 0; then
+                echo "Error: smartctl failed to find mpath"
+                exit 1
+            fi
+
+        else
+            #Bluestore multipath SAS
+            timeout 10 ssh $host smartctl -a -d scsi /dev/mapper/$part > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}';
+
+        fi
+
+    else
+        echo "Multipath: No"
+        #Non-multipath uses the same method to identify SATA/SAS, but the parsing is different
+        part=$(timeout 10 ssh $host pvs --select vg_name=$vg | grep /dev | cut -d '/' -f3 | cut -d ' ' -f1)
+        echo "Path: /dev/$part"
+        disktype=$(timeout 10 ssh $host udevadm info --query=property /dev/$part | awk -F= -v key="ID_BUS" '$1==key {print $2}')
+
+        if [ "$disktype" = 'ata' ]; then
+            #Bluestore non-multipath SATA
+            if ! timeout 10 ssh $host smartctl -a /dev/$part | grep -E '(Serial|Model|Raw_Read_Error_Rate|Current_Pending_Sector|Offline_Uncorrectable)'; then
+                echo "Error: smartctl failed to find path"
+                exit 1
+            fi
+
+        else
+            #Bluestore non-multipath SAS
+            if ! timeout 10 ssh $host smartctl -a -d scsi /dev/$part > /tmp/smartdata_temp; cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' | paste -sd ' ' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'; then
+                echo "Error: smartctl failed to find path"
+                exit 1
+            fi
+
+        fi
+    fi
+else
+    #Exception check
+    echo "Error: OSD is neither filestore nor bluestore"
+    exit 1
+fi

disk-tooling/smartdata_sas.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+#Saves smartdata to a temporary file, then uses multiple searches to parse it.
+
+smartctl -a /dev/$1 > /tmp/smartdata_temp
+
+cat /tmp/smartdata_temp | grep -E 'Serial|Vendor:|Product:|Status:' && cat /tmp/smartdata_temp | grep -B1 -A1 uncorrected | awk '{print $NF}' && cat /tmp/smartdata_temp | grep -A2 read: | awk '{print $1 $NF}'

disk-tooling/ticket-drive.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+#Checks if the host can be connected to
+host=$2
+if ! timeout 10 ssh $host exit; then
+    echo "Error: SSH connection failed"
+    exit 1
+fi
+#Uses osdsearch to find the osd and parsed pathname
+source ~/jamesf/osdsearch $1 $2
+echo "Creating ticket..."
+#Uses report_osd_alltypes to print a report of the osd
+source ~/jamesf/report_osd_alltypes $osd > /tmp/osdreport
+echo "" >> /tmp/osdreport
+#Uses check_osd to verify if osd is ready to be removed
+~/tom/check_osd.sh $2 $parsedpath >> /tmp/osdreport
+
+#Creates the ticket in specific format
+echo ""
+echo "Status: new" > /tmp/driveticket
+echo "Subject: Disk Replacement $1 on $2" >> /tmp/driveticket
+echo "" >> /tmp/driveticket
+echo -e "This drive has been ticketed for replacement using the ticket-drive script.\n" >> /tmp/driveticket
+echo "$(cat /tmp/osdreport)" >> /tmp/driveticket
+cat /tmp/driveticket
+
+#Submits ticket via sendmail
+read -p "Submit this ticket? (Y/N) " -n 1 -r
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+    if ! timeout 10 sendmail ceph-disk@helpdesk.gridpp.rl.ac.uk < /tmp/driveticket
+    then
+        echo -e "\nTicket submission failed. (Ceph-Disk)"
+        exit
+    fi
+    if ! timeout 10 sendmail hardware@helpdesk.gridpp.rl.ac.uk < /tmp/driveticket
+    then
+        echo -e "\nTicket submission failed. (Fabric-Hardware)"
+        exit
+    fi
+    echo -e "\nTicket submitted to Ceph-Disk and Fabric-Hardware successfully!"
+else
+    echo -e "\nTicket was not submitted."
+    exit 1
+fi
+
+#Uses linktickets to find the ticket IDs and link the two tickets together, then print a URL to them.
+echo "Would you like to link the tickets together and receive a URL to them? (Y/N)"
+read -p "Note: This will require RT credentials: " -n 1 -r
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+    subject="Disk Replacement $1 on $2"
+    echo -e "\n"
+    source ~/jamesf/linktickets $subject
+    echo "Ceph-Disk ticket URL: https://helpdesk.gridpp.rl.ac.uk/Ticket/Display.html?id=${cephTicketID}"
+    echo "Fabric-Hardware ticket URL: https://helpdesk.gridpp.rl.ac.uk/Ticket/Display.html?id=${fabricTicketID}"
+else
+    echo -e "\n"
+    exit 1
+fi