gluster-plus-one-scale First Version

.gitignore

@@ -0,0 +1,6 @@
+__pycache__/
+*.pyc
+*.pyo
+*.log
+dist
+

extras/gf_pos_testing.py

@@ -0,0 +1,99 @@
+from test_data.gf_plus_one_test_data import *
+from scale.gf_plus_one_scale import *
+
+# Test code
+def test():
+	print ("***********************************************************")	
+	fname = create_new_node(12,4)
+	v_info = gf_bricks_from_file(fname)
+	for br in v_info:
+		print(br)
+
+	print ("***********************************************************")	
+
+	fname  = create_gluster_v_info(6,4,2)
+	v_info = gf_bricks_from_file(fname)
+
+	for br in v_info:
+		print(br)
+
+	print ("***********************************************************")
+
+	print ("volume_brick_list")
+	node_brick_list = gf_get_volume_bricks_list(v_info)
+	print (node_brick_list)
+	for elm in node_brick_list:
+		print (elm)
+
+
+	print ("***********************************************************")
+	print ("node_brick_dict")
+	node_brick = gf_get_node_bricks_dict(v_info)
+	for key,val in node_brick.items():
+		print (key, val)
+		print ('\n')
+
+	print ("***********************************************************")
+	print ("subvol_brick_list")
+	node_b = gf_get_volume_bricks_list(v_info)
+
+	subvol_brick_list = gf_subvol_bricks_dict(node_b, 6)
+	count = 0
+	for elm in subvol_brick_list:
+		print("ec -subvolume-"+ str (count))
+		print(elm)
+		count += 1 
+
+	print ("***********************************************************")
+'''
+	print ("Volume info")
+	volume_info = gf_get_volume_info ('vol')
+	for elm in volume_info:
+		print (elm)
+
+	print ("***********************************************************")
+
+	print ("Volume config")
+	volume_config = gf_get_volume_config(volume_info)
+	print (volume_config)
+
+
+	print ("***********************************************************")
+
+	print ("volume_brick_list")
+	node_brick_list = gf_get_volume_bricks_list(volume_info)
+	for elm in node_brick_list:
+		print (elm)
+
+
+	print ("***********************************************************")
+
+
+	print ("node_brick_dict")
+	node_brick = gf_get_node_bricks_dict(volume_info)
+	for key,val in node_brick.items():
+		print (f"Host = {key} bricks = {val}")
+
+
+	print ("***********************************************************")
+
+	print ("subvol_brick_list")
+	subvol_brick_list = gf_subvol_bricks_list(volume_info)
+	count = 0
+	for elm in subvol_brick_list:
+		print("ec -subvolume-"+ str (count))
+		print(elm)
+		count += 1 
+	print ("***********************************************************")
+'''
+
+
+if True:
+	test()
+
+
+
+
+
+
+

new-node.txt

@@ -0,0 +1,12 @@
+node-4:/root/brick-new-1
+node-4:/root/brick-new-2
+node-4:/root/brick-new-3
+node-4:/root/brick-new-4
+node-4:/root/brick-new-5
+node-4:/root/brick-new-6
+node-4:/root/brick-new-7
+node-4:/root/brick-new-8
+node-4:/root/brick-new-9
+node-4:/root/brick-new-10
+node-4:/root/brick-new-11
+node-4:/root/brick-new-12

pos_main.py

@@ -0,0 +1,173 @@
+"""This script facilitates plus one scaling for disperse volume."""
+
+
+import os
+import scale.gf_plus_one_scale as sc
+import test_data.gf_plus_one_test_data as test
+from scale.gf_logs import glogger
+from scale.gf_exceptions import (VolumeNotHealthy,
+                                 GfCommandFailed)
+from scale.gf_api import (gluster_volume_is_ready_to_scale,
+                          gf_are_bricks_sufficient,
+                          gluster_volume_is_ready_to_scale,
+                          gluster_create_brick_map_to_swap,
+                          gluster_volume_commit_drive_swap,
+                          gluster_check_volume_health,
+                          gluster_check_bricks_status)
+
+def yes(value):
+    """Return True if value is Y or Yes in any case."""
+    return value.lower() in ["y", "yes"]
+
+
+def gf_get_volume_and_bricks():
+    """Get volume name, new node and file name from user."""
+    volname = input("Enter name of the volume: ")
+    filename = input("Enter file name which contains new bricks: ")
+    hostname = input("Enter IP/hostname of new node: ")
+
+    if not volname or not filename or not hostname:
+        glogger.debug(f'Invalid user input: volname = {volname}, filename = {filename}, hostname = {hostname}')
+        raise ValueError("volname or filename or hostname is not correct")
+
+    return volname, filename, hostname
+
+
+def gluster_volume_commit_all(volname, bricks_migration_map):
+    """Commit physical replacment of all the bricks.
+
+    Input:
+    volname: Name of the volume
+    bricks_migration_map: Physical migration map of old and new bricks.
+    """
+    for old_brick, new_brick in bricks_migration_map:
+        glogger.info(f'Commiting replace bricks for old_brick = {old_brick} and new_brick = {new_brick}')
+        print(f'Commiting replace bricks for old_brick = {old_brick} and new_brick = {new_brick}')
+        try:
+            gluster_volume_commit_drive_swap(volname, old_brick, new_brick)
+        except GfCommandFailed as er:
+            glogger.exception(f'replace brick failed for {old_brick} and {new_brick}')
+            continue
+        else:
+            glogger.info(f'replace brick is done for {old_brick} and {new_brick}')
+
+
+def gf_swap_all_bricks(volname, bricks_migration_map):
+    """Swap all the bricks.
+
+    This function helps to swap all the bricks first and then commit
+    the physical migration in one shot.
+    Input:
+    volname: Name of the volume
+    bricks_migration_map: Physical migration map of old and new bricks.
+    """
+    while True:
+        print("Swap following disks : ")
+        for br in bricks_migration_map:
+            print(br)
+        done = input("Have you finished swapping all the disks: (y/n) ")
+        if yes(done):
+            gluster_volume_commit_all(volname, bricks_migration_map)
+        else:
+            continue
+
+
+def gf_swap_one_by_one_brick(volname, bricks_migration_map):
+    """Swap bricks one by one.
+
+    Ask user to swap a pair of old and new brick and then
+    commit that migration
+    Input:
+    volname: Name of the volume
+    bricks_migration_map: Physical migration map of old and new bricks.
+    """
+    for old_brick, new_brick in bricks_migration_map:
+        while True:
+            try:
+                print(f'Swap {old_brick} and {new_brick}')
+                done = input("Have you finished swapping above disks: (y/n) ")
+                if yes(done):
+                    gluster_volume_commit_drive_swap(volname, old_brick, new_brick)
+                    break
+                else:
+                    continue
+            except GfCommandFailed as er:
+                        glogger.exception(f'replace brick failed for {old_brick} and {new_brick}')
+                        continue
+            else:
+                glogger.info(f'replace brick is done for {old_brick} and {new_brick}')
+
+
+def gf_migrate_bricks_using_swap_map(volname, bricks_migration_map):
+    """Swap bricks using migration map.
+
+    Ask user how the migration should happen.
+    Input:
+    volname: Name of the volume
+    bricks_migration_map: Physical migration map of old and new bricks.
+    """
+    while True:
+        try:
+            print("All together : 1")
+            print("One brick at a time : 0")
+            all = int(input("How do you want to swap device: (0 or 1) "))
+            if all not in [0, 1]:
+                print("Please enter correct option.")
+                continue
+            else:
+                break
+        except:
+            print("Please enter valid option")
+            glogger.debug(f'Invalid swap option, enter an interger (0/1)')
+            done = input("Want to exit? y/N")
+            if yes(done):
+                exit(1)
+
+    if (all):
+        gf_swap_all_bricks(volname, bricks_migration_map)
+    else:
+        gf_swap_one_by_one_brick(volname, bricks_migration_map)
+
+
+def main():
+    """Main function to start complete plus one scaling process."""
+    while True:
+        try:
+            volname, filename, hostname = gf_get_volume_and_bricks()
+        except ValueError as er:
+            print(er)
+            glogger.exception(er)
+        else:
+            print(f'volname = {volname}')
+            print(f'filename = {filename}')
+            print(f'hostname = {hostname}')
+            break
+    try:
+        list_of_empty_drives = sc.gf_get_new_bricks_list(filename)
+        print ("**************New Bricks*****************")
+        for br in list_of_empty_drives:
+            print(br)
+        print ("****************************************")
+    except GfCommandFailed as e:
+        print(e.message)
+        exit(1)
+
+    if not gluster_volume_is_ready_to_scale(volname, list_of_empty_drives):
+        print(f'EC volume, {volname}, can not be scaled. Volume not healthy or all the bricks are not UP')
+        exit(1)
+    else:
+        print(f'{volname} can be scaled')
+
+    print ("****************************************")
+    bricks_migration_map = gluster_create_brick_map_to_swap(
+        vol_name=volname, new_host=hostname,
+        list_of_empty_drives=list_of_empty_drives)
+
+    for old, new in bricks_migration_map:
+        print (f'old_brick={old} and new brick={new}')
+
+    gf_migrate_bricks_using_swap_map(volname, bricks_migration_map)
+
+
+if __name__ == "__main__":
+    main()

scale/brick-disk-map.py

@@ -0,0 +1,52 @@
+"""Script to map a brick to the mounted disk."""
+
+
+import re
+import subprocess as sp
+
+
+def run_gluster_command(cmd):
+    """Execute gluster commands.
+
+    args:
+    cmd: String of command. ex. "gluster volume info <volname>""
+    returns:
+    tuple (ret,output,err)
+    ret = return status of command
+    output = output string
+    err = error message if any
+    """
+    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE, shell=True)
+    (output, err) = p.communicate()
+    ret = p.returncode
+    return (ret, output, err)
+
+
+def get_device_file_dict():
+    """Get the list of bricks and its mount location."""
+    cmd = 'lshw -class disk'
+    desc = "description"
+    log_name = "logical name"
+    serial = "serial"
+
+    dev = []
+    dev_list = []
+
+    ret, output, err = run_gluster_command(cmd)
+    output = output.decode('ASCII')
+    dev_info = output.split('\n')
+    for line in dev_info:
+        if re.search(desc, line):
+            if dev:
+                dev_list.append(dev)
+
+            dev = []
+        if re.search(log_name, line) or re.search(serial, line):
+            temp = line.split(':')
+            temp[1] = temp[1].strip(' ')
+            dev.append(temp[1])
+    dev_list.append(dev)
+    for line in dev_list:
+        print(line)
+
+get_device_file_dict()