diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index 194d090170b4..d1026e08d553 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -273,28 +273,11 @@ def check_down_segments(logger: Any, options: Any, dburl: dbconn.DbURL): conn = dbconn.connect(dburl, encoding='UTF8', allowSystemTableMods=True) dbconn.execSQL(conn, "SELECT gp_request_fts_probe_scan()") cnt_primaries_down = int(dbconn.queryRow(conn, f"SELECT COUNT(1) FROM gp_segment_configuration WHERE role = 'p' AND status = 'd'")[0]) - cnt_mirrors_down = int(dbconn.queryRow(conn, f"SELECT COUNT(1) FROM gp_segment_configuration WHERE role = 'm' AND status = 'd'")[0]) conn.close() if cnt_primaries_down != 0: raise Exception('Detected some primary segments are down, please recover manually') - if cnt_mirrors_down != 0: - logger.info("Some mirrors are down, trying to recover them, it may take some time...") - recoverseg_options = "-a -F" - if options.logfile_directory is not None: - recoverseg_options = recoverseg_options + f' -l "{str(options.logfile_directory)}"' - global cmd_recoverseg - try: - cmd_recoverseg = GpRecoverSeg("Running gprecoverseg", options=recoverseg_options) - cmd_recoverseg.run(validateAfter=True) - except Exception as e: - logger.error(str(e)) - error_msg = f"Failed to execute 'gprecoverseg {recoverseg_options}'" - raise Exception(error_msg) - finally: - cmd_recoverseg = None - def main(options, args, parser): conn = None try: @@ -322,14 +305,14 @@ def main(options, args, parser): dburl = dbconn.DbURL(dbname=DBNAME, port=gpenv.getCoordinatorPort()) + check_down_segments(logger, options, dburl) + check_running_gputils(dburl, options.coordinator_data_directory) create_pid_file(options.coordinator_data_directory) gparray_dump_filename = options.coordinator_data_directory + '/gparraydump' - check_down_segments(logger, options, dburl) - logger.info('Init gparray from catalog') try: gparray = GpArray.initFromCatalog(dburl, utility=True) diff --git a/gpMgmt/bin/gpmovemirrors b/gpMgmt/bin/gpmovemirrors index 463a7d692c99..59a599077722 100755 --- a/gpMgmt/bin/gpmovemirrors +++ b/gpMgmt/bin/gpmovemirrors @@ -85,6 +85,8 @@ def parseargs(): help='show this help message and exit.') parser.add_option('-a', dest="interactive", action='store_false', default=True, help="quiet mode, do not require user input for confirmations") + parser.add_option('--skip-resource-estimation', dest='skip_resource_estimation', metavar='', + action='store_true', default=False, help='Skip resource estimation (storage)') parser.add_option('--usage', action="briefhelp") parser.set_defaults(verbose=False, filters=[], slice=(None, None)) @@ -376,9 +378,10 @@ try: pairs.append(pair) """ Validating Disk Space requirement """ - disk_usage = RelocateDiskUsage(pairs, options.batch_size, options) - if not disk_usage.validate_disk_space(): - raise InsufficientDiskSpaceError("Insufficient disk space on target mirror hosts.") + if not options.skip_resource_estimation: + disk_usage = RelocateDiskUsage(pairs, options.batch_size, options) + if not disk_usage.validate_disk_space(): + raise InsufficientDiskSpaceError("Insufficient disk space on target mirror hosts.") """ Prepare common execution steps for running commands on segments """ oldMirrorsToMove = [mirror for mirror in newConfig.oldMirrorList if not mirror.inPlace] diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index f6b7ed70b42e..5167d9e85b46 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -325,6 +325,28 @@ def __init__(self, datadir, mode, wait, timeout): self.append("stop") +class PgCtlStatusArgs(CmdArgs): + """ + Used by CoordinatorStop, SegmentStop to format the pg_ctl command + to stop a backend postmaster + + >>> str(PgCtlStatusArgs("/data1/coordinator/gpseg-1")) + '$GPHOME/bin/pg_ctl -D /data1/coordinator/gpseg-1 status' + + """ + + def __init__(self, datadir): + """ + @param datadir: database data directory + """ + CmdArgs.__init__(self, [ + "$GPHOME/bin/pg_ctl", + "-D", str(datadir), + ]) + self.append("status") + + + class CoordinatorStart(Command): def __init__(self, name, dataDir, port, era, wrapper, wrapper_args, specialMode=None, restrictedMode=False, timeout=SEGMENT_TIMEOUT_DEFAULT, @@ -441,6 +463,25 @@ def remote(name, hostname, dataDir, mode='smart'): cmd.run(validateAfter=True) return cmd +#----------------------------------------------- +class SegmentStatus(Command): + def __init__(self, name, dataDir, ctxt=LOCAL, remoteHost=None): + + self.cmdStr = str( PgCtlStatusArgs(dataDir) ) + Command.__init__(self, name, self.cmdStr, ctxt, remoteHost) + + @staticmethod + def local(name, dataDir): + cmd=SegmentStatus(name, dataDir) + cmd.run(validateAfter=False) + return cmd + + @staticmethod + def remote(name, hostname, dataDir): + cmd=SegmentStatus(name, dataDir, ctxt=REMOTE, remoteHost=hostname) + cmd.run(validateAfter=False) + return cmd + #----------------------------------------------- class SegmentIsShutDown(Command): """ diff --git a/gpMgmt/bin/gppylib/fault_injection.py b/gpMgmt/bin/gppylib/fault_injection.py index 388097634575..50268764a47d 100755 --- a/gpMgmt/bin/gppylib/fault_injection.py +++ b/gpMgmt/bin/gppylib/fault_injection.py @@ -11,6 +11,7 @@ GPMGMT_FAULT_FILE_FLAG = 'GPMGMT_FAULT_FILE_FLAG' GPMGMT_FAULT_TYPE_SYSPEND = 'suspend' +GPMGMT_FAULT_TYPE_VALUE = 'value' def inject_fault(fault_point): if GPMGMT_FAULT_POINT in os.environ and fault_point == os.environ[GPMGMT_FAULT_POINT]: @@ -32,10 +33,17 @@ def raise_exception(delay: int): else: raise Exception('Fault Injection %s' % os.environ[GPMGMT_FAULT_POINT]) +def inject_fault_get_value() -> str: + if GPMGMT_FAULT_TYPE in os.environ and os.environ[GPMGMT_FAULT_TYPE] == GPMGMT_FAULT_TYPE_VALUE: + if GPMGMT_FAULT_POINT in os.environ: + return os.environ[GPMGMT_FAULT_POINT] + return '' + # decorator for test purposes -def wrap_state_func_with_faults(func): +def wrap_func_with_faults(func): def func_with_faults(*args): inject_fault(f'{func.__name__}_begin') - func(*args) + result = func(*args) inject_fault(f'{func.__name__}_end') + return result return func_with_faults diff --git a/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py b/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py index 9c07e7f2387b..8ef5a305fc86 100644 --- a/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py +++ b/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py @@ -21,6 +21,7 @@ from gppylib.commands.gp import is_pid_postmaster, get_pid_from_remotehost from gppylib.commands.unix import check_pid_on_remotehost from gppylib.programs.clsRecoverSegment_triples import RecoveryTriplet +from gppylib.fault_injection import * logger = gplog.get_default_logger() @@ -313,6 +314,7 @@ def _trigger_fts_probe(self, port=0): dbconn.execSQL(conn,"SELECT gp_request_fts_probe_scan()") conn.close() + @wrap_func_with_faults def _update_config(self, recovery_info_by_host, gpArray): # should use mainUtils.getProgramName but I can't make it work! programName = os.path.split(sys.argv[0])[-1] @@ -587,6 +589,7 @@ def _run_recovery(self, action_name, recovery_info_by_host, gpEnv): self._remove_progress_files(recovery_info_by_host, recovery_results) return recovery_results + @wrap_func_with_faults def _do_recovery(self, recovery_info_by_host, gpEnv): """ # Recover and start segments using gpsegrecovery, which will internally call either diff --git a/gpMgmt/bin/gppylib/operations/rebalanceSegments.py b/gpMgmt/bin/gppylib/operations/rebalanceSegments.py index 3ef843f2d3ae..ffdc46738b6b 100644 --- a/gpMgmt/bin/gppylib/operations/rebalanceSegments.py +++ b/gpMgmt/bin/gppylib/operations/rebalanceSegments.py @@ -6,6 +6,7 @@ from gppylib.commands.gp import GpSegStopCmd from gppylib.commands import base from gppylib import gplog +from gppylib.fault_injection import * from gppylib.operations.segment_reconfigurer import SegmentReconfigurer @@ -116,6 +117,8 @@ def rebalance(self): pool.addCommand(cmd) base.join_and_indicate_progress(pool) + + inject_fault('GpSegmentRebalanceOperation_rebalance_at_seg_stop') failed_count = 0 completed = pool.getCompletedItems() diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py index a1d347d27899..94589de18cc5 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py @@ -145,6 +145,11 @@ def __init__(self, conn: dbconn.Connection, logger: Any, dburl: dbconn.DbURL, op self.plan = None self.main_state_from_prev_run = self.rebalance_schema.getMainStateFromPreviousRun() + self.shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() + self.is_shrink_rollback_in_progress = self.gg_shrink.state_is_from_rollback_flow(self.shrink_state_from_prev_run) + self.prev_shrink_run_was_complete = self.gg_shrink.state_is_final(self.shrink_state_from_prev_run) + + def on_every_state(self) -> None: if self.state in self.states_logged: self.rebalance_schema.storeMainState(self.state) @@ -168,7 +173,7 @@ def shutdown(self) -> None: # state callbacks start here - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_OPTIONS_VALIDATION(self) -> None: if self.options.clean_required: self.trigger('move_to_STATE_CLEANUP') @@ -177,25 +182,37 @@ def on_enter_STATE_OPTIONS_VALIDATION(self) -> None: else: self.trigger('move_to_STATE_PLANNING_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_CLEANUP(self) -> None: if not self.rebalance_schema.schemaExists(): self.logger.info(f"Rebalance schema doesn't exist. Cleanup is not required.") else: - prev_run_was_complete = (self.main_state_from_prev_run == 'STATE_EXECUTOR_DONE' or - self.main_state_from_prev_run == 'STATE_ROLLBACK') - self.gg_shrink.cleanup(prev_run_was_complete) + self.plan = self.rebalance_schema.retrieveSavedPlan() + if isinstance(self.plan, ShrinkPlan): + self.gg_shrink.cleanup(self.prev_shrink_run_was_complete) self.rebalance_schema.dropSchema() self.logger.info('Cleanup is complete') self.trigger('move_to_STATE_END') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_ROLLBACK(self) -> None: - self.plan = self.rebalance_schema.retrieveSavedPlan() - self.gg_shrink.rollback(self.plan) - self.trigger('move_to_STATE_END') - - @wrap_state_func_with_faults + try: + if self.main_state_from_prev_run == 'STATE_EXECUTOR_DONE': + self.logger.info("Previous run was completed successfully. Can't perform rollback.") + return + self.plan = self.rebalance_schema.retrieveSavedPlan() + if isinstance(self.plan, ShrinkPlan): + if self.is_shrink_rollback_in_progress: + self.logger.info("Rollback is already in progress, and was interrupted. Execute 'ggrebalance' without '-r' flag.") + return + if not self.prev_shrink_run_was_complete: + self.gg_shrink.rollback(self.plan) + return + self.gg_rebalance.rollback() + finally: + self.trigger('move_to_STATE_END') + + @wrap_func_with_faults def on_enter_STATE_PLANNING_STARTED(self) -> None: if self.options.target_segment_count != None: self.plan = Planner(self.logger, self.dburl, self.gparray, self.options).plan() @@ -205,11 +222,11 @@ def on_enter_STATE_PLANNING_STARTED(self) -> None: self.trigger('move_to_STATE_PLANNING_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_PLANNING_DONE(self) -> None: self.trigger('move_to_STATE_CHECK_PREVIOUS_RUN') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: if not self.rebalance_schema.schemaExists(): if self.plan == None: @@ -243,57 +260,57 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: self.trigger('move_to_STATE_EXECUTOR_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SETUP_SCHEMA_STARTED(self) -> None: # Create schema and status tables. # It will also save plan in order to use it for recovering after interruption self.rebalance_schema.createSchema(self.plan) self.trigger('move_to_STATE_SETUP_SCHEMA_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SETUP_SCHEMA_DONE(self) -> None: self.logger.info(f'Created "{self.rebalance_schema.getSchemaName()}" schema') self.trigger('move_to_STATE_EXECUTOR_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_EXECUTOR_STARTED(self) -> None: if isinstance(self.plan, ShrinkPlan): - shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() - if not self.gg_shrink.state_is_final(shrink_state_from_prev_run): + if not self.prev_shrink_run_was_complete: self.trigger('move_to_STATE_SHRINK_STARTED') return self.trigger('move_to_STATE_REBALANCE_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_EXECUTOR_DONE(self) -> None: self.trigger('move_to_STATE_END') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_STARTED(self) -> None: self.gg_shrink.run(self.plan) self.trigger('move_to_STATE_SHRINK_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_DONE(self) -> None: self.trigger('move_to_STATE_REBALANCE_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_STARTED(self) -> None: - if self.plan is not None and self.plan.getMoves() is not None: + if (self.plan is not None and + self.plan.getMoves() is not None and + not self.is_shrink_rollback_in_progress): self.gg_rebalance.run(self.plan) - self.logger.info('Rebalance is complete') self.trigger('move_to_STATE_REBALANCE_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_DONE(self) -> None: self.trigger('move_to_STATE_EXECUTOR_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_END(self) -> None: pass - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_ERROR(self) -> None: raise Exception('Main SM entered STATE_ERROR') diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index 3d371fea5d27..223fc3ac2cda 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -7,7 +7,7 @@ from gppylib.commands.unix import * from gppylib.commands.gp import * from gppylib.gplog import * - from gppylib.commands.gp import GpMoveMirrors + from gppylib.commands.gp import GpMoveMirrors, SegmentStatus from gppylib.system.environment import * from gprebalance_modules.planner import * from gprebalance_modules.rebalance_schema import RebalanceSchema, STATE_NOT_DEFINED @@ -36,6 +36,12 @@ class RebalanceSM: 'STATE_REBALANCE_DONE' ] + states_rollback_rebalance_flow = [ + 'STATE_REBALANCE_ROLLBACK_STARTED', + 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED', + 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE' + ] + transitions = [ { 'trigger': 'start', @@ -59,7 +65,10 @@ class RebalanceSM: }, { 'trigger': 'move_to_STATE_REBALANCE_EXECUTION_STARTED', - 'source': ['STATE_REBALANCE_PREPARE_MOVES_DONE', 'STATE_REBALANCE_MOVES_SUCCEEDED', 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE'], + 'source': ['STATE_REBALANCE_PREPARE_MOVES_DONE', + 'STATE_REBALANCE_MOVES_SUCCEEDED', + 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE', + 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE'], 'dest': 'STATE_REBALANCE_EXECUTION_STARTED' }, { @@ -82,6 +91,21 @@ class RebalanceSM: 'source': 'STATE_REBALANCE_EXECUTION_STARTED', 'dest': 'STATE_REBALANCE_EXECUTION_DONE' }, + { + 'trigger': 'rollback', + 'source': 'STATE_REBALANCE_INIT', + 'dest': 'STATE_REBALANCE_ROLLBACK_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED', + 'source': 'STATE_REBALANCE_ROLLBACK_STARTED', + 'dest': 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE', + 'source': 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED', + 'dest': 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE' + }, { 'trigger': 'move_to_STATE_REBALANCE_DONE', 'source': 'STATE_REBALANCE_EXECUTION_DONE', @@ -106,10 +130,11 @@ def __init__(self, conn: dbconn.Connection, schema: RebalanceSchema, logger: Any self.conn = conn self.rebalance_schema = schema self.cmd = None + self.is_rollback_flow = False self.machine = Machine(model = self, queued=True, - states = self.states_main_rebalance_flow + self.states_not_logged, + states = self.states_main_rebalance_flow + self.states_not_logged + self.states_rollback_rebalance_flow, transitions = self.transitions, initial = 'STATE_REBALANCE_INIT', before_state_change = 'on_every_state') @@ -119,7 +144,7 @@ def on_every_state(self) -> None: self.logger.info('Rebalance was interrupted') raise Exception('Rebalance was interrupted') - if self.state in self.states_main_rebalance_flow: + if self.state in self.states_main_rebalance_flow + self.states_rollback_rebalance_flow: self.rebalance_schema.storeRebalanceState(self.state) def run(self, plan: Plan) -> None: @@ -139,12 +164,18 @@ def run(self, plan: Plan) -> None: self.trigger('start') - def process_moves(self, moves: List[LogicalMove]): - if len(moves) == 0: + def rollback(self) -> None: + if self.rebalance_schema.schemaExists(): + self.trigger('rollback') + else: + self.logger.info("Rebalance schema doesn't exist. Can't perform rollback.") + + def process_moves(self, steps: List[RebalanceStepMoveMirror]): + if len(steps) == 0: return - filename = self.create_config_file(moves) - gpmovemirrors_options = f'-a -i {filename}' + filename = self.create_config_file(steps) + gpmovemirrors_options = f'--skip-resource-estimation -a -i {filename}' if self.options.parallel is not None: batch_size = self.options.parallel @@ -281,16 +312,26 @@ def lookup_seg(self, seg: Segment) -> bool: return True return False - def create_config_file(self, moves: List[LogicalMove]) -> str: + def create_config_file(self, steps: List[RebalanceStepMoveMirror]) -> str: filename = f'/tmp/ggrebalance_move_config_pid{os.getpid()}' with open(filename, 'w') as fp: - for move in moves: - segment_current_info = move.seg - if not self.lookup_seg(segment_current_info): - self.logger.info(f'Skip segment for gpmovemirrors: {str(segment_current_info)}') - continue - cfg_line = f'{segment_current_info.getSegmentHostName()}|{segment_current_info.getSegmentPort()}|{segment_current_info.getSegmentDataDirectory()} ' - cfg_line += f'{move.dstHost.hostname}|{move.target_port}|{move.target_datadir}\n' + for step in steps: + assert isinstance(step, RebalanceStepMoveMirror) + move = step.getMove() + if step.isRollback(): + segment_current_info = move.seg + if self.lookup_seg(segment_current_info): + cfg_line = f'{segment_current_info.getSegmentHostName()}|{segment_current_info.getSegmentPort()}|{segment_current_info.getSegmentDataDirectory()} ' + else: + cfg_line = f'{move.dstHost.hostname}|{move.target_port}|{move.target_datadir} ' + cfg_line += f'{segment_current_info.getSegmentHostName()}|{segment_current_info.getSegmentPort()}|{segment_current_info.getSegmentDataDirectory()}\n' + else: + segment_current_info = move.seg + if not self.lookup_seg(segment_current_info): + self.logger.info(f'Skip segment for gpmovemirrors: {str(segment_current_info)}') + continue + cfg_line = f'{segment_current_info.getSegmentHostName()}|{segment_current_info.getSegmentPort()}|{segment_current_info.getSegmentDataDirectory()} ' + cfg_line += f'{move.dstHost.hostname}|{move.target_port}|{move.target_datadir}\n' fp.write(cfg_line) return filename @@ -303,9 +344,14 @@ def state_is_final(self, state: str) -> bool: return state == self.states_main_rebalance_flow[-1] def get_state_after_interrupt(self, prev_state) -> str: - if (prev_state == 'STATE_REBALANCE_EXECUTION_STARTED' or - prev_state == 'STATE_REBALANCE_MOVES_SUCCEEDED' or - prev_state == 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE'): + if prev_state in self.states_rollback_rebalance_flow[:-1]: + prev_idx = self.states_rollback_rebalance_flow.index(prev_state) + return self.states_rollback_rebalance_flow[prev_idx + 1] + + if (prev_state in ['STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE', + 'STATE_REBALANCE_EXECUTION_STARTED', + 'STATE_REBALANCE_MOVES_SUCCEEDED', + 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE']): return 'STATE_REBALANCE_EXECUTION_STARTED' prev_idx = self.states_main_rebalance_flow.index(prev_state) @@ -315,9 +361,199 @@ def get_state_after_interrupt(self, prev_state) -> str: def reset_in_progress_execution_steps(self) -> None: in_progress_steps = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.IN_PROGRESS]) for step in in_progress_steps: - step.setStatus(RebalanceStep.Status.PLANNED) + step.setStatus(RebalanceStep.Status.ERROR, step.isRollback()) + self.rebalance_schema.updateExecutionStep(step) + + def process_error_execution_steps(self) -> None: + dbconn.execSQL(self.conn, "BEGIN") + try: + error_steps = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.ERROR]) + if len(error_steps) == 0: + return + + # All steps in an errored batch should be the same type, so we probe only + # the first one to detect the type and process accordingly. + if isinstance(error_steps[0], RebalanceStepMoveMirror): + self.process_error_execution_steps_mirror_moves(error_steps) + else: + self.process_error_execution_steps_switchovers(error_steps) + finally: + dbconn.execSQL(self.conn, "COMMIT") + + def process_error_execution_steps_mirror_moves(self, error_steps: List[RebalanceStep]) -> None: + self.logger.info('Process failed segment moves...') + steps_left_todo = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) + for step in error_steps: + self.logger.info(f'Checking error status for step: {str(step)}') + dbid = step.getMove().seg.getSegmentDbId() + target_hostname = step.getMove().dstHost.hostname + target_datadir = step.getMove().target_datadir + target_port = step.getMove().target_port + + catalog_segment_info = self.get_catalog_gp_segment_configuration_for_dbid(dbid) + self.logger.info(f'Segment info from catalog: {str(catalog_segment_info)}') + + gp_segment_configuration_updated = (catalog_segment_info.hostname == target_hostname) + + port_updated = False + if gp_segment_configuration_updated: + port_updated = (self.get_postgresql_conf_port(target_hostname, target_datadir) == target_port) + + self.logger.info(f'gp_segment_configuration is updated: {gp_segment_configuration_updated}') + self.logger.info(f'Port is updated: {port_updated}') + + time_waited = 0 + SLEEP_PERIOD_SEC = 1.0 + TIMEOUT_SEC = 120.0 + if port_updated: + self.logger.info(f'Start checking if segment is up with timeout of {TIMEOUT_SEC} sec.') + while time_waited < TIMEOUT_SEC: + # Start polling segment status with timeout + segment_process_started = SegmentStatus.remote('Segment status check', target_hostname, target_datadir).was_successful() + if segment_process_started: + catalog_segment_info = self.get_catalog_gp_segment_configuration_for_dbid(dbid) + if catalog_segment_info.isSegmentUp() and catalog_segment_info.isSegmentModeSynchronized(): + self.logger.info('The step is complete, mark it as done') + step.setStatus(RebalanceStep.Status.DONE, step.isRollback()) + self.rebalance_schema.updateExecutionStep(step) + break + + time.sleep(SLEEP_PERIOD_SEC) + time_waited = time_waited + SLEEP_PERIOD_SEC + if time_waited >= TIMEOUT_SEC and self.interactive_check('Timeout waiting for segment start, wait again?'): + time_waited = 0 + + # Continue with the next step, if we already marked this one + if step.getStatus() == RebalanceStep.Status.DONE: + continue + + if not gp_segment_configuration_updated and self.interactive_check(f'Retry step?'): + if step.isRollback(): + self.logger.info('Plan to retry rollback step') + step.setStatus(RebalanceStep.Status.PLANNED, True) + else: + self.logger.info('Plan to retry step') + step.setStatus(RebalanceStep.Status.PLANNED) + self.rebalance_schema.updateExecutionStep(step) + continue + + if not step.isRollback() and self.interactive_check('Rollback step?'): + self.logger.info('Plan to rollback step') + step.setStatus(RebalanceStep.Status.PLANNED, True) + else: + self.logger.info('Cancel step') + step.setStatus(RebalanceStep.Status.CANCELLED) + self.rebalance_schema.updateExecutionStep(step) + + # Mark dependent steps accordingly + self.mark_dependent_steps_on_error(error_steps, steps_left_todo) + + def process_error_execution_steps_switchovers(self, error_steps: List[RebalanceStep]) -> None: + self.logger.info('Process failed switchovers...') + steps_left_todo = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) + for step in error_steps: + self.logger.info(f'Processing error status for switchover step: {str(step)}') + if self.interactive_check(f'Retry step?'): + if step.isRollback(): + self.logger.info('Plan to retry rollback step') + step.setStatus(RebalanceStep.Status.PLANNED, True) + else: + self.logger.info('Plan to retry step') + step.setStatus(RebalanceStep.Status.PLANNED) + self.rebalance_schema.updateExecutionStep(step) + continue + + if not step.isRollback() and self.interactive_check('Rollback step?'): + self.logger.info('Plan to rollback step') + step.setStatus(RebalanceStep.Status.PLANNED, True) + + # Revert type of switchover + rollback_step_for_switchover = None + if isinstance(step, RebalanceStepSwitchoverToMirror): + rollback_step_for_switchover = RebalanceStepSwitchoverToPrimary(step.getMove()) + elif isinstance(step, RebalanceStepSwitchoverToPrimary): + rollback_step_for_switchover = RebalanceStepSwitchoverToMirror(step.getMove()) + + if rollback_step_for_switchover: + rollback_step_for_switchover.setMoveOrder(step.getMoveOrder()) + rollback_step_for_switchover.setStatus(step.getStatus(), True) + step = rollback_step_for_switchover + else: + self.logger.info('Cancel step') + step.setStatus(RebalanceStep.Status.CANCELLED) self.rebalance_schema.updateExecutionStep(step) + # Mark dependent steps accordingly + self.mark_dependent_steps_on_error(error_steps, steps_left_todo) + + def mark_dependent_steps_on_error(self, error_steps: List[RebalanceStep], steps_left_todo: List[RebalanceStep]) -> None: + # 1. If there are steps planned for ROLLBACK - we mark all left todo steps for the same content as already rolled back + if not self.is_rollback_flow: + for step in error_steps: + if step.getStatus() == RebalanceStep.Status.PLANNED and step.isRollback(): + content_id = step.getMove().seg.getSegmentContentId() + for step_todo in steps_left_todo: + if step_todo.getMove().seg.getSegmentContentId() == content_id: + self.logger.info(f'Mark as already rolled back the dependent step {step_todo}') + step_todo.setStatus(RebalanceStep.Status.DONE, True) + self.rebalance_schema.updateExecutionStep(step_todo) + # 2. If there are any cancelled steps - we need to: + # a. cancel all not yet done steps of the same dbid, + # b. and *ALL* switchovers, + # c. and do cancelation recursively. + # But, actually, it means that we need to cancel everything besides steps revived from the ERROR state just above, + # as left todo steps didn't get into this ERRORed batch, meaning they must have different step type (meaning switchover). + if any(step.getStatus() == RebalanceStep.Status.CANCELLED for step in error_steps): + for step_todo in steps_left_todo: + self.logger.info(f'Mark as CANCELLED the step {step_todo}') + step_todo.setStatus(RebalanceStep.Status.CANCELLED, step_todo.isRollback()) + self.rebalance_schema.updateExecutionStep(step_todo) + + def get_catalog_gp_segment_configuration_for_dbid(self, dbid: int) -> Segment: + row = dbconn.queryRow(self.conn, + f"SELECT dbid||'|'||content||'|'||role||'|'||preferred_role||'|'||mode||'|'||status||'|'||hostname||'|'||address||'|'||port||'|'||datadir " + f"FROM gp_segment_configuration WHERE dbid = {dbid}") + return Segment.initFromString(row[0]) + + def get_postgresql_conf_port(self, hostname: str, datadir: str) -> int: + cmd = Command( + name="get_postgresql_conf_port", + cmdStr=f"grep -E '^port\\s*=' {datadir}/postgresql.conf | sed -E 's/^port\\s*=\\s*([0-9]+).*/\\1/'", + ctxt=REMOTE, + remoteHost=hostname) + cmd.run() + + if not cmd.was_successful(): + self.logger.info(f"Failed to get port from postgresql.conf on {hostname}: {cmd.get_stderr()}") + return -1 + + output = cmd.get_stdout().strip() + if not output or not output.isdigit(): + return -1 + + return int(output) + + # Decorator to overwrite the logic of interactive_check() + # during tests execution. + def wrap_interactive_check_with_faults(fun): + def func_with_faults(self, msg: str): + try: + inject_value = inject_fault_get_value() + injected_answers = json.loads(inject_value) + if injected_answers.get(msg, '') == 'yes': + return True + if injected_answers.get(msg, '') == 'no': + return False + except: + pass + return fun(self, msg) + return func_with_faults + + @wrap_interactive_check_with_faults + def interactive_check(self, msg: str) -> bool: + # TODO: add logic here when implementing interactive mode + return False + @staticmethod def convert_moves_to_rebalance_steps(moves: List[LogicalMove]) -> List[RebalanceStep]: # In the loop below we create a list of rebalance execution steps from the plan's list of moves. @@ -368,16 +604,21 @@ def fill_rebalance_steps(): # state callbacks start here - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: state_from_prev_run = self.rebalance_schema.getRebalanceStateFromPreviousRun() + self.is_rollback_flow = self.rebalance_schema.isRollbackRebalanceFlow(self.states_rollback_rebalance_flow[0]) + if state_from_prev_run == STATE_NOT_DEFINED: self.trigger('move_to_STATE_REBALANCE_STARTED') elif self.state_is_final(state_from_prev_run): self.logger.info('Cluster is already rebalanced...') else: - self.logger.info('Continue interrupted rebalance operation...') + if self.is_rollback_flow: + self.logger.info('Continue interrupted rebalance rollback operation...') + else: + self.logger.info('Continue interrupted rebalance operation...') self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") try: next_state = self.get_state_after_interrupt(state_from_prev_run) @@ -388,11 +629,11 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: # use auto to_«state» method to recover self.trigger(f'to_{next_state}') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_STARTED(self) -> None: self.trigger('move_to_STATE_REBALANCE_PREPARE_MOVES_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED(self) -> None: if not self.rebalance_plan.getMoves(): raise Exception('Rebalance executor was launched with a plan without segment movements') @@ -413,22 +654,23 @@ def on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED(self) -> None: self.trigger('move_to_STATE_REBALANCE_PREPARE_MOVES_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE(self) -> None: self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_EXECUTION_STARTED(self) -> None: - if self.rebalance_schema.allExecutionStepsAreDone(): self.trigger('move_to_STATE_REBALANCE_EXECUTION_DONE') return # In normal execution we shouldn't have IN_PROGRESS steps at this moment. # If they are presented, it means they are left from previous interrupted run. - # Bring them back to PLANNED state, so we can try to process them again. + # Set them to ERROR state. self.reset_in_progress_execution_steps() + self.process_error_execution_steps() + steps_to_execute = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) if len(steps_to_execute) > 0: @@ -446,16 +688,15 @@ def on_enter_STATE_REBALANCE_EXECUTION_STARTED(self) -> None: (len(current_batch) > 0 and (type(current_batch[0]) is not type(step)))): break - step.setStatus(RebalanceStep.Status.IN_PROGRESS) + step.setStatus(RebalanceStep.Status.IN_PROGRESS, step.isRollback()) self.rebalance_schema.updateExecutionStep(step) current_batch.append(step) if isinstance(current_batch[0], RebalanceStepMoveMirror): self.logger.info('Rebalance - start moving segments:') - moves = [step.getMove() for step in current_batch] - for move in moves: - self.logger.info(str(move)) - self.process_moves(moves) + for step in current_batch: + self.logger.info(str(step)) + self.process_moves(current_batch) self.logger.info('Rebalance - end moving segments') else: direction = self.RoleSwapDirection.PRIMARY_TO_MIRROR @@ -468,25 +709,22 @@ def on_enter_STATE_REBALANCE_EXECUTION_STARTED(self) -> None: self.execute_role_swaps(segments, direction) self.logger.info('Rebalance - end role swap') - # TODO: check the errored segments here, once we implement rollback for the rebalance. - # For now if some error happened, the entire tool will halt its work, so if we reached this point - # just mark all steps as done. for step in steps_to_execute: if step.getStatus() == RebalanceStep.Status.IN_PROGRESS: - step.setStatus(RebalanceStep.Status.DONE) + step.setStatus(RebalanceStep.Status.DONE, step.isRollback()) self.rebalance_schema.updateExecutionStep(step) self.trigger('move_to_STATE_REBALANCE_MOVES_SUCCEEDED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_MOVES_SUCCEEDED(self) -> None: self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_EXECUTION_DONE(self) -> None: self.trigger('move_to_STATE_REBALANCE_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED(self) -> None: # Approve all consequent steps that require approval @@ -500,20 +738,91 @@ def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED(self) # TODO: we'll need to add logic here to get approval from the user in the interactive mode, # once we start implementing the interactive mode. # In non-interactive mode we assume that the switchover is always approved. - step.setStatus(RebalanceStep.Status.PLANNED) + step.setStatus(RebalanceStep.Status.PLANNED, step.isRollback()) self.rebalance_schema.updateExecutionStep(step) self.trigger('move_to_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE(self) -> None: self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults + def on_enter_STATE_REBALANCE_ROLLBACK_STARTED(self) -> None: + self.is_rollback_flow = True + self.logger.info('Starting rebalance rollback') + self.trigger('move_to_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED') + + @wrap_func_with_faults + def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED(self) -> None: + + self.logger.info('Start preparing steps for rollback...') + actual_rollback_steps_cnt = 0 + rollback_steps = self.rebalance_schema.getExecutionSteps([]) + + if len(rollback_steps) > 0: + move_order = rollback_steps[-1].getMoveOrder() + + for i, step in enumerate(rollback_steps): + # reverse the move order + step.setMoveOrder(move_order) + move_order -= 1 + if step.isRollback(): + continue + + # convert not yet executed steps as already rolled back + if step.getStatus() in [RebalanceStep.Status.APPROVE_REQUIRED, RebalanceStep.Status.PLANNED]: + step.setStatus(RebalanceStep.Status.DONE, True) + actual_rollback_steps_cnt += 1 + elif step.getStatus() in [RebalanceStep.Status.IN_PROGRESS, RebalanceStep.Status.DONE, RebalanceStep.Status.ERROR]: + step.setStatus(RebalanceStep.Status.PLANNED, True) + actual_rollback_steps_cnt += 1 + elif step.getStatus() == RebalanceStep.Status.CANCELLED: + # We do nothing for CANCELLED steps - for now they can be processed only if run ggrebalance from scratch. + self.logger.warning(f'Step {str(step)} is marked as CANCELLED, and skipped during ROLLBACK processing.') + continue + + rollback_step_for_switchover = None + + # Revert type of switchover + if isinstance(step, RebalanceStepSwitchoverToMirror): + rollback_step_for_switchover = RebalanceStepSwitchoverToPrimary(step.getMove()) + elif isinstance(step, RebalanceStepSwitchoverToPrimary): + rollback_step_for_switchover = RebalanceStepSwitchoverToMirror(step.getMove()) + + if rollback_step_for_switchover: + rollback_step_for_switchover.setMoveOrder(step.getMoveOrder()) + rollback_step_for_switchover.setStatus(step.getStatus(), True) + if step.getStatus() == RebalanceStep.Status.PLANNED: + rollback_step_for_switchover.setStatus(RebalanceStep.Status.APPROVE_REQUIRED, True) + rollback_steps[i] = rollback_step_for_switchover + + rollback_steps.sort(key=lambda x: x.getMoveOrder()) + + if actual_rollback_steps_cnt > 0: + self.logger.info('Saving following rollback rebalance execution steps:') + for step in rollback_steps: + self.logger.info(str(step)) + self.rebalance_schema.saveExecutionSteps(rollback_steps) + self.logger.info('Saved rollback rebalance execution steps') + else: + self.logger.info('No steps to rollback found for rebalance') + + self.trigger('move_to_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE') + + @wrap_func_with_faults + def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') + + @wrap_func_with_faults def on_enter_STATE_REBALANCE_DONE(self) -> None: - pass + if self.is_rollback_flow: + self.rebalance_schema.dropSchema() + self.logger.info('Rebalance rollback is complete') + else: + self.logger.info('Rebalance is complete') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_ERROR(self) -> None: raise Exception('Rebalance execution entered STATE_ERROR') diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py index a7bdbdf35814..b7c1d4df040f 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py @@ -46,6 +46,11 @@ def createSchema(self, plan: Plan) -> None: (plan BYTEA) DISTRIBUTED REPLICATED''') + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.segment_move_steps} + (move_order INT NOT NULL UNIQUE, status TEXT, is_rollback BOOL, step BYTEA) + DISTRIBUTED REPLICATED''') + self.savePlan(plan) dbconn.execSQL(self.conn, 'COMMIT') @@ -95,6 +100,15 @@ def getRebalanceStateFromPreviousRun(self) -> str: def getMainStateFromPreviousRun(self) -> str: return self.getStateFromPreviousRun(self.STATE_CATEGORY_MAIN) + def isRollbackRebalanceFlow(self, rollback_start_state: str) -> bool: + if self.schemaExists(): + row = dbconn.queryRow(self.conn, + f"SELECT COUNT(1) FROM {self.schema_name}.{self.rebalance_status} " + f"WHERE state_category = '{self.STATE_CATEGORY_REBALANCE}' " + f"AND state = '{rollback_start_state}'") + return int(row[0]) != 0 + return False + def rebalanceSchema(self, target_segment_count: int) -> None: # Before rebalancing check if the tables are already rebalanced # (in case we re-enter after interruption that happened after COMMIT but before new state) @@ -113,6 +127,11 @@ def rebalanceSchema(self, target_segment_count: int) -> None: f'''ALTER TABLE "{self.schema_name}"."{self.saved_plan}" REBALANCE {target_segment_count}''') + if get_table_distr_segment_count(self.conn, self.schema_name, self.segment_move_steps) > target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.schema_name}"."{self.segment_move_steps}" + REBALANCE {target_segment_count}''') + def storeState(self, state: str, state_category: str) -> None: if self.schemaExists(): dbconn.execSQL(self.conn, @@ -150,17 +169,12 @@ def getTablesToRebalanceWithStatus(self, status: str) -> cursor: def saveExecutionSteps(self, steps: List[RebalanceStep]) -> None: dbconn.execSQL(self.conn, 'BEGIN') - dbconn.execSQL(self.conn, f'DROP TABLE IF EXISTS {self.schema_name}.{self.segment_move_steps}') - - dbconn.execSQL(self.conn, - f'''CREATE TABLE {self.schema_name}.{self.segment_move_steps} - (move_order INT NOT NULL UNIQUE, status TEXT, step BYTEA) - DISTRIBUTED REPLICATED''') + dbconn.execSQL(self.conn, f'TRUNCATE TABLE {self.schema_name}.{self.segment_move_steps}') for step in steps: dbconn.execSQL(self.conn, f'''INSERT INTO {self.schema_name}.{self.segment_move_steps} - VALUES ({step.getMoveOrder()}, '{step.getStatus().name}', '\\x{step.serializeStep().hex()}')''') + VALUES ({step.getMoveOrder()}, '{step.getStatus().name}', '{step.isRollback()}', '\\x{step.serializeStep().hex()}')''') dbconn.execSQL(self.conn, 'COMMIT') @@ -171,7 +185,8 @@ def updateExecutionStep(self, step: RebalanceStep) -> None: def allExecutionStepsAreDone(self) -> bool: row = dbconn.queryRow(self.conn, - f"SELECT count(1) FROM {self.schema_name}.{self.segment_move_steps} WHERE status <> '{RebalanceStep.Status.DONE.name}'") + f"SELECT count(1) FROM {self.schema_name}.{self.segment_move_steps} " + f"WHERE status NOT IN ('{RebalanceStep.Status.DONE.name}', '{RebalanceStep.Status.CANCELLED.name}')") not_done_count = int(row[0]) return not_done_count == 0 diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_step.py b/gpMgmt/bin/gprebalance_modules/rebalance_step.py index 39bc176d1616..73c2008eabe5 100755 --- a/gpMgmt/bin/gprebalance_modules/rebalance_step.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_step.py @@ -10,39 +10,45 @@ class Status(Enum): PLANNED = 2 IN_PROGRESS = 3 ERROR = 4 - ROLLBACK_PLANNED = 5 - ROLLED_BACK = 6 - CANCELLED = 7 - DONE = 8 + CANCELLED = 5 + DONE = 6 def __init__(self, move: LogicalMove): self.move_order = -1 self.move = move self.status = self.Status.PLANNED + self.rollback = False - def __str__(self): + def __str__(self) -> str: + rollback_label = '' + if self.isRollback(): + rollback_label = '[ROLLBACK] ' return ( - f"Rebalance step with move_order: {self.getMoveOrder()}, status: {self.getStatus()}" + f"{rollback_label}Rebalance step with move_order: {self.getMoveOrder()}, status: {self.getStatus()}" ) - def getMoveOrder(self): + def getMoveOrder(self) -> int: return self.move_order - def setMoveOrder(self, move_order: int): + def setMoveOrder(self, move_order: int) -> None: self.move_order = move_order - def getStatus(self): + def getStatus(self) -> Status: return self.status - def getMove(self): + def getMove(self) -> LogicalMove: return self.move - def setStatus(self, status: Status): + def setStatus(self, status: Status, rollback: bool = False) -> None: self.status = status + self.rollback = rollback def serializeStep(self) -> bytes: return pickle.dumps(self) + def isRollback(self) -> bool: + return self.rollback + class RebalanceStepMoveMirror(RebalanceStep): def __init__(self, move: LogicalMove): super().__init__(move) @@ -58,7 +64,7 @@ def __init__(self, move: LogicalMove): super().__init__(move) self.status = self.Status.APPROVE_REQUIRED - def __str__(self): + def __str__(self) -> str: return ( f"{super().__str__()}, type: RebalanceStepSwitchoverToMirror, DBID {str(self.move.seg.getSegmentDbId())}" ) @@ -68,7 +74,7 @@ def __init__(self, move: LogicalMove): super().__init__(move) self.status = self.Status.APPROVE_REQUIRED - def __str__(self): + def __str__(self) -> str: return ( f"{super().__str__()}, type: RebalanceStepSwitchoverToPrimary, DBID {str(self.move.seg.getSegmentDbId())}" ) diff --git a/gpMgmt/bin/gprebalance_modules/shrink.py b/gpMgmt/bin/gprebalance_modules/shrink.py index 48798267ed6c..48a1f37c0f96 100644 --- a/gpMgmt/bin/gprebalance_modules/shrink.py +++ b/gpMgmt/bin/gprebalance_modules/shrink.py @@ -324,9 +324,12 @@ def cleanup(self, prev_run_was_complete: bool) -> None: def state_is_final(self, state: str) -> bool: return state == self.states_main_shrink_flow[-1] + def state_is_from_rollback_flow(self, state: str) -> bool: + return state in self.states_rollback_flow + # state callbacks start here - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: assert self.rebalance_schema.schemaExists() # check whether we can get the state where we stopped in previous run @@ -338,7 +341,7 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: self.logger.info(f"Previous run was completed successfully. Can't perform rollback.") self.trigger('move_to_STATE_END_FROM_ROLLBACK') else: - if state_from_prev_run in self.states_rollback_flow: + if self.state_is_from_rollback_flow(state_from_prev_run): self.logger.info('Continue interrupted shrink rollback operation...') self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") try: @@ -369,7 +372,7 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: # use auto to_«state» method to recover self.trigger(f'to_{next_state}') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED(self) -> None: dbconn.execSQL(self.conn, 'BEGIN') dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') @@ -385,34 +388,34 @@ def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED(self) self.trigger('move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE(self) -> None: self.logger.info(f'Updated target segment count to {self.shrink_plan.getTargetSegmentCount()}') self.trigger('move_to_STATE_PREPARE_SHRINK_SCHEMA_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED(self) -> None: self.prepare_shrink_schema(False) self.trigger('move_to_STATE_PREPARE_SHRINK_SCHEMA_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE(self) -> None: self.logger.info(f'Initiated list of tables to rebalance') self.trigger('move_to_STATE_SHRINK_TABLES_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_TABLES_STARTED(self) -> None: self.logger.info('Start tables rebalance for shrink') # perform 'ALTER TABLE REBALANCE' for all not yet processed tables self.rebalance_tables('none', 'done', self.shrink_plan.getTargetSegmentCount()) self.trigger('move_to_STATE_SHRINK_TABLES_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_TABLES_DONE(self) -> None: self.logger.info('Tables rebalance complete') self.trigger('move_to_STATE_SHRINK_CATALOG_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_CATALOG_STARTED(self) -> None: self.logger.info('Start catalog shrink') @@ -427,12 +430,12 @@ def on_enter_STATE_SHRINK_CATALOG_STARTED(self) -> None: self.trigger('move_to_STATE_SHRINK_CATALOG_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_CATALOG_DONE(self) -> None: self.logger.info('Catalog shrink complete') self.trigger('move_to_STATE_SHRINK_SEGMENTS_STOP_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED(self) -> None: self.logger.info('Stopping shrinked segments...') @@ -469,18 +472,18 @@ def on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED(self) -> None: self.trigger('move_to_STATE_SHRINK_SEGMENTS_STOP_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_SEGMENTS_STOP_DONE(self) -> None: self.logger.info('Shrinked segments were stopped') self.trigger('move_to_STATE_SHRINK_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_DONE(self) -> None: os.remove(self.gparray_dump_file) self.logger.info('Shrink is complete') self.trigger('move_to_STATE_END') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START(self) -> None: dbconn.execSQL(self.conn, 'BEGIN') dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') @@ -492,51 +495,51 @@ def on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START(self) -> N self.trigger('move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE(self) -> None: self.trigger('move_to_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START(self) -> None: self.prepare_shrink_schema(True) self.trigger('move_to_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE(self) -> None: self.trigger('move_to_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START(self) -> None: self.logger.info('Start tables rebalance for rollback') # perform 'ALTER TABLE REBALANCE' for all not yet processed tables self.rebalance_tables('done', 'none', self.gparray.get_segment_count()) self.trigger('move_to_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE(self) -> None: self.trigger('move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START(self) -> None: if os.path.exists(self.gparray_dump_file): os.remove(self.gparray_dump_file) self.rebalance_schema.dropSchema() self.trigger('move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE(self) -> None: self.logger.info('Rollback is complete.') self.trigger('move_to_STATE_END_FROM_ROLLBACK') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_END_FROM_ROLLBACK(self) -> None: self.trigger('move_to_STATE_END') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_END(self) -> None: pass - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_ERROR(self) -> None: raise Exception('Shrink entered STATE_ERROR') diff --git a/gpMgmt/sbin/gpsegrecovery.py b/gpMgmt/sbin/gpsegrecovery.py index 74c70ef8e92b..c41bc6ced644 100644 --- a/gpMgmt/sbin/gpsegrecovery.py +++ b/gpMgmt/sbin/gpsegrecovery.py @@ -18,6 +18,7 @@ from gppylib.operations.segment_tablespace_locations import get_segment_tablespace_oid_locations from gppylib.commands.unix import terminate_proc_tree from gppylib.commands.unix import get_remote_link_path +from gppylib.fault_injection import * class FullRecovery(Command): @@ -343,6 +344,7 @@ def sync_tablespaces(self): os.symlink(targetPath, targetOidPath) +@wrap_func_with_faults def start_segment(recovery_info, logger, era): seg = Segment(None, None, None, None, None, None, None, None, recovery_info.target_port, recovery_info.target_datadir) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature index 5300a340ca10..d7b84a8225a6 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature @@ -81,3 +81,35 @@ Feature: ggrebalance behave tests Then ggrebalance should return a return code of 0 And ggrebalance should print "Reset numsegments to default is done." to logfile with latest timestamp And ggrebalance should print "Cleanup is complete" to logfile with latest timestamp + + Scenario: test 4. check cleanup after shrink is complete, and rebalance was interrupted + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_REBALANCE_DONE_begin" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Cleanup is complete" to logfile with latest timestamp + # some mirrors are definitely down, so do not check them + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index a66020dc26b0..07b943c14ea6 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -128,12 +128,10 @@ Feature: ggrebalance behave tests (rebalance scenarios) And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows And all files in gpAdminLogs directory are deleted And set fault inject "" - And set fault inject delay ms When the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject - And unset fault inject delay And all files in gpAdminLogs directory are deleted And the gprecoverseg lock directory is removed When the user runs "ggrebalance" @@ -153,32 +151,25 @@ Feature: ggrebalance behave tests (rebalance scenarios) Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 Examples: - | fault_name | fault_delay_ms | - | on_enter_STATE_REBALANCE_STARTED_begin | 0 | - | on_enter_STATE_REBALANCE_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_begin | 0 | - | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE_begin | 0 | - | on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE_end | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_begin | 0 | - | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_end | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_begin | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_begin | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_end | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_DONE_begin | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_DONE_end | 0 | - | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 0 | - | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 0 | - | on_enter_STATE_REBALANCE_DONE_begin | 0 | - | on_enter_STATE_REBALANCE_DONE_end | 0 | - | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 1500 | - | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 3000 | - | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 1500 | - | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 3000 | - | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | 3000 | + | fault_name | + | on_enter_STATE_REBALANCE_STARTED_begin | + | on_enter_STATE_REBALANCE_STARTED_end | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_begin | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_end | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE_begin | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE_end | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_end | + | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_begin | + | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_end | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_end | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_end | + | on_enter_STATE_REBALANCE_EXECUTION_DONE_begin | + | on_enter_STATE_REBALANCE_EXECUTION_DONE_end | + | on_enter_STATE_REBALANCE_DONE_begin | + | on_enter_STATE_REBALANCE_DONE_end | Scenario: 4. rebalance - check rebalance after interrupted shrink. Given the database is not running @@ -251,3 +242,822 @@ Feature: ggrebalance behave tests (rebalance scenarios) And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 + + Scenario Outline: 6.1.1 rebalance - interrupt during mirror move before gp_segment_configuration update, continue and retry failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "yes" to the prompt "Retry step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: False" to logfile with latest timestamp + And ggrebalance should print "Port is updated: False" to logfile with latest timestamp + And ggrebalance should print "Plan to retry step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | _update_config_begin | + + Scenario Outline: 6.1.2 rebalance - interrupt during mirror move before gp_segment_configuration update, continue and rollback failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Retry step?" + And user will answer "yes" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: False" to logfile with latest timestamp + And ggrebalance should print "Port is updated: False" to logfile with latest timestamp + And ggrebalance should print "Plan to rollback step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | _update_config_begin | + + Scenario Outline: 6.1.3 rebalance - interrupt during mirror move before gp_segment_configuration update, continue and cancel failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Retry step?" + And user will answer "no" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: False" to logfile with latest timestamp + And ggrebalance should print "Port is updated: False" to logfile with latest timestamp + And ggrebalance should print "Cancel step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'd'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | _update_config_begin | + + Scenario Outline: 6.2.1. rebalance - interrupt during switchover step (before invocation of 'gprecoverseg'), continue and retry failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "yes" to the prompt "Retry step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Processing error status for switchover step" to logfile with latest timestamp + And ggrebalance should print "Plan to retry step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | + | GpSegmentRebalanceOperation_rebalance_at_seg_stop | + + Scenario Outline: 6.2.2. rebalance - interrupt during switchover P->M step (before invocation of 'gprecoverseg'), continue and rollback failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Retry step?" + And user will answer "yes" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Processing error status for switchover step" to logfile with latest timestamp + And ggrebalance should print "Plan to rollback step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 4 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | + | GpSegmentRebalanceOperation_rebalance_at_seg_stop | + + Scenario Outline: 6.2.3. rebalance - interrupt during switchover M->P step (before invocation of 'gprecoverseg'), continue and rollback failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Retry step?" + And user will answer "yes" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Processing error status for switchover step" to logfile with latest timestamp + And ggrebalance should print "Plan to rollback step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 4 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | + + Scenario Outline: 6.2.4. rebalance - interrupt during switchover P->M step (before invocation of 'gprecoverseg'), continue and cancel failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Retry step?" + And user will answer "no" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Processing error status for switchover step" to logfile with latest timestamp + And ggrebalance should print "Cancel step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 4 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | + + Scenario Outline: 6.3.1. rebalance - interrupt during mirror move after gp_segment_configuration update, but before port update, continue and cancel failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: True" to logfile with latest timestamp + And ggrebalance should print "Port is updated: False" to logfile with latest timestamp + And ggrebalance should print "Cancel step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + # some mirrors are definitely down, so do not check them + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | _update_config_end | + + Scenario Outline: 6.3.2. rebalance - interrupt during mirror move after gp_segment_configuration update, but before port update, continue and rollback failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "yes" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: True" to logfile with latest timestamp + And ggrebalance should print "Port is updated: False" to logfile with latest timestamp + And ggrebalance should print "Plan to rollback step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | _update_config_end | + + Scenario Outline: 6.4.1. rebalance - interrupt during mirror move after port update (when the mirror is actually started), and continue. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "Start checking if segment is up with timeout" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: True" to logfile with latest timestamp + And ggrebalance should print "Port is updated: True" to logfile with latest timestamp + And ggrebalance should print "The step is complete, mark it as done" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | _do_recovery_end | + + Scenario Outline: 6.4.2. rebalance - interrupt during mirror move after port update (but before the mirror is actually started), and continue (with step rollback). + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And on host "sdw1" set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And on host "sdw1" unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Timeout waiting for segment start, wait again?" + And user will answer "yes" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "Start checking if segment is up with timeout" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: True" to logfile with latest timestamp + And ggrebalance should print "Port is updated: True" to logfile with latest timestamp + And ggrebalance should print "Plan to rollback step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | start_segment_begin | + + Scenario Outline: 6.4.3. rebalance - interrupt during mirror move after port update (but before the mirror is actually started), and continue (with step cancel). + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And on host "sdw1" set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And on host "sdw1" unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Timeout waiting for segment start, wait again?" + And user will answer "no" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "Start checking if segment is up with timeout" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: True" to logfile with latest timestamp + And ggrebalance should print "Port is updated: True" to logfile with latest timestamp + And ggrebalance should print "Cancel step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + # some mirrors are definitely down, so do not check them + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | start_segment_begin | + + Scenario Outline: 7.1. rebalance - rebalance interrupt and full rollback. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And the gp_segment_configuration have been saved + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance rollback is complete" to logfile with latest timestamp + And verify the gp_segment_configuration has been restored + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_end | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_end | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | + | on_enter_STATE_REBALANCE_DONE_begin | + + Scenario Outline: 7.2. rebalance - rebalance interrupt, rollback (and interrupt again) and continue. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And the gp_segment_configuration have been saved + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_REBALANCE_DONE_begin" + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + And set fault inject "" + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "yes" to the prompt "Retry step?" + And the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance rollback is complete" to logfile with latest timestamp + And verify the gp_segment_configuration has been restored + And clear user's answers + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED_begin | + | on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED_end | + | on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE_begin | + | on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE_end | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_end | + | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_begin | + | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_end | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_end | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_end | + | on_enter_STATE_REBALANCE_EXECUTION_DONE_begin | + | on_enter_STATE_REBALANCE_EXECUTION_DONE_end | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | + | on_enter_STATE_REBALANCE_DONE_begin | + + Scenario: test 7.3.1. rebalance - interrupt during shrink, and full rollback. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_SHRINK_TABLES_STARTED_begin" + And the gp_segment_configuration have been saved + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is complete" to logfile with latest timestamp + And verify the gp_segment_configuration has been restored + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Scenario: test 7.3.2. rebalance - interrupt during shrink, and full rollback, interrupt during shrink rollback, and continue. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_SHRINK_CATALOG_STARTED_begin" + And the gp_segment_configuration have been saved + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_end" + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is already in progress, and was interrupted. Execute 'ggrebalance' without '-r' flag." to logfile with latest timestamp + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is complete" to logfile with latest timestamp + And verify the gp_segment_configuration has been restored + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Scenario: test 7.4. rebalance - interrupt after shrink, but before rebalance start, and full rollback. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_REBALANCE_STARTED_begin" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance rollback is complete" to logfile with latest timestamp + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 + + Scenario: test 7.4. rebalance - shrink, rebalance (and interrupt during it) and full rollback. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_REBALANCE_DONE_begin" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance rollback is complete" to logfile with latest timestamp + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 3d6f3936aa3f..e09f4eecde3c 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -4585,6 +4585,30 @@ def impl(context): if hasattr(context, 'fault_flag_filename') and os.path.exists(context.fault_flag_filename): os.remove(context.fault_flag_filename) +@given('on host "{host}" set fault inject "{fault}"') +@then('on host "{host}" set fault inject "{fault}"') +@when('on host "{host}" set fault inject "{fault}"') +def impl(context, fault, host): + os.environ[fault_injection.GPMGMT_FAULT_POINT] = fault + cmd = f""" + ssh {host} " + echo 'export {fault_injection.GPMGMT_FAULT_POINT}={fault}' >> ~/.bashrc" + export {fault_injection.GPMGMT_FAULT_POINT}={fault} + """ + run_command(context, cmd.strip()) + +@given('on host "{host}" unset fault inject') +@then('on host "{host}" unset fault inject') +@when('on host "{host}" unset fault inject') +def impl(context, host): + cmd = f""" + ssh {host} " + sed -i '/{fault_injection.GPMGMT_FAULT_POINT}=/d' ~/.bashrc + unset {fault_injection.GPMGMT_FAULT_POINT} + " + """ + run_command(context, cmd.strip()) + @given('set fault inject delay {delay} ms') @then('set fault inject delay {delay} ms') @when('set fault inject delay {delay} ms') @@ -4607,6 +4631,25 @@ def impl(context): def impl(context): os.environ[fault_injection.GPMGMT_FAULT_DELAY_MS] = "" +@given('user will answer "{answer}" to the prompt "{prompt}"') +@then('user will answer "{answer}" to the prompt "{prompt}"') +@when('user will answer "{answer}" to the prompt "{prompt}"') +def impl(context, answer, prompt): + assert answer == 'yes' or answer == 'no' + if not hasattr(context, 'fault_injected_answers'): + context.fault_injected_answers = {} + context.fault_injected_answers[prompt] = answer + os.environ[fault_injection.GPMGMT_FAULT_TYPE] = fault_injection.GPMGMT_FAULT_TYPE_VALUE + os.environ[fault_injection.GPMGMT_FAULT_POINT] = json.dumps(context.fault_injected_answers) + +@given("clear user's answers") +@then("clear user's answers") +@when("clear user's answers") +def impl(context): + context.fault_injected_answers = {} + os.environ[fault_injection.GPMGMT_FAULT_TYPE] = '' + os.environ[fault_injection.GPMGMT_FAULT_POINT] = '' + @given('stub') def impl(context): pass