From 48822fc225dffabcd677f86aa626947e68c5afea Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 27 Feb 2026 13:41:03 +1000 Subject: [PATCH 01/28] Add initial error handling --- gpMgmt/bin/ggrebalance | 2 + gpMgmt/bin/gppylib/commands/gp.py | 41 +++++ gpMgmt/bin/gppylib/fault_injection.py | 9 +- .../gppylib/operations/buildMirrorSegments.py | 5 + .../bin/gprebalance_modules/ggrebalance_sm.py | 144 +++++++++++++++++- .../gprebalance_modules/rebalance_schema.py | 3 +- .../bin/gprebalance_modules/rebalance_step.py | 8 + .../mgmt_utils/ggrebalance_rebalance.feature | 144 ++++++++++++++++++ .../behave/mgmt_utils/steps/mgmt_utils.py | 11 ++ 9 files changed, 359 insertions(+), 8 deletions(-) diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index 194d090170b4..b6f655f6223e 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -279,6 +279,7 @@ def check_down_segments(logger: Any, options: Any, dburl: dbconn.DbURL): if cnt_primaries_down != 0: raise Exception('Detected some primary segments are down, please recover manually') + """ if cnt_mirrors_down != 0: logger.info("Some mirrors are down, trying to recover them, it may take some time...") recoverseg_options = "-a -F" @@ -294,6 +295,7 @@ def check_down_segments(logger: Any, options: Any, dburl: dbconn.DbURL): raise Exception(error_msg) finally: cmd_recoverseg = None + """ def main(options, args, parser): conn = None diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index f6b7ed70b42e..5167d9e85b46 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -325,6 +325,28 @@ def __init__(self, datadir, mode, wait, timeout): self.append("stop") +class PgCtlStatusArgs(CmdArgs): + """ + Used by CoordinatorStop, SegmentStop to format the pg_ctl command + to stop a backend postmaster + + >>> str(PgCtlStatusArgs("/data1/coordinator/gpseg-1")) + '$GPHOME/bin/pg_ctl -D /data1/coordinator/gpseg-1 status' + + """ + + def __init__(self, datadir): + """ + @param datadir: database data directory + """ + CmdArgs.__init__(self, [ + "$GPHOME/bin/pg_ctl", + "-D", str(datadir), + ]) + self.append("status") + + + class CoordinatorStart(Command): def __init__(self, name, dataDir, port, era, wrapper, wrapper_args, specialMode=None, restrictedMode=False, timeout=SEGMENT_TIMEOUT_DEFAULT, @@ -441,6 +463,25 @@ def remote(name, hostname, dataDir, mode='smart'): cmd.run(validateAfter=True) return cmd +#----------------------------------------------- +class SegmentStatus(Command): + def __init__(self, name, dataDir, ctxt=LOCAL, remoteHost=None): + + self.cmdStr = str( PgCtlStatusArgs(dataDir) ) + Command.__init__(self, name, self.cmdStr, ctxt, remoteHost) + + @staticmethod + def local(name, dataDir): + cmd=SegmentStatus(name, dataDir) + cmd.run(validateAfter=False) + return cmd + + @staticmethod + def remote(name, hostname, dataDir): + cmd=SegmentStatus(name, dataDir, ctxt=REMOTE, remoteHost=hostname) + cmd.run(validateAfter=False) + return cmd + #----------------------------------------------- class SegmentIsShutDown(Command): """ diff --git a/gpMgmt/bin/gppylib/fault_injection.py b/gpMgmt/bin/gppylib/fault_injection.py index 142ab17dfcfd..ed511bee3e00 100755 --- a/gpMgmt/bin/gppylib/fault_injection.py +++ b/gpMgmt/bin/gppylib/fault_injection.py @@ -24,10 +24,17 @@ def raise_exception(delay: int): else: raise Exception('Fault Injection %s' % os.environ[GPMGMT_FAULT_POINT]) +def inject_fault_check(fault_point) -> bool: + if GPMGMT_FAULT_POINT in os.environ and fault_point == os.environ[GPMGMT_FAULT_POINT]: + return True + else: + return False + # decorator for test purposes def wrap_state_func_with_faults(func): def func_with_faults(*args): inject_fault(f'{func.__name__}_begin') - func(*args) + result = func(*args) inject_fault(f'{func.__name__}_end') + return result return func_with_faults diff --git a/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py b/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py index 9c07e7f2387b..b7d629bd7709 100644 --- a/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py +++ b/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py @@ -21,6 +21,7 @@ from gppylib.commands.gp import is_pid_postmaster, get_pid_from_remotehost from gppylib.commands.unix import check_pid_on_remotehost from gppylib.programs.clsRecoverSegment_triples import RecoveryTriplet +from gppylib.fault_injection import * logger = gplog.get_default_logger() @@ -313,6 +314,7 @@ def _trigger_fts_probe(self, port=0): dbconn.execSQL(conn,"SELECT gp_request_fts_probe_scan()") conn.close() + @wrap_state_func_with_faults def _update_config(self, recovery_info_by_host, gpArray): # should use mainUtils.getProgramName but I can't make it work! programName = os.path.split(sys.argv[0])[-1] @@ -587,6 +589,7 @@ def _run_recovery(self, action_name, recovery_info_by_host, gpEnv): self._remove_progress_files(recovery_info_by_host, recovery_results) return recovery_results + @wrap_state_func_with_faults def _do_recovery(self, recovery_info_by_host, gpEnv): """ # Recover and start segments using gpsegrecovery, which will internally call either @@ -687,6 +690,7 @@ def _get_failed_reachable_segments(self): failed_reachable_segments.append(failed) return failed_reachable_segments + @wrap_state_func_with_faults def _stop_failed_segments(self, gpEnv): failed_reachable_segments = self._get_failed_reachable_segments() if len(failed_reachable_segments) == 0: @@ -715,6 +719,7 @@ def _stop_failed_segments(self, gpEnv): # self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, suppressErrorCheck=True) + @wrap_state_func_with_faults def _wait_fts_to_mark_down_segments(self, gpEnv, segments_to_mark_down): """Waits for FTS prober to mark segments as down""" diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index 3d371fea5d27..599fb4aed704 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -7,7 +7,7 @@ from gppylib.commands.unix import * from gppylib.commands.gp import * from gppylib.gplog import * - from gppylib.commands.gp import GpMoveMirrors + from gppylib.commands.gp import GpMoveMirrors, SegmentStatus from gppylib.system.environment import * from gprebalance_modules.planner import * from gprebalance_modules.rebalance_schema import RebalanceSchema, STATE_NOT_DEFINED @@ -315,9 +315,142 @@ def get_state_after_interrupt(self, prev_state) -> str: def reset_in_progress_execution_steps(self) -> None: in_progress_steps = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.IN_PROGRESS]) for step in in_progress_steps: - step.setStatus(RebalanceStep.Status.PLANNED) + step.setStatus(RebalanceStep.Status.ERROR) self.rebalance_schema.updateExecutionStep(step) + def process_error_execution_steps(self) -> None: + dbconn.execSQL(self.conn, "BEGIN") + try: + error_steps = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.ERROR]) + steps_left_todo = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) + any_step_cancelled = False + # TODO: what about switchover steps? + for step in error_steps: + self.logger.info(f'Checking error status for step: {str(step)}') + dbid = step.getMove().seg.getSegmentDbId() + target_hostname = step.getMove().dstHost.hostname + target_datadir = step.getMove().target_datadir + target_port = step.getMove().target_port + + catalog_segment_info = self.get_catalog_gp_segment_configuration_for_dbid(dbid) + self.logger.info(f'Segment info from catalog: {str(catalog_segment_info)}') + + gp_segment_configuration_updated = (catalog_segment_info.hostname == target_hostname) + + port_updated = False + if gp_segment_configuration_updated: + port_updated = (self.get_postgresql_conf_port(target_hostname, target_datadir) == target_port) + + self.logger.info(f'gp_segment_configuration is updated: {gp_segment_configuration_updated}') + self.logger.info(f'Port is updated: {port_updated}') + + time_waited = 0 + SLEEP_PERIOD_SEC = 1.0 + TIMEOUT_SEC = 120.0 + if port_updated: + self.logger.info(f'Start checking if segment is up with timeout of {TIMEOUT_SEC} sec.') + while time_waited < TIMEOUT_SEC: + # Start polling segment status with timeout + segment_process_started = SegmentStatus.remote('Segment status check', target_hostname, target_datadir).was_successful() + if segment_process_started: + catalog_segment_info = self.get_catalog_gp_segment_configuration_for_dbid(dbid) + if catalog_segment_info.isSegmentUp() and catalog_segment_info.isSegmentModeSynchronized(): + self.logger.info('The step is complete, mark it as done') + step.setStatus(RebalanceStep.Status.DONE) + self.rebalance_schema.updateExecutionStep(step) + return + + time.sleep(SLEEP_PERIOD_SEC) + time_waited = time_waited + SLEEP_PERIOD_SEC + if self.interactive_check('Timeout waiting for segment start, wait again?'): + time_waited = 0 + + if not gp_segment_configuration_updated and self.interactive_check(f'Retry step?'): + if step.isRollback(): + self.logger.info('Plan to retry rollback step') + step.setStatus(RebalanceStep.Status.ROLLBACK_PLANNED) + else: + self.logger.info('Plan to retry step') + step.setStatus(RebalanceStep.Status.PLANNED) + self.rebalance_schema.updateExecutionStep(step) + return + + if not step.isRollback() and self.interactive_check('Rollback step?'): + self.logger.info('Plan to rollback step') + step.setStatus(RebalanceStep.Status.ROLLBACK_PLANNED) + else: + # TODO: need to cancel all dependent steps + self.logger.info('Cancel step') + step.setStatus(RebalanceStep.Status.CANCELLED) + any_step_cancelled = True + self.rebalance_schema.updateExecutionStep(step) + + # Mark dependent steps accordingly + # 1. If there are steps planned for ROLLBACK - we mark all left todo steps for the same content as already ROLLED_BACK + for step in error_steps: + if step.getStatus() == RebalanceStep.Status.ROLLBACK_PLANNED: + content_id = step.getMove().seg.getSegmentContentId() + # TODO: DOUBLE check the full rollback flow here!!!!!!!!!!! + for step_todo in steps_left_todo: + if step_todo.getMove().seg.getSegmentContentId() == content_id: + self.logger.info(f'Mark as already ROLLED_BACK the dependent step {step_todo}') + step_todo.setStatus(RebalanceStep.Status.ROLLED_BACK) + self.rebalance_schema.updateExecutionStep(step_todo) + # 2. If there are any cancelled steps - we need to: + # a. cancel all not yet done steps of the same dbid, + # b. and *ALL* switchovers, + # c. and do cancelation recursively. + # But, actually, it means that we need to cancel everything besides steps revived from the ERROR state just above, + # as left todo steps didn't get into this ERRORed batch, meaning they must have different step type (meaning switchover). + if any_step_cancelled: + for step_todo in steps_left_todo: + self.logger.info(f'Mark as CANCELLED the step {step_todo}') + step_todo.setStatus(RebalanceStep.Status.CANCELLED) + self.rebalance_schema.updateExecutionStep(step_todo) + + finally: + dbconn.execSQL(self.conn, "COMMIT") + + def get_catalog_gp_segment_configuration_for_dbid(self, dbid: int) -> Segment: + row = dbconn.queryRow(self.conn, + f"SELECT dbid||'|'||content||'|'||role||'|'||preferred_role||'|'||mode||'|'||status||'|'||hostname||'|'||address||'|'||port||'|'||datadir " + f"FROM gp_segment_configuration WHERE dbid = {dbid}") + return Segment.initFromString(row[0]) + + def get_postgresql_conf_port(self, hostname: str, datadir: str) -> int: + #TODO: recheck cmd + cmd = Command( + name="get_postgresql_conf_port", + cmdStr=f"grep -E '^port\\s*=' {datadir}/postgresql.conf | sed -E 's/^port\\s*=\\s*([0-9]+).*/\\1/'", + ctxt=REMOTE, + remoteHost=hostname) + cmd.run() + + if not cmd.was_successful(): + self.logger.info(f"Failed to get port from postgresql.conf on {hostname}: {cmd.get_stderr()}") + return -1 + + output = cmd.get_stdout().strip() + if not output or not output.isdigit(): + return -1 + + return int(output) + + # decorator to TODO: add comments + def wrap_interactive_check_with_faults(fun): + def func_with_faults(self, msg: str): + if inject_fault_check(msg + 'yes'): + return True + if inject_fault_check(msg + 'no'): + return False + return fun(self, msg) + return func_with_faults + + @wrap_interactive_check_with_faults + def interactive_check(self, msg: str) -> bool: + #Currently only a stub + return False + @staticmethod def convert_moves_to_rebalance_steps(moves: List[LogicalMove]) -> List[RebalanceStep]: # In the loop below we create a list of rebalance execution steps from the plan's list of moves. @@ -426,9 +559,11 @@ def on_enter_STATE_REBALANCE_EXECUTION_STARTED(self) -> None: # In normal execution we shouldn't have IN_PROGRESS steps at this moment. # If they are presented, it means they are left from previous interrupted run. - # Bring them back to PLANNED state, so we can try to process them again. + # Set them to ERROR state. self.reset_in_progress_execution_steps() + self.process_error_execution_steps() + steps_to_execute = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) if len(steps_to_execute) > 0: @@ -468,9 +603,6 @@ def on_enter_STATE_REBALANCE_EXECUTION_STARTED(self) -> None: self.execute_role_swaps(segments, direction) self.logger.info('Rebalance - end role swap') - # TODO: check the errored segments here, once we implement rollback for the rebalance. - # For now if some error happened, the entire tool will halt its work, so if we reached this point - # just mark all steps as done. for step in steps_to_execute: if step.getStatus() == RebalanceStep.Status.IN_PROGRESS: step.setStatus(RebalanceStep.Status.DONE) diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py index a7bdbdf35814..12c938e187c7 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py @@ -171,7 +171,8 @@ def updateExecutionStep(self, step: RebalanceStep) -> None: def allExecutionStepsAreDone(self) -> bool: row = dbconn.queryRow(self.conn, - f"SELECT count(1) FROM {self.schema_name}.{self.segment_move_steps} WHERE status <> '{RebalanceStep.Status.DONE.name}'") + f"SELECT count(1) FROM {self.schema_name}.{self.segment_move_steps} " + f"WHERE status NOT IN ('{RebalanceStep.Status.DONE.name}', '{RebalanceStep.Status.CANCELLED.name}')") not_done_count = int(row[0]) return not_done_count == 0 diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_step.py b/gpMgmt/bin/gprebalance_modules/rebalance_step.py index 39bc176d1616..66e4914f7c2b 100755 --- a/gpMgmt/bin/gprebalance_modules/rebalance_step.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_step.py @@ -19,6 +19,7 @@ def __init__(self, move: LogicalMove): self.move_order = -1 self.move = move self.status = self.Status.PLANNED + self.rollback = False def __str__(self): return ( @@ -39,10 +40,17 @@ def getMove(self): def setStatus(self, status: Status): self.status = status + if self.status == self.Status.ROLLBACK_PLANNED: + #For a step, when it is planned to rollback, + #there is no turning back. + self.rollback = True def serializeStep(self) -> bytes: return pickle.dumps(self) + def isRollback(self) -> bool: + return self.rollback + class RebalanceStepMoveMirror(RebalanceStep): def __init__(self, move: LogicalMove): super().__init__(move) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index a66020dc26b0..99e69dadac99 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -251,3 +251,147 @@ Feature: ggrebalance behave tests (rebalance scenarios) And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 + + Scenario Outline: 3.10 rebalance - interrupt (with retry) and continue. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "yes" to the prompt "Retry step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: False" to logfile with latest timestamp + And ggrebalance should print "Port is updated: False" to logfile with latest timestamp + And ggrebalance should print "Plan to retry step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + #| _stop_failed_segments_begin | + #| _stop_failed_segments_end | + #| _wait_fts_to_mark_down_segments_begin | + #| _wait_fts_to_mark_down_segments_end | + | _update_config_begin | + + Scenario Outline: 3.11 rebalance - interrupt (with cancel) and continue. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: True" to logfile with latest timestamp + And ggrebalance should print "Port is updated: False" to logfile with latest timestamp + And ggrebalance should print "Cancel step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + # some mirrors are definitely down, so do not check them + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | _update_config_end | + + + Scenario Outline: 3.12 rebalance - interrupt (when the mirror is actually started) and continue. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "Start checking if segment is up with timeout" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: True" to logfile with latest timestamp + And ggrebalance should print "Port is updated: True" to logfile with latest timestamp + And ggrebalance should print "The step is complete, mark it as done" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | _do_recovery_end | diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 059f6e27c8f5..ed18ea79285a 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -4546,6 +4546,17 @@ def impl(context, delay): def impl(context): os.environ['GPMGMT_FAULT_DELAY_MS'] = "" +@given('user will answer "{answer}" to the prompt "{prompt}"') +@then('user will answer "{answer}" to the prompt "{prompt}"') +@when('user will answer "{answer}" to the prompt "{prompt}"') +def impl(context, answer, prompt): + assert answer == 'yes' or answer == 'no' + #TODO: change to defines + if 'GPMGMT_FAULT_POINT' in os.environ and os.environ['GPMGMT_FAULT_POINT'] != "": + raise Exception('Need to unset fault injection before using this step') + fault = prompt + answer + os.environ['GPMGMT_FAULT_POINT'] = fault + @given('stub') def impl(context): pass From b34be7fb4f24613ad2719579d4fd6b37e660dac5 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Mon, 2 Mar 2026 16:35:34 +1000 Subject: [PATCH 02/28] Implement full rollback initial draft --- .../ggrebalance_main_sm.py | 14 +- .../bin/gprebalance_modules/ggrebalance_sm.py | 169 ++++++++++++++---- .../gprebalance_modules/rebalance_schema.py | 13 +- .../bin/gprebalance_modules/rebalance_step.py | 32 ++-- .../mgmt_utils/ggrebalance_rebalance.feature | 44 +++++ 5 files changed, 219 insertions(+), 53 deletions(-) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py index a1d347d27899..a3f5a762bd46 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py @@ -191,9 +191,16 @@ def on_enter_STATE_CLEANUP(self) -> None: @wrap_state_func_with_faults def on_enter_STATE_ROLLBACK(self) -> None: - self.plan = self.rebalance_schema.retrieveSavedPlan() - self.gg_shrink.rollback(self.plan) - self.trigger('move_to_STATE_END') + try: + self.plan = self.rebalance_schema.retrieveSavedPlan() + if isinstance(self.plan, ShrinkPlan): + shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() + if not self.gg_shrink.state_is_final(shrink_state_from_prev_run): + self.gg_shrink.rollback(self.plan) + return + self.gg_rebalance.rollback() + finally: + self.trigger('move_to_STATE_END') @wrap_state_func_with_faults def on_enter_STATE_PLANNING_STARTED(self) -> None: @@ -281,7 +288,6 @@ def on_enter_STATE_SHRINK_DONE(self) -> None: def on_enter_STATE_REBALANCE_STARTED(self) -> None: if self.plan is not None and self.plan.getMoves() is not None: self.gg_rebalance.run(self.plan) - self.logger.info('Rebalance is complete') self.trigger('move_to_STATE_REBALANCE_DONE') diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index 599fb4aed704..34e47f151581 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -36,6 +36,13 @@ class RebalanceSM: 'STATE_REBALANCE_DONE' ] + states_rollback_rebalance_flow = [ + 'STATE_REBALANCE_ROLLBACK_STARTED', + 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED', + 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE' + # TODO: Add schema drop state??? + ] + transitions = [ { 'trigger': 'start', @@ -59,7 +66,10 @@ class RebalanceSM: }, { 'trigger': 'move_to_STATE_REBALANCE_EXECUTION_STARTED', - 'source': ['STATE_REBALANCE_PREPARE_MOVES_DONE', 'STATE_REBALANCE_MOVES_SUCCEEDED', 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE'], + 'source': ['STATE_REBALANCE_PREPARE_MOVES_DONE', + 'STATE_REBALANCE_MOVES_SUCCEEDED', + 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE', + 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE'], 'dest': 'STATE_REBALANCE_EXECUTION_STARTED' }, { @@ -82,6 +92,21 @@ class RebalanceSM: 'source': 'STATE_REBALANCE_EXECUTION_STARTED', 'dest': 'STATE_REBALANCE_EXECUTION_DONE' }, + { + 'trigger': 'rollback', + 'source': 'STATE_REBALANCE_INIT', + 'dest': 'STATE_REBALANCE_ROLLBACK_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED', + 'source': 'STATE_REBALANCE_ROLLBACK_STARTED', + 'dest': 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE', + 'source': 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED', + 'dest': 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE' + }, { 'trigger': 'move_to_STATE_REBALANCE_DONE', 'source': 'STATE_REBALANCE_EXECUTION_DONE', @@ -106,10 +131,11 @@ def __init__(self, conn: dbconn.Connection, schema: RebalanceSchema, logger: Any self.conn = conn self.rebalance_schema = schema self.cmd = None + self.is_rebalance_flow = False self.machine = Machine(model = self, queued=True, - states = self.states_main_rebalance_flow + self.states_not_logged, + states = self.states_main_rebalance_flow + self.states_not_logged + self.states_rollback_rebalance_flow, transitions = self.transitions, initial = 'STATE_REBALANCE_INIT', before_state_change = 'on_every_state') @@ -119,7 +145,7 @@ def on_every_state(self) -> None: self.logger.info('Rebalance was interrupted') raise Exception('Rebalance was interrupted') - if self.state in self.states_main_rebalance_flow: + if self.state in self.states_main_rebalance_flow + self.states_rollback_rebalance_flow: self.rebalance_schema.storeRebalanceState(self.state) def run(self, plan: Plan) -> None: @@ -139,11 +165,17 @@ def run(self, plan: Plan) -> None: self.trigger('start') - def process_moves(self, moves: List[LogicalMove]): - if len(moves) == 0: + def rollback(self) -> None: + if self.rebalance_schema.schemaExists(): + self.trigger('rollback') + else: + self.logger.info("Rebalance schema doesn't exist. Can't perform rollback.") + + def process_moves(self, steps: List[RebalanceStepMoveMirror]): + if len(steps) == 0: return - filename = self.create_config_file(moves) + filename = self.create_config_file(steps) gpmovemirrors_options = f'-a -i {filename}' if self.options.parallel is not None: @@ -281,16 +313,24 @@ def lookup_seg(self, seg: Segment) -> bool: return True return False - def create_config_file(self, moves: List[LogicalMove]) -> str: + def create_config_file(self, steps: List[RebalanceStepMoveMirror]) -> str: filename = f'/tmp/ggrebalance_move_config_pid{os.getpid()}' with open(filename, 'w') as fp: - for move in moves: - segment_current_info = move.seg - if not self.lookup_seg(segment_current_info): - self.logger.info(f'Skip segment for gpmovemirrors: {str(segment_current_info)}') - continue - cfg_line = f'{segment_current_info.getSegmentHostName()}|{segment_current_info.getSegmentPort()}|{segment_current_info.getSegmentDataDirectory()} ' - cfg_line += f'{move.dstHost.hostname}|{move.target_port}|{move.target_datadir}\n' + for step in steps: + assert isinstance(step, RebalanceStepMoveMirror) + move = step.getMove() + if step.isRollback(): + segment_current_info = move.seg + # TODO: check lookup_seg here + cfg_line = f'{move.dstHost.hostname}|{move.target_port}|{move.target_datadir} ' + cfg_line += f'{segment_current_info.getSegmentHostName()}|{segment_current_info.getSegmentPort()}|{segment_current_info.getSegmentDataDirectory()}\n' + else: + segment_current_info = move.seg + if not self.lookup_seg(segment_current_info): + self.logger.info(f'Skip segment for gpmovemirrors: {str(segment_current_info)}') + continue + cfg_line = f'{segment_current_info.getSegmentHostName()}|{segment_current_info.getSegmentPort()}|{segment_current_info.getSegmentDataDirectory()} ' + cfg_line += f'{move.dstHost.hostname}|{move.target_port}|{move.target_datadir}\n' fp.write(cfg_line) return filename @@ -303,6 +343,7 @@ def state_is_final(self, state: str) -> bool: return state == self.states_main_rebalance_flow[-1] def get_state_after_interrupt(self, prev_state) -> str: + # TODO: rollback if (prev_state == 'STATE_REBALANCE_EXECUTION_STARTED' or prev_state == 'STATE_REBALANCE_MOVES_SUCCEEDED' or prev_state == 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE'): @@ -315,7 +356,7 @@ def get_state_after_interrupt(self, prev_state) -> str: def reset_in_progress_execution_steps(self) -> None: in_progress_steps = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.IN_PROGRESS]) for step in in_progress_steps: - step.setStatus(RebalanceStep.Status.ERROR) + step.setStatus(RebalanceStep.Status.ERROR, step.isRollback()) self.rebalance_schema.updateExecutionStep(step) def process_error_execution_steps(self) -> None: @@ -356,6 +397,7 @@ def process_error_execution_steps(self) -> None: catalog_segment_info = self.get_catalog_gp_segment_configuration_for_dbid(dbid) if catalog_segment_info.isSegmentUp() and catalog_segment_info.isSegmentModeSynchronized(): self.logger.info('The step is complete, mark it as done') + #TODO: rollback handling step.setStatus(RebalanceStep.Status.DONE) self.rebalance_schema.updateExecutionStep(step) return @@ -368,7 +410,7 @@ def process_error_execution_steps(self) -> None: if not gp_segment_configuration_updated and self.interactive_check(f'Retry step?'): if step.isRollback(): self.logger.info('Plan to retry rollback step') - step.setStatus(RebalanceStep.Status.ROLLBACK_PLANNED) + step.setStatus(RebalanceStep.Status.PLANNED, True) else: self.logger.info('Plan to retry step') step.setStatus(RebalanceStep.Status.PLANNED) @@ -377,7 +419,7 @@ def process_error_execution_steps(self) -> None: if not step.isRollback() and self.interactive_check('Rollback step?'): self.logger.info('Plan to rollback step') - step.setStatus(RebalanceStep.Status.ROLLBACK_PLANNED) + step.setStatus(RebalanceStep.Status.PLANNED, True) else: # TODO: need to cancel all dependent steps self.logger.info('Cancel step') @@ -386,15 +428,15 @@ def process_error_execution_steps(self) -> None: self.rebalance_schema.updateExecutionStep(step) # Mark dependent steps accordingly - # 1. If there are steps planned for ROLLBACK - we mark all left todo steps for the same content as already ROLLED_BACK + # 1. If there are steps planned for ROLLBACK - we mark all left todo steps for the same content as already rolled back for step in error_steps: - if step.getStatus() == RebalanceStep.Status.ROLLBACK_PLANNED: + if step.getStatus() == RebalanceStep.Status.PLANNED and step.isRollback(): content_id = step.getMove().seg.getSegmentContentId() # TODO: DOUBLE check the full rollback flow here!!!!!!!!!!! for step_todo in steps_left_todo: if step_todo.getMove().seg.getSegmentContentId() == content_id: - self.logger.info(f'Mark as already ROLLED_BACK the dependent step {step_todo}') - step_todo.setStatus(RebalanceStep.Status.ROLLED_BACK) + self.logger.info(f'Mark as already rolled back the dependent step {step_todo}') + step_todo.setStatus(RebalanceStep.Status.DONE, True) self.rebalance_schema.updateExecutionStep(step_todo) # 2. If there are any cancelled steps - we need to: # a. cancel all not yet done steps of the same dbid, @@ -504,6 +546,9 @@ def fill_rebalance_steps(): @wrap_state_func_with_faults def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: state_from_prev_run = self.rebalance_schema.getRebalanceStateFromPreviousRun() + self.is_rebalance_flow = self.rebalance_schema.isRollbackRebalanceFlow(self.states_rollback_rebalance_flow[0]) + if self.is_rebalance_flow: + self.logger.info('Continue rebalance rollback.') if state_from_prev_run == STATE_NOT_DEFINED: self.trigger('move_to_STATE_REBALANCE_STARTED') @@ -552,7 +597,6 @@ def on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE(self) -> None: @wrap_state_func_with_faults def on_enter_STATE_REBALANCE_EXECUTION_STARTED(self) -> None: - if self.rebalance_schema.allExecutionStepsAreDone(): self.trigger('move_to_STATE_REBALANCE_EXECUTION_DONE') return @@ -581,16 +625,15 @@ def on_enter_STATE_REBALANCE_EXECUTION_STARTED(self) -> None: (len(current_batch) > 0 and (type(current_batch[0]) is not type(step)))): break - step.setStatus(RebalanceStep.Status.IN_PROGRESS) + step.setStatus(RebalanceStep.Status.IN_PROGRESS, step.isRollback()) self.rebalance_schema.updateExecutionStep(step) current_batch.append(step) if isinstance(current_batch[0], RebalanceStepMoveMirror): self.logger.info('Rebalance - start moving segments:') - moves = [step.getMove() for step in current_batch] - for move in moves: - self.logger.info(str(move)) - self.process_moves(moves) + for step in current_batch: + self.logger.info(str(step)) + self.process_moves(current_batch) self.logger.info('Rebalance - end moving segments') else: direction = self.RoleSwapDirection.PRIMARY_TO_MIRROR @@ -605,7 +648,7 @@ def on_enter_STATE_REBALANCE_EXECUTION_STARTED(self) -> None: for step in steps_to_execute: if step.getStatus() == RebalanceStep.Status.IN_PROGRESS: - step.setStatus(RebalanceStep.Status.DONE) + step.setStatus(RebalanceStep.Status.DONE, step.isRollback()) self.rebalance_schema.updateExecutionStep(step) self.trigger('move_to_STATE_REBALANCE_MOVES_SUCCEEDED') @@ -632,7 +675,7 @@ def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED(self) # TODO: we'll need to add logic here to get approval from the user in the interactive mode, # once we start implementing the interactive mode. # In non-interactive mode we assume that the switchover is always approved. - step.setStatus(RebalanceStep.Status.PLANNED) + step.setStatus(RebalanceStep.Status.PLANNED, step.isRollback()) self.rebalance_schema.updateExecutionStep(step) self.trigger('move_to_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE') @@ -641,9 +684,75 @@ def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED(self) def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE(self) -> None: self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_ROLLBACK_STARTED(self) -> None: + self.is_rebalance_flow = True + self.logger.info('Starting rebalance rollback.') + self.trigger('move_to_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED(self) -> None: + + actual_rollback_steps_cnt = 0 + rollback_steps = self.rebalance_schema.getExecutionSteps([]) + + if len(rollback_steps) > 0: + move_order = rollback_steps[-1].getMoveOrder() + + for i, step in enumerate(rollback_steps): + # reverse the move order + step.setMoveOrder(move_order) + move_order -= 1 + if step.isRollback(): + continue + + # convert not yet executed steps as already rolled back + if step.getStatus() in [RebalanceStep.Status.APPROVE_REQUIRED, RebalanceStep.Status.PLANNED]: + step.setStatus(RebalanceStep.Status.DONE, True) + actual_rollback_steps_cnt += 1 + elif step.getStatus() in [RebalanceStep.Status.IN_PROGRESS, RebalanceStep.Status.DONE]: + step.setStatus(RebalanceStep.Status.PLANNED, True) + actual_rollback_steps_cnt += 1 + # TODO: what about errored or cancelled steps + + rollback_step_for_switchover = None + + # Revert type of switchover + if isinstance(step, RebalanceStepSwitchoverToMirror): + rollback_step_for_switchover = RebalanceStepSwitchoverToPrimary(step.getMove()) + elif isinstance(step, RebalanceStepSwitchoverToPrimary): + rollback_step_for_switchover = RebalanceStepSwitchoverToMirror(step.getMove()) + + if rollback_step_for_switchover: + rollback_step_for_switchover.setMoveOrder(step.getMoveOrder()) + rollback_step_for_switchover.setStatus(step.getStatus(), True) + if step.getStatus() == RebalanceStep.Status.PLANNED: + rollback_step_for_switchover.setStatus(RebalanceStep.Status.APPROVE_REQUIRED, True) + rollback_steps[i] = rollback_step_for_switchover + + rollback_steps.sort(key=lambda x: x.getMoveOrder()) + + if actual_rollback_steps_cnt > 0: + self.logger.info('Saving following rollback rebalance execution steps:') + for step in rollback_steps: + self.logger.info(str(step)) + self.rebalance_schema.saveExecutionSteps(rollback_steps) + self.logger.info('Saved rollback rebalance execution steps.') + else: + self.logger.info('No steps to rollback found for rebalance') + + self.trigger('move_to_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') + @wrap_state_func_with_faults def on_enter_STATE_REBALANCE_DONE(self) -> None: - pass + if self.is_rebalance_flow: + self.logger.info('Rebalance rollback is complete') + else: + self.logger.info('Rebalance is complete') @wrap_state_func_with_faults def on_enter_STATE_ERROR(self) -> None: diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py index 12c938e187c7..ccd71d9c225a 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py @@ -95,6 +95,15 @@ def getRebalanceStateFromPreviousRun(self) -> str: def getMainStateFromPreviousRun(self) -> str: return self.getStateFromPreviousRun(self.STATE_CATEGORY_MAIN) + def isRollbackRebalanceFlow(self, rollback_start_state: str) -> bool: + if self.schemaExists(): + row = dbconn.queryRow(self.conn, + f"SELECT COUNT(1) FROM {self.schema_name}.{self.rebalance_status} " + f"WHERE state_category = '{self.STATE_CATEGORY_REBALANCE}' " + f"AND state = '{rollback_start_state}'") + return int(row[0]) != 0 + return False + def rebalanceSchema(self, target_segment_count: int) -> None: # Before rebalancing check if the tables are already rebalanced # (in case we re-enter after interruption that happened after COMMIT but before new state) @@ -154,13 +163,13 @@ def saveExecutionSteps(self, steps: List[RebalanceStep]) -> None: dbconn.execSQL(self.conn, f'''CREATE TABLE {self.schema_name}.{self.segment_move_steps} - (move_order INT NOT NULL UNIQUE, status TEXT, step BYTEA) + (move_order INT NOT NULL UNIQUE, status TEXT, is_rollback BOOL, step BYTEA) DISTRIBUTED REPLICATED''') for step in steps: dbconn.execSQL(self.conn, f'''INSERT INTO {self.schema_name}.{self.segment_move_steps} - VALUES ({step.getMoveOrder()}, '{step.getStatus().name}', '\\x{step.serializeStep().hex()}')''') + VALUES ({step.getMoveOrder()}, '{step.getStatus().name}', '{step.isRollback()}', '\\x{step.serializeStep().hex()}')''') dbconn.execSQL(self.conn, 'COMMIT') diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_step.py b/gpMgmt/bin/gprebalance_modules/rebalance_step.py index 66e4914f7c2b..73c2008eabe5 100755 --- a/gpMgmt/bin/gprebalance_modules/rebalance_step.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_step.py @@ -10,10 +10,8 @@ class Status(Enum): PLANNED = 2 IN_PROGRESS = 3 ERROR = 4 - ROLLBACK_PLANNED = 5 - ROLLED_BACK = 6 - CANCELLED = 7 - DONE = 8 + CANCELLED = 5 + DONE = 6 def __init__(self, move: LogicalMove): self.move_order = -1 @@ -21,29 +19,29 @@ def __init__(self, move: LogicalMove): self.status = self.Status.PLANNED self.rollback = False - def __str__(self): + def __str__(self) -> str: + rollback_label = '' + if self.isRollback(): + rollback_label = '[ROLLBACK] ' return ( - f"Rebalance step with move_order: {self.getMoveOrder()}, status: {self.getStatus()}" + f"{rollback_label}Rebalance step with move_order: {self.getMoveOrder()}, status: {self.getStatus()}" ) - def getMoveOrder(self): + def getMoveOrder(self) -> int: return self.move_order - def setMoveOrder(self, move_order: int): + def setMoveOrder(self, move_order: int) -> None: self.move_order = move_order - def getStatus(self): + def getStatus(self) -> Status: return self.status - def getMove(self): + def getMove(self) -> LogicalMove: return self.move - def setStatus(self, status: Status): + def setStatus(self, status: Status, rollback: bool = False) -> None: self.status = status - if self.status == self.Status.ROLLBACK_PLANNED: - #For a step, when it is planned to rollback, - #there is no turning back. - self.rollback = True + self.rollback = rollback def serializeStep(self) -> bytes: return pickle.dumps(self) @@ -66,7 +64,7 @@ def __init__(self, move: LogicalMove): super().__init__(move) self.status = self.Status.APPROVE_REQUIRED - def __str__(self): + def __str__(self) -> str: return ( f"{super().__str__()}, type: RebalanceStepSwitchoverToMirror, DBID {str(self.move.seg.getSegmentDbId())}" ) @@ -76,7 +74,7 @@ def __init__(self, move: LogicalMove): super().__init__(move) self.status = self.Status.APPROVE_REQUIRED - def __str__(self): + def __str__(self) -> str: return ( f"{super().__str__()}, type: RebalanceStepSwitchoverToPrimary, DBID {str(self.move.seg.getSegmentDbId())}" ) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 99e69dadac99..bdd28d1f1dec 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -395,3 +395,47 @@ Feature: ggrebalance behave tests (rebalance scenarios) Examples: | fault_name | | _do_recovery_end | + + Scenario Outline: 4.1 rebalance - interrupt and rollback. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And the gp_segment_configuration have been saved + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance rollback is complete" to logfile with latest timestamp + And verify the gp_segment_configuration has been restored + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_end | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_end | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | + | on_enter_STATE_REBALANCE_DONE_begin | From 211e6bfd5b2c0b8f5942d519d4a40769bf0cc80c Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Tue, 3 Mar 2026 10:19:08 +1000 Subject: [PATCH 03/28] Tests and fixes for rollback --- .../bin/gprebalance_modules/ggrebalance_sm.py | 207 ++++++++++-------- .../gprebalance_modules/rebalance_schema.py | 12 +- .../mgmt_utils/ggrebalance_rebalance.feature | 60 +++++ 3 files changed, 185 insertions(+), 94 deletions(-) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index 34e47f151581..abaf1593010c 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -343,10 +343,14 @@ def state_is_final(self, state: str) -> bool: return state == self.states_main_rebalance_flow[-1] def get_state_after_interrupt(self, prev_state) -> str: - # TODO: rollback - if (prev_state == 'STATE_REBALANCE_EXECUTION_STARTED' or - prev_state == 'STATE_REBALANCE_MOVES_SUCCEEDED' or - prev_state == 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE'): + if prev_state in self.states_rollback_rebalance_flow[:-1]: + prev_idx = self.states_rollback_rebalance_flow.index(prev_state) + return self.states_rollback_rebalance_flow[prev_idx + 1] + + if (prev_state in ['STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE', + 'STATE_REBALANCE_EXECUTION_STARTED', + 'STATE_REBALANCE_MOVES_SUCCEEDED', + 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE']): return 'STATE_REBALANCE_EXECUTION_STARTED' prev_idx = self.states_main_rebalance_flow.index(prev_state) @@ -363,95 +367,114 @@ def process_error_execution_steps(self) -> None: dbconn.execSQL(self.conn, "BEGIN") try: error_steps = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.ERROR]) - steps_left_todo = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) - any_step_cancelled = False - # TODO: what about switchover steps? - for step in error_steps: - self.logger.info(f'Checking error status for step: {str(step)}') - dbid = step.getMove().seg.getSegmentDbId() - target_hostname = step.getMove().dstHost.hostname - target_datadir = step.getMove().target_datadir - target_port = step.getMove().target_port - - catalog_segment_info = self.get_catalog_gp_segment_configuration_for_dbid(dbid) - self.logger.info(f'Segment info from catalog: {str(catalog_segment_info)}') - - gp_segment_configuration_updated = (catalog_segment_info.hostname == target_hostname) - - port_updated = False - if gp_segment_configuration_updated: - port_updated = (self.get_postgresql_conf_port(target_hostname, target_datadir) == target_port) - - self.logger.info(f'gp_segment_configuration is updated: {gp_segment_configuration_updated}') - self.logger.info(f'Port is updated: {port_updated}') - - time_waited = 0 - SLEEP_PERIOD_SEC = 1.0 - TIMEOUT_SEC = 120.0 - if port_updated: - self.logger.info(f'Start checking if segment is up with timeout of {TIMEOUT_SEC} sec.') - while time_waited < TIMEOUT_SEC: - # Start polling segment status with timeout - segment_process_started = SegmentStatus.remote('Segment status check', target_hostname, target_datadir).was_successful() - if segment_process_started: - catalog_segment_info = self.get_catalog_gp_segment_configuration_for_dbid(dbid) - if catalog_segment_info.isSegmentUp() and catalog_segment_info.isSegmentModeSynchronized(): - self.logger.info('The step is complete, mark it as done') - #TODO: rollback handling - step.setStatus(RebalanceStep.Status.DONE) - self.rebalance_schema.updateExecutionStep(step) - return - - time.sleep(SLEEP_PERIOD_SEC) - time_waited = time_waited + SLEEP_PERIOD_SEC - if self.interactive_check('Timeout waiting for segment start, wait again?'): - time_waited = 0 - - if not gp_segment_configuration_updated and self.interactive_check(f'Retry step?'): - if step.isRollback(): - self.logger.info('Plan to retry rollback step') - step.setStatus(RebalanceStep.Status.PLANNED, True) - else: - self.logger.info('Plan to retry step') - step.setStatus(RebalanceStep.Status.PLANNED) - self.rebalance_schema.updateExecutionStep(step) - return + if len(error_steps) == 0: + return + + # All steps in an errored batch should be the same type, so we probe only + # the first one to detect the type and process accordingly. + if isinstance(error_steps[0], RebalanceStepMoveMirror): + self.process_error_execution_steps_mirror_moves(error_steps) + else: + self.process_error_execution_steps_switchovers(error_steps) + + finally: + dbconn.execSQL(self.conn, "COMMIT") - if not step.isRollback() and self.interactive_check('Rollback step?'): - self.logger.info('Plan to rollback step') + def process_error_execution_steps_mirror_moves(self, error_steps: List[RebalanceStep]) -> None: + steps_left_todo = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) + any_step_cancelled = False + self.logger.info('Process failed segment moves...') + for step in error_steps: + self.logger.info(f'Checking error status for step: {str(step)}') + dbid = step.getMove().seg.getSegmentDbId() + target_hostname = step.getMove().dstHost.hostname + target_datadir = step.getMove().target_datadir + target_port = step.getMove().target_port + + catalog_segment_info = self.get_catalog_gp_segment_configuration_for_dbid(dbid) + self.logger.info(f'Segment info from catalog: {str(catalog_segment_info)}') + + gp_segment_configuration_updated = (catalog_segment_info.hostname == target_hostname) + + port_updated = False + if gp_segment_configuration_updated: + port_updated = (self.get_postgresql_conf_port(target_hostname, target_datadir) == target_port) + + self.logger.info(f'gp_segment_configuration is updated: {gp_segment_configuration_updated}') + self.logger.info(f'Port is updated: {port_updated}') + + time_waited = 0 + SLEEP_PERIOD_SEC = 1.0 + TIMEOUT_SEC = 120.0 + if port_updated: + self.logger.info(f'Start checking if segment is up with timeout of {TIMEOUT_SEC} sec.') + while time_waited < TIMEOUT_SEC: + # Start polling segment status with timeout + segment_process_started = SegmentStatus.remote('Segment status check', target_hostname, target_datadir).was_successful() + if segment_process_started: + catalog_segment_info = self.get_catalog_gp_segment_configuration_for_dbid(dbid) + if catalog_segment_info.isSegmentUp() and catalog_segment_info.isSegmentModeSynchronized(): + self.logger.info('The step is complete, mark it as done') + #TODO: rollback handling + step.setStatus(RebalanceStep.Status.DONE) + self.rebalance_schema.updateExecutionStep(step) + return + + time.sleep(SLEEP_PERIOD_SEC) + time_waited = time_waited + SLEEP_PERIOD_SEC + if self.interactive_check('Timeout waiting for segment start, wait again?'): + time_waited = 0 + + if not gp_segment_configuration_updated and self.interactive_check(f'Retry step?'): + if step.isRollback(): + self.logger.info('Plan to retry rollback step') step.setStatus(RebalanceStep.Status.PLANNED, True) else: - # TODO: need to cancel all dependent steps - self.logger.info('Cancel step') - step.setStatus(RebalanceStep.Status.CANCELLED) - any_step_cancelled = True + self.logger.info('Plan to retry step') + step.setStatus(RebalanceStep.Status.PLANNED) self.rebalance_schema.updateExecutionStep(step) + return - # Mark dependent steps accordingly - # 1. If there are steps planned for ROLLBACK - we mark all left todo steps for the same content as already rolled back - for step in error_steps: - if step.getStatus() == RebalanceStep.Status.PLANNED and step.isRollback(): - content_id = step.getMove().seg.getSegmentContentId() - # TODO: DOUBLE check the full rollback flow here!!!!!!!!!!! - for step_todo in steps_left_todo: - if step_todo.getMove().seg.getSegmentContentId() == content_id: - self.logger.info(f'Mark as already rolled back the dependent step {step_todo}') - step_todo.setStatus(RebalanceStep.Status.DONE, True) - self.rebalance_schema.updateExecutionStep(step_todo) - # 2. If there are any cancelled steps - we need to: - # a. cancel all not yet done steps of the same dbid, - # b. and *ALL* switchovers, - # c. and do cancelation recursively. - # But, actually, it means that we need to cancel everything besides steps revived from the ERROR state just above, - # as left todo steps didn't get into this ERRORed batch, meaning they must have different step type (meaning switchover). - if any_step_cancelled: - for step_todo in steps_left_todo: - self.logger.info(f'Mark as CANCELLED the step {step_todo}') - step_todo.setStatus(RebalanceStep.Status.CANCELLED) - self.rebalance_schema.updateExecutionStep(step_todo) + if not step.isRollback() and self.interactive_check('Rollback step?'): + self.logger.info('Plan to rollback step') + step.setStatus(RebalanceStep.Status.PLANNED, True) + else: + self.logger.info('Cancel step') + step.setStatus(RebalanceStep.Status.CANCELLED) + any_step_cancelled = True + self.rebalance_schema.updateExecutionStep(step) - finally: - dbconn.execSQL(self.conn, "COMMIT") + # Mark dependent steps accordingly + # 1. If there are steps planned for ROLLBACK - we mark all left todo steps for the same content as already rolled back + for step in error_steps: + if step.getStatus() == RebalanceStep.Status.PLANNED and step.isRollback(): + content_id = step.getMove().seg.getSegmentContentId() + # TODO: DOUBLE check the full rollback flow here!!!!!!!!!!! + for step_todo in steps_left_todo: + if step_todo.getMove().seg.getSegmentContentId() == content_id: + self.logger.info(f'Mark as already rolled back the dependent step {step_todo}') + step_todo.setStatus(RebalanceStep.Status.DONE, True) + self.rebalance_schema.updateExecutionStep(step_todo) + # 2. If there are any cancelled steps - we need to: + # a. cancel all not yet done steps of the same dbid, + # b. and *ALL* switchovers, + # c. and do cancelation recursively. + # But, actually, it means that we need to cancel everything besides steps revived from the ERROR state just above, + # as left todo steps didn't get into this ERRORed batch, meaning they must have different step type (meaning switchover). + if any_step_cancelled: + for step_todo in steps_left_todo: + self.logger.info(f'Mark as CANCELLED the step {step_todo}') + step_todo.setStatus(RebalanceStep.Status.CANCELLED) + self.rebalance_schema.updateExecutionStep(step_todo) + + def process_error_execution_steps_switchovers(self, error_steps: List[RebalanceStep]) -> None: + self.logger.info('Process failed switchovers...') + # TODO: proper error handling? + for step in error_steps: + self.logger.info(f'Checking error status for step: {str(step)}') + self.logger.info('Plan to retry step') + step.setStatus(RebalanceStep.Status.PLANNED, step.isRollback()) + self.rebalance_schema.updateExecutionStep(step) def get_catalog_gp_segment_configuration_for_dbid(self, dbid: int) -> Segment: row = dbconn.queryRow(self.conn, @@ -547,15 +570,17 @@ def fill_rebalance_steps(): def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: state_from_prev_run = self.rebalance_schema.getRebalanceStateFromPreviousRun() self.is_rebalance_flow = self.rebalance_schema.isRollbackRebalanceFlow(self.states_rollback_rebalance_flow[0]) - if self.is_rebalance_flow: - self.logger.info('Continue rebalance rollback.') + if state_from_prev_run == STATE_NOT_DEFINED: self.trigger('move_to_STATE_REBALANCE_STARTED') elif self.state_is_final(state_from_prev_run): self.logger.info('Cluster is already rebalanced...') else: - self.logger.info('Continue interrupted rebalance operation...') + if self.is_rebalance_flow: + self.logger.info('Continue interrupted rebalance rollback operation...') + else: + self.logger.info('Continue interrupted rebalance operation...') self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") try: next_state = self.get_state_after_interrupt(state_from_prev_run) @@ -687,7 +712,7 @@ def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE(self) -> @wrap_state_func_with_faults def on_enter_STATE_REBALANCE_ROLLBACK_STARTED(self) -> None: self.is_rebalance_flow = True - self.logger.info('Starting rebalance rollback.') + self.logger.info('Starting rebalance rollback') self.trigger('move_to_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED') @wrap_state_func_with_faults diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py index ccd71d9c225a..74d11cc6bc2d 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py @@ -48,6 +48,8 @@ def createSchema(self, plan: Plan) -> None: self.savePlan(plan) + self.recreateExecutionStepsTable() + dbconn.execSQL(self.conn, 'COMMIT') def dropSchema(self) -> None: @@ -156,15 +158,19 @@ def getTablesToRebalanceWithStatus(self, status: str) -> cursor: return dbconn.query(self.conn, f"""SELECT db_name, schema_name, rel_name FROM {self.schema_name}.{self.table_rebalance_status_detail} WHERE status = '{status}'""") - def saveExecutionSteps(self, steps: List[RebalanceStep]) -> None: - dbconn.execSQL(self.conn, 'BEGIN') - + def recreateExecutionStepsTable(self) -> None: dbconn.execSQL(self.conn, f'DROP TABLE IF EXISTS {self.schema_name}.{self.segment_move_steps}') dbconn.execSQL(self.conn, f'''CREATE TABLE {self.schema_name}.{self.segment_move_steps} (move_order INT NOT NULL UNIQUE, status TEXT, is_rollback BOOL, step BYTEA) DISTRIBUTED REPLICATED''') + + def saveExecutionSteps(self, steps: List[RebalanceStep]) -> None: + dbconn.execSQL(self.conn, 'BEGIN') + + # TODO: just truncate here? + self.recreateExecutionStepsTable() for step in steps: dbconn.execSQL(self.conn, diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index bdd28d1f1dec..3751cb9590b6 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -439,3 +439,63 @@ Feature: ggrebalance behave tests (rebalance scenarios) | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | | on_enter_STATE_REBALANCE_DONE_begin | + + Scenario Outline: 4.2 rebalance - interrupt, rollback (and interrupt again) and continue. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And the gp_segment_configuration have been saved + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin" + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + And set fault inject "" + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance rollback is complete" to logfile with latest timestamp + And verify the gp_segment_configuration has been restored + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED_begin | + | on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED_end | + | on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE_begin | + | on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE_end | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_end | + | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_begin | + | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_end | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_end | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_end | + | on_enter_STATE_REBALANCE_EXECUTION_DONE_begin | + | on_enter_STATE_REBALANCE_EXECUTION_DONE_end | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | + | on_enter_STATE_REBALANCE_DONE_begin | + | on_enter_STATE_REBALANCE_DONE_end | From fb1467e23647a4ffdb4213993778c853ab2136c2 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Tue, 3 Mar 2026 10:22:05 +1000 Subject: [PATCH 04/28] Fix test --- gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 3751cb9590b6..ee80e5625926 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -454,7 +454,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows And all files in gpAdminLogs directory are deleted - And set fault inject "on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin" + And set fault inject "on_enter_STATE_REBALANCE_DONE_begin" When the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp From 35a8f4e3035b73ceef873a0f4e718d48463f1464 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Tue, 3 Mar 2026 13:58:36 +1000 Subject: [PATCH 05/28] Refactoring and improvements --- .../bin/gprebalance_modules/ggrebalance_sm.py | 80 ++++++++++++------- .../mgmt_utils/ggrebalance_rebalance.feature | 6 +- .../behave/mgmt_utils/steps/mgmt_utils.py | 5 +- 3 files changed, 56 insertions(+), 35 deletions(-) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index abaf1593010c..8ab073513689 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -131,7 +131,7 @@ def __init__(self, conn: dbconn.Connection, schema: RebalanceSchema, logger: Any self.conn = conn self.rebalance_schema = schema self.cmd = None - self.is_rebalance_flow = False + self.is_rollback_flow = False self.machine = Machine(model = self, queued=True, @@ -376,13 +376,10 @@ def process_error_execution_steps(self) -> None: self.process_error_execution_steps_mirror_moves(error_steps) else: self.process_error_execution_steps_switchovers(error_steps) - finally: dbconn.execSQL(self.conn, "COMMIT") def process_error_execution_steps_mirror_moves(self, error_steps: List[RebalanceStep]) -> None: - steps_left_todo = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) - any_step_cancelled = False self.logger.info('Process failed segment moves...') for step in error_steps: self.logger.info(f'Checking error status for step: {str(step)}') @@ -418,7 +415,7 @@ def process_error_execution_steps_mirror_moves(self, error_steps: List[Rebalance #TODO: rollback handling step.setStatus(RebalanceStep.Status.DONE) self.rebalance_schema.updateExecutionStep(step) - return + continue time.sleep(SLEEP_PERIOD_SEC) time_waited = time_waited + SLEEP_PERIOD_SEC @@ -433,7 +430,7 @@ def process_error_execution_steps_mirror_moves(self, error_steps: List[Rebalance self.logger.info('Plan to retry step') step.setStatus(RebalanceStep.Status.PLANNED) self.rebalance_schema.updateExecutionStep(step) - return + continue if not step.isRollback() and self.interactive_check('Rollback step?'): self.logger.info('Plan to rollback step') @@ -441,41 +438,62 @@ def process_error_execution_steps_mirror_moves(self, error_steps: List[Rebalance else: self.logger.info('Cancel step') step.setStatus(RebalanceStep.Status.CANCELLED) - any_step_cancelled = True self.rebalance_schema.updateExecutionStep(step) # Mark dependent steps accordingly - # 1. If there are steps planned for ROLLBACK - we mark all left todo steps for the same content as already rolled back + self.mark_dependent_steps_on_error(error_steps) + + def process_error_execution_steps_switchovers(self, error_steps: List[RebalanceStep]) -> None: + self.logger.info('Process failed switchovers...') for step in error_steps: - if step.getStatus() == RebalanceStep.Status.PLANNED and step.isRollback(): - content_id = step.getMove().seg.getSegmentContentId() - # TODO: DOUBLE check the full rollback flow here!!!!!!!!!!! - for step_todo in steps_left_todo: - if step_todo.getMove().seg.getSegmentContentId() == content_id: - self.logger.info(f'Mark as already rolled back the dependent step {step_todo}') - step_todo.setStatus(RebalanceStep.Status.DONE, True) - self.rebalance_schema.updateExecutionStep(step_todo) + # TODO: add comments and update the log below + self.logger.info(f'Checking error status for step: {str(step)}') + if self.interactive_check(f'Retry step?'): + if step.isRollback(): + self.logger.info('Plan to retry rollback step') + step.setStatus(RebalanceStep.Status.PLANNED, True) + else: + self.logger.info('Plan to retry step') + step.setStatus(RebalanceStep.Status.PLANNED) + self.rebalance_schema.updateExecutionStep(step) + continue + + if not step.isRollback() and self.interactive_check('Rollback step?'): + self.logger.info('Plan to rollback step') + step.setStatus(RebalanceStep.Status.PLANNED, True) + else: + self.logger.info('Cancel step') + step.setStatus(RebalanceStep.Status.CANCELLED) + self.rebalance_schema.updateExecutionStep(step) + + # Mark dependent steps accordingly + self.mark_dependent_steps_on_error(error_steps) + + def mark_dependent_steps_on_error(self, error_steps: List[RebalanceStep]) -> None: + # Mark dependent steps accordingly + steps_left_todo = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) + # 1. If there are steps planned for ROLLBACK - we mark all left todo steps for the same content as already rolled back + if not self.is_rollback_flow: + for step in error_steps: + if step.getStatus() == RebalanceStep.Status.PLANNED and step.isRollback(): + content_id = step.getMove().seg.getSegmentContentId() + for step_todo in steps_left_todo: + if step_todo.getMove().seg.getSegmentContentId() == content_id: + self.logger.info(f'Mark as already rolled back the dependent step {step_todo}') + step_todo.setStatus(RebalanceStep.Status.DONE, True) + self.rebalance_schema.updateExecutionStep(step_todo) # 2. If there are any cancelled steps - we need to: # a. cancel all not yet done steps of the same dbid, # b. and *ALL* switchovers, # c. and do cancelation recursively. # But, actually, it means that we need to cancel everything besides steps revived from the ERROR state just above, # as left todo steps didn't get into this ERRORed batch, meaning they must have different step type (meaning switchover). - if any_step_cancelled: + if any(step.getStatus() == RebalanceStep.Status.CANCELLED for step in error_steps): for step_todo in steps_left_todo: self.logger.info(f'Mark as CANCELLED the step {step_todo}') - step_todo.setStatus(RebalanceStep.Status.CANCELLED) + step_todo.setStatus(RebalanceStep.Status.CANCELLED, step_todo.isRollback()) self.rebalance_schema.updateExecutionStep(step_todo) - def process_error_execution_steps_switchovers(self, error_steps: List[RebalanceStep]) -> None: - self.logger.info('Process failed switchovers...') - # TODO: proper error handling? - for step in error_steps: - self.logger.info(f'Checking error status for step: {str(step)}') - self.logger.info('Plan to retry step') - step.setStatus(RebalanceStep.Status.PLANNED, step.isRollback()) - self.rebalance_schema.updateExecutionStep(step) - def get_catalog_gp_segment_configuration_for_dbid(self, dbid: int) -> Segment: row = dbconn.queryRow(self.conn, f"SELECT dbid||'|'||content||'|'||role||'|'||preferred_role||'|'||mode||'|'||status||'|'||hostname||'|'||address||'|'||port||'|'||datadir " @@ -569,7 +587,7 @@ def fill_rebalance_steps(): @wrap_state_func_with_faults def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: state_from_prev_run = self.rebalance_schema.getRebalanceStateFromPreviousRun() - self.is_rebalance_flow = self.rebalance_schema.isRollbackRebalanceFlow(self.states_rollback_rebalance_flow[0]) + self.is_rollback_flow = self.rebalance_schema.isRollbackRebalanceFlow(self.states_rollback_rebalance_flow[0]) if state_from_prev_run == STATE_NOT_DEFINED: @@ -577,7 +595,7 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: elif self.state_is_final(state_from_prev_run): self.logger.info('Cluster is already rebalanced...') else: - if self.is_rebalance_flow: + if self.is_rollback_flow: self.logger.info('Continue interrupted rebalance rollback operation...') else: self.logger.info('Continue interrupted rebalance operation...') @@ -711,7 +729,7 @@ def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE(self) -> @wrap_state_func_with_faults def on_enter_STATE_REBALANCE_ROLLBACK_STARTED(self) -> None: - self.is_rebalance_flow = True + self.is_rollback_flow = True self.logger.info('Starting rebalance rollback') self.trigger('move_to_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED') @@ -774,7 +792,7 @@ def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE(self) -> None: @wrap_state_func_with_faults def on_enter_STATE_REBALANCE_DONE(self) -> None: - if self.is_rebalance_flow: + if self.is_rollback_flow: self.logger.info('Rebalance rollback is complete') else: self.logger.info('Rebalance is complete') diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index ee80e5625926..ebd0deda9fe9 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -282,6 +282,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) And ggrebalance should print "Port is updated: False" to logfile with latest timestamp And ggrebalance should print "Plan to retry step" to logfile with latest timestamp And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And unset fault inject And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" @@ -333,6 +334,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) And ggrebalance should print "Port is updated: False" to logfile with latest timestamp And ggrebalance should print "Cancel step" to logfile with latest timestamp And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And unset fault inject # some mirrors are definitely down, so do not check them And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" @@ -468,10 +470,12 @@ Feature: ggrebalance behave tests (rebalance scenarios) And unset fault inject And all files in gpAdminLogs directory are deleted And the gprecoverseg lock directory is removed - When the user runs "ggrebalance" + When user will answer "yes" to the prompt "Retry step?" + And the user runs "ggrebalance" Then ggrebalance should return a return code of 0 And ggrebalance should print "Rebalance rollback is complete" to logfile with latest timestamp And verify the gp_segment_configuration has been restored + And unset fault inject And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index eb5036b6e52d..b6389fcf4087 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -4612,11 +4612,10 @@ def impl(context): @when('user will answer "{answer}" to the prompt "{prompt}"') def impl(context, answer, prompt): assert answer == 'yes' or answer == 'no' - #TODO: change to defines - if 'GPMGMT_FAULT_POINT' in os.environ and os.environ['GPMGMT_FAULT_POINT'] != "": + if fault_injection.GPMGMT_FAULT_POINT in os.environ and os.environ[fault_injection.GPMGMT_FAULT_POINT] != "": raise Exception('Need to unset fault injection before using this step') fault = prompt + answer - os.environ['GPMGMT_FAULT_POINT'] = fault + os.environ[fault_injection.GPMGMT_FAULT_POINT] = fault @given('stub') def impl(context): From 75496788c1520ee11ae7695e1a49824f61b64ac7 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Tue, 3 Mar 2026 14:07:50 +1000 Subject: [PATCH 06/28] Update check_down_segments --- gpMgmt/bin/ggrebalance | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index b6f655f6223e..2817f39612ca 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -273,30 +273,11 @@ def check_down_segments(logger: Any, options: Any, dburl: dbconn.DbURL): conn = dbconn.connect(dburl, encoding='UTF8', allowSystemTableMods=True) dbconn.execSQL(conn, "SELECT gp_request_fts_probe_scan()") cnt_primaries_down = int(dbconn.queryRow(conn, f"SELECT COUNT(1) FROM gp_segment_configuration WHERE role = 'p' AND status = 'd'")[0]) - cnt_mirrors_down = int(dbconn.queryRow(conn, f"SELECT COUNT(1) FROM gp_segment_configuration WHERE role = 'm' AND status = 'd'")[0]) conn.close() if cnt_primaries_down != 0: raise Exception('Detected some primary segments are down, please recover manually') - """ - if cnt_mirrors_down != 0: - logger.info("Some mirrors are down, trying to recover them, it may take some time...") - recoverseg_options = "-a -F" - if options.logfile_directory is not None: - recoverseg_options = recoverseg_options + f' -l "{str(options.logfile_directory)}"' - global cmd_recoverseg - try: - cmd_recoverseg = GpRecoverSeg("Running gprecoverseg", options=recoverseg_options) - cmd_recoverseg.run(validateAfter=True) - except Exception as e: - logger.error(str(e)) - error_msg = f"Failed to execute 'gprecoverseg {recoverseg_options}'" - raise Exception(error_msg) - finally: - cmd_recoverseg = None - """ - def main(options, args, parser): conn = None try: From d706583241d3ae553e4041b612b6fba5ff0ea790 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Tue, 3 Mar 2026 15:54:52 +1000 Subject: [PATCH 07/28] Add tests and fixes --- .../bin/gprebalance_modules/ggrebalance_sm.py | 11 ++-- .../gprebalance_modules/rebalance_schema.py | 5 ++ .../mgmt_utils/ggrebalance_rebalance.feature | 64 +++++++++++++++++++ 3 files changed, 76 insertions(+), 4 deletions(-) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index 8ab073513689..c5387a329225 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -412,16 +412,19 @@ def process_error_execution_steps_mirror_moves(self, error_steps: List[Rebalance catalog_segment_info = self.get_catalog_gp_segment_configuration_for_dbid(dbid) if catalog_segment_info.isSegmentUp() and catalog_segment_info.isSegmentModeSynchronized(): self.logger.info('The step is complete, mark it as done') - #TODO: rollback handling - step.setStatus(RebalanceStep.Status.DONE) + step.setStatus(RebalanceStep.Status.DONE, step.isRollback()) self.rebalance_schema.updateExecutionStep(step) - continue + break time.sleep(SLEEP_PERIOD_SEC) time_waited = time_waited + SLEEP_PERIOD_SEC - if self.interactive_check('Timeout waiting for segment start, wait again?'): + if time_waited >= TIMEOUT_SEC and self.interactive_check('Timeout waiting for segment start, wait again?'): time_waited = 0 + # Continue with the next step, if we already marked this one + if step.getStatus() == RebalanceStep.Status.DONE: + continue + if not gp_segment_configuration_updated and self.interactive_check(f'Retry step?'): if step.isRollback(): self.logger.info('Plan to retry rollback step') diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py index 74d11cc6bc2d..def60e821feb 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py @@ -124,6 +124,11 @@ def rebalanceSchema(self, target_segment_count: int) -> None: f'''ALTER TABLE "{self.schema_name}"."{self.saved_plan}" REBALANCE {target_segment_count}''') + if get_table_distr_segment_count(self.conn, self.schema_name, self.segment_move_steps) > target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.schema_name}"."{self.segment_move_steps}" + REBALANCE {target_segment_count}''') + def storeState(self, state: str, state_category: str) -> None: if self.schemaExists(): dbconn.execSQL(self.conn, diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index ebd0deda9fe9..571be9b59cc0 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -503,3 +503,67 @@ Feature: ggrebalance behave tests (rebalance scenarios) | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | | on_enter_STATE_REBALANCE_DONE_begin | | on_enter_STATE_REBALANCE_DONE_end | + + Scenario: test 4.3. rebalance - interrupt during shrink, and rollback. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_SHRINK_TABLES_STARTED_begin" + And the gp_segment_configuration have been saved + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 3" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is complete" to logfile with latest timestamp + And verify the gp_segment_configuration has been restored + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Scenario: test 4.3. rebalance - shrink, rebalance (and interrupt during it) and rollback. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_REBALANCE_DONE_begin" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance rollback is complete" to logfile with latest timestamp + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 From f777e6978c588ade2be0d36d7e0212dda999362b Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 4 Mar 2026 10:25:35 +1000 Subject: [PATCH 08/28] Fix tests, drop schema at the end of the rollback --- .../bin/gprebalance_modules/ggrebalance_sm.py | 5 +- .../mgmt_utils/ggrebalance_rebalance.feature | 106 ++++++++++++------ 2 files changed, 74 insertions(+), 37 deletions(-) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index c5387a329225..433a9e360e38 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -40,7 +40,6 @@ class RebalanceSM: 'STATE_REBALANCE_ROLLBACK_STARTED', 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED', 'STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE' - # TODO: Add schema drop state??? ] transitions = [ @@ -450,7 +449,7 @@ def process_error_execution_steps_switchovers(self, error_steps: List[RebalanceS self.logger.info('Process failed switchovers...') for step in error_steps: # TODO: add comments and update the log below - self.logger.info(f'Checking error status for step: {str(step)}') + self.logger.info(f'Processing error status for switchover step: {str(step)}') if self.interactive_check(f'Retry step?'): if step.isRollback(): self.logger.info('Plan to retry rollback step') @@ -504,7 +503,6 @@ def get_catalog_gp_segment_configuration_for_dbid(self, dbid: int) -> Segment: return Segment.initFromString(row[0]) def get_postgresql_conf_port(self, hostname: str, datadir: str) -> int: - #TODO: recheck cmd cmd = Command( name="get_postgresql_conf_port", cmdStr=f"grep -E '^port\\s*=' {datadir}/postgresql.conf | sed -E 's/^port\\s*=\\s*([0-9]+).*/\\1/'", @@ -796,6 +794,7 @@ def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE(self) -> None: @wrap_state_func_with_faults def on_enter_STATE_REBALANCE_DONE(self) -> None: if self.is_rollback_flow: + self.rebalance_schema.dropSchema() self.logger.info('Rebalance rollback is complete') else: self.logger.info('Rebalance is complete') diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 571be9b59cc0..fc3f0bb21cad 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -128,12 +128,10 @@ Feature: ggrebalance behave tests (rebalance scenarios) And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows And all files in gpAdminLogs directory are deleted And set fault inject "" - And set fault inject delay ms When the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject - And unset fault inject delay And all files in gpAdminLogs directory are deleted And the gprecoverseg lock directory is removed When the user runs "ggrebalance" @@ -153,32 +151,25 @@ Feature: ggrebalance behave tests (rebalance scenarios) Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 Examples: - | fault_name | fault_delay_ms | - | on_enter_STATE_REBALANCE_STARTED_begin | 0 | - | on_enter_STATE_REBALANCE_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_begin | 0 | - | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE_begin | 0 | - | on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE_end | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_begin | 0 | - | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_end | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_begin | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_begin | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_end | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_DONE_begin | 0 | - | on_enter_STATE_REBALANCE_EXECUTION_DONE_end | 0 | - | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 0 | - | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 0 | - | on_enter_STATE_REBALANCE_DONE_begin | 0 | - | on_enter_STATE_REBALANCE_DONE_end | 0 | - | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 1500 | - | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 3000 | - | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 1500 | - | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 3000 | - | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | 3000 | + | fault_name | + | on_enter_STATE_REBALANCE_STARTED_begin | + | on_enter_STATE_REBALANCE_STARTED_end | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_begin | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_end | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE_begin | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE_end | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_end | + | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_begin | + | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_end | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_end | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_begin | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_end | + | on_enter_STATE_REBALANCE_EXECUTION_DONE_begin | + | on_enter_STATE_REBALANCE_EXECUTION_DONE_end | + | on_enter_STATE_REBALANCE_DONE_begin | + | on_enter_STATE_REBALANCE_DONE_end | Scenario: 4. rebalance - check rebalance after interrupted shrink. Given the database is not running @@ -298,12 +289,59 @@ Feature: ggrebalance behave tests (rebalance scenarios) Examples: | fault_name | - #| _stop_failed_segments_begin | - #| _stop_failed_segments_end | - #| _wait_fts_to_mark_down_segments_begin | - #| _wait_fts_to_mark_down_segments_end | + | _stop_failed_segments_begin | + | _stop_failed_segments_end | + | _wait_fts_to_mark_down_segments_begin | + | _wait_fts_to_mark_down_segments_end | | _update_config_begin | + Scenario Outline: 3.10.1 rebalance - interrupt (with retry) and continue. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "yes" to the prompt "Retry step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Processing error status for switchover step" to logfile with latest timestamp + And ggrebalance should print "Plan to retry step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And unset fault inject + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | + Scenario Outline: 3.11 rebalance - interrupt (with cancel) and continue. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" @@ -519,7 +557,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When the user runs "ggrebalance -x 3" + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject @@ -534,7 +572,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 - Scenario: test 4.3. rebalance - shrink, rebalance (and interrupt during it) and rollback. + Scenario: test 4.4. rebalance - shrink, rebalance (and interrupt during it) and rollback. Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" From 6e4bcd840ba478517b8bf9efe11daf2d2b6a9e1b Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 4 Mar 2026 10:38:48 +1000 Subject: [PATCH 09/28] Update test descriptions --- .../mgmt_utils/ggrebalance_rebalance.feature | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index fc3f0bb21cad..a82787e18193 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -243,7 +243,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 - Scenario Outline: 3.10 rebalance - interrupt (with retry) and continue. + Scenario Outline: 6.1. rebalance - interrupt during mirror move before gp_segment_configuration update, continue and retry failed step. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" @@ -295,7 +295,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | _wait_fts_to_mark_down_segments_end | | _update_config_begin | - Scenario Outline: 3.10.1 rebalance - interrupt (with retry) and continue. + Scenario Outline: 6.2. rebalance - interrupt during switchover step (before invocation of 'gprecoverseg'), continue and retry failed step. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" @@ -342,7 +342,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | - Scenario Outline: 3.11 rebalance - interrupt (with cancel) and continue. + Scenario Outline: 6.3. rebalance - interrupt during mirror move after gp_segment_configuration update, but before port update, continue and cancel failed step. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" @@ -389,7 +389,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | _update_config_end | - Scenario Outline: 3.12 rebalance - interrupt (when the mirror is actually started) and continue. + Scenario Outline: 6.4. rebalance - interrupt during mirror move after before port update (when the mirror is actually started), and continue. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" @@ -436,7 +436,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | fault_name | | _do_recovery_end | - Scenario Outline: 4.1 rebalance - interrupt and rollback. + Scenario Outline: 7.1. rebalance - rebalance interrupt and full rollback. Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" @@ -480,7 +480,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | | on_enter_STATE_REBALANCE_DONE_begin | - Scenario Outline: 4.2 rebalance - interrupt, rollback (and interrupt again) and continue. + Scenario Outline: 7.2. rebalance - rebalance interrupt, rollback (and interrupt again) and continue. Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" @@ -542,7 +542,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | on_enter_STATE_REBALANCE_DONE_begin | | on_enter_STATE_REBALANCE_DONE_end | - Scenario: test 4.3. rebalance - interrupt during shrink, and rollback. + Scenario: test 7.3. rebalance - interrupt during shrink, and full rollback. Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" @@ -572,7 +572,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 - Scenario: test 4.4. rebalance - shrink, rebalance (and interrupt during it) and rollback. + Scenario: test 7.4. rebalance - shrink, rebalance (and interrupt during it) and full rollback. Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" From 197c187099632731f85b2d9adb77f4446a2f584c Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 4 Mar 2026 10:58:01 +1000 Subject: [PATCH 10/28] Add test --- .../mgmt_utils/ggrebalance_rebalance.feature | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index a82787e18193..05991317cc91 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -572,6 +572,40 @@ Feature: ggrebalance behave tests (rebalance scenarios) When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + Scenario: test 7.4. rebalance - interrupt after shrink, but before rebalance start, and full rollback. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_REBALANCE_STARTED_begin" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance rollback is complete" to logfile with latest timestamp + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 + Scenario: test 7.4. rebalance - shrink, rebalance (and interrupt during it) and full rollback. Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' From 41c56958dd29c159b390a9bcd47a4c68a9824d54 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 4 Mar 2026 14:55:38 +1000 Subject: [PATCH 11/28] Add 6.1.2 test --- gpMgmt/bin/gppylib/fault_injection.py | 11 ++-- .../bin/gprebalance_modules/ggrebalance_sm.py | 30 ++++++---- .../mgmt_utils/ggrebalance_rebalance.feature | 60 +++++++++++++++++-- .../behave/mgmt_utils/steps/mgmt_utils.py | 17 ++++-- 4 files changed, 93 insertions(+), 25 deletions(-) diff --git a/gpMgmt/bin/gppylib/fault_injection.py b/gpMgmt/bin/gppylib/fault_injection.py index 72427d009351..6d22104ec5d8 100755 --- a/gpMgmt/bin/gppylib/fault_injection.py +++ b/gpMgmt/bin/gppylib/fault_injection.py @@ -11,6 +11,7 @@ GPMGMT_FAULT_FILE_FLAG = 'GPMGMT_FAULT_FILE_FLAG' GPMGMT_FAULT_TYPE_SYSPEND = 'suspend' +GPMGMT_FAULT_TYPE_VALUE = 'value' def inject_fault(fault_point): if GPMGMT_FAULT_POINT in os.environ and fault_point == os.environ[GPMGMT_FAULT_POINT]: @@ -32,11 +33,11 @@ def raise_exception(delay: int): else: raise Exception('Fault Injection %s' % os.environ[GPMGMT_FAULT_POINT]) -def inject_fault_check(fault_point) -> bool: - if GPMGMT_FAULT_POINT in os.environ and fault_point == os.environ[GPMGMT_FAULT_POINT]: - return True - else: - return False +def inject_fault_get_value() -> str: + if GPMGMT_FAULT_TYPE in os.environ and os.environ[GPMGMT_FAULT_TYPE] == GPMGMT_FAULT_TYPE_VALUE: + if GPMGMT_FAULT_POINT in os.environ: + return os.environ[GPMGMT_FAULT_POINT] + return '' # decorator for test purposes def wrap_state_func_with_faults(func): diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index 433a9e360e38..97d20a5ebfdb 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -320,8 +320,10 @@ def create_config_file(self, steps: List[RebalanceStepMoveMirror]) -> str: move = step.getMove() if step.isRollback(): segment_current_info = move.seg - # TODO: check lookup_seg here - cfg_line = f'{move.dstHost.hostname}|{move.target_port}|{move.target_datadir} ' + if self.lookup_seg(segment_current_info): + cfg_line = f'{segment_current_info.getSegmentHostName()}|{segment_current_info.getSegmentPort()}|{segment_current_info.getSegmentDataDirectory()} ' + else: + cfg_line = f'{move.dstHost.hostname}|{move.target_port}|{move.target_datadir} ' cfg_line += f'{segment_current_info.getSegmentHostName()}|{segment_current_info.getSegmentPort()}|{segment_current_info.getSegmentDataDirectory()}\n' else: segment_current_info = move.seg @@ -380,6 +382,7 @@ def process_error_execution_steps(self) -> None: def process_error_execution_steps_mirror_moves(self, error_steps: List[RebalanceStep]) -> None: self.logger.info('Process failed segment moves...') + steps_left_todo = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) for step in error_steps: self.logger.info(f'Checking error status for step: {str(step)}') dbid = step.getMove().seg.getSegmentDbId() @@ -443,12 +446,12 @@ def process_error_execution_steps_mirror_moves(self, error_steps: List[Rebalance self.rebalance_schema.updateExecutionStep(step) # Mark dependent steps accordingly - self.mark_dependent_steps_on_error(error_steps) + self.mark_dependent_steps_on_error(error_steps, steps_left_todo) def process_error_execution_steps_switchovers(self, error_steps: List[RebalanceStep]) -> None: self.logger.info('Process failed switchovers...') + steps_left_todo = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) for step in error_steps: - # TODO: add comments and update the log below self.logger.info(f'Processing error status for switchover step: {str(step)}') if self.interactive_check(f'Retry step?'): if step.isRollback(): @@ -469,11 +472,10 @@ def process_error_execution_steps_switchovers(self, error_steps: List[RebalanceS self.rebalance_schema.updateExecutionStep(step) # Mark dependent steps accordingly - self.mark_dependent_steps_on_error(error_steps) + self.mark_dependent_steps_on_error(error_steps, steps_left_todo) - def mark_dependent_steps_on_error(self, error_steps: List[RebalanceStep]) -> None: + def mark_dependent_steps_on_error(self, error_steps: List[RebalanceStep], steps_left_todo: List[RebalanceStep]) -> None: # Mark dependent steps accordingly - steps_left_todo = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) # 1. If there are steps planned for ROLLBACK - we mark all left todo steps for the same content as already rolled back if not self.is_rollback_flow: for step in error_steps: @@ -523,10 +525,16 @@ def get_postgresql_conf_port(self, hostname: str, datadir: str) -> int: # decorator to TODO: add comments def wrap_interactive_check_with_faults(fun): def func_with_faults(self, msg: str): - if inject_fault_check(msg + 'yes'): - return True - if inject_fault_check(msg + 'no'): - return False + try: + inject_value = inject_fault_get_value() + injected_answers = json.loads(inject_value) + self.logger.info(f'[RELOG DBG] injected value = "{inject_value}"') + if injected_answers.get(msg, '') == 'yes': + return True + if injected_answers.get(msg, '') == 'no': + return False + except: + pass return fun(self, msg) return func_with_faults diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 05991317cc91..6d0598b0e0dd 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -243,7 +243,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 - Scenario Outline: 6.1. rebalance - interrupt during mirror move before gp_segment_configuration update, continue and retry failed step. + Scenario Outline: 6.1.1 rebalance - interrupt during mirror move before gp_segment_configuration update, continue and retry failed step. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" @@ -273,7 +273,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) And ggrebalance should print "Port is updated: False" to logfile with latest timestamp And ggrebalance should print "Plan to retry step" to logfile with latest timestamp And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp - And unset fault inject + And clear user's answers And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" @@ -295,6 +295,56 @@ Feature: ggrebalance behave tests (rebalance scenarios) | _wait_fts_to_mark_down_segments_end | | _update_config_begin | + Scenario Outline: 6.1.2 rebalance - interrupt during mirror move before gp_segment_configuration update, continue and rollback failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Retry step?" + And user will answer "yes" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: False" to logfile with latest timestamp + And ggrebalance should print "Port is updated: False" to logfile with latest timestamp + And ggrebalance should print "Plan to rollback step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + # TODO: add to all cases where needed + And clear user's answers + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | _update_config_begin | + Scenario Outline: 6.2. rebalance - interrupt during switchover step (before invocation of 'gprecoverseg'), continue and retry failed step. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" @@ -323,7 +373,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) And ggrebalance should print "Processing error status for switchover step" to logfile with latest timestamp And ggrebalance should print "Plan to retry step" to logfile with latest timestamp And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp - And unset fault inject + And clear user's answers And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" @@ -372,7 +422,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) And ggrebalance should print "Port is updated: False" to logfile with latest timestamp And ggrebalance should print "Cancel step" to logfile with latest timestamp And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp - And unset fault inject + And clear user's answers # some mirrors are definitely down, so do not check them And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" @@ -513,7 +563,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) Then ggrebalance should return a return code of 0 And ggrebalance should print "Rebalance rollback is complete" to logfile with latest timestamp And verify the gp_segment_configuration has been restored - And unset fault inject + And clear user's answers And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index b6389fcf4087..3d57f31fa80b 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -4612,10 +4612,19 @@ def impl(context): @when('user will answer "{answer}" to the prompt "{prompt}"') def impl(context, answer, prompt): assert answer == 'yes' or answer == 'no' - if fault_injection.GPMGMT_FAULT_POINT in os.environ and os.environ[fault_injection.GPMGMT_FAULT_POINT] != "": - raise Exception('Need to unset fault injection before using this step') - fault = prompt + answer - os.environ[fault_injection.GPMGMT_FAULT_POINT] = fault + if not hasattr(context, 'fault_injected_answers'): + context.fault_injected_answers = {} + context.fault_injected_answers[prompt] = answer + os.environ[fault_injection.GPMGMT_FAULT_TYPE] = fault_injection.GPMGMT_FAULT_TYPE_VALUE + os.environ[fault_injection.GPMGMT_FAULT_POINT] = json.dumps(context.fault_injected_answers) + +@given("clear user's answers") +@then("clear user's answers") +@when("clear user's answers") +def impl(context): + context.fault_injected_answers = {} + os.environ[fault_injection.GPMGMT_FAULT_TYPE] = '' + os.environ[fault_injection.GPMGMT_FAULT_POINT] = '' @given('stub') def impl(context): From 682b12c175e5ed00864197edd11f58ffffbce7d4 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 4 Mar 2026 15:17:32 +1000 Subject: [PATCH 12/28] Add test 6.1.3. --- .../mgmt_utils/ggrebalance_rebalance.feature | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 6d0598b0e0dd..d3edb38d7ca0 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -326,7 +326,6 @@ Feature: ggrebalance behave tests (rebalance scenarios) And ggrebalance should print "Port is updated: False" to logfile with latest timestamp And ggrebalance should print "Plan to rollback step" to logfile with latest timestamp And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp - # TODO: add to all cases where needed And clear user's answers And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" @@ -345,6 +344,55 @@ Feature: ggrebalance behave tests (rebalance scenarios) | fault_name | | _update_config_begin | + Scenario Outline: 6.1.3 rebalance - interrupt during mirror move before gp_segment_configuration update, continue and cancel failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Retry step?" + And user will answer "no" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: False" to logfile with latest timestamp + And ggrebalance should print "Port is updated: False" to logfile with latest timestamp + And ggrebalance should print "Cancel step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'd'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | _update_config_begin | + Scenario Outline: 6.2. rebalance - interrupt during switchover step (before invocation of 'gprecoverseg'), continue and retry failed step. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" From 50a0033e94125110acac26c73b8ac11b0b3f942b Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 4 Mar 2026 16:51:14 +1000 Subject: [PATCH 13/28] Add test 6.3.2 --- gpMgmt/bin/gpmovemirrors | 9 ++-- .../bin/gprebalance_modules/ggrebalance_sm.py | 2 +- .../mgmt_utils/ggrebalance_rebalance.feature | 49 ++++++++++++++++++- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/gpMgmt/bin/gpmovemirrors b/gpMgmt/bin/gpmovemirrors index 463a7d692c99..59a599077722 100755 --- a/gpMgmt/bin/gpmovemirrors +++ b/gpMgmt/bin/gpmovemirrors @@ -85,6 +85,8 @@ def parseargs(): help='show this help message and exit.') parser.add_option('-a', dest="interactive", action='store_false', default=True, help="quiet mode, do not require user input for confirmations") + parser.add_option('--skip-resource-estimation', dest='skip_resource_estimation', metavar='', + action='store_true', default=False, help='Skip resource estimation (storage)') parser.add_option('--usage', action="briefhelp") parser.set_defaults(verbose=False, filters=[], slice=(None, None)) @@ -376,9 +378,10 @@ try: pairs.append(pair) """ Validating Disk Space requirement """ - disk_usage = RelocateDiskUsage(pairs, options.batch_size, options) - if not disk_usage.validate_disk_space(): - raise InsufficientDiskSpaceError("Insufficient disk space on target mirror hosts.") + if not options.skip_resource_estimation: + disk_usage = RelocateDiskUsage(pairs, options.batch_size, options) + if not disk_usage.validate_disk_space(): + raise InsufficientDiskSpaceError("Insufficient disk space on target mirror hosts.") """ Prepare common execution steps for running commands on segments """ oldMirrorsToMove = [mirror for mirror in newConfig.oldMirrorList if not mirror.inPlace] diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index 97d20a5ebfdb..c4a76895e15d 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -175,7 +175,7 @@ def process_moves(self, steps: List[RebalanceStepMoveMirror]): return filename = self.create_config_file(steps) - gpmovemirrors_options = f'-a -i {filename}' + gpmovemirrors_options = f'--skip-resource-estimation -a -i {filename}' if self.options.parallel is not None: batch_size = self.options.parallel diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index d3edb38d7ca0..301f240c8329 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -440,7 +440,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | - Scenario Outline: 6.3. rebalance - interrupt during mirror move after gp_segment_configuration update, but before port update, continue and cancel failed step. + Scenario Outline: 6.3.1. rebalance - interrupt during mirror move after gp_segment_configuration update, but before port update, continue and cancel failed step. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" @@ -486,6 +486,53 @@ Feature: ggrebalance behave tests (rebalance scenarios) | fault_name | | _update_config_end | + Scenario Outline: 6.3.2. rebalance - interrupt during mirror move after gp_segment_configuration update, but before port update, continue and rollback failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "yes" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: True" to logfile with latest timestamp + And ggrebalance should print "Port is updated: False" to logfile with latest timestamp + And ggrebalance should print "Plan to rollback step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | _update_config_end | Scenario Outline: 6.4. rebalance - interrupt during mirror move after before port update (when the mirror is actually started), and continue. Given the database is not running From f018d2bb796fbaf9f0de856b09fb075e6e6b3e11 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 4 Mar 2026 22:11:05 +1000 Subject: [PATCH 14/28] Add test 6.4.2. --- gpMgmt/sbin/gpsegrecovery.py | 2 + .../mgmt_utils/ggrebalance_rebalance.feature | 52 ++++++++++++++++++- .../behave/mgmt_utils/steps/mgmt_utils.py | 25 +++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/gpMgmt/sbin/gpsegrecovery.py b/gpMgmt/sbin/gpsegrecovery.py index 74c70ef8e92b..45ce5869d12c 100644 --- a/gpMgmt/sbin/gpsegrecovery.py +++ b/gpMgmt/sbin/gpsegrecovery.py @@ -18,6 +18,7 @@ from gppylib.operations.segment_tablespace_locations import get_segment_tablespace_oid_locations from gppylib.commands.unix import terminate_proc_tree from gppylib.commands.unix import get_remote_link_path +from gppylib.fault_injection import * class FullRecovery(Command): @@ -343,6 +344,7 @@ def sync_tablespaces(self): os.symlink(targetPath, targetOidPath) +@wrap_state_func_with_faults def start_segment(recovery_info, logger, era): seg = Segment(None, None, None, None, None, None, None, None, recovery_info.target_port, recovery_info.target_datadir) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 301f240c8329..7ed246d83ff3 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -534,7 +534,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | fault_name | | _update_config_end | - Scenario Outline: 6.4. rebalance - interrupt during mirror move after before port update (when the mirror is actually started), and continue. + Scenario Outline: 6.4.1. rebalance - interrupt during mirror move after port update (when the mirror is actually started), and continue. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" @@ -581,6 +581,56 @@ Feature: ggrebalance behave tests (rebalance scenarios) | fault_name | | _do_recovery_end | + Scenario Outline: 6.4.2. rebalance - interrupt during mirror move after port update (but before the mirror is actually started), and continue. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And on host "sdw1" set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And on host "sdw1" unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Timeout waiting for segment start, wait again?" + And user will answer "yes" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "Start checking if segment is up with timeout" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: True" to logfile with latest timestamp + And ggrebalance should print "Port is updated: True" to logfile with latest timestamp + And ggrebalance should print "Plan to rollback step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | start_segment_begin | + Scenario Outline: 7.1. rebalance - rebalance interrupt and full rollback. Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 3d57f31fa80b..aa11a4674799 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -4585,6 +4585,31 @@ def impl(context): if hasattr(context, 'fault_flag_filename') and os.path.exists(context.fault_flag_filename): os.remove(context.fault_flag_filename) +@given('on host "{host}" set fault inject "{fault}"') +@then('on host "{host}" set fault inject "{fault}"') +@when('on host "{host}" set fault inject "{fault}"') +def impl(context, fault, host): + os.environ[fault_injection.GPMGMT_FAULT_POINT] = fault + cmd = f""" + ssh {host} " + echo 'export {fault_injection.GPMGMT_FAULT_POINT}={fault}' >> ~/.bashrc" + export {fault_injection.GPMGMT_FAULT_POINT}={fault} + """ + #print(f'RELOG = {cmd.strip()}') + run_command(context, cmd.strip()) + +@given('on host "{host}" unset fault inject') +@then('on host "{host}" unset fault inject') +@when('on host "{host}" unset fault inject') +def impl(context, host): + cmd = f""" + ssh {host} " + sed -i '/{fault_injection.GPMGMT_FAULT_POINT}=/d' ~/.bashrc + unset {fault_injection.GPMGMT_FAULT_POINT} + " + """ + run_command(context, cmd.strip()) + @given('set fault inject delay {delay} ms') @then('set fault inject delay {delay} ms') @when('set fault inject delay {delay} ms') From e971872b0a62923f0d2f5ca4eaf1a4af70846171 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 5 Mar 2026 09:46:23 +1000 Subject: [PATCH 15/28] Update comments, remove dbg logs --- gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py | 7 +++---- gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index c4a76895e15d..70d05cd23e39 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -475,7 +475,6 @@ def process_error_execution_steps_switchovers(self, error_steps: List[RebalanceS self.mark_dependent_steps_on_error(error_steps, steps_left_todo) def mark_dependent_steps_on_error(self, error_steps: List[RebalanceStep], steps_left_todo: List[RebalanceStep]) -> None: - # Mark dependent steps accordingly # 1. If there are steps planned for ROLLBACK - we mark all left todo steps for the same content as already rolled back if not self.is_rollback_flow: for step in error_steps: @@ -522,13 +521,13 @@ def get_postgresql_conf_port(self, hostname: str, datadir: str) -> int: return int(output) - # decorator to TODO: add comments + # Decorator to overwrite the logic of interactive_check() + # during tests execution. def wrap_interactive_check_with_faults(fun): def func_with_faults(self, msg: str): try: inject_value = inject_fault_get_value() injected_answers = json.loads(inject_value) - self.logger.info(f'[RELOG DBG] injected value = "{inject_value}"') if injected_answers.get(msg, '') == 'yes': return True if injected_answers.get(msg, '') == 'no': @@ -540,7 +539,7 @@ def func_with_faults(self, msg: str): @wrap_interactive_check_with_faults def interactive_check(self, msg: str) -> bool: - #Currently only a stub + # TODO: add logic here when implementing interactive mode return False @staticmethod diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index aa11a4674799..e09f4eecde3c 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -4595,7 +4595,6 @@ def impl(context, fault, host): echo 'export {fault_injection.GPMGMT_FAULT_POINT}={fault}' >> ~/.bashrc" export {fault_injection.GPMGMT_FAULT_POINT}={fault} """ - #print(f'RELOG = {cmd.strip()}') run_command(context, cmd.strip()) @given('on host "{host}" unset fault inject') From 8379e87a68e7baf3c7e168ee8cad423b025fcff4 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 5 Mar 2026 09:50:49 +1000 Subject: [PATCH 16/28] Update rollback prepare --- gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index 70d05cd23e39..807f5235bb66 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -761,10 +761,10 @@ def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED(self) -> None: if step.getStatus() in [RebalanceStep.Status.APPROVE_REQUIRED, RebalanceStep.Status.PLANNED]: step.setStatus(RebalanceStep.Status.DONE, True) actual_rollback_steps_cnt += 1 - elif step.getStatus() in [RebalanceStep.Status.IN_PROGRESS, RebalanceStep.Status.DONE]: + elif step.getStatus() in [RebalanceStep.Status.IN_PROGRESS, RebalanceStep.Status.DONE, RebalanceStep.Status.ERROR]: step.setStatus(RebalanceStep.Status.PLANNED, True) actual_rollback_steps_cnt += 1 - # TODO: what about errored or cancelled steps + # We do nothing for CANCELLED steps - for now they can be processed only if run ggrebalance from scratch. rollback_step_for_switchover = None From 07d3dcafba9f5394515c843a7fefb3d28ded2bd2 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 5 Mar 2026 09:56:24 +1000 Subject: [PATCH 17/28] Truncate steps table --- .../gprebalance_modules/rebalance_schema.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py index def60e821feb..b7c1d4df040f 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py @@ -46,9 +46,12 @@ def createSchema(self, plan: Plan) -> None: (plan BYTEA) DISTRIBUTED REPLICATED''') - self.savePlan(plan) + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.segment_move_steps} + (move_order INT NOT NULL UNIQUE, status TEXT, is_rollback BOOL, step BYTEA) + DISTRIBUTED REPLICATED''') - self.recreateExecutionStepsTable() + self.savePlan(plan) dbconn.execSQL(self.conn, 'COMMIT') @@ -163,19 +166,10 @@ def getTablesToRebalanceWithStatus(self, status: str) -> cursor: return dbconn.query(self.conn, f"""SELECT db_name, schema_name, rel_name FROM {self.schema_name}.{self.table_rebalance_status_detail} WHERE status = '{status}'""") - def recreateExecutionStepsTable(self) -> None: - dbconn.execSQL(self.conn, f'DROP TABLE IF EXISTS {self.schema_name}.{self.segment_move_steps}') - - dbconn.execSQL(self.conn, - f'''CREATE TABLE {self.schema_name}.{self.segment_move_steps} - (move_order INT NOT NULL UNIQUE, status TEXT, is_rollback BOOL, step BYTEA) - DISTRIBUTED REPLICATED''') - def saveExecutionSteps(self, steps: List[RebalanceStep]) -> None: dbconn.execSQL(self.conn, 'BEGIN') - # TODO: just truncate here? - self.recreateExecutionStepsTable() + dbconn.execSQL(self.conn, f'TRUNCATE TABLE {self.schema_name}.{self.segment_move_steps}') for step in steps: dbconn.execSQL(self.conn, From 67c24cd20226fd43a55b5acd2a81fccee17ef753 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 5 Mar 2026 10:06:14 +1000 Subject: [PATCH 18/28] Improve logging --- gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index 807f5235bb66..85e50dc21bc6 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -744,6 +744,7 @@ def on_enter_STATE_REBALANCE_ROLLBACK_STARTED(self) -> None: @wrap_state_func_with_faults def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED(self) -> None: + self.logger.info('Start preparing steps for rollback...') actual_rollback_steps_cnt = 0 rollback_steps = self.rebalance_schema.getExecutionSteps([]) @@ -764,7 +765,10 @@ def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED(self) -> None: elif step.getStatus() in [RebalanceStep.Status.IN_PROGRESS, RebalanceStep.Status.DONE, RebalanceStep.Status.ERROR]: step.setStatus(RebalanceStep.Status.PLANNED, True) actual_rollback_steps_cnt += 1 - # We do nothing for CANCELLED steps - for now they can be processed only if run ggrebalance from scratch. + elif step.getStatus() == RebalanceStep.Status.CANCELLED: + # We do nothing for CANCELLED steps - for now they can be processed only if run ggrebalance from scratch. + self.logger.warning(f'Step {str(step)} is marked as CANCELLED, and skipped during ROLLBACK processing.') + continue rollback_step_for_switchover = None @@ -788,7 +792,7 @@ def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED(self) -> None: for step in rollback_steps: self.logger.info(str(step)) self.rebalance_schema.saveExecutionSteps(rollback_steps) - self.logger.info('Saved rollback rebalance execution steps.') + self.logger.info('Saved rollback rebalance execution steps') else: self.logger.info('No steps to rollback found for rebalance') From 7deaba54feb898a70e4919715d5478720fe32762 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 5 Mar 2026 13:00:29 +1000 Subject: [PATCH 19/28] Add test 6.4.3., update test 7.2 --- .../mgmt_utils/ggrebalance_rebalance.feature | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 7ed246d83ff3..142d22c0a317 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -581,7 +581,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | fault_name | | _do_recovery_end | - Scenario Outline: 6.4.2. rebalance - interrupt during mirror move after port update (but before the mirror is actually started), and continue. + Scenario Outline: 6.4.2. rebalance - interrupt during mirror move after port update (but before the mirror is actually started), and continue (with step rollback). Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" @@ -631,6 +631,54 @@ Feature: ggrebalance behave tests (rebalance scenarios) | fault_name | | start_segment_begin | + Scenario Outline: 6.4.3. rebalance - interrupt during mirror move after port update (but before the mirror is actually started), and continue (with step cancel). + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And on host "sdw1" set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And on host "sdw1" unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Timeout waiting for segment start, wait again?" + And user will answer "no" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Checking error status for step" to logfile with latest timestamp + And ggrebalance should print "Start checking if segment is up with timeout" to logfile with latest timestamp + And ggrebalance should print "gp_segment_configuration is updated: True" to logfile with latest timestamp + And ggrebalance should print "Port is updated: True" to logfile with latest timestamp + And ggrebalance should print "Cancel step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + # some mirrors are definitely down, so do not check them + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | start_segment_begin | + Scenario Outline: 7.1. rebalance - rebalance interrupt and full rollback. Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' @@ -735,7 +783,6 @@ Feature: ggrebalance behave tests (rebalance scenarios) | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | | on_enter_STATE_REBALANCE_DONE_begin | - | on_enter_STATE_REBALANCE_DONE_end | Scenario: test 7.3. rebalance - interrupt during shrink, and full rollback. Given the database is not running From 0a5fbb0ce2215a08376527562cfb83451a9219b4 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 5 Mar 2026 13:06:02 +1000 Subject: [PATCH 20/28] Rename wrap_state_func_with_faults() to wrap_func_with_faults() --- gpMgmt/bin/gppylib/fault_injection.py | 2 +- .../gppylib/operations/buildMirrorSegments.py | 8 ++-- .../ggrebalance_main_sm.py | 32 ++++++------- .../bin/gprebalance_modules/ggrebalance_sm.py | 28 +++++------ gpMgmt/bin/gprebalance_modules/shrink.py | 46 +++++++++---------- gpMgmt/sbin/gpsegrecovery.py | 2 +- 6 files changed, 59 insertions(+), 59 deletions(-) diff --git a/gpMgmt/bin/gppylib/fault_injection.py b/gpMgmt/bin/gppylib/fault_injection.py index 6d22104ec5d8..50268764a47d 100755 --- a/gpMgmt/bin/gppylib/fault_injection.py +++ b/gpMgmt/bin/gppylib/fault_injection.py @@ -40,7 +40,7 @@ def inject_fault_get_value() -> str: return '' # decorator for test purposes -def wrap_state_func_with_faults(func): +def wrap_func_with_faults(func): def func_with_faults(*args): inject_fault(f'{func.__name__}_begin') result = func(*args) diff --git a/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py b/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py index b7d629bd7709..1bf3556e4212 100644 --- a/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py +++ b/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py @@ -314,7 +314,7 @@ def _trigger_fts_probe(self, port=0): dbconn.execSQL(conn,"SELECT gp_request_fts_probe_scan()") conn.close() - @wrap_state_func_with_faults + @wrap_func_with_faults def _update_config(self, recovery_info_by_host, gpArray): # should use mainUtils.getProgramName but I can't make it work! programName = os.path.split(sys.argv[0])[-1] @@ -589,7 +589,7 @@ def _run_recovery(self, action_name, recovery_info_by_host, gpEnv): self._remove_progress_files(recovery_info_by_host, recovery_results) return recovery_results - @wrap_state_func_with_faults + @wrap_func_with_faults def _do_recovery(self, recovery_info_by_host, gpEnv): """ # Recover and start segments using gpsegrecovery, which will internally call either @@ -690,7 +690,7 @@ def _get_failed_reachable_segments(self): failed_reachable_segments.append(failed) return failed_reachable_segments - @wrap_state_func_with_faults + @wrap_func_with_faults def _stop_failed_segments(self, gpEnv): failed_reachable_segments = self._get_failed_reachable_segments() if len(failed_reachable_segments) == 0: @@ -719,7 +719,7 @@ def _stop_failed_segments(self, gpEnv): # self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, suppressErrorCheck=True) - @wrap_state_func_with_faults + @wrap_func_with_faults def _wait_fts_to_mark_down_segments(self, gpEnv, segments_to_mark_down): """Waits for FTS prober to mark segments as down""" diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py index a3f5a762bd46..157c66d86ff6 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py @@ -168,7 +168,7 @@ def shutdown(self) -> None: # state callbacks start here - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_OPTIONS_VALIDATION(self) -> None: if self.options.clean_required: self.trigger('move_to_STATE_CLEANUP') @@ -177,7 +177,7 @@ def on_enter_STATE_OPTIONS_VALIDATION(self) -> None: else: self.trigger('move_to_STATE_PLANNING_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_CLEANUP(self) -> None: if not self.rebalance_schema.schemaExists(): self.logger.info(f"Rebalance schema doesn't exist. Cleanup is not required.") @@ -189,7 +189,7 @@ def on_enter_STATE_CLEANUP(self) -> None: self.logger.info('Cleanup is complete') self.trigger('move_to_STATE_END') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_ROLLBACK(self) -> None: try: self.plan = self.rebalance_schema.retrieveSavedPlan() @@ -202,7 +202,7 @@ def on_enter_STATE_ROLLBACK(self) -> None: finally: self.trigger('move_to_STATE_END') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_PLANNING_STARTED(self) -> None: if self.options.target_segment_count != None: self.plan = Planner(self.logger, self.dburl, self.gparray, self.options).plan() @@ -212,11 +212,11 @@ def on_enter_STATE_PLANNING_STARTED(self) -> None: self.trigger('move_to_STATE_PLANNING_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_PLANNING_DONE(self) -> None: self.trigger('move_to_STATE_CHECK_PREVIOUS_RUN') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: if not self.rebalance_schema.schemaExists(): if self.plan == None: @@ -250,19 +250,19 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: self.trigger('move_to_STATE_EXECUTOR_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SETUP_SCHEMA_STARTED(self) -> None: # Create schema and status tables. # It will also save plan in order to use it for recovering after interruption self.rebalance_schema.createSchema(self.plan) self.trigger('move_to_STATE_SETUP_SCHEMA_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SETUP_SCHEMA_DONE(self) -> None: self.logger.info(f'Created "{self.rebalance_schema.getSchemaName()}" schema') self.trigger('move_to_STATE_EXECUTOR_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_EXECUTOR_STARTED(self) -> None: if isinstance(self.plan, ShrinkPlan): shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() @@ -271,35 +271,35 @@ def on_enter_STATE_EXECUTOR_STARTED(self) -> None: return self.trigger('move_to_STATE_REBALANCE_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_EXECUTOR_DONE(self) -> None: self.trigger('move_to_STATE_END') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_STARTED(self) -> None: self.gg_shrink.run(self.plan) self.trigger('move_to_STATE_SHRINK_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_DONE(self) -> None: self.trigger('move_to_STATE_REBALANCE_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_STARTED(self) -> None: if self.plan is not None and self.plan.getMoves() is not None: self.gg_rebalance.run(self.plan) self.trigger('move_to_STATE_REBALANCE_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_DONE(self) -> None: self.trigger('move_to_STATE_EXECUTOR_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_END(self) -> None: pass - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_ERROR(self) -> None: raise Exception('Main SM entered STATE_ERROR') diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index 85e50dc21bc6..f89a03e0ed74 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -592,7 +592,7 @@ def fill_rebalance_steps(): # state callbacks start here - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: state_from_prev_run = self.rebalance_schema.getRebalanceStateFromPreviousRun() self.is_rollback_flow = self.rebalance_schema.isRollbackRebalanceFlow(self.states_rollback_rebalance_flow[0]) @@ -617,11 +617,11 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: # use auto to_«state» method to recover self.trigger(f'to_{next_state}') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_STARTED(self) -> None: self.trigger('move_to_STATE_REBALANCE_PREPARE_MOVES_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED(self) -> None: if not self.rebalance_plan.getMoves(): raise Exception('Rebalance executor was launched with a plan without segment movements') @@ -642,11 +642,11 @@ def on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED(self) -> None: self.trigger('move_to_STATE_REBALANCE_PREPARE_MOVES_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE(self) -> None: self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_EXECUTION_STARTED(self) -> None: if self.rebalance_schema.allExecutionStepsAreDone(): self.trigger('move_to_STATE_REBALANCE_EXECUTION_DONE') @@ -704,15 +704,15 @@ def on_enter_STATE_REBALANCE_EXECUTION_STARTED(self) -> None: self.trigger('move_to_STATE_REBALANCE_MOVES_SUCCEEDED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_MOVES_SUCCEEDED(self) -> None: self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_EXECUTION_DONE(self) -> None: self.trigger('move_to_STATE_REBALANCE_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED(self) -> None: # Approve all consequent steps that require approval @@ -731,17 +731,17 @@ def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED(self) self.trigger('move_to_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE(self) -> None: self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_ROLLBACK_STARTED(self) -> None: self.is_rollback_flow = True self.logger.info('Starting rebalance rollback') self.trigger('move_to_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED(self) -> None: self.logger.info('Start preparing steps for rollback...') @@ -798,11 +798,11 @@ def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_STARTED(self) -> None: self.trigger('move_to_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_ROLLBACK_PREPARE_MOVES_DONE(self) -> None: self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_REBALANCE_DONE(self) -> None: if self.is_rollback_flow: self.rebalance_schema.dropSchema() @@ -810,7 +810,7 @@ def on_enter_STATE_REBALANCE_DONE(self) -> None: else: self.logger.info('Rebalance is complete') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_ERROR(self) -> None: raise Exception('Rebalance execution entered STATE_ERROR') diff --git a/gpMgmt/bin/gprebalance_modules/shrink.py b/gpMgmt/bin/gprebalance_modules/shrink.py index 48798267ed6c..d5b8fc8b8652 100644 --- a/gpMgmt/bin/gprebalance_modules/shrink.py +++ b/gpMgmt/bin/gprebalance_modules/shrink.py @@ -326,7 +326,7 @@ def state_is_final(self, state: str) -> bool: # state callbacks start here - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: assert self.rebalance_schema.schemaExists() # check whether we can get the state where we stopped in previous run @@ -369,7 +369,7 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: # use auto to_«state» method to recover self.trigger(f'to_{next_state}') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED(self) -> None: dbconn.execSQL(self.conn, 'BEGIN') dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') @@ -385,34 +385,34 @@ def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED(self) self.trigger('move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE(self) -> None: self.logger.info(f'Updated target segment count to {self.shrink_plan.getTargetSegmentCount()}') self.trigger('move_to_STATE_PREPARE_SHRINK_SCHEMA_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED(self) -> None: self.prepare_shrink_schema(False) self.trigger('move_to_STATE_PREPARE_SHRINK_SCHEMA_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE(self) -> None: self.logger.info(f'Initiated list of tables to rebalance') self.trigger('move_to_STATE_SHRINK_TABLES_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_TABLES_STARTED(self) -> None: self.logger.info('Start tables rebalance for shrink') # perform 'ALTER TABLE REBALANCE' for all not yet processed tables self.rebalance_tables('none', 'done', self.shrink_plan.getTargetSegmentCount()) self.trigger('move_to_STATE_SHRINK_TABLES_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_TABLES_DONE(self) -> None: self.logger.info('Tables rebalance complete') self.trigger('move_to_STATE_SHRINK_CATALOG_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_CATALOG_STARTED(self) -> None: self.logger.info('Start catalog shrink') @@ -427,12 +427,12 @@ def on_enter_STATE_SHRINK_CATALOG_STARTED(self) -> None: self.trigger('move_to_STATE_SHRINK_CATALOG_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_CATALOG_DONE(self) -> None: self.logger.info('Catalog shrink complete') self.trigger('move_to_STATE_SHRINK_SEGMENTS_STOP_STARTED') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED(self) -> None: self.logger.info('Stopping shrinked segments...') @@ -469,18 +469,18 @@ def on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED(self) -> None: self.trigger('move_to_STATE_SHRINK_SEGMENTS_STOP_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_SEGMENTS_STOP_DONE(self) -> None: self.logger.info('Shrinked segments were stopped') self.trigger('move_to_STATE_SHRINK_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_DONE(self) -> None: os.remove(self.gparray_dump_file) self.logger.info('Shrink is complete') self.trigger('move_to_STATE_END') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START(self) -> None: dbconn.execSQL(self.conn, 'BEGIN') dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') @@ -492,51 +492,51 @@ def on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START(self) -> N self.trigger('move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE(self) -> None: self.trigger('move_to_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START(self) -> None: self.prepare_shrink_schema(True) self.trigger('move_to_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE(self) -> None: self.trigger('move_to_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START(self) -> None: self.logger.info('Start tables rebalance for rollback') # perform 'ALTER TABLE REBALANCE' for all not yet processed tables self.rebalance_tables('done', 'none', self.gparray.get_segment_count()) self.trigger('move_to_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE(self) -> None: self.trigger('move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START(self) -> None: if os.path.exists(self.gparray_dump_file): os.remove(self.gparray_dump_file) self.rebalance_schema.dropSchema() self.trigger('move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE(self) -> None: self.logger.info('Rollback is complete.') self.trigger('move_to_STATE_END_FROM_ROLLBACK') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_END_FROM_ROLLBACK(self) -> None: self.trigger('move_to_STATE_END') - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_END(self) -> None: pass - @wrap_state_func_with_faults + @wrap_func_with_faults def on_enter_STATE_ERROR(self) -> None: raise Exception('Shrink entered STATE_ERROR') diff --git a/gpMgmt/sbin/gpsegrecovery.py b/gpMgmt/sbin/gpsegrecovery.py index 45ce5869d12c..c41bc6ced644 100644 --- a/gpMgmt/sbin/gpsegrecovery.py +++ b/gpMgmt/sbin/gpsegrecovery.py @@ -344,7 +344,7 @@ def sync_tablespaces(self): os.symlink(targetPath, targetOidPath) -@wrap_state_func_with_faults +@wrap_func_with_faults def start_segment(recovery_info, logger, era): seg = Segment(None, None, None, None, None, None, None, None, recovery_info.target_port, recovery_info.target_datadir) From 3e7d4f80e1ca9278a5379a9226c93fb0662498fe Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 5 Mar 2026 14:52:34 +1000 Subject: [PATCH 21/28] Update basic tests, and fix code for them --- .../ggrebalance_main_sm.py | 12 +++++-- .../mgmt_utils/ggrebalance_basics.feature | 32 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py index 157c66d86ff6..08e25da98f30 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py @@ -182,9 +182,12 @@ def on_enter_STATE_CLEANUP(self) -> None: if not self.rebalance_schema.schemaExists(): self.logger.info(f"Rebalance schema doesn't exist. Cleanup is not required.") else: - prev_run_was_complete = (self.main_state_from_prev_run == 'STATE_EXECUTOR_DONE' or - self.main_state_from_prev_run == 'STATE_ROLLBACK') - self.gg_shrink.cleanup(prev_run_was_complete) + self.plan = self.rebalance_schema.retrieveSavedPlan() + prev_shrink_run_was_complete = True + if isinstance(self.plan, ShrinkPlan): + shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() + prev_shrink_run_was_complete = self.gg_shrink.state_is_final(shrink_state_from_prev_run) + self.gg_shrink.cleanup(prev_shrink_run_was_complete) self.rebalance_schema.dropSchema() self.logger.info('Cleanup is complete') self.trigger('move_to_STATE_END') @@ -192,6 +195,9 @@ def on_enter_STATE_CLEANUP(self) -> None: @wrap_func_with_faults def on_enter_STATE_ROLLBACK(self) -> None: try: + if self.main_state_from_prev_run == 'STATE_EXECUTOR_DONE': + self.logger.info("Previous run was completed successfully. Can't perform rollback.") + return self.plan = self.rebalance_schema.retrieveSavedPlan() if isinstance(self.plan, ShrinkPlan): shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature index 5300a340ca10..d7b84a8225a6 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature @@ -81,3 +81,35 @@ Feature: ggrebalance behave tests Then ggrebalance should return a return code of 0 And ggrebalance should print "Reset numsegments to default is done." to logfile with latest timestamp And ggrebalance should print "Cleanup is complete" to logfile with latest timestamp + + Scenario: test 4. check cleanup after shrink is complete, and rebalance was interrupted + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_REBALANCE_DONE_begin" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Cleanup is complete" to logfile with latest timestamp + # some mirrors are definitely down, so do not check them + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 From 43915d08a515d59a406030d1b176565bdb1fd02d Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 5 Mar 2026 16:23:54 +1000 Subject: [PATCH 22/28] Add case for 6.2 test, and related fix --- gpMgmt/bin/ggrebalance | 4 ++-- gpMgmt/bin/gppylib/operations/rebalanceSegments.py | 3 +++ gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index 2817f39612ca..d1026e08d553 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -305,14 +305,14 @@ def main(options, args, parser): dburl = dbconn.DbURL(dbname=DBNAME, port=gpenv.getCoordinatorPort()) + check_down_segments(logger, options, dburl) + check_running_gputils(dburl, options.coordinator_data_directory) create_pid_file(options.coordinator_data_directory) gparray_dump_filename = options.coordinator_data_directory + '/gparraydump' - check_down_segments(logger, options, dburl) - logger.info('Init gparray from catalog') try: gparray = GpArray.initFromCatalog(dburl, utility=True) diff --git a/gpMgmt/bin/gppylib/operations/rebalanceSegments.py b/gpMgmt/bin/gppylib/operations/rebalanceSegments.py index 3ef843f2d3ae..ffdc46738b6b 100644 --- a/gpMgmt/bin/gppylib/operations/rebalanceSegments.py +++ b/gpMgmt/bin/gppylib/operations/rebalanceSegments.py @@ -6,6 +6,7 @@ from gppylib.commands.gp import GpSegStopCmd from gppylib.commands import base from gppylib import gplog +from gppylib.fault_injection import * from gppylib.operations.segment_reconfigurer import SegmentReconfigurer @@ -116,6 +117,8 @@ def rebalance(self): pool.addCommand(cmd) base.join_and_indicate_progress(pool) + + inject_fault('GpSegmentRebalanceOperation_rebalance_at_seg_stop') failed_count = 0 completed = pool.getCompletedItems() diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 142d22c0a317..4a04739bf870 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -439,6 +439,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | fault_name | | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | + | GpSegmentRebalanceOperation_rebalance_at_seg_stop | Scenario Outline: 6.3.1. rebalance - interrupt during mirror move after gp_segment_configuration update, but before port update, continue and cancel failed step. Given the database is not running From 934ad5fe97a96c5041a4203b7888e61812e728a8 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 5 Mar 2026 18:59:01 +1000 Subject: [PATCH 23/28] Add 6.2.2. test --- .../bin/gprebalance_modules/ggrebalance_sm.py | 12 +++++ .../mgmt_utils/ggrebalance_rebalance.feature | 51 ++++++++++++++++++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index f89a03e0ed74..223fc3ac2cda 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -466,6 +466,18 @@ def process_error_execution_steps_switchovers(self, error_steps: List[RebalanceS if not step.isRollback() and self.interactive_check('Rollback step?'): self.logger.info('Plan to rollback step') step.setStatus(RebalanceStep.Status.PLANNED, True) + + # Revert type of switchover + rollback_step_for_switchover = None + if isinstance(step, RebalanceStepSwitchoverToMirror): + rollback_step_for_switchover = RebalanceStepSwitchoverToPrimary(step.getMove()) + elif isinstance(step, RebalanceStepSwitchoverToPrimary): + rollback_step_for_switchover = RebalanceStepSwitchoverToMirror(step.getMove()) + + if rollback_step_for_switchover: + rollback_step_for_switchover.setMoveOrder(step.getMoveOrder()) + rollback_step_for_switchover.setStatus(step.getStatus(), True) + step = rollback_step_for_switchover else: self.logger.info('Cancel step') step.setStatus(RebalanceStep.Status.CANCELLED) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 4a04739bf870..9c54f74d6183 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -393,7 +393,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | fault_name | | _update_config_begin | - Scenario Outline: 6.2. rebalance - interrupt during switchover step (before invocation of 'gprecoverseg'), continue and retry failed step. + Scenario Outline: 6.2.1. rebalance - interrupt during switchover step (before invocation of 'gprecoverseg'), continue and retry failed step. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" @@ -441,6 +441,55 @@ Feature: ggrebalance behave tests (rebalance scenarios) | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | | GpSegmentRebalanceOperation_rebalance_at_seg_stop | + Scenario Outline: 6.2.2. rebalance - interrupt during switchover step (before invocation of 'gprecoverseg'), continue and rollback failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -p -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Retry step?" + And user will answer "yes" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Processing error status for switchover step" to logfile with latest timestamp + And ggrebalance should print "Plan to rollback step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + #And the cluster configuration has 4 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + #And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + #And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | + | GpSegmentRebalanceOperation_rebalance_at_seg_stop | + Scenario Outline: 6.3.1. rebalance - interrupt during mirror move after gp_segment_configuration update, but before port update, continue and cancel failed step. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" From a58de48cb71f6c02920408ed0c8bc142a1a61344 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 5 Mar 2026 20:25:40 +1000 Subject: [PATCH 24/28] Add test 7.3.2. --- .../ggrebalance_main_sm.py | 9 +++- gpMgmt/bin/gprebalance_modules/shrink.py | 8 +++- .../mgmt_utils/ggrebalance_rebalance.feature | 43 ++++++++++++++++++- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py index 08e25da98f30..acedf31b0b29 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py @@ -144,6 +144,7 @@ def __init__(self, conn: dbconn.Connection, logger: Any, dburl: dbconn.DbURL, op self.plan = None self.main_state_from_prev_run = self.rebalance_schema.getMainStateFromPreviousRun() + self.is_shrink_rollback_in_progress = False def on_every_state(self) -> None: if self.state in self.states_logged: @@ -201,6 +202,9 @@ def on_enter_STATE_ROLLBACK(self) -> None: self.plan = self.rebalance_schema.retrieveSavedPlan() if isinstance(self.plan, ShrinkPlan): shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() + if self.gg_shrink.state_is_from_rollback_flow(shrink_state_from_prev_run): + self.logger.info("Rollback is already in progress, and was interrupted. Execute 'ggrebalance' without '-r' flag.") + return if not self.gg_shrink.state_is_final(shrink_state_from_prev_run): self.gg_shrink.rollback(self.plan) return @@ -272,6 +276,7 @@ def on_enter_STATE_SETUP_SCHEMA_DONE(self) -> None: def on_enter_STATE_EXECUTOR_STARTED(self) -> None: if isinstance(self.plan, ShrinkPlan): shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() + self.is_shrink_rollback_in_progress = self.gg_shrink.state_is_from_rollback_flow(shrink_state_from_prev_run) if not self.gg_shrink.state_is_final(shrink_state_from_prev_run): self.trigger('move_to_STATE_SHRINK_STARTED') return @@ -292,7 +297,9 @@ def on_enter_STATE_SHRINK_DONE(self) -> None: @wrap_func_with_faults def on_enter_STATE_REBALANCE_STARTED(self) -> None: - if self.plan is not None and self.plan.getMoves() is not None: + if (self.plan is not None and + self.plan.getMoves() is not None and + not self.is_shrink_rollback_in_progress): self.gg_rebalance.run(self.plan) self.trigger('move_to_STATE_REBALANCE_DONE') diff --git a/gpMgmt/bin/gprebalance_modules/shrink.py b/gpMgmt/bin/gprebalance_modules/shrink.py index d5b8fc8b8652..358217ac33ef 100644 --- a/gpMgmt/bin/gprebalance_modules/shrink.py +++ b/gpMgmt/bin/gprebalance_modules/shrink.py @@ -324,6 +324,9 @@ def cleanup(self, prev_run_was_complete: bool) -> None: def state_is_final(self, state: str) -> bool: return state == self.states_main_shrink_flow[-1] + def state_is_from_rollback_flow(self, state: str) -> bool: + return state in self.states_rollback_flow + # state callbacks start here @wrap_func_with_faults @@ -338,7 +341,7 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: self.logger.info(f"Previous run was completed successfully. Can't perform rollback.") self.trigger('move_to_STATE_END_FROM_ROLLBACK') else: - if state_from_prev_run in self.states_rollback_flow: + if self.state_is_from_rollback_flow(state_from_prev_run): self.logger.info('Continue interrupted shrink rollback operation...') self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") try: @@ -520,7 +523,8 @@ def on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE(self) -> None: def on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START(self) -> None: if os.path.exists(self.gparray_dump_file): os.remove(self.gparray_dump_file) - self.rebalance_schema.dropSchema() + if self.options.skip_rebalance: + self.rebalance_schema.dropSchema() self.trigger('move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE') @wrap_func_with_faults diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 9c54f74d6183..7a9648764ec4 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -834,7 +834,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | | on_enter_STATE_REBALANCE_DONE_begin | - Scenario: test 7.3. rebalance - interrupt during shrink, and full rollback. + Scenario: test 7.3.1. rebalance - interrupt during shrink, and full rollback. Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" @@ -864,6 +864,47 @@ Feature: ggrebalance behave tests (rebalance scenarios) When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + Scenario: test 7.3.2. rebalance - interrupt during shrink, and full rollback, interrupt during shrink rollback, and continue. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_SHRINK_CATALOG_STARTED_begin" + And the gp_segment_configuration have been saved + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_end" + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is already in progress, and was interrupted. Execute 'ggrebalance' without '-r' flag." to logfile with latest timestamp + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is complete" to logfile with latest timestamp + And verify the gp_segment_configuration has been restored + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + Scenario: test 7.4. rebalance - interrupt after shrink, but before rebalance start, and full rollback. Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' From 45d69d8c8be0dc2d87078a08499d58d15258d45a Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 6 Mar 2026 10:47:04 +1000 Subject: [PATCH 25/28] Refactor --- .../ggrebalance_main_sm.py | 20 +++++++++---------- gpMgmt/bin/gprebalance_modules/shrink.py | 3 +-- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py index acedf31b0b29..94589de18cc5 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py @@ -144,7 +144,11 @@ def __init__(self, conn: dbconn.Connection, logger: Any, dburl: dbconn.DbURL, op self.plan = None self.main_state_from_prev_run = self.rebalance_schema.getMainStateFromPreviousRun() - self.is_shrink_rollback_in_progress = False + + self.shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() + self.is_shrink_rollback_in_progress = self.gg_shrink.state_is_from_rollback_flow(self.shrink_state_from_prev_run) + self.prev_shrink_run_was_complete = self.gg_shrink.state_is_final(self.shrink_state_from_prev_run) + def on_every_state(self) -> None: if self.state in self.states_logged: @@ -184,11 +188,8 @@ def on_enter_STATE_CLEANUP(self) -> None: self.logger.info(f"Rebalance schema doesn't exist. Cleanup is not required.") else: self.plan = self.rebalance_schema.retrieveSavedPlan() - prev_shrink_run_was_complete = True if isinstance(self.plan, ShrinkPlan): - shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() - prev_shrink_run_was_complete = self.gg_shrink.state_is_final(shrink_state_from_prev_run) - self.gg_shrink.cleanup(prev_shrink_run_was_complete) + self.gg_shrink.cleanup(self.prev_shrink_run_was_complete) self.rebalance_schema.dropSchema() self.logger.info('Cleanup is complete') self.trigger('move_to_STATE_END') @@ -201,11 +202,10 @@ def on_enter_STATE_ROLLBACK(self) -> None: return self.plan = self.rebalance_schema.retrieveSavedPlan() if isinstance(self.plan, ShrinkPlan): - shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() - if self.gg_shrink.state_is_from_rollback_flow(shrink_state_from_prev_run): + if self.is_shrink_rollback_in_progress: self.logger.info("Rollback is already in progress, and was interrupted. Execute 'ggrebalance' without '-r' flag.") return - if not self.gg_shrink.state_is_final(shrink_state_from_prev_run): + if not self.prev_shrink_run_was_complete: self.gg_shrink.rollback(self.plan) return self.gg_rebalance.rollback() @@ -275,9 +275,7 @@ def on_enter_STATE_SETUP_SCHEMA_DONE(self) -> None: @wrap_func_with_faults def on_enter_STATE_EXECUTOR_STARTED(self) -> None: if isinstance(self.plan, ShrinkPlan): - shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() - self.is_shrink_rollback_in_progress = self.gg_shrink.state_is_from_rollback_flow(shrink_state_from_prev_run) - if not self.gg_shrink.state_is_final(shrink_state_from_prev_run): + if not self.prev_shrink_run_was_complete: self.trigger('move_to_STATE_SHRINK_STARTED') return self.trigger('move_to_STATE_REBALANCE_STARTED') diff --git a/gpMgmt/bin/gprebalance_modules/shrink.py b/gpMgmt/bin/gprebalance_modules/shrink.py index 358217ac33ef..48a1f37c0f96 100644 --- a/gpMgmt/bin/gprebalance_modules/shrink.py +++ b/gpMgmt/bin/gprebalance_modules/shrink.py @@ -523,8 +523,7 @@ def on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE(self) -> None: def on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START(self) -> None: if os.path.exists(self.gparray_dump_file): os.remove(self.gparray_dump_file) - if self.options.skip_rebalance: - self.rebalance_schema.dropSchema() + self.rebalance_schema.dropSchema() self.trigger('move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE') @wrap_func_with_faults From ee3f19907da91a4be2d3727de3ce08f0e7a96013 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 6 Mar 2026 12:31:59 +1000 Subject: [PATCH 26/28] Split 6.2.2. test --- .../mgmt_utils/ggrebalance_rebalance.feature | 52 +++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 7a9648764ec4..032405b52a3d 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -441,7 +441,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | | GpSegmentRebalanceOperation_rebalance_at_seg_stop | - Scenario Outline: 6.2.2. rebalance - interrupt during switchover step (before invocation of 'gprecoverseg'), continue and rollback failed step. + Scenario Outline: 6.2.2. rebalance - interrupt during switchover P->M step (before invocation of 'gprecoverseg'), continue and rollback failed step. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" @@ -457,7 +457,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows And all files in gpAdminLogs directory are deleted And set fault inject "" - When the user runs "ggrebalance -p -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject @@ -487,9 +487,55 @@ Feature: ggrebalance behave tests (rebalance scenarios) Examples: | fault_name | | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | - | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | | GpSegmentRebalanceOperation_rebalance_at_seg_stop | + Scenario Outline: 6.2.3. rebalance - interrupt during switchover M->P step (before invocation of 'gprecoverseg'), continue and rollback failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Retry step?" + And user will answer "yes" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Processing error status for switchover step" to logfile with latest timestamp + And ggrebalance should print "Plan to rollback step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + #And the cluster configuration has 4 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + #And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + #And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | + Scenario Outline: 6.3.1. rebalance - interrupt during mirror move after gp_segment_configuration update, but before port update, continue and cancel failed step. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" From 0e3020f7e1059f4551e43720d4fab60c276a5491 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 6 Mar 2026 13:15:29 +1000 Subject: [PATCH 27/28] Uncomment some checks --- .../behave/mgmt_utils/ggrebalance_rebalance.feature | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 032405b52a3d..05e2287d3293 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -472,11 +472,11 @@ Feature: ggrebalance behave tests (rebalance scenarios) And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp And clear user's answers And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" - #And the cluster configuration has 4 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 4 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" - #And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" And the cluster configuration has 1 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" - #And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 @@ -520,11 +520,11 @@ Feature: ggrebalance behave tests (rebalance scenarios) And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp And clear user's answers And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" - #And the cluster configuration has 4 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 4 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" - #And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" - #And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 From ecf1efc273c7e9e72767673a53c2ad247eacbe8f Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 6 Mar 2026 13:38:31 +1000 Subject: [PATCH 28/28] Minor test updates --- .../gppylib/operations/buildMirrorSegments.py | 2 - .../mgmt_utils/ggrebalance_rebalance.feature | 51 +++++++++++++++++-- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py b/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py index 1bf3556e4212..8ef5a305fc86 100644 --- a/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py +++ b/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py @@ -690,7 +690,6 @@ def _get_failed_reachable_segments(self): failed_reachable_segments.append(failed) return failed_reachable_segments - @wrap_func_with_faults def _stop_failed_segments(self, gpEnv): failed_reachable_segments = self._get_failed_reachable_segments() if len(failed_reachable_segments) == 0: @@ -719,7 +718,6 @@ def _stop_failed_segments(self, gpEnv): # self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, suppressErrorCheck=True) - @wrap_func_with_faults def _wait_fts_to_mark_down_segments(self, gpEnv, segments_to_mark_down): """Waits for FTS prober to mark segments as down""" diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index 05e2287d3293..07b943c14ea6 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -289,10 +289,6 @@ Feature: ggrebalance behave tests (rebalance scenarios) Examples: | fault_name | - | _stop_failed_segments_begin | - | _stop_failed_segments_end | - | _wait_fts_to_mark_down_segments_begin | - | _wait_fts_to_mark_down_segments_end | | _update_config_begin | Scenario Outline: 6.1.2 rebalance - interrupt during mirror move before gp_segment_configuration update, continue and rollback failed step. @@ -536,6 +532,53 @@ Feature: ggrebalance behave tests (rebalance scenarios) | fault_name | | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | + Scenario Outline: 6.2.4. rebalance - interrupt during switchover P->M step (before invocation of 'gprecoverseg'), continue and cancel failed step. + Given the database is not running + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + When the user runs "ggrebalance -n 1 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When user will answer "no" to the prompt "Retry step?" + And user will answer "no" to the prompt "Rollback step?" + And the user runs "ggrebalance -n 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Processing error status for switchover step" to logfile with latest timestamp + And ggrebalance should print "Cancel step" to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And clear user's answers + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 4 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | + Scenario Outline: 6.3.1. rebalance - interrupt during mirror move after gp_segment_configuration update, but before port update, continue and cancel failed step. Given the database is not running And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast'"