From 9fdd5dd95a0601e70ce2e4aebdc0d1e6ca58008c Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Fri, 20 Dec 2024 05:53:13 +0300 Subject: [PATCH 01/22] Initial commit From dc117b2653d38ee886f82cb933f74c2b9f20f9c5 Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Wed, 29 Jan 2025 14:48:45 +0300 Subject: [PATCH 02/22] [ADBDEV-6847] Implement cluster validation possibility (#1164) Implement cluster validation possibility This is the first commit for building an MVP for new rebalance utility - gprebalance. This utility is intended to be used for the situation, when after cluster resize (after expand, shrink) is in unbalanced state. Balanced state is defined very simple: if number of segments per host is equal across all the hosts, then cluster is balanced. There are a lot of other aspects for proper implementation of optimal rebalance algorithm, which will be implemented in the next patches. This patch adds the skeleton of future utility, providing initial validation of rebalance possibility. It includes checks, that validate some basic aspects: whether segments can be distributed uniformly and can target mirroring strategy be achieved. Decided to provide validation through separate classes, which is different approach from gpexpand utility. Also, some unit tests have been added. Validation of available disk space is not implemented since cannot be achieved at this initial validation step --- gpMgmt/bin/Makefile | 5 +- .../data/gprebalance/balanced_grouped.array | 16 + .../gprebalance/balanced_no_mirrors.array | 11 + .../data/gprebalance/balanced_spread.array | 22 ++ .../unit/data/gprebalance/segments_down.array | 12 + .../data/gprebalance/unbalanced_neg.array | 9 + .../data/gprebalance/unbalanced_pos.array | 8 + .../unbalanced_to_grouped_pos.array | 12 + .../unbalanced_to_spread_neg.array | 29 ++ .../unbalanced_to_spread_pos.array | 17 + .../gprebalance/unbalanced_unpreferred.array | 12 + .../test/unit/test_unit_gprebalance.py | 191 ++++++++++ gpMgmt/bin/gprebalance | 360 ++++++++++++++++++ gpMgmt/bin/gprebalance_modules/Makefile | 22 ++ gpMgmt/bin/gprebalance_modules/__init__.py | 0 gpMgmt/bin/gprebalance_modules/rebalance.py | 99 +++++ .../rebalance_validator.py | 78 ++++ 17 files changed, 901 insertions(+), 2 deletions(-) create mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_grouped.array create mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_no_mirrors.array create mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_spread.array create mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/segments_down.array create mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_neg.array create mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_pos.array create mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_grouped_pos.array create mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_neg.array create mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_pos.array create mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_unpreferred.array create mode 100644 gpMgmt/bin/gppylib/test/unit/test_unit_gprebalance.py create mode 100755 gpMgmt/bin/gprebalance create mode 100644 gpMgmt/bin/gprebalance_modules/Makefile create mode 100644 gpMgmt/bin/gprebalance_modules/__init__.py create mode 100644 gpMgmt/bin/gprebalance_modules/rebalance.py create mode 100644 gpMgmt/bin/gprebalance_modules/rebalance_validator.py diff --git a/gpMgmt/bin/Makefile b/gpMgmt/bin/Makefile index c05171049828..f65048d56cab 100644 --- a/gpMgmt/bin/Makefile +++ b/gpMgmt/bin/Makefile @@ -7,7 +7,7 @@ ifneq "$(wildcard $(top_builddir)/src/Makefile.global)" "" include $(top_builddir)/src/Makefile.global endif -SUBDIRS = stream gpcheckcat_modules gpconfig_modules gpssh_modules gppylib lib +SUBDIRS = stream gpcheckcat_modules gpconfig_modules gpssh_modules gppylib lib gprebalance_modules SUBDIRS += ifaddrs $(recurse) @@ -16,7 +16,8 @@ PROGRAMS= analyzedb gpactivatestandby gpaddmirrors gpcheckcat gpcheckperf \ gpcheckresgroupimpl gpconfig gpdeletesystem gpexpand gpinitstandby \ gpinitsystem gpload gpload.py gplogfilter gpmovemirrors \ gprecoverseg gpreload gpsync gpsd gpssh gpssh-exkeys gpstart \ - gpstate gpstop minirepro gpmemwatcher gpmemreport gpcheckresgroupv2impl + gpstate gpstop minirepro gpmemwatcher gpmemreport gpcheckresgroupv2impl \ + gprebalance installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)/lib' diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_grouped.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_grouped.array new file mode 100644 index 000000000000..605e0a6afd16 --- /dev/null +++ b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_grouped.array @@ -0,0 +1,16 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 +5|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 +6|4|m|m|s|u|sdw1|sdw1|50004|/data/mirror4 +7|5|m|m|s|u|sdw1|sdw1|50005|/data/mirror5 +# SDW2 +8|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 +9|4|p|p|s|u|sdw2|sdw2|40004|/data/primary4 +10|5|p|p|s|u|sdw2|sdw2|40005|/data/primary5 +11|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 +12|1|m|m|s|u|sdw2|sdw2|50001|/data/mirror1 +13|2|m|m|s|u|sdw2|sdw2|50002|/data/mirror2 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_no_mirrors.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_no_mirrors.array new file mode 100644 index 000000000000..23b1219c8b16 --- /dev/null +++ b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_no_mirrors.array @@ -0,0 +1,11 @@ +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|40003|/data/primary3 +# SDW2 +6|4|p|p|s|u|sdw2|sdw2|40004|/data/primary4 +7|5|p|p|s|u|sdw2|sdw2|40005|/data/primary5 +8|6|p|p|s|u|sdw2|sdw2|40006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|40007|/data/primary7 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_spread.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_spread.array new file mode 100644 index 000000000000..61bb5538c581 --- /dev/null +++ b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_spread.array @@ -0,0 +1,22 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +4|5|m|m|s|u|sdw1|sdw1|50005|/data/mirror5 +5|6|m|m|s|u|sdw1|sdw1|50006|/data/mirror6 +# SDW2 +6|2|p|p|s|u|sdw2|sdw2|40002|/data/primary2 +7|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 +8|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 +9|7|m|m|s|u|sdw2|sdw2|50007|/data/mirror7 +# SDW3 +10|4|p|p|s|u|sdw3|sdw3|40004|/data/primary4 +11|5|p|p|s|u|sdw3|sdw3|40005|/data/primary5 +12|1|m|m|s|u|sdw3|sdw3|50001|/data/mirror1 +13|2|m|m|s|u|sdw3|sdw3|50002|/data/mirror2 +# SDW4 +14|6|p|p|s|u|sdw4|sdw4|40006|/data/primary6 +15|7|p|p|s|u|sdw4|sdw4|40007|/data/primary7 +16|3|m|m|s|u|sdw4|sdw4|50003|/data/mirror3 +17|4|m|m|s|u|sdw4|sdw4|50004|/data/mirror4 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/segments_down.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/segments_down.array new file mode 100644 index 000000000000..6faa871528ad --- /dev/null +++ b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/segments_down.array @@ -0,0 +1,12 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|d|sdw1|sdw1|40001|/data/primary1 +4|2|m|m|s|u|sdw1|sdw1|50002|/data/mirror2 +5|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 +# SDW2 +6|2|p|p|s|u|sdw2|sdw2|40002|/data/primary2 +7|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 +8|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 +9|1|m|m|s|u|sdw2|sdw2|50001|/data/mirror1 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_neg.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_neg.array new file mode 100644 index 000000000000..0ef9b91a4e8f --- /dev/null +++ b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_neg.array @@ -0,0 +1,9 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 (3 primaries) +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 +# SDW2 (2 primary) +5|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 +6|4|p|p|s|u|sdw2|sdw2|40004|/data/primary4 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_pos.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_pos.array new file mode 100644 index 000000000000..b6fc095d2984 --- /dev/null +++ b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_pos.array @@ -0,0 +1,8 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 (3 primaries) +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 +# SDW2 (1 primary) +5|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_grouped_pos.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_grouped_pos.array new file mode 100644 index 000000000000..0afdfa70aee7 --- /dev/null +++ b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_grouped_pos.array @@ -0,0 +1,12 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 (3 primaries) +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 +5|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 +# SDW2 (1 primary) +6|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 +7|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 +8|1|m|m|s|u|sdw2|sdw2|50001|/data/mirror1 +9|2|m|m|s|u|sdw2|sdw2|50002|/data/mirror2 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_neg.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_neg.array new file mode 100644 index 000000000000..9710b89543a1 --- /dev/null +++ b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_neg.array @@ -0,0 +1,29 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW0 (5 primaries + 3 mirrors) +2|0|p|p|s|u|sdw0|sdw0|40000|/data/primary0 +3|1|p|p|s|u|sdw0|sdw0|40001|/data/primary1 +4|2|p|p|s|u|sdw0|sdw0|40002|/data/primary2 +5|3|p|p|s|u|sdw0|sdw0|40003|/data/primary3 +6|4|p|p|s|u|sdw0|sdw0|40004|/data/primary4 +7|8|m|m|s|u|sdw0|sdw0|50008|/data/mirror8 +8|9|m|m|s|u|sdw0|sdw0|50009|/data/mirror9 +9|10|m|m|s|u|sdw0|sdw0|50010|/data/mirror10 +# SDW1 (4 primaries + 5 mirrors) +10|5|p|p|s|u|sdw1|sdw1|40005|/data/primary5 +11|6|p|p|s|u|sdw1|sdw1|40006|/data/primary6 +12|7|p|p|s|u|sdw1|sdw1|40007|/data/primary7 +13|8|p|p|s|u|sdw1|sdw1|40008|/data/primary8 +14|0|m|m|s|u|sdw1|sdw1|50000|/data/mirror0 +15|1|m|m|s|u|sdw1|sdw1|50001|/data/mirror1 +16|2|m|m|s|u|sdw1|sdw1|50002|/data/mirror2 +17|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 +18|4|m|m|s|u|sdw1|sdw1|50004|/data/mirror4 +# SDW2 (3 primaries + 4 mirrors) +19|9|p|p|s|u|sdw2|sdw2|40009|/data/primary9 +20|10|p|p|s|u|sdw2|sdw2|40010|/data/primary10 +21|11|p|p|s|u|sdw2|sdw2|40011|/data/primary11 +22|5|m|m|s|u|sdw2|sdw2|50005|/data/mirror5 +23|6|m|m|s|u|sdw2|sdw2|50006|/data/mirror6 +24|7|m|m|s|u|sdw2|sdw2|50007|/data/mirror7 +25|11|m|m|s|u|sdw2|sdw2|50011|/data/mirror11 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_pos.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_pos.array new file mode 100644 index 000000000000..8edd92d5c9fd --- /dev/null +++ b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_pos.array @@ -0,0 +1,17 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW0 +2|0|p|p|s|u|sdw0|sdw0|40000|/data/primary0 +3|1|p|p|s|u|sdw0|sdw0|40001|/data/primary1 +4|2|p|p|s|u|sdw0|sdw0|40002|/data/primary2 +5|3|p|p|s|u|sdw0|sdw0|40003|/data/primary3 +6|4|m|m|s|u|sdw0|sdw0|50004|/data/mirror4 +7|5|m|m|s|u|sdw0|sdw0|50005|/data/mirror5 +# SDW1 +8|4|p|p|s|u|sdw1|sdw1|40004|/data/primary4 +9|0|m|m|s|u|sdw1|sdw1|50000|/data/mirror0 +10|1|m|m|s|u|sdw1|sdw1|50001|/data/mirror1 +# SDW2 +11|5|p|p|s|u|sdw2|sdw2|40005|/data/primary5 +12|2|m|m|s|u|sdw2|sdw2|50002|/data/mirror2 +13|3|m|m|s|u|sdw2|sdw2|50003|/data/mirror3 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_unpreferred.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_unpreferred.array new file mode 100644 index 000000000000..c34ab72f1bbe --- /dev/null +++ b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_unpreferred.array @@ -0,0 +1,12 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|m|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 +5|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 +# SDW2 +8|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 +11|0|p|m|s|u|sdw2|sdw2|50000|/data/mirror0 +12|1|m|m|s|u|sdw2|sdw2|50001|/data/mirror1 +13|2|m|m|s|u|sdw2|sdw2|50002|/data/mirror2 diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gprebalance.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gprebalance.py new file mode 100644 index 000000000000..9ef921dbd42f --- /dev/null +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gprebalance.py @@ -0,0 +1,191 @@ +import os +import imp + +from .gp_unittest import * +from mock import * +from gppylib.gparray import Segment, GpArray +from gppylib.db.dbconn import DbURL +from gppylib.db import catalog +from gppylib.gplog import * +from gppylib.system.configurationInterface import GpConfigurationProvider +from gppylib.system.environment import GpCoordinatorEnvironment +from gppylib.db import dbconn +import io +import sys + + +def initGparrayFromFile(basename): + filename = os.path.dirname(__file__) + \ + "/data/gprebalance/" + basename + ".array" + segdbs = [] + with open(filename, 'r') as fp: + for line in fp: + if not line.lstrip().startswith('#'): + segdbs.append(Segment.initFromString(line)) + return GpArray(segdbs, segdbs) + + +class GpTestRebalance(GpTestCase): + def setUp(self): + gprebalance_file = os.path.abspath( + os.path.dirname(__file__) + "/../../../gprebalance") + self.subject = imp.load_source('gprebalance', gprebalance_file) + self.old_sys_argv = sys.argv + sys.argv = [] + self.options, self.args, self.parser = self.subject.parseargs() + self.options.mirroring = 'grouped' + + self.subject.logger = Mock( + spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal', 'exception']) + + self.subject.check_running_gputils = Mock(return_value=False) + + self.apply_patches([ + patch('builtins.open', mock_open(), create=True), + patch('builtins.input'), + patch('gprebalance.dbconn.DbURL', return_value=Mock(), spec=DbURL), + patch('gprebalance.dbconn.connect', return_value=Mock()), + patch('gprebalance.GpCoordinatorEnvironment', + return_value=Mock(), spec=GpCoordinatorEnvironment), + patch('gprebalance.configurationInterface.getConfigurationProvider'), + patch('os.path.exists', return_value=Mock()), + patch('gprebalance.get_default_logger', + return_value=self.subject.logger), + ]) + self.input_mock = self.get_mock_from_apply_patch("input") + self.getConfigProviderFunctionMock = self.get_mock_from_apply_patch( + "getConfigurationProvider") + self.gpCoordinatorEnvironmentMock = self.get_mock_from_apply_patch( + "GpCoordinatorEnvironment") + self.previous_coordinator_data_directory = os.getenv( + 'COORDINATOR_DATA_DIRECTORY', '') + os.environ["COORDINATOR_DATA_DIRECTORY"] = '/tmp/dirdoesnotexist' + configProviderMock = Mock(spec=GpConfigurationProvider) + self.getConfigProviderFunctionMock.return_value = configProviderMock + configProviderMock.initializeProvider.return_value = configProviderMock + self.gpCoordinatorEnvironmentMock.return_value.getCoordinatorPort.return_value = 123456 + + def tearDown(self): + os.environ['COORDINATOR_DATA_DIRECTORY'] = self.previous_coordinator_data_directory + sys.argv = self.old_sys_argv + super(GpTestRebalance, self).tearDown() + + @patch('gprebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("balanced_grouped")) + def test_already_balanced_grouped(self, mock1): + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.info.assert_any_call( + "Cluster is already balanced") + + @patch('gprebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("balanced_spread")) + def test_already_balanced_spread(self, mock1): + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.info.assert_any_call( + "Cluster is already balanced") + + @patch('gprebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("balanced_no_mirrors")) + def test_no_mirrors(self, mock1): + self.input_mock.return_value = "N" + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.error.assert_any_call( + "User Aborted. Exiting...") + self.input_mock.return_value = "Y" + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.info.assert_any_call( + "Cluster is already balanced") + + @patch('gprebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("balanced_no_mirrors")) + def test_no_mirrors_allow_mirrorless(self, mock1): + saved = self.options.allow_mirrorless + self.options.allow_mirrorless = True + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.info.assert_any_call( + "Cluster is already balanced") + self.options.allow_mirrorless = saved + + @patch('gprebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("segments_down")) + def test_segments_down(self, mock1): + self.input_mock.return_value = "N" + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.error.assert_any_call( + "User Aborted. Exiting...") + self.input_mock.return_value = "Y" + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.info.assert_any_call( + "Cluster is already balanced") + + @patch('gprebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("unbalanced_pos")) + def test_unbalanced_pos(self, mock1): + self.input_mock.return_value = "Y" + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.info.assert_any_call( + "Validation passed. Preparing rebalance...") + + @patch('gprebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("unbalanced_neg")) + def test_unbalanced_neg(self, mock1): + self.input_mock.return_value = "Y" + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.error.assert_called_with("gprebalance failed: Cannot evenly distribute 5 segments across 2 hosts. " + "\n\nExiting...") + + @patch('gprebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("unbalanced_to_grouped_pos")) + def test_unbalanced_to_grouped_pos(self, mock1): + saved = self.options.mirroring + self.options.mirroring = 'grouped' + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.info.assert_any_call( + "Validation passed. Preparing rebalance...") + self.options.mirroring = saved + + @patch('gprebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("unbalanced_to_spread_pos")) + def test_unbalanced_to_spread_pos(self, mock1): + saved = self.options.mirroring + self.options.mirroring = 'spread' + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.info.assert_any_call( + "Validation passed. Preparing rebalance...") + self.options.mirroring = saved + + @patch('gprebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("unbalanced_to_spread_neg")) + def test_unbalanced_to_spread_neg(self, mock1): + saved = self.options.mirroring + self.options.mirroring = 'spread' + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.error.assert_called_with( + "gprebalance failed: Cannot support spread mirroring strategy on given configuration." + " Use cluster utilities like gpresize or gpexpand to get desired cluster configuration \n\nExiting...") + self.options.mirroring = saved + + @patch('gprebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("unbalanced_unpreferred")) + def test_unbalanced_unpreferred(self, mock1): + self.input_mock.return_value = "Y" + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.info.assert_any_call( + "Validation passed. Preparing rebalance...") + + +if __name__ == '__main__': + run_tests() diff --git a/gpMgmt/bin/gprebalance b/gpMgmt/bin/gprebalance new file mode 100755 index 000000000000..d08d24824091 --- /dev/null +++ b/gpMgmt/bin/gprebalance @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 + +from gppylib.mainUtils import getProgramName + +import psycopg2 +import yaml +from psycopg2 import DatabaseError, OperationalError +try: + from gppylib.commands.unix import * + from gppylib.commands.gp import * + from gppylib.gparray import GpArray, Segment + from gppylib.gpparseopts import OptParser, OptChecker + from gppylib.gplog import * + from gppylib.db import dbconn + from gppylib.userinput import * + from gppylib.system.environment import GpCoordinatorEnvironment + from gppylib.system import configurationInterface, configurationImplGpdb + from gppylib.programs.clsRecoverSegment_triples import get_segments_with_running_basebackup + from gppylib.parseutils import line_reader, check_values, canonicalize_address + from gppylib.operations.update_pg_hba_on_segments import update_pg_hba_on_segments + from gprebalance_modules.rebalance import GPRebalance, MirrorStrategy + from gprebalance_modules.rebalance_validator import ClusterValidator, StateValidationError +except ImportError as e: + sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) + + +DBNAME = 'postgres' + +_description = (""" +Rebalances the existing segments configuration for getting +optimal performance of whole cluster. +""") + +_help = [""" +[-m spread|grouped] +Mode is the desirable mirroring strategy after rebalance. Makes sense only when mirroring +is enabled for the cluster. Default value is "grouped". +[-f ] +The hosts configuration file (YAML format) defines the target hosts on which +the rebalance procedure will distribute segments from existing configuration. +Existing info can be generated from gp_segment_configuration through -g option. +hosts: +- hostname: # Machine hostname + address:
# Network address for connections + primary_datadirs: # Directories for primary segments + - /datadir1 + - /datadir1 + mirror_datadirs: # Directories for mirror segments + - /mdatadir1 + - /mdatadir2 +You can include empty hosts in the configuration or shrink the existing number +of hosts, the utility will try to balance the segments strictly across hosts +from the file. +"""] + +_usage = """ + gprebalance -g + gprebalance [-m ] [-c] [-f ] [-s] [-v] + gprebalance -c + gprebalance -? | -h | --help | --verbose | -v +""" + +EXECNAME = os.path.split(__file__)[-1] + + +def parseargs(): + parser = OptParser(option_class=OptChecker, + description=' '.join(_description.split()), + version='%prog version $Revision$') + parser.setHelp(_help) + parser.set_usage('%prog ' + _usage) + parser.remove_option('-h') + + parser.add_option('-m', '--mirror-mode', dest='mirroring', + help='desirable mirroring strategy') + parser.add_option('-c', '--clean', action='store_true', + help='remove the rebalance schema.') + parser.add_option('-f', '--target-file', metavar='', dest='filename', + help='yaml containing target hosts configuration') + parser.add_option('--allow-mirrorless', dest='allow_mirrorless', action='store_true', + help='Allow to rebalance a cluster without mirrors', default=False) + parser.add_option('-g', '--gen-hosts', action='store_true', dest='genconf', + help='dump cluster hosts configuration in yaml format') + parser.add_option('-s', '--silent', action='store_true', default=False, + help='Do not prompt for confirmation to proceed on warnings') + parser.add_option('-v', '--verbose', action='store_true', + help='debug output.') + parser.add_option('-h', '-?', '--help', action='help', + help='show this help message and exit.') + parser.add_option('--usage', action="briefhelp") + + parser.set_defaults(verbose=False) + + # Parse the command line arguments + (options, args) = parser.parse_args() + return options, args, parser + + +def validate_options(options, args, parser): + if len(args) > 0: + logger.error(f'Unknown argument {args[0]}') + parser.exit(1) + + if options.mirroring and options.mirroring not in ('grouped', 'spread'): + logger.error('Mirroring strategy %s is not supported' % + options.mirroring) + parser.print_help() + parser.exit(1) + + try: + options.coordinator_data_directory = get_coordinatordatadir() + options.gphome = get_gphome() + except GpError as msg: + logger.error(msg) + parser.exit(1) + + if not os.path.exists(options.coordinator_data_directory): + logger.error('Coordinator data directory %s does not exist.' % + options.coordinator_data_directory) + parser.exit(1) + + if options.genconf: + for arg in sys.argv[1:]: + if arg not in ('-g', '--gen-hosts'): + logger.error('-g or --gen-hosts flag must be used alone') + parser.print_usage() + parser.exit(1) + + return options, args + + +def check_running_gputils(dburl: dbconn.DbURL, coordinator_data_directory: str): + """ + Checks if there are any running instances of + gprebalance/gpbackup/gpexpand/gpshrink/gpresize + """ + for util in ('gprebalance', 'gpexpand', 'gpshrink', 'gpresize'): + try: + with open(f'{coordinator_data_directory}/{util}.pid', 'r') as fp: + pid = int(fp.readline().strip()) + if check_pid(pid): + logger.error(f'{util} is already running.') + sys.exit(1) + except IOError: + pass + + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + cursor = dbconn.query(conn, '''SELECT datid + FROM pg_stat_activity + WHERE application_name LIKE 'gpbackup%' OR + application_name LIKE 'gprestore%' ''') + if cursor.rowcount > 0: + logger.error( + '''gpbackup/gprestore utility is already running.''') + sys.exit(1) + + if is_gprecoverseg_running(): + logger.error( + '''gprecoverseg is already running.''') + sys.exit(1) + + +def create_pid_file(coordinator_data_directory: str): + """ + Creates gprebalance pid file + """ + with open(coordinator_data_directory + '/gprebalance.pid', 'w') as fp: + fp.write(str(os.getpid())) + + +def remove_pid_file(coordinator_data_directory: str): + """ + Removes gprebalance pid file + """ + try: + os.unlink(coordinator_data_directory + '/gprebalance.pid') + except IOError: + pass + + +def dump_hosts_info(hosts): + """ + Converts info about hosts to YAML format + """ + yaml_entries = [] + for (hostname, address), val in hosts.items(): + host_entry = { + 'hostname': hostname, + 'address': address, + 'primary_datadirs': list(val.primary_datadirs), + 'mirror_datadirs': list(val.mirror_datadirs), + } + yaml_entries.append(host_entry) + + yaml_entries.sort(key=lambda x: x['hostname']) + config = {'hosts': yaml_entries} + config_yaml = yaml.dump(config, default_flow_style=False, sort_keys=False) + outputfile = os.getcwd() + "/gprebalance_hosts.yaml" + with open('gprebalance_hosts.yaml', 'w') as fp: + fp.write(config_yaml) + logger.info('Generating hosts config in %s' % outputfile) + + +def possibility_checks(gpreb: GPRebalance): + logger.info('Validation of rebalance possibility...') + + hosts_list = gpreb.current_hosts + + if gpreb.unpreferred_segments: + if not gpreb.options.silent and ask_yesno('', "Current role does not match preferred role for several segments.\n" + "Are you sure you want " + "to continue with this gprebalance session?", "N"): + hstmap = gpreb.getHostsFromGpArray() + for prim, mir in gpreb.unpreferred_segments: + hstmap[(prim.hostname, prim.address) + ].primary_segments.discard(prim.content) + hstmap[(prim.hostname, prim.address) + ].mirror_segments.add(prim.content) + hstmap[(mir.hostname, mir.address) + ].primary_segments.add(mir.content) + hstmap[(mir.hostname, mir.address) + ].mirror_segments.discard(mir.content) + hosts_list = list(hstmap.values()) + + startup_validator = ClusterValidator( + hosts_list, + gpreb.target_hosts, + gpreb.original_gparray.getSegmentsAsLoadedFromDb(), + gpreb.original_gparray.get_mirroring_enabled(), + gpreb.target_strategy) + try: + startup_validator.validate_segment_status() + except StateValidationError as e: + if not gpreb.options.silent and not ask_yesno('', " %s Are you sure you want " + "to continue with this gprebalance session?" % str(e), "N"): + logger.error('User Aborted. Exiting...') + sys.exit(1) + elif gpreb.options.silent: + logger.error(f'{str(e)} Exiting...') + sys.exit(1) + if not gpreb.original_gparray.get_mirroring_enabled() and not gpreb.options.allow_mirrorless: + if not gpreb.options.silent and not ask_yesno('', + "Mirroring is disabled. \n" + "During rebalance the whole cluster may " + "be not available. \n" + "Are you sure you want " + "to continue with this gprebalance session?", "N"): + logger.error('User Aborted. Exiting...') + sys.exit(1) + elif gpreb.options.silent: + logger.error('Mirroring is disabled. Exiting...') + sys.exit(1) + + if gpreb.original_gparray.get_mirroring_enabled() and startup_validator.mirror_strategy == MirrorStrategy.MIRRORLESS: + if not gpreb.options.silent: + mirror_type = ask_string( + "\nYou haven't specified desirable mirroring strategy. Spread mirroring places\n" + "a given hosts mirrored segments each on a separate host. You must be \n" + "using more hosts than the number of segments per host for spread mirroring. \n" + "Grouped mirroring places all of a given hosts segments on a single \n" + "mirrored host. You must be using at least 2 hosts for grouped strategy.\n\n", + "What type of mirroring strategy would you like?", + 'grouped', ['spread', 'grouped']) + else: + mirror_type = 'grouped' + strat = MirrorStrategy.SPREAD if mirror_type == 'spread' else MirrorStrategy.GROUPED + gpreb.setMirroringStrategy(strat) + startup_validator.mirror_strategy = strat + + already_bal, _ = startup_validator.validate_existing_configuration() + if already_bal: + # case of hosts shrinkage, expansion + if len(set(startup_validator.existing_hosts) & + set(startup_validator.target_hosts)) < len(startup_validator.existing_hosts) or \ + len(startup_validator.target_hosts) > len(startup_validator.existing_hosts): + try: + startup_validator.prevalidate_segment_distribution() + startup_validator.prevalidate_mirror_strategy() + except StateValidationError: + logger.error( + 'Cluster is already balanced. Cannot additionally rebalance to newly added hosts') + sys.exit(1) + else: + logger.info('Cluster is already balanced') + sys.exit(0) + else: + startup_validator.prevalidate_segment_distribution() + startup_validator.prevalidate_mirror_strategy() + + +def sig_handler(sig, arg): + + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGHUP, signal.SIG_DFL) + + # raise sig + os.kill(os.getpid(), sig) + + +def main(options, args, parser): + try: + signal.signal(signal.SIGTERM, sig_handler) + signal.signal(signal.SIGHUP, sig_handler) + + logger = get_default_logger() + setup_tool_logging(EXECNAME, getLocalHostname(), getUserName()) + + options, args = validate_options(options, args, parser) + + if options.verbose: + enable_verbose_logging() + + gpEnv = GpCoordinatorEnvironment( + options.coordinator_data_directory, True) + configurationInterface.registerConfigurationProvider( + configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) + configurationInterface.getConfigurationProvider( + ).initializeProvider(gpEnv.getCoordinatorPort()) + dburl = dbconn.DbURL(dbname=DBNAME, port=gpEnv.getCoordinatorPort()) + # check parallel execution of other cluster utilities + check_running_gputils(dburl, options.coordinator_data_directory) + # create pid file + create_pid_file(options.coordinator_data_directory) + + try: + gparray = GpArray.initFromCatalog(dburl, utility=True) + + except DatabaseError as ex: + logger.error("Failed to connect to database. Make sure the" + " Greenplum instance you wish to expand is running" + " and that your environment is correct, then rerun" + " gprebalance" + ' '.join(sys.argv[1:])) + sys.exit(1) + + gprebalance = GPRebalance(logger, gparray, dburl, options) + + # dump existing hosts configuration + if options.genconf: + dump_hosts_info(gprebalance.getHostsFromGpArray()) + sys.exit(0) + + possibility_checks(gprebalance) + logger.info('Validation passed. Preparing rebalance...') + logger.info('Exiting...') + sys.exit(0) + except Exception as e: + if options and options.verbose: + logger.exception('gprebalance failed. exiting...') + else: + logger.error(f'gprebalance failed: {e} \n\nExiting...') + sys.exit(1) + except KeyboardInterrupt: + sys.exit('\nUser Interrupted') + finally: + remove_pid_file(get_coordinatordatadir()) + + +if __name__ == '__main__': + options, args, parser = parseargs() + main(options, args, parser) diff --git a/gpMgmt/bin/gprebalance_modules/Makefile b/gpMgmt/bin/gprebalance_modules/Makefile new file mode 100644 index 000000000000..f959ba450ba2 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/Makefile @@ -0,0 +1,22 @@ +# gpMgmt/bin/gprebalance_modules/Makefile + +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +PROGRAMS= __init__.py rebalance_validator.py rebalance.py + +installdirs: + $(MKDIR_P) '$(DESTDIR)$(bindir)/gprebalance_modules' + +install: installdirs + for file in $(PROGRAMS); do \ + $(INSTALL_SCRIPT) $$file '$(DESTDIR)$(bindir)/gprebalance_modules/'$$file ; \ + done + +uninstall: + for file in $(PROGRAMS); do \ + rm -f '$(DESTDIR)$(bindir)/gprebalance_modules/'$$file ; \ + done + +clean distclean: + rm -f *.pyc diff --git a/gpMgmt/bin/gprebalance_modules/__init__.py b/gpMgmt/bin/gprebalance_modules/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/gpMgmt/bin/gprebalance_modules/rebalance.py b/gpMgmt/bin/gprebalance_modules/rebalance.py new file mode 100644 index 000000000000..137d13fe3c97 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance.py @@ -0,0 +1,99 @@ +import yaml +import os +from dataclasses import dataclass +from typing import List, Set, Dict +from enum import Enum +from gppylib.db import dbconn +from gppylib.gparray import GpArray, Segment, MODE_NOT_SYNC, STATUS_DOWN + + +@dataclass(eq=False) +class Host: + hostname: str + address: str + primary_datadirs: Set[str] + mirror_datadirs: Set[str] + # set of content ids + primary_segments: Set[int] + mirror_segments: Set[int] + + def __eq__(self, other): + return self.hostname == other.hostname and \ + self.address == other.address + + def __hash__(self): + return hash((self.hostname, self.address)) + + +class MirrorStrategy(Enum): + MIRRORLESS = "none" + GROUPED = "grouped" + SPREAD = "spread" + + +class GPRebalance: + def __init__(self, logger, gparray, dburl, options): + self.logger = logger + self.dburl = dburl + self.options = options + self.original_gparray = gparray + self.conn = dbconn.connect( + self.dburl, utility=True, encoding='UTF8', allowSystemTableMods=True) + if options.mirroring == 'spread': + self.target_strategy = MirrorStrategy.SPREAD + elif options.mirroring == 'grouped': + self.target_strategy = MirrorStrategy.GROUPED + else: + self.target_strategy = MirrorStrategy.MIRRORLESS + + self.current_hosts = list(self.getHostsFromGpArray().values()) + self.target_hosts = self.current_hosts + if options.filename: + with open(options.filename, 'r') as fp: + hosts = {} + config = yaml.safe_load(fp) + for host_config in config['hosts']: + key = (host_config['hostname'], host_config['address']) + hosts[key] = Host(hostname=host_config['hostname'], + address=host_config['address'], + primary_datadirs=set( + host_config['primary_datadirs']), + mirror_datadirs=set( + host_config['mirror_datadirs']), + primary_segments=set(), mirror_segments=set()) + self.target_hosts = list(hosts.values()) + + self.unpreferred_segments = self.getSegmentsUnpreferredRole() + + def getSegmentsUnpreferredRole(self) -> List[tuple[Segment, Segment]]: + segs = [] + for pair in self.original_gparray.segmentPairs: + prim = pair.primaryDB + mir = pair.mirrorDB + if prim.role != prim.preferred_role and mir.role != mir.preferred_role: + segs.append((prim, mir)) + return segs + + def setMirroringStrategy(self, strategy: MirrorStrategy): + self.target_strategy = strategy + + def getHostsFromGpArray(self) -> Dict[tuple[str, str], Host]: + hosts = {} + for seg in self.original_gparray.getSegmentsAsLoadedFromDb(): + if seg.content >= 0: + hosts[(seg.hostname, seg.address)] = Host( + hostname=seg.hostname, address=seg.address, primary_datadirs=set(), mirror_datadirs=set(), + primary_segments=set(), mirror_segments=set()) + for pair in self.original_gparray.segmentPairs: + primary = pair.primaryDB + mirror = pair.mirrorDB + key_pr = (primary.hostname, primary.address) + hosts[key_pr].primary_datadirs.add( + os.path.dirname(primary.datadir)) + hosts[key_pr].primary_segments.add(primary.content) + if mirror: + key_mr = (mirror.hostname, mirror.address) + hosts[key_mr].mirror_datadirs.add( + os.path.dirname(mirror.datadir)) + hosts[key_mr].mirror_segments.add(mirror.content) + return hosts diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_validator.py b/gpMgmt/bin/gprebalance_modules/rebalance_validator.py new file mode 100644 index 000000000000..c2d3a2718548 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_validator.py @@ -0,0 +1,78 @@ +from typing import List +from gprebalance_modules.rebalance import Host, MirrorStrategy, dbconn, GpArray, Segment, MODE_NOT_SYNC, STATUS_DOWN + + +class StateValidationError(Exception): + pass + + +class ClusterValidator: + def __init__(self, existing_hosts: List[Host], target_hosts: List[Host], segarray: List[Segment], has_mirrors: bool, mirror_strategy: MirrorStrategy): + self.mirror_strategy = mirror_strategy + self.existing_hosts = existing_hosts + self.target_hosts = target_hosts + self.segarray = segarray + self.has_mirrors = has_mirrors + + def validate_segment_status(self): + inv = [seg.content for seg in self.segarray if not seg.valid] + if len(inv) > 0: + raise StateValidationError( + f"The {[c for c in inv]} segments are down") + + def validate_existing_configuration(self) -> tuple[bool, MirrorStrategy]: + arr = GpArray(self.segarray) + total_primaries = arr.get_primary_count() + total_hosts = len(self.existing_hosts) + expected_primaries = total_primaries // total_hosts + strat = None + + if arr.guessIsSpreadMirror(): + strat = MirrorStrategy.SPREAD + elif arr.hasMirrors: + strat = MirrorStrategy.GROUPED + for host in self.existing_hosts: + if host.primary_segments & host.mirror_segments: + strat = None + break + + for host in self.existing_hosts: + if len(host.primary_segments) != expected_primaries: + return False, strat + + return True, strat + + def prevalidate_segment_distribution(self): + """ + Validate whether segments can be uniformly distributed across target hosts + """ + total_primary_segments = sum(len(h.primary_segments) + for h in self.existing_hosts) + total_hosts = len(self.target_hosts) + + if total_primary_segments % total_hosts != 0: + raise StateValidationError( + f"Cannot evenly distribute {total_primary_segments} segments across {total_hosts} hosts." + ) + + def prevalidate_mirror_strategy(self): + """ + Validate whether the specified mirroring strategy can be achieved + """ + if not self.has_mirrors: + return + total_hosts = len(self.target_hosts) + total_primary_segments = sum(len(h.primary_segments) + for h in self.existing_hosts) + if total_hosts < 2: + raise StateValidationError( + """Cannot support target mirroring strategy on given configuration. All + primaries will be at single host.""" + ) + + primaries_per_host = total_primary_segments // total_hosts + if self.mirror_strategy == MirrorStrategy.SPREAD and primaries_per_host >= total_hosts: + raise StateValidationError( + "Cannot support spread mirroring strategy on given configuration. " + "Use cluster utilities like gpresize or gpexpand to get desired cluster configuration" + ) From 9a64377092cbf4a118702327d51de1dc5fe66b32 Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Mon, 3 Feb 2025 08:52:30 +0300 Subject: [PATCH 03/22] gprebalance skeleton (#1201) gprebalance skeleton is complemented with additional options from mvp specification. --- gpMgmt/bin/gprebalance | 57 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/gpMgmt/bin/gprebalance b/gpMgmt/bin/gprebalance index d08d24824091..f4ed43693ae1 100755 --- a/gpMgmt/bin/gprebalance +++ b/gpMgmt/bin/gprebalance @@ -4,6 +4,7 @@ from gppylib.mainUtils import getProgramName import psycopg2 import yaml +import datetime from psycopg2 import DatabaseError, OperationalError try: from gppylib.commands.unix import * @@ -26,6 +27,8 @@ except ImportError as e: DBNAME = 'postgres' +MAX_PARALLEL_WORKERS = 4 + _description = (""" Rebalances the existing segments configuration for getting optimal performance of whole cluster. @@ -55,8 +58,17 @@ from the file. _usage = """ gprebalance -g + gprebalance [-m ] [-c] [-f ] [-s] [-v] + + gprebalance [-d duration[hh][:mm[:ss]] | [-e 'YYYY-MM-DD hh:mm:ss']] + + gprebalance -r + gprebalance -c + + gprebalance -p + gprebalance -? | -h | --help | --verbose | -v """ @@ -75,8 +87,18 @@ def parseargs(): help='desirable mirroring strategy') parser.add_option('-c', '--clean', action='store_true', help='remove the rebalance schema.') - parser.add_option('-f', '--target-file', metavar='', dest='filename', + parser.add_option('-f', '--target-hosts', metavar='', dest='filename', help='yaml containing target hosts configuration') + parser.add_option('-p', '--show-plan', dest='show_plan', action='store_true', default=False, + help='show rebalance plan') + parser.add_option('-r', '--rollback', action='store_true', + help='remove the rebalance schema.') + parser.add_option('-d', '--duration', type='duration', metavar='[h][:m[:s]]', + help='duration from beginning to end.') + parser.add_option('-e', '--end', type='datetime', metavar='datetime', + help="ending date and time in the format 'YYYY-MM-DD hh:mm:ss'.") + parser.add_option('-n', '--parallel', type="int", default=1, metavar="", + help='number of workerks performing segment movements at a time. Valid values are 1-%d.' % MAX_PARALLEL_WORKERS) parser.add_option('--allow-mirrorless', dest='allow_mirrorless', action='store_true', help='Allow to rebalance a cluster without mirrors', default=False) parser.add_option('-g', '--gen-hosts', action='store_true', dest='genconf', @@ -88,6 +110,12 @@ def parseargs(): parser.add_option('-h', '-?', '--help', action='help', help='show this help message and exit.') parser.add_option('--usage', action="briefhelp") + parser.add_option('-S', '--simple-progress', action='store_true', + help='show simple progress.') + parser.add_option('', '--hba-hostnames', action='store_true', default=False, + help='use hostnames instead of CIDR in pg_hba.conf') + parser.add_option('--allow-intermediate-mixture', dest='allow_mixture', action='store_true', + help='Allow primary and mirror from one pair to be at the same host during balancing', default=False) parser.set_defaults(verbose=False) @@ -126,6 +154,33 @@ def validate_options(options, args, parser): parser.print_usage() parser.exit(1) + if options.parallel > MAX_PARALLEL_WORKERS or options.parallel < 1: + logger.error( + 'Invalid argument. parallel value must be >= 1 and <= %d' % MAX_PARALLEL_WORKERS) + parser.print_help() + parser.exit(1) + + if options.end: + if not isinstance(options.end, datetime.datetime): + options.end = datetime.datetime.combine( + options.end, datetime.time(0)) + if options.end < datetime.datetime.now(): + logger.error('End time occurs in the past') + parser.print_help() + parser.exit(1) + if options.duration: + if options.end > datetime.datetime.now() + options.duration: + options.end = datetime.datetime.now() + options.duration + elif options.duration: + options.end = datetime.datetime.now() + options.duration + + if options.rollback and options.clean: + rollbackOpt = "--rollback" if "--rollback" in sys.argv else "-r" + cleanOpt = "--clean" if "--clean" in sys.argv else "-c" + logger.error("%s and %s options cannot be specified together." % + (rollbackOpt, cleanOpt)) + parser.exit(1) + return options, args From 47b225abea9670e9fb07928274a58c68ac906cbb Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Tue, 18 Feb 2025 17:17:23 +0300 Subject: [PATCH 04/22] Rebalance algorithm (#1204) This code proposes the rebalance algorithm. GpRebalance.createPlan() returns a Plan represented by the list of Moves. The algorithm itself produces an intiutive greed solution by manual setting the final balanced state. --- gpMgmt/bin/gprebalance | 9 +- gpMgmt/bin/gprebalance_modules/Makefile | 2 +- gpMgmt/bin/gprebalance_modules/rebalance.py | 140 +++++++- .../bin/gprebalance_modules/rebalance_plan.py | 323 ++++++++++++++++++ .../rebalance_validator.py | 6 +- 5 files changed, 456 insertions(+), 24 deletions(-) create mode 100644 gpMgmt/bin/gprebalance_modules/rebalance_plan.py diff --git a/gpMgmt/bin/gprebalance b/gpMgmt/bin/gprebalance index f4ed43693ae1..4ad48ceba79b 100755 --- a/gpMgmt/bin/gprebalance +++ b/gpMgmt/bin/gprebalance @@ -105,6 +105,8 @@ def parseargs(): help='dump cluster hosts configuration in yaml format') parser.add_option('-s', '--silent', action='store_true', default=False, help='Do not prompt for confirmation to proceed on warnings') + parser.add_option('-p', '--show-plan', dest='show_plan', action='store_true', default=False, + help='show rebalance plan') parser.add_option('-v', '--verbose', action='store_true', help='debug output.') parser.add_option('-h', '-?', '--help', action='help', @@ -325,8 +327,8 @@ def possibility_checks(gpreb: GPRebalance): already_bal, _ = startup_validator.validate_existing_configuration() if already_bal: # case of hosts shrinkage, expansion - if len(set(startup_validator.existing_hosts) & - set(startup_validator.target_hosts)) < len(startup_validator.existing_hosts) or \ + if len(startup_validator.existing_hosts & + startup_validator.target_hosts) < len(startup_validator.existing_hosts) or \ len(startup_validator.target_hosts) > len(startup_validator.existing_hosts): try: startup_validator.prevalidate_segment_distribution() @@ -396,6 +398,9 @@ def main(options, args, parser): possibility_checks(gprebalance) logger.info('Validation passed. Preparing rebalance...') + plan = gprebalance.createPlan() + if options.show_plan: + logger.info(f"Resulting plan: \n {str(plan)}") logger.info('Exiting...') sys.exit(0) except Exception as e: diff --git a/gpMgmt/bin/gprebalance_modules/Makefile b/gpMgmt/bin/gprebalance_modules/Makefile index f959ba450ba2..0125529264f9 100644 --- a/gpMgmt/bin/gprebalance_modules/Makefile +++ b/gpMgmt/bin/gprebalance_modules/Makefile @@ -3,7 +3,7 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -PROGRAMS= __init__.py rebalance_validator.py rebalance.py +PROGRAMS= __init__.py rebalance_validator.py rebalance.py rebalance_plan.py installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)/gprebalance_modules' diff --git a/gpMgmt/bin/gprebalance_modules/rebalance.py b/gpMgmt/bin/gprebalance_modules/rebalance.py index 137d13fe3c97..09758204ce21 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance.py @@ -1,21 +1,39 @@ import yaml import os from dataclasses import dataclass -from typing import List, Set, Dict +from typing import List, Set, Dict, NamedTuple from enum import Enum from gppylib.db import dbconn from gppylib.gparray import GpArray, Segment, MODE_NOT_SYNC, STATUS_DOWN +from gppylib.commands.base import * + + +class SegmentId(NamedTuple): + dbid: int + contentid: int + + +class HostStatus(Enum): + ACTIVE = "active" + DECOMMISSIONING = "decommissioning" + NEW = "new" + REPLACEMENT = "replacement" -@dataclass(eq=False) class Host: - hostname: str - address: str - primary_datadirs: Set[str] - mirror_datadirs: Set[str] - # set of content ids - primary_segments: Set[int] - mirror_segments: Set[int] + def __init__(self, hostname: str, address: str, primary_datadirs: Set[str] = set(), + mirror_datadirs: Set[str] = set(), primary_segments: Set[SegmentId] = set(), + mirror_segments: Set[SegmentId] = set(), status: HostStatus = HostStatus.ACTIVE): + self.hostname = hostname + self.address = address + # set of strings + self.primary_datadirs = primary_datadirs + self.mirror_datadirs = mirror_datadirs + # set of SegmentId(dbid, content id) + self.primary_segments = primary_segments + self.mirror_segments = mirror_segments + self.replacement_for = None + self.status = status def __eq__(self, other): return self.hostname == other.hostname and \ @@ -24,6 +42,42 @@ def __eq__(self, other): def __hash__(self): return hash((self.hostname, self.address)) + def __len__(self): + return len(self.primary_segments) + + def is_overloaded(self, target_load: int) -> bool: + return len(self.primary_segments) > target_load + + def is_underloaded(self, target_load: int) -> bool: + return len(self.primary_segments) < target_load + + def add_primary(self, seg): + self.primary_segments.add(seg) + + def remove_primary(self, seg): + self.primary_segments.discard(seg) + + def add_mirror(self, seg): + self.mirror_segments.add(seg) + + def remove_mirror(self, seg): + self.mirror_segments.discard(seg) + + def __str__(self): + mirrors = [] + prims = [] + for m in self.mirror_segments: + mirrors.append(f"M{m.contentid}") + for p in self.primary_segments: + prims.append(f"S{p.contentid}") + + ps = " ".join(prims) + ms = " ".join(mirrors) + return f"H_{self.hostname}:[{ps} {ms}]" + + +ClusterState = Dict[tuple[str, str], Host] + class MirrorStrategy(Enum): MIRRORLESS = "none" @@ -31,6 +85,9 @@ class MirrorStrategy(Enum): SPREAD = "spread" +from gprebalance_modules.rebalance_plan import ClusterBalancer, Move, Plan # nopep8 + + class GPRebalance: def __init__(self, logger, gparray, dburl, options): self.logger = logger @@ -38,7 +95,7 @@ def __init__(self, logger, gparray, dburl, options): self.options = options self.original_gparray = gparray self.conn = dbconn.connect( - self.dburl, utility=True, encoding='UTF8', allowSystemTableMods=True) + self.dburl, encoding='UTF8', allowSystemTableMods=True) if options.mirroring == 'spread': self.target_strategy = MirrorStrategy.SPREAD elif options.mirroring == 'grouped': @@ -46,7 +103,8 @@ def __init__(self, logger, gparray, dburl, options): else: self.target_strategy = MirrorStrategy.MIRRORLESS - self.current_hosts = list(self.getHostsFromGpArray().values()) + self.current_conf = self.getHostsFromGpArray() + self.current_hosts = set(list(self.current_conf.values())) self.target_hosts = self.current_hosts if options.filename: with open(options.filename, 'r') as fp: @@ -58,10 +116,26 @@ def __init__(self, logger, gparray, dburl, options): address=host_config['address'], primary_datadirs=set( host_config['primary_datadirs']), + primary_segments=set(), mirror_datadirs=set( host_config['mirror_datadirs']), - primary_segments=set(), mirror_segments=set()) - self.target_hosts = list(hosts.values()) + mirror_segments=set()) + # User can explicitly mark the host to be replacement for existing one + if "replace" in host_config: + rep = tuple(host_config['replace'].split(', ')) + if rep in self.current_conf: + hosts[key].replacement_for = rep + hosts[key].status = HostStatus.REPLACEMENT + self.current_conf[rep].status = HostStatus.DECOMMISSIONING + else: + raise ValueError(f"\'replace\' field of host {host_config['hostname']} contains" + "does not belong to current configuration ") + + self.target_hosts = set(list(hosts.values())) + if len(self.current_hosts & self.target_hosts) < len(self.current_hosts): + for k, h in self.current_conf.items(): + if k not in hosts: + h.status = HostStatus.DECOMMISSIONING self.unpreferred_segments = self.getSegmentsUnpreferredRole() @@ -77,23 +151,53 @@ def getSegmentsUnpreferredRole(self) -> List[tuple[Segment, Segment]]: def setMirroringStrategy(self, strategy: MirrorStrategy): self.target_strategy = strategy - def getHostsFromGpArray(self) -> Dict[tuple[str, str], Host]: + def getHostsFromGpArray(self) -> ClusterState: hosts = {} for seg in self.original_gparray.getSegmentsAsLoadedFromDb(): if seg.content >= 0: hosts[(seg.hostname, seg.address)] = Host( - hostname=seg.hostname, address=seg.address, primary_datadirs=set(), mirror_datadirs=set(), - primary_segments=set(), mirror_segments=set()) + hostname=seg.hostname, address=seg.address, primary_datadirs=set(), primary_segments=set(), mirror_datadirs=set(), mirror_segments=set()) for pair in self.original_gparray.segmentPairs: primary = pair.primaryDB mirror = pair.mirrorDB key_pr = (primary.hostname, primary.address) hosts[key_pr].primary_datadirs.add( os.path.dirname(primary.datadir)) - hosts[key_pr].primary_segments.add(primary.content) + hosts[key_pr].primary_segments.add( + SegmentId(primary.dbid, primary.content)) if mirror: key_mr = (mirror.hostname, mirror.address) hosts[key_mr].mirror_datadirs.add( os.path.dirname(mirror.datadir)) - hosts[key_mr].mirror_segments.add(mirror.content) + hosts[key_mr].mirror_segments.add( + SegmentId(mirror.dbid, mirror.content)) return hosts + + def getNewHosts(self) -> tuple[Set, Set]: + new_hosts = self.target_hosts.difference( + self.current_hosts & self.target_hosts) + replacement_hosts = set() + other = set() + for h in new_hosts: + if h.status == HostStatus.REPLACEMENT: + replacement_hosts.add(h) + else: + h.status = HostStatus.NEW + other.add(h) + return other if len(other) > 0 else None, replacement_hosts if len(replacement_hosts) > 0 else None + + def createPlan(self) -> Plan: + primaries_count = self.original_gparray.get_primary_count() + total_hosts = len(self.target_hosts) + assert (primaries_count % total_hosts == 0) + self.logger.info('Planning rebalance...') + + new_hosts, replacement_hosts = self.getNewHosts() + target_load = primaries_count // total_hosts + original_segments_map = {SegmentId( + seg.dbid, seg.content): seg for seg in self.original_gparray.getSegmentsAsLoadedFromDb()} + + balancer = ClusterBalancer(self.current_conf, (new_hosts, replacement_hosts), + original_segments_map, target_load, self.target_strategy) + + return balancer.getPlan(balancer.balance()) diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_plan.py b/gpMgmt/bin/gprebalance_modules/rebalance_plan.py new file mode 100644 index 000000000000..703d719e6582 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_plan.py @@ -0,0 +1,323 @@ +import copy +import random +import math +from typing import List, Set, Dict +from dataclasses import dataclass +from collections import defaultdict +from gprebalance_modules.rebalance import Host, HostStatus, SegmentId, ClusterState, MirrorStrategy, Segment + + +@dataclass +class Move: + segid: SegmentId + srcHost: Host + dstHost: Host + is_mirror: bool + final_datadir: str + + +class Plan: + def __init__(self): + self.in_conf = None + self.moves = [] + self.out_conf = None + self.roles = [] + self.number_of_moves = 0 + + def __str__(self): + finalStr = "" + for i in range(len(self.moves)): + m = self.moves[i] + if self.roles[m.segid] == 'p': + finalStr += f"{i} :S{m.segid.contentid} :" + else: + finalStr += f"{i} :M{m.segid.contentid} :" + finalStr += f"H{m.srcHost.hostname} -> H{m.dstHost.hostname}\n" + return finalStr + + +class NoValidMovesError(Exception): + def __init__(self, msg="No valid targets for selected segment"): + super().__init__(msg) + + +class ClusterBalancer(): + def __init__(self, initial_conf: ClusterState, + new_hosts: tuple[Set[Host], Set[Host]], + segmentMap: Dict[SegmentId, Segment], + target_load: int, + strat: MirrorStrategy + ): + self.in_conf = initial_conf + self.new_hosts, self.replacement_hosts = new_hosts + self.initialSegmentMap = segmentMap + self.target_load = target_load + self.target_strategy = strat + self.initial_mirror_mapping = {} + for k, v in segmentMap.items(): + if v.isSegmentMirror(): + self.initial_mirror_mapping[k.contentid] = v + + def _get_working_conf(self) -> ClusterState: + """Get host-centric representation of cluster configuration""" + working_conf = copy.deepcopy(self.in_conf) + if self.new_hosts: + for h in self.new_hosts: + assert (h.status == HostStatus.NEW) + working_conf[(h.hostname, h.address)] = copy.deepcopy(h) + if self.replacement_hosts: + for h in self.replacement_hosts: + assert (h.status == HostStatus.REPLACEMENT) + working_conf[(h.hostname, h.address)] = copy.deepcopy(h) + assert ( + working_conf[h.replacement_for].status == HostStatus.DECOMMISSIONING) + return working_conf + + def _getCurrentMirrorMapping(self, state: ClusterState) -> Dict[int, tuple[str, str]]: + mirror_mapping = {} + for k, v in state.items(): + for mirid in v.mirror_segments: + mirror_mapping[mirid.contentid] = (v.hostname, v.address) + return mirror_mapping + + def _getCurrentSegmentMapping(self, state: ClusterState) -> Dict[int, tuple[str, str]]: + segment_mapping = {} + for k, v in state.items(): + for segid in v.primary_segments: + segment_mapping[segid.contentid] = (v.hostname, v.address) + return segment_mapping + + def _satisfies_strategy(self, state: ClusterState) -> bool: + mirror_mapping = self._getCurrentMirrorMapping(state) + if self.target_strategy == MirrorStrategy.GROUPED: + for _, host in state.items(): + if host.status == HostStatus.DECOMMISSIONING: + continue + mirror_hosts = set() + for s in host.primary_segments: + mirror_hosts.add(mirror_mapping[s.contentid]) + if len(mirror_hosts) > 1: + return False + elif len(mirror_hosts & set([(host.hostname, host.address)])) > 0: + return False + elif len(host.mirror_segments) == 0: + return False + elif self.target_strategy == MirrorStrategy.SPREAD: + for _, host in state.items(): + if host.status == HostStatus.DECOMMISSIONING: + continue + mirror_hosts_counts: Dict[Host, int] = defaultdict(int) + for s in host.primary_segments: + mirror_hosts_counts[mirror_mapping[s.contentid]] += 1 + for t_h, count in mirror_hosts_counts.items(): + if t_h == (host.hostname, host.address): + return False + elif count > 1: + return False + return True + + def _check_balance(self, state: ClusterState) -> bool: + for _, host in state.items(): + load = len(host.primary_segments) + deviation = abs(load - self.target_load) + if host.status == HostStatus.DECOMMISSIONING and (host.primary_segments or host.mirror_segments): + return False + elif host.status != HostStatus.DECOMMISSIONING and deviation != 0: + return False + return True + + def _move_from_decomissioning(self, state: ClusterState): + if self.replacement_hosts: + for rep in self.replacement_hosts: + # Host from which we must move all segments + decom_host_id = rep.replacement_for + replacement_host_id = (rep.hostname, rep.address) + assert (state[decom_host_id].status == + HostStatus.DECOMMISSIONING) + primaries = state[decom_host_id].primary_segments + mirrors = state[decom_host_id].mirror_segments + for segid in primaries: + state[replacement_host_id].add_primary(segid) + for segid in mirrors: + state[replacement_host_id].add_mirror(segid) + state[decom_host_id].primary_segments = set() + state[decom_host_id].mirror_segments = set() + + def _check_constraints(self, state: ClusterState): + return self._check_balance(state) and self._satisfies_strategy(state) + + def balance(self) -> ClusterState: + working_conf = self._get_working_conf() + # First, check if there are hosts explicitly marked for replacement + if self.replacement_hosts: + self._move_from_decomissioning(working_conf) + if self._check_constraints(working_conf): + return working_conf + + conf = self.rough_balance(working_conf) + assert (self._check_constraints(conf)) + return conf + + def _is_move_valid(self, segmentid: SegmentId, target_host: Host, state: ClusterState) -> bool: + segment_locations = self._getCurrentSegmentMapping(state) + + current_host = segment_locations[segmentid.contentid] + + if current_host == (target_host.hostname, target_host.address): + return False + + # Check target host status + if target_host.status == HostStatus.DECOMMISSIONING: + return False + + # For primary moves, only check capacity + if state[current_host].status != HostStatus.DECOMMISSIONING: + return len(target_host.primary_segments) + 1 <= self.target_load + + return True + + def rough_balance(self, working_conf: ClusterState) -> ClusterState: + + def calculate_surplus_deficit(): + loads = {(h.hostname, h.address): len(h.primary_segments) + for _, h in working_conf.items() + if h.status != HostStatus.DECOMMISSIONING} + surplus = {} + deficit = {} + for host_id, load in loads.items(): + surplus[host_id] = max(0, load - self.target_load) + deficit[host_id] = max(0, self.target_load - load) + return surplus, deficit + # Phase 1: Redistribute Primary Segments + while True: + # Priority 1: Clear decommissioning hosts + decom_hosts = [h for _, h in working_conf.items() + if h.status == HostStatus.DECOMMISSIONING] + + moved = False + # Move from decommissioning hosts first + for source in decom_hosts: + if not source.primary_segments: + continue + + potential_targets = sorted( + [h for _, h in working_conf.items() + if h.status not in (HostStatus.DECOMMISSIONING,)], + key=lambda h: len(h.primary_segments) + ) + + for segment in sorted(list(source.primary_segments), key=lambda x: x.contentid, reverse=True): + for target in potential_targets: + if self._is_move_valid(segment, target, working_conf): + source.remove_primary(segment) + target.add_primary(segment) + moved = True + break + + surplus, deficit = calculate_surplus_deficit() + if sum(surplus.values()) == 0: + break + # If no decom moves, balance regular hosts + if not moved: + surplus_hosts = [(s, h) for h, s in surplus.items() if s > 0] + deficit_hosts = [(d, h) for h, d in deficit.items() if d > 0] + surplus_hosts.sort(reverse=True) + deficit_hosts.sort(reverse=True) + + for _, source_id in surplus_hosts: + source = working_conf[source_id] + for _, target_id in deficit_hosts: + target = working_conf[target_id] + + for segment in sorted(list(source.primary_segments), key=lambda x: x.contentid, reverse=True): + if self._is_move_valid(segment, target, working_conf): + source.remove_primary(segment) + target.add_primary(segment) + moved = True + break + if moved: + break + if moved: + break + + if not moved: + break + + # Phase 2: Deterministic Mirror Placement + for _, host in working_conf.items(): + host.mirror_segments = set() + + active_hosts = [(h.hostname, h.address) for _, h in working_conf.items() + if h.status != HostStatus.DECOMMISSIONING] + active_hosts.sort() # ensure consistent ordering + + if self.target_strategy == MirrorStrategy.GROUPED: + # For each host, place all its primaries' mirrors on the next host + for i, host_id in enumerate(active_hosts): + next_host_id = active_hosts[(i + 1) % len(active_hosts)] + primaries = working_conf[host_id].primary_segments + for segment in primaries: + mirror = self.initial_mirror_mapping[segment.contentid] + working_conf[next_host_id].add_mirror( + SegmentId(mirror.dbid, mirror.content)) + + elif self.target_strategy == MirrorStrategy.SPREAD: + # For each host, distribute mirrors across other hosts + for host_id in active_hosts: + other_hosts = [h for h in active_hosts if h != host_id] + primaries = list(working_conf[host_id].primary_segments) + + # Distribute mirrors evenly across other hosts + for i, segment in enumerate(primaries): + mirror_host = other_hosts[i % len(other_hosts)] + mirror = self.initial_mirror_mapping[segment.contentid] + working_conf[mirror_host].add_mirror( + SegmentId(mirror.dbid, mirror.content)) + + return working_conf + + def getPlan(self, finalState: ClusterState) -> Plan: + in_conf = self._get_working_conf() + moves, roles = self.get_moves_between_states(in_conf, finalState) + plan = Plan() + plan.moves = moves + plan.number_of_moves = len(moves) + plan.in_conf = in_conf + plan.out_conf = finalState + plan.roles = roles + return plan + + def get_moves_between_states(self, state1: ClusterState, state2: ClusterState): + moves = [] + roles = {} + primary_map1 = self._getCurrentSegmentMapping(state1) + mirror_map1 = self._getCurrentMirrorMapping(state1) + primary_map2 = self._getCurrentSegmentMapping(state2) + mirror_map2 = self._getCurrentMirrorMapping(state2) + + # Find primary segment moves + for contentid, target_location in primary_map2.items(): + current_location = primary_map1.get(contentid) + if current_location and current_location != target_location: + source_host = state1[current_location] + target_host = state1[target_location] + segid = next(seg for seg in source_host.primary_segments + if seg.contentid == contentid) + moves.append( + Move(segid, source_host, target_host, False, None)) + roles[segid] = 'p' + + # Find mirror segment moves + for contentid, target_location in mirror_map2.items(): + current_location = mirror_map1.get(contentid) + if current_location and current_location != target_location: + source_host = state1[current_location] + target_host = state1[target_location] + segid = next(seg for seg in source_host.mirror_segments + if seg.contentid == contentid) + moves.append(Move(segid, source_host, target_host, True, None)) + roles[segid] = 'm' + + moves.sort(key=lambda m: 1 if m.is_mirror else 0) + + return moves, roles diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_validator.py b/gpMgmt/bin/gprebalance_modules/rebalance_validator.py index c2d3a2718548..8c10fb142737 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_validator.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_validator.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Set from gprebalance_modules.rebalance import Host, MirrorStrategy, dbconn, GpArray, Segment, MODE_NOT_SYNC, STATUS_DOWN @@ -7,7 +7,7 @@ class StateValidationError(Exception): class ClusterValidator: - def __init__(self, existing_hosts: List[Host], target_hosts: List[Host], segarray: List[Segment], has_mirrors: bool, mirror_strategy: MirrorStrategy): + def __init__(self, existing_hosts: Set[Host], target_hosts: Set[Host], segarray: List[Segment], has_mirrors: bool, mirror_strategy: MirrorStrategy): self.mirror_strategy = mirror_strategy self.existing_hosts = existing_hosts self.target_hosts = target_hosts @@ -32,7 +32,7 @@ def validate_existing_configuration(self) -> tuple[bool, MirrorStrategy]: elif arr.hasMirrors: strat = MirrorStrategy.GROUPED for host in self.existing_hosts: - if host.primary_segments & host.mirror_segments: + if set([s.contentid for s in host.primary_segments]) & set([s.contentid for s in host.mirror_segments]): strat = None break From 5dc85b0d74b24380414d6ff3466e3b8d2257b102 Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Thu, 13 Mar 2025 14:34:27 +0300 Subject: [PATCH 05/22] Rebalance automation (#1236) The proposed code contains main framework for rebalance execution. Some options are not implemented fully and are expected to be finished in next tasks. The code describes the following segment movement approach. Firstly, we creating a movements plan: simple steps telling which segment to which host to move. Steps in plan can be different: Mirror only moves. Both primary and mirror are moved to different hosts. Primary only moves. Primary and mirror are swapped. For each type of movement we clarify the target dirs and ports at target hosts, able to contain the size of moved segment. To do that the DiskFree and DiskUsage commands are used. The movements, in its turn, are composite and imply extra actions including segment switching. Mirror only moves use only single gprecoverseg call to perform movement. If we move primary and mirror pair, the strategy is following. The mirror is firstly moved via gprecoverseg to primary's target host. Then the roles are switched. Then ex-primary (new mirror) is moved to mirror's target host. Primary only moves imply 2 role switches. Switch.Move.Switch. Primary mirror swap is executed similar to 2nd type. Mirror is moved to primary dir in its own host. Switch. Ex-primary is moved to mirror dir in its own host. The status management is written in general and may contain errors. Cleanup is prepared by RekGRpth Co-authored-by: Georgy Shelkovy --- gpMgmt/bin/gprebalance | 118 +- gpMgmt/bin/gprebalance_modules/Makefile | 2 +- gpMgmt/bin/gprebalance_modules/rebalance.py | 142 ++- .../gprebalance_modules/rebalance_executor.py | 1039 +++++++++++++++++ .../bin/gprebalance_modules/rebalance_plan.py | 28 +- .../gprebalance_modules/rebalance_status.py | 208 ++++ .../behave/mgmt_utils/gprebalance.feature | 15 + .../behave/mgmt_utils/steps/mgmt_utils.py | 18 + 8 files changed, 1523 insertions(+), 47 deletions(-) create mode 100644 gpMgmt/bin/gprebalance_modules/rebalance_executor.py create mode 100644 gpMgmt/bin/gprebalance_modules/rebalance_status.py create mode 100644 gpMgmt/test/behave/mgmt_utils/gprebalance.feature diff --git a/gpMgmt/bin/gprebalance b/gpMgmt/bin/gprebalance index 4ad48ceba79b..e613505026df 100755 --- a/gpMgmt/bin/gprebalance +++ b/gpMgmt/bin/gprebalance @@ -27,8 +27,6 @@ except ImportError as e: DBNAME = 'postgres' -MAX_PARALLEL_WORKERS = 4 - _description = (""" Rebalances the existing segments configuration for getting optimal performance of whole cluster. @@ -73,6 +71,8 @@ _usage = """ """ EXECNAME = os.path.split(__file__)[-1] +MAX_BATCH_SIZE = 128 +MAX_PARALLEL_WORKERS = 96 def parseargs(): @@ -89,15 +89,13 @@ def parseargs(): help='remove the rebalance schema.') parser.add_option('-f', '--target-hosts', metavar='', dest='filename', help='yaml containing target hosts configuration') - parser.add_option('-p', '--show-plan', dest='show_plan', action='store_true', default=False, - help='show rebalance plan') parser.add_option('-r', '--rollback', action='store_true', help='remove the rebalance schema.') parser.add_option('-d', '--duration', type='duration', metavar='[h][:m[:s]]', help='duration from beginning to end.') parser.add_option('-e', '--end', type='datetime', metavar='datetime', help="ending date and time in the format 'YYYY-MM-DD hh:mm:ss'.") - parser.add_option('-n', '--parallel', type="int", default=1, metavar="", + parser.add_option('-n', '--parallel', type="int", default=4, metavar="", help='number of workerks performing segment movements at a time. Valid values are 1-%d.' % MAX_PARALLEL_WORKERS) parser.add_option('--allow-mirrorless', dest='allow_mirrorless', action='store_true', help='Allow to rebalance a cluster without mirrors', default=False) @@ -111,10 +109,12 @@ def parseargs(): help='debug output.') parser.add_option('-h', '-?', '--help', action='help', help='show this help message and exit.') + parser.add_option('-B', '--batch-size', type='int', default=16, metavar="", + help='Rebalance configuration batch size. Valid values are 1-%d' % MAX_BATCH_SIZE) parser.add_option('--usage', action="briefhelp") parser.add_option('-S', '--simple-progress', action='store_true', help='show simple progress.') - parser.add_option('', '--hba-hostnames', action='store_true', default=False, + parser.add_option('', '--hba-hostnames', action='store_true', default=False, dest='hba_hostnames', help='use hostnames instead of CIDR in pg_hba.conf') parser.add_option('--allow-intermediate-mixture', dest='allow_mixture', action='store_true', help='Allow primary and mirror from one pair to be at the same host during balancing', default=False) @@ -263,22 +263,6 @@ def possibility_checks(gpreb: GPRebalance): hosts_list = gpreb.current_hosts - if gpreb.unpreferred_segments: - if not gpreb.options.silent and ask_yesno('', "Current role does not match preferred role for several segments.\n" - "Are you sure you want " - "to continue with this gprebalance session?", "N"): - hstmap = gpreb.getHostsFromGpArray() - for prim, mir in gpreb.unpreferred_segments: - hstmap[(prim.hostname, prim.address) - ].primary_segments.discard(prim.content) - hstmap[(prim.hostname, prim.address) - ].mirror_segments.add(prim.content) - hstmap[(mir.hostname, mir.address) - ].primary_segments.add(mir.content) - hstmap[(mir.hostname, mir.address) - ].mirror_segments.discard(mir.content) - hosts_list = list(hstmap.values()) - startup_validator = ClusterValidator( hosts_list, gpreb.target_hosts, @@ -295,6 +279,11 @@ def possibility_checks(gpreb: GPRebalance): elif gpreb.options.silent: logger.error(f'{str(e)} Exiting...') sys.exit(1) + if gpreb.unpreferred_segments: + logger.error( + "Current role does not match preferred role for several segments. " + "Call gprecoverseg -r to restore the original roles") + sys.exit(1) if not gpreb.original_gparray.get_mirroring_enabled() and not gpreb.options.allow_mirrorless: if not gpreb.options.silent and not ask_yesno('', "Mirroring is disabled. \n" @@ -345,19 +334,24 @@ def possibility_checks(gpreb: GPRebalance): startup_validator.prevalidate_mirror_strategy() -def sig_handler(sig, arg): +gprebalance = None - signal.signal(signal.SIGTERM, signal.SIG_DFL) - signal.signal(signal.SIGHUP, signal.SIG_DFL) - # raise sig - os.kill(os.getpid(), sig) +def sig_handler(sig, arg): + if gprebalance is not None: + gprebalance.shutdown() + if gprebalance.executor is None: + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGHUP, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_DFL) + os.kill(os.getpid(), sig) def main(options, args, parser): try: signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGHUP, sig_handler) + signal.signal(signal.SIGINT, sig_handler) logger = get_default_logger() setup_tool_logging(EXECNAME, getLocalHostname(), getUserName()) @@ -374,21 +368,30 @@ def main(options, args, parser): configurationInterface.getConfigurationProvider( ).initializeProvider(gpEnv.getCoordinatorPort()) dburl = dbconn.DbURL(dbname=DBNAME, port=gpEnv.getCoordinatorPort()) - # check parallel execution of other cluster utilities - check_running_gputils(dburl, options.coordinator_data_directory) - # create pid file - create_pid_file(options.coordinator_data_directory) try: + # check parallel execution of other cluster utilities + check_running_gputils(dburl, options.coordinator_data_directory) gparray = GpArray.initFromCatalog(dburl, utility=True) - except DatabaseError as ex: logger.error("Failed to connect to database. Make sure the" " Greenplum instance you wish to expand is running" " and that your environment is correct, then rerun" " gprebalance" + ' '.join(sys.argv[1:])) sys.exit(1) + + # create pid file + create_pid_file(options.coordinator_data_directory) + + gprebalance_db_status = None + try: + gprebalance_db_status = GPRebalance.prepare_gpdb_state( + logger, dburl, options) + except: + pass + + global gprebalance gprebalance = GPRebalance(logger, gparray, dburl, options) # dump existing hosts configuration @@ -396,12 +399,49 @@ def main(options, args, parser): dump_hosts_info(gprebalance.getHostsFromGpArray()) sys.exit(0) - possibility_checks(gprebalance) - logger.info('Validation passed. Preparing rebalance...') - plan = gprebalance.createPlan() - if options.show_plan: - logger.info(f"Resulting plan: \n {str(plan)}") - logger.info('Exiting...') + gprebalance_file_status = gprebalance.get_state_from_file() + + if options.clean: + gprebalance.remove_status_file() + gprebalance.cleanup_schema() + gprebalance.cleanup_directory() + logger.info('Cleanup Finished. exiting...') + sys.exit(0) + + from gprebalance_modules.rebalance_status import RebalanceStatus # nopep8 + + if (gprebalance_db_status == RebalanceStatus.COMPLETED or + gprebalance_file_status == 'EXECUTION_DONE'): + logger.info('Rebalance has already completed') + logger.info( + 'If you want to rebalance again, run gprebalance -c to perform cleanup') + elif gprebalance_file_status is None and gprebalance_db_status is None: + gprebalance.statusManager.create_status_file() + possibility_checks(gprebalance) + logger.info('Validation passed. Preparing rebalance...') + gprebalance.statusManager.set_status('VALIDATED', os.path.abspath( + options.filename) if options.filename else None) + plan = gprebalance.createPlan() + gprebalance.save_plan(plan) + if options.show_plan: + logger.info(f"Resulting plan: \n{str(plan)}") + sys.exit(0) + gprebalance.setup_schema() + gprebalance.execute_plan(plan) + elif gprebalance.statusManager.conf_dir: + plan = gprebalance.load_plan(gprebalance.statusManager.conf_dir) + if options.show_plan: + logger.info(f"Resulting plan: \n{str(plan)}") + sys.exit(0) + if gprebalance_file_status == 'PLANNED': + gprebalance.setup_schema() + gprebalance.execute_plan(plan) + sys.exit(0) + elif gprebalance_file_status == 'EXECUTION_FAILED': + logger.info('Previous rebalance session was interrupted due to an error. Trying to resume...') + elif gprebalance_file_status == 'EXECUTION_STOPPED': + logger.info('Previous rebalance session was interrupted due to an timeout or user signal. Trying to resume...') + gprebalance.resume() sys.exit(0) except Exception as e: if options and options.verbose: @@ -412,6 +452,8 @@ def main(options, args, parser): except KeyboardInterrupt: sys.exit('\nUser Interrupted') finally: + if gprebalance: + gprebalance.shutdown() remove_pid_file(get_coordinatordatadir()) diff --git a/gpMgmt/bin/gprebalance_modules/Makefile b/gpMgmt/bin/gprebalance_modules/Makefile index 0125529264f9..692d727c7073 100644 --- a/gpMgmt/bin/gprebalance_modules/Makefile +++ b/gpMgmt/bin/gprebalance_modules/Makefile @@ -3,7 +3,7 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -PROGRAMS= __init__.py rebalance_validator.py rebalance.py rebalance_plan.py +PROGRAMS= __init__.py rebalance_validator.py rebalance.py rebalance_plan.py rebalance_executor.py rebalance_status.py installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)/gprebalance_modules' diff --git a/gpMgmt/bin/gprebalance_modules/rebalance.py b/gpMgmt/bin/gprebalance_modules/rebalance.py index 09758204ce21..c5503241122d 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance.py @@ -1,3 +1,5 @@ +from collections import defaultdict +from datetime import datetime import yaml import os from dataclasses import dataclass @@ -6,6 +8,7 @@ from gppylib.db import dbconn from gppylib.gparray import GpArray, Segment, MODE_NOT_SYNC, STATUS_DOWN from gppylib.commands.base import * +from gppylib.commands.gp import * class SegmentId(NamedTuple): @@ -13,6 +16,12 @@ class SegmentId(NamedTuple): contentid: int +@dataclass +class SegmentSize(): + source_data_dir_usage: int + source_tablespace_usage: Dict[str, int] + + class HostStatus(Enum): ACTIVE = "active" DECOMMISSIONING = "decommissioning" @@ -86,6 +95,8 @@ class MirrorStrategy(Enum): from gprebalance_modules.rebalance_plan import ClusterBalancer, Move, Plan # nopep8 +from gprebalance_modules.rebalance_executor import RebalanceExecutor, CONF_DIR # nopep8 +from gprebalance_modules.rebalance_status import StatusManager, RebalanceStatus # nopep8 class GPRebalance: @@ -112,7 +123,7 @@ def __init__(self, logger, gparray, dburl, options): config = yaml.safe_load(fp) for host_config in config['hosts']: key = (host_config['hostname'], host_config['address']) - hosts[key] = Host(hostname=host_config['hostname'], + file_host = Host(hostname=host_config['hostname'], address=host_config['address'], primary_datadirs=set( host_config['primary_datadirs']), @@ -120,6 +131,10 @@ def __init__(self, logger, gparray, dburl, options): mirror_datadirs=set( host_config['mirror_datadirs']), mirror_segments=set()) + hosts[key] = file_host + if key in self.current_conf: + self.current_conf[key].primary_datadirs |= file_host.primary_datadirs + self.current_conf[key].mirror_datadirs |= file_host.mirror_datadirs # User can explicitly mark the host to be replacement for existing one if "replace" in host_config: rep = tuple(host_config['replace'].split(', ')) @@ -138,6 +153,12 @@ def __init__(self, logger, gparray, dburl, options): h.status = HostStatus.DECOMMISSIONING self.unpreferred_segments = self.getSegmentsUnpreferredRole() + self.segmentMap = {SegmentId( + seg.dbid, seg.content): seg for seg in self.original_gparray.getSegmentsAsLoadedFromDb()} + self.statusManager = StatusManager( + self.conn, self.options, self.logger) + + self.executor = None def getSegmentsUnpreferredRole(self) -> List[tuple[Segment, Segment]]: segs = [] @@ -201,3 +222,122 @@ def createPlan(self) -> Plan: original_segments_map, target_load, self.target_strategy) return balancer.getPlan(balancer.balance()) + + def save_plan(self, plan: Plan): + # pickle the plan in conf directory + datadir = self.options.coordinator_data_directory + CONF_DIR + os.makedirs(datadir, exist_ok=True) + plan.save_to_file(datadir, "plan") + self.statusManager.set_status('PLANNED', datadir) + + def load_plan(self, datadir): + filename = datadir + "/plan.pkl" + if not os.path.exists(filename): + raise FileNotFoundError(f"No pickle file found at {filename}") + with open(filename, 'rb') as f: + plan = pickle.load(f) + return plan + + def execute_plan(self, plan: Plan): + self.executor = RebalanceExecutor(plan, + self.original_gparray, + self.segmentMap, + plan.in_conf, + self.logger, + self.statusManager, + self.conn, + self.dburl, + self.options) + self.executor.execute_moves() + + @staticmethod + def prepare_gpdb_state(logger, dburl, options) -> str: + gprebalance_db_status = None + + try: + gprebalance_db_status = GPRebalance.get_status_from_db( + dburl, options) + logger.debug('Expansion status returned is %s' % + gprebalance_db_status) + except Exception as e: + raise Exception( + 'Error while trying to query the gprebalance schema: %s' % e) + + return gprebalance_db_status + + @staticmethod + def get_status_from_db(dburl, options) -> RebalanceStatus: + """Gets gprebalance status from the gprebalance schema""" + status_conn = None + gprebalance_db_status = None + if get_local_db_mode(options.coordinator_data_directory) == 'NORMAL': + try: + status_conn = dbconn.connect(dburl, encoding='UTF8') + # Get the last status entry + cursor = dbconn.query( + status_conn, 'SELECT status FROM gprebalance.status ORDER BY updated DESC LIMIT 1') + if cursor.rowcount == 1: + gprebalance_db_status = cursor.fetchone()[0] + + except Exception: + # rebalance schema doesn't exists or there was a connection failure. + pass + finally: + if status_conn: + status_conn.close() + + # make sure gprebalance schema doesn't exist since it wasn't in DB provided + if not gprebalance_db_status: + conn = dbconn.connect(dburl, encoding='UTF8', utility=True) + try: + count = dbconn.querySingleton(conn, + "SELECT count(n.nspname) FROM pg_catalog.pg_namespace n WHERE n.nspname = 'gprebalance'") + if count > 0: + raise Exception( + "Existing rebalance state could not be determined, but a gprebalance schema already exists. Cannot proceed.") + finally: + conn.close() + + return RebalanceStatus(gprebalance_db_status) + + def get_state_from_file(self): + """Returns expansion state from status file""" + return self.statusManager.get_current_status()[0] + + def setup_schema(self): + self.statusManager.set_status('REBALANCE_PREPARE_SCHEMA_STARTED') + self.logger.info('Creating expansion schema') + self.statusManager.setup_schema() + self.statusManager.set_status('REBALANCE_PREPARE_SCHEMA_DONE') + self.statusManager.set_db_status(RebalanceStatus.INITIALIZED) + + def cleanup_schema(self): + if not self.conn: + return + self.logger.info('Dropping rebalance schema') + self.statusManager.cleanup_schema() + + def cleanup_directory(self): + self.logger.info('Dropping rebalance directory') + datadir = self.options.coordinator_data_directory + CONF_DIR + cmd = RemoveDirectory("Dropping rebalance directory", datadir) + cmd.run(validateAfter=True) + + def remove_status_file(self): + self.logger.info('Dropping status file') + if self.statusManager: + self.statusManager.remove_all() + + def resume(self): + """TODO: implement proper state handling and provide + possibility to perform rebalance after fails""" + raise NotImplementedError('Resuming operation is not implemented. Call gprebalance -c and rerun') + + def shutdown(self): + self.logger.info('Shutting down gprebalance...') + if self.executor: + self.executor.shutdown() + self.executor = None + if self.conn: + self.conn.close() + self.conn = None diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_executor.py b/gpMgmt/bin/gprebalance_modules/rebalance_executor.py new file mode 100644 index 000000000000..985a285e3064 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_executor.py @@ -0,0 +1,1039 @@ +import base64 +import fcntl +import multiprocessing +import time +from collections import defaultdict +import pickle +from typing import List, Dict, Optional, Set, Tuple +from gprebalance_modules.rebalance_plan import Move, Plan # nopep8 +from gprebalance_modules.rebalance_status import StatusManager, RebalanceStatus, MoveStatus, SqlError +from gprebalance_modules.rebalance import ClusterState, SegmentId, SegmentSize, Host +from gppylib.gparray import GpArray, Segment +from gppylib.db import dbconn +from gppylib.commands.base import * +from gppylib.commands.gp import * +from gppylib.commands.unix import DiskFree, DiskUsage, RemoveDirectory, getLocalHostname, getUserName +from gppylib.operations.validate_disk_space import FileSystem +from gppylib.parseutils import * +from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram +from gppylib.system import configurationInterface, configurationImplGpdb, fileSystemInterface, \ + fileSystemImplOs, osInterface, osImplNative, faultProberInterface, faultProberImplGpdb +from gppylib.userinput import * + +MAX_BATCH_SIZE = 128 +FILENAME = "/move_" +CONF_DIR = "/rebalance" +DEFAULT_PRIMARY_PREF = "/data/primary" +DEFAULT_MIRROR_PREF = "/data/mirror" +GPRECOVERSEG_DIR = 'gpAdminLogs/rebalance' + +begining_timestamp = None +segment_prefix = "gpseg" + +class InsufficientDiskSpaceError(Exception): + pass + + +class NoValidDataDirectories(Exception): + pass + + +class RecoveryProcess: + @staticmethod + def run_recovery(cmd_args: list, result_queue: multiprocessing.Queue, log_file: str): + try: + #prevent signal propagation from parent + os.setpgrp() + signal.signal(signal.SIGINT, signal.SIG_IGN) + + log_fd = os.open(log_file, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + flags = fcntl.fcntl(log_fd, fcntl.F_GETFL) + fcntl.fcntl(log_fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + os.dup2(log_fd, 1) # stdout + os.dup2(log_fd, 2) # stderr + os.close(log_fd) + + import gppylib.gplog as gplog + if gplog._LOGGER: + for handler in gplog._LOGGER.handlers[:]: + handler.close() + gplog._LOGGER.removeHandler(handler) + gplog._LOGGER = None + + gplog._FILENAME = None + gplog._DEFAULT_FORMATTER = None + gplog._LITERAL_FORMATTER = None + gplog._SOUT_HANDLER = None + gplog._FILE_HANDLER = None + + # Register all necessary interfaces to run a gprecoverseg + # in a separate process + configurationInterface.registerConfigurationProvider( + configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) + fileSystemInterface.registerFileSystemProvider( + fileSystemImplOs.GpFileSystemProviderUsingOs()) + osInterface.registerOsProvider( + osImplNative.GpOsProviderUsingNative()) + faultProberInterface.registerFaultProber( + faultProberImplGpdb.GpFaultProberImplGpdb()) + + local_parser = GpRecoverSegmentProgram.createParser() + local_options, args = local_parser.parse_args(cmd_args) + + gplog.setup_tool_logging("gprecoverseg", getLocalHostname(), + getUserName(), + logdir=local_options.logfileDirectory) + + # Create and run the program + cmd = GpRecoverSegmentProgram.createProgram(local_options, args) + cmd.run() + + except SystemExit as e: + error_msg = None + if e.code != 0: + error_msg = f"Gprecoverseg failed with exit code: {e.code}. See the log in {log_file}" + result_queue.put({ + "status": "FAILED" if e.code != 0 else "SUCCESS", + "error": error_msg + }) + except Exception as e: + error_msg = f"Error in gprecoverseg process: {str(e)}" + result_queue.put({ + "status": "FAILED", + "error": error_msg + }) + finally: + cmd.cleanup() + + +class SingleMoveCommand(SQLCommand): + def __init__(self, name: str, status_url: dbconn.DbURL, step_details, logger): + self.status_url = status_url + self.logger = logger + (self.segment, self.move, self.segmentSize, + self.conf_dir, self.needs_switch, self.options) = step_details + + self.move_error = False + + SQLCommand.__init__(self, name) + + def write_gprecoverseg_config(self): + filename = self.conf_dir + FILENAME + "dbid" + str(self.segment.dbid) + with open(filename, 'w') as fp: + line = (f"{canonicalize_address(self.segment.address)}|" + f"{self.segment.port}|{self.segment.datadir} " + f"{canonicalize_address(self.move.dstHost.address)}|" + f"{self.move.target_port}|" + f"{self.move.target_datadir}") + self.logger.info( + "About to run gprecoverseg for mirror move " + f"(dbid = {self.segment.dbid}, content = {self.segment.content}) {line}") + fp.write(line) + return filename + + def run(self, validateAfter=False): + status_conn = None + try: + status_conn = dbconn.connect(self.status_url, encoding='UTF8') + segsize = self.segmentSize.source_data_dir_usage + if self.segmentSize.source_tablespace_usage: + segsize += sum(self.segmentSize.source_tablespace_usage.values()) + StatusManager.record_move(status_conn, self.move, self.segment, + segsize, datetime.datetime.now()) + + filename = self.write_gprecoverseg_config() + + strtime=datetime.datetime.now().strftime('%Y%m%d_%H%M%S') + + # in order to run gprecoverseg processes separately and avoid any races + # for resources gprecoverseg generates, we create separate log directories. + log_dir = f"{os.path.join(os.environ.get('HOME', '.'),GPRECOVERSEG_DIR)}/gprecoverseg_dbid{self.segment.dbid}_{strtime}" + mkdirCmd = MakeDirectory("rebalance log dir", log_dir) + mkdirCmd.run(validateAfter=True) + + mkdirCmd = MakeDirectory("rebalance log dir", log_dir, ctxt=REMOTE, remoteHost=self.move.srcHost.hostname) + mkdirCmd.run(validateAfter=True) + mkdirCmd = MakeDirectory("rebalance log dir", log_dir, ctxt=REMOTE, remoteHost=self.move.dstHost.hostname) + mkdirCmd.run(validateAfter=True) + + log_file = f"{self.conf_dir}/gprecoverseg_dbid{self.segment.dbid}_{strtime}" + # Prepare command arguments + cmd_args = [ + '-i', filename, + '-B', '1', + '-v', '-a', + '-l', log_dir + ] + if self.options.hba_hostnames: + cmd_args.append('--hba-hostnames') + try: + StatusManager.update_record_status( + status_conn, [self.segment.dbid], MoveStatus.IN_PROGRESS) + except SqlError: + raise Exception( + f"Could not update status for move with dbid={self.segment.dbid}: ") + + result_queue = multiprocessing.Queue() + recovery_process = multiprocessing.Process( + target=RecoveryProcess.run_recovery, + args=(cmd_args, result_queue, log_file) + ) + recovery_process.start() + result = result_queue.get() + recovery_process.join() + if result["status"] == "FAILED": + self.logger.error( + f"Could not perform mirror dbid={self.segment.dbid} " + f"move with content {self.segment.content} due to " + f"recoverseg error: {result['error']}\n" + "Check the gprecoverseg l og file, fix any problems, and re-run" + ) + try: + StatusManager.update_record_status( + status_conn, [self.segment.dbid], MoveStatus.FAILED) + except SqlError: + raise Exception( + f"Could not update status for move with dbid={self.segment.dbid}: ") + + self.move_error = True + status_conn.close() + return + if self.needs_switch: + try: + StatusManager.update_record_status( + status_conn, [self.segment.dbid], MoveStatus.AWAITS_SWITCH) + except SqlError: + raise Exception( + f"Could not update status for move with dbid={self.segment.dbid}: ") + + try: + StatusManager.update_record_status( + status_conn, [self.segment.dbid], MoveStatus.COMPLETED) + except SqlError: + raise Exception( + f"Could not update status for move with dbid={self.segment.dbid}: ") + + self.logger.info("Removing old segment's datadir (dbidi = %d): %s", + self.segment.dbid, self.segment.datadir) + cmd = RemoveDirectory("remove old mirror segment directories", self.segment.datadir, + ctxt=REMOTE, remoteHost=self.segment.address) + cmd.run(validateAfter=True) + if self.segmentSize.source_tablespace_usage: + for tblspdir in self.segmentSize.source_tablespace_usage: + cmd = RemoveDirectory("remove old mirror segment directories", tblspdir, + ctxt=REMOTE, remoteHost=self.segment.address) + self.logger.info("Removing old segment's tablespace datadir (dbidi = %d): %s", + self.segment.dbid, tblspdir) + cmd.run(validateAfter=True) + status_conn.close() + + except Exception as ex: + self.logger.error(ex.__str__().strip()) + self.move_error = True + status_conn.close() + return + + + +class FilesystemSpace: + def __init__(self, filesystem: FileSystem, directories: Set[str]): + self.filesystem = filesystem + self.directories = directories # dirs on this filesystem + self.available_space = filesystem.disk_free + self.planned_usage = 0 + + def reserve_space(self, size: int): + self.planned_usage += size + + def can_accommodate(self, size: int) -> bool: + return (self.available_space - self.planned_usage) >= size + + +def _target_filesystems(addr: str, directories: List[str], batch_size) -> List[FileSystem]: + filesystems = [] # list of FileSystem() + pool = WorkerPool(numWorkers=min(len(directories), batch_size)) + try: + cmd = DiskFree(addr, directories) + pool.addCommand(cmd) + pool.join() + finally: + pool.haltWork() + pool.joinWorkers() + for cmd in pool.getCompletedItems(): + if not cmd.was_successful(): + raise Exception("Failed to check disk free on target segment: {}" .format( + cmd.get_results().stderr)) + filesystems = pickle.loads( + base64.urlsafe_b64decode(cmd.get_results().stdout)) + return filesystems + + +class HostResources: + def __init__(self, host: Host, ports: tuple[Set[int], Set[int]]): + self.host_address = host.address + self.primary_datadirs = host.primary_datadirs + self.mirror_datadirs = host.mirror_datadirs + self.used_primary_ports, self.used_mirror_ports = ports + self.filesystem_spaces: List[FilesystemSpace] = [] + self.hostname = host.hostname + + # Initialize filesystem tracking for all directories + all_dirs = host.primary_datadirs.union(host.mirror_datadirs) + self._init_filesystem_spaces(all_dirs) + self.base_port = self._determine_base_port() + + def _init_filesystem_spaces(self, directories: Set[str]): + """Initialize filesystem space tracking for all directories""" + filesystems = _target_filesystems( + self.host_address, list(directories), MAX_BATCH_SIZE) + + # Group directories by filesystem + for fs in filesystems: + dirs_on_fs = {d for d in directories if d in fs.directories} + self.filesystem_spaces.append(FilesystemSpace(fs, dirs_on_fs)) + + def accommodate_segment(self, segment_size: SegmentSize, target_datadir: str): + """Check if segment can be accommodated considering all its space requirements""" + + # Find filesystem for main datadir and add space requirement + datadir_fs = self._get_filesystem_for_dir(target_datadir) + if not datadir_fs: + raise Exception(f"Host {self.hostname} does not have any valid primary " + f"datadirs for segment") + if datadir_fs.can_accommodate(segment_size.source_data_dir_usage): + datadir_fs.reserve_space(segment_size.source_data_dir_usage) + + # Add tablespace requirements to respective filesystems + if segment_size.source_tablespace_usage: + for tblspc_dir, usage in segment_size.source_tablespace_usage.items(): + tblspc_fs = self._get_filesystem_for_dir(tblspc_dir) + if not tblspc_fs: + raise Exception(f"Host {self.hostname} does not have any valid primary " + f"datadirs for segment") + if tblspc_fs.can_accommodate(usage): + tblspc_fs.reserve_space(usage) + + def _get_filesystem_for_dir(self, directory: str) -> Optional[FilesystemSpace]: + """Find FilesystemSpace object containing given directory""" + # First check existing filesystem mappings + for fs_space in self.filesystem_spaces: + if directory in fs_space.directories: + return fs_space + + # If not found, fetch filesystem info for this directory + filesystems = _target_filesystems( + self.host_address, [directory], MAX_BATCH_SIZE) + + if not filesystems: + return None + + # Check if the filesystem already exists in our list + fs = filesystems[0] + for existing_fs in self.filesystem_spaces: + if existing_fs.filesystem.name == fs.name: + existing_fs.directories.add(directory) + return existing_fs + + # If not, create new FilesystemSpace + new_fs_space = FilesystemSpace(fs, {directory}) + self.filesystem_spaces.append(new_fs_space) + return new_fs_space + + def _determine_base_port(self) -> int: + """Determine base port from existing port assignments""" + all_ports = self.used_primary_ports | self.used_mirror_ports + + # Find the most common base port + port_bases = defaultdict(int) + for port in all_ports: + # For each port, calculate what base port it might correspond to + # assuming port = base + (content * 2) [+ 1 for mirrors] + for content in range(0, 128): # reasonable content_id range + if port % 2 == 0: # primary + possible_base = port - (content * 2) + else: # mirror + possible_base = port - (content * 2) - 1 + + if possible_base > 0: + port_bases[possible_base] += 1 + + if not port_bases: + return 7000 + + # Return the most frequently occurring base port + return max(port_bases.items(), key=lambda x: x[1])[0] + + def can_accommodate_port(self, is_mirror: bool, content_id: int) -> Optional[int]: + """ + Find available port for segment using existing base port pattern + Returns suitable port number or None if no port available + """ + used_ports = self.used_primary_ports | self.used_mirror_ports + + # Calculate port based on content_id and base port + port = self.base_port + (content_id * 2) + if is_mirror: + port += 1 + + if port not in used_ports: + return port + + # If standard port not available, try finding next available port + # maintaining the same even/odd pattern + start_port = max(used_ports) + 2 if used_ports else self.base_port + if start_port % 2 != (0 if not is_mirror else 1): + start_port += 1 + + current_port = start_port + while current_port < 65536: # Max TCP port + if current_port not in used_ports: + return current_port + current_port += 2 + + return None + + def reserve_port(self, port: int, is_mirror: bool): + """Reserve port for segment""" + if is_mirror: + self.used_mirror_ports.add(port) + else: + self.used_primary_ports.add(port) + + +class RebalanceExecutor: + def __init__(self, + plan: Plan, + original_array: GpArray, + segmentMap: Dict[SegmentId, Segment], + cluster_state: ClusterState, + logger, + statusManager: StatusManager, + conn: dbconn.Connection, + dburl: dbconn.DbURL, + options, + ): + self.moves = plan.moves + self.plan = plan + self.logger = logger + self.gparr = original_array + self.cluster_state = cluster_state + self.segmentMap = segmentMap + self.conn = conn + self.statusManager = statusManager + self.options = options + self.dburl = dburl + segids = [] + for m in plan.moves: + segids.append(m.segid) + self.segmentSizes = self.estimateSegmentSizes(segids) + self.resources = self.initializeHostResources(plan.moves) + self.queue = None + self.shutdown_requested = False + + self.define_datadir_prefix() + + def define_datadir_prefix(self): + first_source_dir = None + for _, segment in self.segmentMap.items(): + if segment.content >= 0: + first_source_dir = segment.datadir + break + + basename = os.path.basename(first_source_dir) + global segment_prefix + segment_prefix = ''.join(c for c in basename if not c.isdigit()) + + def initializeHostResources(self, moves: List[Move]): + def datadir_validator(input_value, default, *args): + if not input_value and not default: + return None + elif not input_value or input_value == '': + input_value = default + if not input_value or input_value.find(' ') != -1 or input_value == '': + return None + else: + return input_value + resources = {} + for m in moves: + prim_ports = set() + mir_ports = set() + if m.dstHost not in resources: + for psid in self.cluster_state[(m.dstHost.hostname, m.dstHost.address)].primary_segments: + prim_ports.add(self.segmentMap[psid].port) + for msid in self.cluster_state[(m.dstHost.hostname, m.dstHost.address)].mirror_segments: + mir_ports.add(self.segmentMap[msid].port) + if len(m.dstHost.primary_datadirs) == 0: + prirmary_prefix = DEFAULT_PRIMARY_PREF + if not self.options.silent: + prirmary_prefix = ask_input(f"\nThe segment (dbid={m.segid.dbid}, content={m.segid.contentid}) " + f"is about to be moved to host {m.dstHost.hostname}, but no primary datadirs " + "are specified for the host.", "Enter the primary datadir prefix",f" (default={DEFAULT_PRIMARY_PREF})", + DEFAULT_PRIMARY_PREF, datadir_validator, None) + m.dstHost.primary_datadirs.add(prirmary_prefix.strip()) + if self.gparr.hasMirrors and len(m.dstHost.mirror_datadirs) == 0: + mirror_prefix = DEFAULT_MIRROR_PREF + if not self.options.silent: + mirror_prefix = ask_input(f"\nThe segment (dbid={m.segid.dbid}, content={m.segid.contentid}) " + f"is about to be moved to host {m.dstHost.hostname}, but no mirror datadirs " + "are specified for the host.", "Enter the mirror datadir prefix",f" (default={DEFAULT_MIRROR_PREF})", + DEFAULT_MIRROR_PREF, datadir_validator, None) + m.dstHost.mirror_datadirs.add(mirror_prefix.strip()) + + resources[m.dstHost] = HostResources( + m.dstHost, (prim_ports, mir_ports)) + return resources + + def _disk_usage(self, hostaddr: str, dirs: List[str]) -> Dict[str, int]: + """ + Get the Disk usage for the given set of directories to the targeted host + input: hostaddr , host from which the disk usage is fetched + input: dirs, list of directories to fetch the details + output: dictionary containing directories with it's disk usage stats in kb(kilo byte) + """ + dirs_disk_usage = {} # map of directories to disk usage + + if len(dirs) <= 0: + return dirs_disk_usage + + pool = WorkerPool(numWorkers=min(len(dirs), self.options.batch_size)) + try: + for directory in dirs: + cmd = DiskUsage('check source segments disk space used', + directory, ctxt=REMOTE, remoteHostAddr=hostaddr) + pool.addCommand(cmd) + pool.join() + finally: + pool.haltWork() + pool.joinWorkers() + + for cmd in pool.getCompletedItems(): + if not cmd.was_successful(): + raise Exception("Unable to check disk usage on source segment: {}" .format( + cmd.get_results().stderr)) + + dirs_disk_usage[cmd.directory] = cmd.kbytes_used() + + return dirs_disk_usage + + def estimateSegmentSizes(self, seglist: List[SegmentId]) -> Dict[SegmentId, SegmentSize]: + if not seglist: + return {} + oid_subq = """ (SELECT * + FROM ( + SELECT oid FROM pg_tablespace + WHERE spcname NOT IN ('pg_default', 'pg_global') + ) AS _q1, + LATERAL gp_tablespace_location(_q1.oid) + ) AS t """ + segment_dbids = ','.join(f'({seg.dbid})' for seg in seglist) + tablespace_location_sql = """ + SELECT c.dbid, c.content, t.tblspc_loc||'/'||c.dbid tblspc_loc + FROM {oid_subq} + JOIN gp_segment_configuration AS c + ON t.gp_segment_id = c.content WHERE c.dbid in (VALUES {segment_ids_str}) + """ .format(oid_subq=oid_subq, segment_ids_str=segment_dbids) + cursor = dbconn.query(self.conn, tablespace_location_sql) + tablespaces = defaultdict(list) + for dbid, content, loc in cursor: + tablespaces[SegmentId(dbid, content)].append(loc) + + segmentSizes = {} + for segid in seglist: + sourceSeg = self.segmentMap[segid] + source_data_dir_usage = self._disk_usage( + sourceSeg.address, [sourceSeg.datadir]) + segmentSizes[segid] = SegmentSize( + source_data_dir_usage[sourceSeg.datadir], None) + for segid, tblspace_dirs in tablespaces.items(): + sourceSeg = self.segmentMap[segid] + source_tblsps_usage = self._disk_usage( + sourceSeg.address, tblspace_dirs) + segmentSizes[segid].source_tablespace_usage = source_tblsps_usage + + return segmentSizes + + def _prepare_swaps(self, swaps: List[Tuple[Move, Move]]): + """ + Choose the target directory for swap case: + 1. primary is moved to mirror dir in its own host + 2. mirror is moved to primary dir in its own host + 3. role switching takes place + """ + for primary_move, mirror_move in swaps: + #important notice. srcHost and dstHost are different + # objects even if they are describing the same host. + # This happens due to code in get_moves_between_states(state1, state2) + # srcHost and dstHost are taken from state1 and state2 correspondingly + primary_host = mirror_move.dstHost + mirror_host = primary_move.dstHost + + primary_id = primary_move.segid + + # define datadir + for datadir in primary_host.mirror_datadirs: + try: + self.resources[primary_host].accommodate_segment( + self.segmentSizes[primary_id], datadir) + primary_move.target_datadir = datadir + f"/{segment_prefix}{primary_move.segid.contentid}" + break + except: + continue + if primary_move.target_datadir == None: + raise NoValidDataDirectories(f"Host {primary_host.hostname} does not have any valid mirror " + f"datadirs for segment {primary_move.segid}. None of the " + f"{primary_host.mirror_datadirs} either exists or has " + "enough free space for segment movement") + primary_move.dstHost = primary_host + for datadir in mirror_host.primary_datadirs: + try: + self.resources[mirror_host].accommodate_segment( + self.segmentSizes[primary_id], datadir) + mirror_move.target_datadir = datadir + f"/{segment_prefix}{mirror_move.segid.contentid}" + break + except: + continue + if mirror_move.target_datadir == None: + raise NoValidDataDirectories(f"Host {mirror_host.hostname} does not have any valid primary " + f"datadirs for segment {mirror_move.segid}. None of the " + f"{mirror_host.primary_datadirs} either exists or has " + "enough free space for segment movement") + mirror_move.dstHost = mirror_host + + primary_move.target_port = self.resources[primary_host].can_accommodate_port( + True, primary_id.contentid) + if not primary_move.target_port: + raise Exception("Cannot accomodate port") + self.resources[primary_host].reserve_port(primary_move.target_port, True) + mirror_move.target_port = self.resources[mirror_host].can_accommodate_port( + False, primary_id.contentid) + if not mirror_move.target_port: + raise Exception("Cannot accomodate port") + self.resources[mirror_host].reserve_port(mirror_move.target_port,False) + + def _prepare_pms(self, primary_mirrors: List[Tuple[Move, Move]]): + """ + Choose the target directory for primary-mirror move case: + 1. mirror is moved to primary dir in primary's target host + 2. role switching takes place + 2. primary is moved to mirror dir in mirror's target host + """ + for primary_move, mirror_move in primary_mirrors: + primary_host = primary_move.dstHost + mirror_host = mirror_move.dstHost + + primary_id = primary_move.segid + mirror_id = mirror_move.segid + + # define datadir + for datadir in mirror_host.mirror_datadirs: + try: + self.resources[primary_host].accommodate_segment( + self.segmentSizes[primary_id], datadir) + primary_move.target_datadir = datadir + f"/{segment_prefix}{primary_move.segid.contentid}" + break + except Exception as e: + self.logger.error(str(e)) + continue + primary_move.dstHost = mirror_host + if primary_move.target_datadir == None: + raise NoValidDataDirectories(f"Host {mirror_host.hostname} does not have any valid mirror " + f"datadirs for segment {primary_move.segid}. None of the " + f"{mirror_host.mirror_datadirs} either exists or has " + "enough free space for segment movement") + for datadir in primary_host.primary_datadirs: + try: + self.resources[mirror_host].accommodate_segment( + self.segmentSizes[primary_id], datadir) + mirror_move.target_datadir = datadir + f"/{segment_prefix}{mirror_move.segid.contentid}" + break + except: + continue + if mirror_move.target_datadir == None: + raise NoValidDataDirectories(f"Host {primary_host.hostname} does not have any valid primary " + f"datadirs for segment {mirror_move.segid}. None of the " + f"{primary_host.primary_datadirs} either exists or has " + "enough free space for segment movement") + mirror_move.dstHost = primary_host + + primary_move.target_port = self.resources[mirror_host].can_accommodate_port( + True, primary_id.contentid) + if not primary_move.target_port: + raise Exception("Cannot accomodate port") + self.resources[mirror_host].reserve_port(primary_move.target_port,True) + mirror_move.target_port = self.resources[primary_host].can_accommodate_port( + False, primary_id.contentid) + if not mirror_move.target_port: + raise Exception("Cannot accomodate port") + self.resources[primary_host].reserve_port(mirror_move.target_port,False) + + def _prepare_ps(self, primaries: List[Move]): + """ + Choose the target directory for primary-only move case: + 1. role switch takes place + 2. primary is moved to target dir + 3. role switch + """ + for primary_move in primaries: + primary_host = primary_move.dstHost + + primary_id = primary_move.segid + # define datadir + for datadir in primary_host.primary_datadirs: + try: + self.resources[primary_host].accommodate_segment( + self.segmentSizes[primary_id], datadir) + primary_move.target_datadir = datadir + f"/{segment_prefix}{primary_move.segid.contentid}" + break + except: + continue + if primary_move.target_datadir == None: + raise NoValidDataDirectories(f"Host {primary_host.hostname} does not have any valid primary " + f"datadirs for segment {primary_move.segid}. None of the " + f"{primary_host.primary_datadirs} either exists or has " + "enough free space for segment movement") + + primary_move.target_port = self.resources[primary_host].can_accommodate_port( + True, primary_id.contentid) + if not primary_move.target_port: + raise Exception("Cannot accomodate port") + self.resources[primary_host].reserve_port(primary_move.target_port ,True) + + def _prepare_ms(self, mirrors: List[Move]): + """ + Choose the target directory for mirror-only move case: + 1. mirror is moved to mirror dir in mirror's target host + """ + for mirror_move in mirrors: + mirror_host = mirror_move.dstHost + + mirror_id = mirror_move.segid + # define datadir + for datadir in mirror_host.mirror_datadirs: + try: + self.resources[mirror_host].accommodate_segment( + self.segmentSizes[mirror_id], datadir) + mirror_move.target_datadir = datadir + f"/{segment_prefix}{mirror_move.segid.contentid}" + break + except: + continue + if mirror_move.target_datadir == None: + raise NoValidDataDirectories(f"Host {mirror_host.hostname} does not have any valid mirror " + f"datadirs for segment {mirror_move.segid}. None of the " + f"{mirror_host.mirror_datadirs} either exists or has " + "enough free space for segment movement") + + mirror_move.target_port = self.resources[mirror_host].can_accommodate_port( + True, mirror_id.contentid) + if not mirror_move.target_port: + raise Exception("Cannot accomodate port") + self.resources[mirror_host].reserve_port(mirror_move.target_port ,True) + + + def _classify_moves(self) -> Tuple[List[Tuple[Move, Move]], List[Tuple[Move, Move]], List[Move], List[Move]]: + """ + Classify moves into: + - pure_swaps: pairs of moves where primary and mirror just switch places + - primary_moves_with_mirrors: pairs of moves where we move both primary and mirror + - primary_moves: independent primary moves + - mirror_moves: independent mirror moves + """ + # Group moves by contentid + moves_by_content = defaultdict(list) + for move in self.moves: + moves_by_content[move.segid.contentid].append(move) + + primary_moves_with_mirrors = [] + primary_moves = [] + mirror_moves = [] + pure_swaps = [] + + for contentid, moves in moves_by_content.items(): + if len(moves) == 2: + primary_move = next( + (m for m in moves if not m.is_mirror), None) + mirror_move = next((m for m in moves if m.is_mirror), None) + + if (primary_move and mirror_move and + primary_move.srcHost == mirror_move.dstHost and + primary_move.dstHost == mirror_move.srcHost): + pure_swaps.append((primary_move, mirror_move)) + elif primary_move and mirror_move: + # This is a primary move with corresponding mirror move + primary_moves_with_mirrors.append( + (primary_move, mirror_move)) + + elif len(moves) == 1: + move = moves[0] + if move.is_mirror: + mirror_moves.append(move) + else: + primary_moves.append(move) + + return pure_swaps, primary_moves_with_mirrors, primary_moves, mirror_moves + + def _create_move_sequence(self) -> Tuple[List[List[Move]], Set[Move], Set[Move]]: + """Create full sequence of moves""" + sequences = [] + current_batch = [] + + pure_swaps, primary_mirrors, primaries, mirrors = self._classify_moves() + self._prepare_swaps(pure_swaps) + self._prepare_pms(primary_mirrors) + self._prepare_ps(primaries) + self._prepare_ms(mirrors) + + former_switches = set() + latter_switches = set() + + current_batch = [] + for mirror_move in mirrors: + if len(current_batch) >= self.options.batch_size: + sequences.append(current_batch) + current_batch = [] + current_batch.append(mirror_move) + + # Pure swaps: move mirrors to primary dirs + for primary_move, mirror_move in pure_swaps: + if len(current_batch) >= self.options.batch_size: + sequences.append(current_batch) + current_batch = [] + current_batch.append(mirror_move) + + # Primary-mirror pairs: move mirrors to primary's target + for primary_move, mirror_move in primary_mirrors: + if len(current_batch) >= self.options.batch_size: + sequences.append(current_batch) + current_batch = [] + current_batch.append(mirror_move) + + if current_batch: + sequences.append(current_batch) + + # First switch point - affects: + # - Pure swaps + # - Primary-mirror pairs + # - First switch for primary-only moves + segments_for_switch1 = [] + segments_for_switch1.extend([pm[0].segid for pm in pure_swaps]) + segments_for_switch1.extend([pm[0].segid for pm in primary_mirrors]) + # First switch for primaries + segments_for_switch1.extend([pm.segid for pm in primaries]) + + if segments_for_switch1: + sequences.append(['SWITCH', segments_for_switch1]) + for seg in segments_for_switch1: + former_switches.add(seg.contentid) + # Phase 3: Post-first-switch moves + current_batch = [] + + # Pure swaps: move ex-primaries to mirror dirs + for primary_move, _ in pure_swaps: + if len(current_batch) >= self.options.batch_size: + sequences.append(current_batch) + current_batch = [] + current_batch.append(primary_move) + + # Primary-mirror pairs: move ex-primaries to target mirror dirs + for primary_move, _ in primary_mirrors: + if len(current_batch) >= self.options.batch_size: + sequences.append(current_batch) + current_batch = [] + current_batch.append(primary_move) + + # Primary-only: move ex-primaries to target primary dirs + for primary_move in primaries: + if len(current_batch) >= self.options.batch_size: + sequences.append(current_batch) + current_batch = [] + current_batch.append(primary_move) + + if current_batch: + sequences.append(current_batch) + + # Second switch point - affects: + # - Second switch for primary-only moves + if primaries: + sequences.append(['SWITCH', [pm.segid for pm in primaries]]) + for seg in primaries: + latter_switches.add(seg.segid.contentid) + + return sequences, former_switches, latter_switches + + def execute_moves(self, firstRun=True): + """Main execution method""" + try: + if not firstRun: + raise NotImplementedError( + "rebalance rerun is not implemented properly yet") + + move_sequences, former_switches, latter_switches = self._create_move_sequence() + + global begining_timestamp + begining_timestamp = datetime.datetime.now() + + conf_dir = self.options.coordinator_data_directory + CONF_DIR + hosts = set() + for move in self.moves: + hosts.add(move.srcHost.hostname) + hosts.add(move.dstHost.hostname) + + if firstRun: + # in order to run gprecoverseg processes separately and avoid any races + # for resources gprecoverseg generates, we create separate log directories. + mkdirCmd = MakeDirectory("rebalance log dir", GPRECOVERSEG_DIR) + mkdirCmd.run(validateAfter=True) + for host in hosts: + mkdirCmd = MakeDirectory("rebalance log dir", GPRECOVERSEG_DIR, ctxt=REMOTE, remoteHost=host) + mkdirCmd.run(validateAfter=True) + self.statusManager.set_status('EXECUTION_PREPARED') + self.statusManager.set_db_status( + RebalanceStatus.PREPARED, begining_timestamp) + self.plan.save_to_file(conf_dir, "plan") + + self.statusManager.set_db_status( + RebalanceStatus.IN_PROGRESS) + + self.queue = WorkerPool(self.options.parallel) + + stopTime = None + stoppedEarly = False + had_error = False + if self.options.end: + stopTime = self.options.end + self.statusManager.set_status('EXECUTION_STARTED') + for sequence in move_sequences: + if self.shutdown_requested: + break + if isinstance(sequence[0], str) and sequence[0] == 'SWITCH': + + while not self.queue.isDone(): + if stopTime and datetime.datetime.now() >= stopTime: + stoppedEarly = True + break + time.sleep(5) + + for moveCommand in self.queue.getCompletedItems(): + if moveCommand.move_error: + had_error = True + break + + if stoppedEarly or had_error: + break + + if self.shutdown_requested: + break + + self.statusManager.set_db_status( + RebalanceStatus.AWAITS_SWITCH) + self.logger.info( + f"Executing role swaps for {len(sequence[1])} segments") + try: + self._execute_role_swaps(sequence[1]) + except Exception as e: + had_error = True + self.logger.error(f"Could not execute role swaps:{str(e)}") + break + + self.statusManager.set_db_status( + RebalanceStatus.IN_PROGRESS) + else: + for move in sequence: + if self.shutdown_requested: + break + segid = move.segid + needs_switch = False + if segid.contentid in former_switches or segid.contentid in latter_switches: + needs_switch = True + step_details = (self.segmentMap[segid], + move, + self.segmentSizes[segid], + conf_dir, + needs_switch, + self.options + ) + + cmd = SingleMoveCommand( + "name", self.dburl, step_details, self.logger) + self.queue.addCommand(cmd) + + while not self.queue.isDone(): + if stopTime and datetime.datetime.now() >= stopTime: + stoppedEarly = True + break + time.sleep(5) + if stoppedEarly: + self.logger.info("Execution timeout is reached. Waiting the existing jobs to finish " + "and stopping rebalance.") + break + for moveCommand in self.queue.getCompletedItems(): + if moveCommand.move_error: + had_error = True + break + + if self.queue: + self.queue.haltWork() + self.queue.joinWorkers() + self.queue = None + + if stoppedEarly or self.shutdown_requested: + self.statusManager.set_db_status( + RebalanceStatus.STOPPED) + self.statusManager.set_status('EXECUTION_STOPPED') + if not self.shutdown_requested: + self.logger.info("Rebalance stopped due to timeout") + elif had_error: + self.statusManager.set_db_status( + RebalanceStatus.FAILED) + raise Exception("execution encountered movement erorrs") + else: + self.statusManager.set_status('EXECUTION_DONE') + self.statusManager.set_db_status( + RebalanceStatus.COMPLETED) + rmdirCmd = RemoveDirectory("remove recoverseg log dir", f"{os.path.join(os.environ.get('HOME', '.'),GPRECOVERSEG_DIR)}") + rmdirCmd.run() + for host in hosts: + rmdirCmd = RemoveDirectory("remove recoverseg log dir", f"{os.path.join(os.environ.get('HOME', '.'),GPRECOVERSEG_DIR)}", ctxt=REMOTE, remoteHost=host) + rmdirCmd.run() + + + except Exception as e: + self.statusManager.set_status('EXECUTION_FAILED') + raise + + def _execute_role_swaps(self, segids: List[SegmentId]): + """Execute multiple role swaps in single gprecoverseg -r call""" + if not segids: + return + try: + with self.conn.cursor() as cur: + cur.execute("BEGIN") + cur.execute("SET allow_system_table_mods=1;") + data = tuple([segid.contentid for segid in segids]) + cur.execute("UPDATE gp_segment_configuration SET preferred_role = 't' WHERE " + "content IN %s AND preferred_role = 'm'", (data,)) + cur.execute("UPDATE gp_segment_configuration SET preferred_role = 'm' WHERE " + "content IN %s AND preferred_role = 'p'", (data,)) + cur.execute("UPDATE gp_segment_configuration SET preferred_role = 'p' WHERE " + "content IN %s AND preferred_role = 't'", (data,)) + cur.execute("UPDATE gprebalance.status_detail SET status = 'PENDING' WHERE " + "status = 'awaits_switch'") + cur.execute("COMMIT") + except Exception as e: + raise Exception('could not execute SQL : %s' % str(e)) + + recoversegOptions = f"-r -a -l {os.path.join(os.environ.get('HOME', '.'),GPRECOVERSEG_DIR)}" + if self.options.hba_hostnames: + recoversegOptions += " --hba-hostnames" + cmd = GpRecoverSeg("Running gprecverseg", options=recoversegOptions) + cmd.run(validateAfter=True) + + + def shutdown(self): + # the execution shutdown assumes finishing + # current jobs in queue + if self.queue: + self.logger.info("Shutdown requested, will complete current jobs...") + self.shutdown_requested = True + self.queue.haltWork() + self.queue.joinWorkers() + self.queue = None diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_plan.py b/gpMgmt/bin/gprebalance_modules/rebalance_plan.py index 703d719e6582..51caf218e1da 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_plan.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_plan.py @@ -1,4 +1,6 @@ import copy +import os +import pickle import random import math from typing import List, Set, Dict @@ -13,7 +15,8 @@ class Move: srcHost: Host dstHost: Host is_mirror: bool - final_datadir: str + target_datadir: str + target_port: int class Plan: @@ -21,8 +24,8 @@ def __init__(self): self.in_conf = None self.moves = [] self.out_conf = None + self.segmentMap = None self.roles = [] - self.number_of_moves = 0 def __str__(self): finalStr = "" @@ -35,6 +38,13 @@ def __str__(self): finalStr += f"H{m.srcHost.hostname} -> H{m.dstHost.hostname}\n" return finalStr + def save_to_file(self, directory: str, filename: str) -> str: + file_path = os.path.join(directory, f"{filename}.pkl") + + # Pickle and save the plan + with open(file_path, 'wb') as f: + pickle.dump(self, f) + class NoValidMovesError(Exception): def __init__(self, msg="No valid targets for selected segment"): @@ -281,9 +291,9 @@ def getPlan(self, finalState: ClusterState) -> Plan: moves, roles = self.get_moves_between_states(in_conf, finalState) plan = Plan() plan.moves = moves - plan.number_of_moves = len(moves) plan.in_conf = in_conf plan.out_conf = finalState + plan.segmentMap = self.initialSegmentMap plan.roles = roles return plan @@ -299,12 +309,15 @@ def get_moves_between_states(self, state1: ClusterState, state2: ClusterState): for contentid, target_location in primary_map2.items(): current_location = primary_map1.get(contentid) if current_location and current_location != target_location: + #important notice. srcHost and dstHost are different + # objects even if they are describing the same host. + # srcHost and dstHost are taken from state1 and state2 correspondingly source_host = state1[current_location] - target_host = state1[target_location] + target_host = state2[target_location] segid = next(seg for seg in source_host.primary_segments if seg.contentid == contentid) moves.append( - Move(segid, source_host, target_host, False, None)) + Move(segid, source_host, target_host, False, None, None)) roles[segid] = 'p' # Find mirror segment moves @@ -312,10 +325,11 @@ def get_moves_between_states(self, state1: ClusterState, state2: ClusterState): current_location = mirror_map1.get(contentid) if current_location and current_location != target_location: source_host = state1[current_location] - target_host = state1[target_location] + target_host = state2[target_location] segid = next(seg for seg in source_host.mirror_segments if seg.contentid == contentid) - moves.append(Move(segid, source_host, target_host, True, None)) + moves.append( + Move(segid, source_host, target_host, True, None, None)) roles[segid] = 'm' moves.sort(key=lambda m: 1 if m.is_mirror else 0) diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_status.py b/gpMgmt/bin/gprebalance_modules/rebalance_status.py new file mode 100644 index 000000000000..a203a843b3b2 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_status.py @@ -0,0 +1,208 @@ +from datetime import datetime +import os +from gppylib.db import dbconn +from enum import Enum +from typing import Dict, List + +from gprebalance_modules.rebalance_plan import Move +from gprebalance_modules.rebalance import SegmentId, Segment, SegmentSize + +create_schema_sql = "CREATE SCHEMA gprebalance" +drop_schema_sql = "DROP SCHEMA IF EXISTS gprebalance CASCADE" + +status_table_sql = """CREATE TABLE gprebalance.status + ( status varchar(255), + updated timestamp ) """ + +status_detail_table_sql = """CREATE TABLE gprebalance.status_detail + ( step_id serial, + dbid smallint, + content smallint, + role char, + preferred_role char, + source_hostname varchar(255), + source_port integer, + source_datadir varchar(255), + dest_hostname varchar(255), + target_datadir varchar(255), + target_port integer, + status varchar(255), + rebalance_started timestamp, + rebalance_finished timestamp, + source_kbytes numeric )""" + + +class MoveStatus(Enum): + PENDING = "pending" + IN_PROGRESS = "in_progress" + AWAITS_SWITCH = "awaits_switch" + COMPLETED = "completed" + FAILED = "failed" + STOPPED = "stopped" + REVERTED = "reverted" + + +class RebalanceStatus(Enum): + INITIALIZED = "initialized" + PREPARED = "prepared" + IN_PROGRESS = "in_progress" + COMPLETED = "completed" + FAILED = "failed" + STOPPED = "stopped" + AWAITS_SWITCH = "awaits_switch" + + +class InvalidStatusError(Exception): + pass + + +class SqlError(Exception): + pass + + +class StatusManager: + def __init__(self, conn: dbconn.Connection, options, logger): + + self.main_conn = conn + self.options = options + self.logger = logger + + self._status_filename = self.options.coordinator_data_directory + '/gprebalance.status' + self._fp = None + self._status = [] + self._status_info = [] + self._status_values = {'UNINITIALIZED': 1, + 'VALIDATED': 2, + 'PLANNED': 3, + 'REBALANCE_PREPARE_SCHEMA_STARTED': 4, + 'REBALANCE_PREPARE_SCHEMA_DONE': 5, + 'EXECUTION_PREPARED': 6, + 'EXECUTION_STARTED': 7, + 'EXECUTION_FAILED' : 8, + 'EXECUTION_STOPPED' : 9, + 'EXECUTION_DONE': 10, + } + self.conf_dir = None + self.target_hosts_filename = None + if os.path.exists(self._status_filename): + self._read_status_file() + + def _read_status_file(self): + self.logger.debug( + "Trying to read in a pre-existing gprebalance status file") + try: + self._fp = open(self._status_filename, 'a+') + self._fp.seek(0) + + for line in self._fp: + (status, status_info) = line.rstrip().split(':') + if status == 'PLANNED': + self.conf_dir = status_info + elif status == 'VALIDATED': + self.target_hosts_filename = status_info + self._status.append(status) + self._status_info.append(status_info) + except IOError: + raise + + if self._status[-1] not in self._status_values: + raise InvalidStatusError( + 'Invalid status file. Unknown status %s' % self._status) + + def create_status_file(self): + """Creates a new gpexpand status file""" + try: + self._fp = open(self._status_filename, 'w') + self._fp.write('UNINITIALIZED:None\n') + self._fp.flush() + os.fsync(self._fp) + self._status.append('UNINITIALIZED') + self._status_info.append('None') + except IOError: + raise + + def get_current_status(self): + """Gets the current status that has been written to the gpexpand + status file""" + if (len(self._status) > 0 and len(self._status_info) > 0): + return (self._status[-1], self._status_info[-1]) + else: + return (None, None) + + def set_status(self, status, status_info=None): + if not self._fp: + raise InvalidStatusError( + 'The status file is invalid and cannot be written to') + if status not in self._status_values: + raise InvalidStatusError( + '%s is an invalid gpexpand status' % status) + self._fp.write('%s:%s\n' % (status, status_info)) + self._fp.flush() + os.fsync(self._fp) + self._status.append(status) + self._status_info.append(status_info) + + def setup_schema(self): + dbconn.execSQL(self.main_conn, create_schema_sql) + dbconn.execSQL(self.main_conn, status_table_sql) + dbconn.execSQL(self.main_conn, status_detail_table_sql) + + def cleanup_schema(self): + dbconn.execSQL(self.main_conn, drop_schema_sql) + + def set_db_status(self, status: RebalanceStatus, stamp: datetime = None): + if not stamp: + stamp = datetime.now() + sql = "INSERT INTO gprebalance.status VALUES ('%s', '%s')" % ( + status.value, stamp) + dbconn.execSQL(self.main_conn, sql) + + @staticmethod + def record_move(conn: dbconn.Connection, move: Move, segment: Segment, source_kbytes: int, start_time: datetime): + """Record initial move details""" + + sql = """ + INSERT INTO gprebalance.status_detail ( + dbid, content, role, preferred_role, + source_hostname, source_port, source_datadir, + dest_hostname, target_datadir, target_port, status, rebalance_started, source_kbytes + ) VALUES ( + %d, %d, '%s', '%s', '%s', %d, '%s', '%s', '%s', %d, '%s', '%s', %d + ) + """ % ( + segment.dbid, + segment.content, + segment.role, + segment.preferred_role, + move.srcHost.hostname, + segment.port, + segment.datadir, + move.dstHost.hostname, + move.target_datadir, + move.target_port, + MoveStatus.PENDING.value, + str(start_time), + source_kbytes + ) + dbconn.execSQL(conn, sql) + + @staticmethod + def update_record_status(conn: dbconn.Connection, dbids: List[int], status: MoveStatus): + """Record initial move details""" + dbid_str = ','.join(str(dbid) for dbid in dbids) + if status == MoveStatus.COMPLETED: + sql = "UPDATE gprebalance.status_detail SET status = '%s' , rebalance_finished = '%s' WHERE dbid IN (%s)" % ( + status.value, datetime.now(), dbid_str + ) + else: + sql = "UPDATE gprebalance.status_detail SET status = '%s' WHERE dbid IN (%s)" % ( + status.value, dbid_str + ) + dbconn.execSQL(conn, sql) + + def remove_all(self): + if self._fp: + self._fp.close() + self._fp = None + if os.path.exists(self._status_filename): + os.unlink(self._status_filename) diff --git a/gpMgmt/test/behave/mgmt_utils/gprebalance.feature b/gpMgmt/test/behave/mgmt_utils/gprebalance.feature new file mode 100644 index 000000000000..0e4872e3e103 --- /dev/null +++ b/gpMgmt/test/behave/mgmt_utils/gprebalance.feature @@ -0,0 +1,15 @@ +@gprebalance +Feature: Tests for gprebalance + Scenario: gprebalance balances cluster after expansion on fixed hosts + Given the database is not running + And a working directory of the test as '/data/gpdata/gpexpand' + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into + And a cluster is created with mirrors on "cdw" and "sdw1,sdw2" + And the cluster is setup for an expansion on hosts "cdw,sdw1,sdw2" + And the number of segments have been saved + When the user runs gpexpand with segment imbalance for a two-node cluster with mirrors + Then verify that the cluster has 4 new segments + When the user runs "gprebalance -s" + Then gprebalance should return a return code of 0 + When the user runs "gprebalance -s" + Then gprebalance should print "[INFO]:-Cluster is already balanced" escaped to stdout diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 378e82c7aa9c..022d51ebb69e 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -3029,6 +3029,24 @@ def impl(context): gpexpand = Gpexpand(context, working_directory=context.working_directory) gpexpand.initialize_segments() +@given('the user runs gpexpand with segment imbalance for a two-node cluster with mirrors') +@when('the user runs gpexpand with segment imbalance for a two-node cluster with mirrors') +def impl(context): + inputfile_contents = """ +sdw2|sdw2|20502|/tmp/gpexpand_behave/two_nodes/data/primary/gpseg4|10|4|p +sdw2|sdw2|21502|/tmp/gpexpand_behave/two_nodes/data/mirror/gpseg4|11|4|m +sdw2|sdw2|20503|/tmp/gpexpand_behave/two_nodes/data/primary/gpseg5|12|5|p +sdw2|sdw2|21503|/tmp/gpexpand_behave/two_nodes/data/mirror/gpseg5|13|5|m""" + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + inputfile_name = "%s/gpexpand_inputfile_%s" % (context.working_directory, timestamp) + with open(inputfile_name, 'w') as fd: + fd.write(inputfile_contents) + + gpexpand = Gpexpand(context, working_directory=context.working_directory) + ret_code, std_err, std_out = gpexpand.initialize_segments() + if ret_code != 0: + raise Exception("gpexpand exited with return code: %d.\nstderr=%s\nstdout=%s" % (ret_code, std_err, std_out)) + @given('the coordinator pid has been saved') def impl(context): data_dir = os.path.join(context.working_directory, From fcc28c4f6efc41a95145b83bdd200d9e6badb7c4 Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Thu, 20 Mar 2025 09:01:07 +0300 Subject: [PATCH 06/22] Rollback handler (#1265) This PR intoduces the rollback handler in gprebalance MVP. The rollback function creates new plan of movements by calculating the difference between current configuration and original state loaded from previously pickled plan. --- gpMgmt/bin/gprebalance | 39 ++++++++++--- gpMgmt/bin/gprebalance_modules/rebalance.py | 50 ++++++++++++++++- .../gprebalance_modules/rebalance_executor.py | 56 +++++++++++++++---- .../gprebalance_modules/rebalance_status.py | 9 ++- 4 files changed, 131 insertions(+), 23 deletions(-) diff --git a/gpMgmt/bin/gprebalance b/gpMgmt/bin/gprebalance index e613505026df..74f569eb0385 100755 --- a/gpMgmt/bin/gprebalance +++ b/gpMgmt/bin/gprebalance @@ -90,7 +90,7 @@ def parseargs(): parser.add_option('-f', '--target-hosts', metavar='', dest='filename', help='yaml containing target hosts configuration') parser.add_option('-r', '--rollback', action='store_true', - help='remove the rebalance schema.') + help='bring the cluster back to the state before rebalance') parser.add_option('-d', '--duration', type='duration', metavar='[h][:m[:s]]', help='duration from beginning to end.') parser.add_option('-e', '--end', type='datetime', metavar='datetime', @@ -367,6 +367,7 @@ def main(options, args, parser): configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) configurationInterface.getConfigurationProvider( ).initializeProvider(gpEnv.getCoordinatorPort()) + dburl = dbconn.DbURL(dbname=DBNAME, port=gpEnv.getCoordinatorPort()) try: @@ -379,6 +380,9 @@ def main(options, args, parser): " and that your environment is correct, then rerun" " gprebalance" + ' '.join(sys.argv[1:])) sys.exit(1) + except ConnectionError as ex: + logger.error(f"{str(ex)}") + sys.exit(1) # create pid file create_pid_file(options.coordinator_data_directory) @@ -411,11 +415,19 @@ def main(options, args, parser): from gprebalance_modules.rebalance_status import RebalanceStatus # nopep8 if (gprebalance_db_status == RebalanceStatus.COMPLETED or - gprebalance_file_status == 'EXECUTION_DONE'): + gprebalance_file_status == 'EXECUTION_DONE') and not options.rollback: logger.info('Rebalance has already completed') logger.info( 'If you want to rebalance again, run gprebalance -c to perform cleanup') - elif gprebalance_file_status is None and gprebalance_db_status is None: + elif options.rollback and gprebalance_file_status == 'ROLLBACK_DONE': + logger.info('Rollback has already completed') + logger.info( + 'If you want to rebalance again, run gprebalance -c to perform cleanup') + elif (gprebalance_file_status is None and gprebalance_db_status is None) or \ + gprebalance_file_status == 'UNINITIALIZED': + if options.rollback: + logger.error("Cannot perform rollback operation. Rebalance session status files are absent.") + sys.exit(1) gprebalance.statusManager.create_status_file() possibility_checks(gprebalance) logger.info('Validation passed. Preparing rebalance...') @@ -429,7 +441,7 @@ def main(options, args, parser): gprebalance.setup_schema() gprebalance.execute_plan(plan) elif gprebalance.statusManager.conf_dir: - plan = gprebalance.load_plan(gprebalance.statusManager.conf_dir) + plan = gprebalance.load_plan(gprebalance.statusManager.conf_dir, False) if options.show_plan: logger.info(f"Resulting plan: \n{str(plan)}") sys.exit(0) @@ -437,11 +449,24 @@ def main(options, args, parser): gprebalance.setup_schema() gprebalance.execute_plan(plan) sys.exit(0) + elif gprebalance_file_status == 'ROLLBACK_FAILED' or \ + gprebalance_file_status == 'ROLLBACK_STOPPED': + logger.info('Previous rollback session was interrupted due to an error. Trying to resume...') + plan = gprebalance.load_plan(gprebalance.statusManager.conf_dir, True) + gprebalance.resume(plan) + sys.exit(0) elif gprebalance_file_status == 'EXECUTION_FAILED': - logger.info('Previous rebalance session was interrupted due to an error. Trying to resume...') + if not options.rollback: + logger.info('Previous rebalance session was interrupted due to an error. Trying to resume...') + gprebalance.resume(plan) + sys.exit(0) elif gprebalance_file_status == 'EXECUTION_STOPPED': - logger.info('Previous rebalance session was interrupted due to an timeout or user signal. Trying to resume...') - gprebalance.resume() + if not options.rollback: + logger.info('Previous rebalance session was interrupted due to an timeout or user signal. Trying to resume...') + gprebalance.resume(plan) + sys.exit(0) + if options.rollback: + gprebalance.rollback(plan) sys.exit(0) except Exception as e: if options and options.verbose: diff --git a/gpMgmt/bin/gprebalance_modules/rebalance.py b/gpMgmt/bin/gprebalance_modules/rebalance.py index c5503241122d..66a40e38ff50 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance.py @@ -230,8 +230,12 @@ def save_plan(self, plan: Plan): plan.save_to_file(datadir, "plan") self.statusManager.set_status('PLANNED', datadir) - def load_plan(self, datadir): - filename = datadir + "/plan.pkl" + def load_plan(self, datadir, rollback): + filename = datadir + if rollback: + filename += "/rollback_plan.pkl" + else: + filename += "/plan.pkl" if not os.path.exists(filename): raise FileNotFoundError(f"No pickle file found at {filename}") with open(filename, 'rb') as f: @@ -327,8 +331,48 @@ def remove_status_file(self): self.logger.info('Dropping status file') if self.statusManager: self.statusManager.remove_all() + + + def rollback(self, plan:Plan): + new_plan = Plan() + cb = ClusterBalancer(self.current_conf, (set(), set()), + self.segmentMap, 0, MirrorStrategy.MIRRORLESS) + new_moves, rls = cb.get_moves_between_states(self.current_conf, plan.in_conf) + for new_move in new_moves: + seg = plan.segmentMap[new_move.segid] + if self.segmentMap[new_move.segid].role == seg.role: + new_move.target_datadir = seg.datadir + new_move.target_port = seg.port + else: + for id, pair_seg in plan.segmentMap.items(): + if new_move.segid.contentid == id.contentid and new_move.segid.dbid != id.dbid: + assert(pair_seg.role == self.segmentMap[new_move.segid].role) + new_move.target_datadir = pair_seg.datadir + new_move.target_port = pair_seg.port + break + for segid, current_seg in self.segmentMap.items(): + target_seg = plan.segmentMap[segid] + if (current_seg.hostname, current_seg.address) == (target_seg.hostname, target_seg.address) \ + and current_seg.role == target_seg.role and current_seg.datadir != target_seg.datadir: + # Are there any cases besides swap when previously rebalance moved segment + # to the same host but to different dir? + assert(current_seg.role == 'm') + move = Move(segid, self.current_conf[(current_seg.hostname, current_seg.address)], + self.current_conf[(current_seg.hostname, current_seg.address)], True, + target_seg.datadir, target_seg.port) + new_moves.append(move) + + new_plan.moves = new_moves + new_plan.segmentMap = self.segmentMap + new_plan.in_conf = self.current_conf + new_plan.out_conf = plan.in_conf + + self.executor = RebalanceExecutor(new_plan, self.original_gparray, self.segmentMap, + self.current_conf, self.logger, self.statusManager, + self.conn, self.dburl, self.options) + self.executor.execute_moves() - def resume(self): + def resume(self, plan): """TODO: implement proper state handling and provide possibility to perform rebalance after fails""" raise NotImplementedError('Resuming operation is not implemented. Call gprebalance -c and rerun') diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_executor.py b/gpMgmt/bin/gprebalance_modules/rebalance_executor.py index 985a285e3064..70d2efeebaf9 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_executor.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_executor.py @@ -186,7 +186,7 @@ def run(self, validateAfter=False): f"Could not perform mirror dbid={self.segment.dbid} " f"move with content {self.segment.content} due to " f"recoverseg error: {result['error']}\n" - "Check the gprecoverseg l og file, fix any problems, and re-run" + f"Check the gprecoverseg log file {log_file}, fix any problems, and re-run" ) try: StatusManager.update_record_status( @@ -426,7 +426,7 @@ def __init__(self, for m in plan.moves: segids.append(m.segid) self.segmentSizes = self.estimateSegmentSizes(segids) - self.resources = self.initializeHostResources(plan.moves) + self.resources = self.initializeHostResources(plan.moves) if not options.rollback else None self.queue = None self.shutdown_requested = False @@ -569,6 +569,12 @@ def _prepare_swaps(self, swaps: List[Tuple[Move, Move]]): primary_id = primary_move.segid + if self.options.rollback: + mirror_move.dstHost = mirror_host + primary_move.dstHost = primary_host + primary_move.target_datadir, mirror_move.target_datadir = mirror_move.target_datadir, primary_move.target_datadir + primary_move.target_port, mirror_move.target_port = mirror_move.target_port, primary_move.target_port + continue # define datadir for datadir in primary_host.mirror_datadirs: try: @@ -624,6 +630,12 @@ def _prepare_pms(self, primary_mirrors: List[Tuple[Move, Move]]): primary_id = primary_move.segid mirror_id = mirror_move.segid + if self.options.rollback: + primary_move.dstHost = mirror_host + mirror_move.dstHost = primary_host + primary_move.target_datadir, mirror_move.target_datadir = mirror_move.target_datadir, primary_move.target_datadir + primary_move.target_port, mirror_move.target_port = mirror_move.target_port, primary_move.target_port + continue # define datadir for datadir in mirror_host.mirror_datadirs: try: @@ -673,6 +685,9 @@ def _prepare_ps(self, primaries: List[Move]): 2. primary is moved to target dir 3. role switch """ + if self.options.rollback: + return + for primary_move in primaries: primary_host = primary_move.dstHost @@ -703,6 +718,8 @@ def _prepare_ms(self, mirrors: List[Move]): Choose the target directory for mirror-only move case: 1. mirror is moved to mirror dir in mirror's target host """ + if self.options.rollback: + return for mirror_move in mirrors: mirror_host = mirror_move.dstHost @@ -885,10 +902,16 @@ def execute_moves(self, firstRun=True): for host in hosts: mkdirCmd = MakeDirectory("rebalance log dir", GPRECOVERSEG_DIR, ctxt=REMOTE, remoteHost=host) mkdirCmd.run(validateAfter=True) - self.statusManager.set_status('EXECUTION_PREPARED') - self.statusManager.set_db_status( + + if self.options.rollback: + self.statusManager.set_status('ROLLBACK_PREPARED') + self.plan.save_to_file(conf_dir, "rollback_plan") + else: + self.statusManager.set_status('EXECUTION_PREPARED') + self.statusManager.set_db_status( RebalanceStatus.PREPARED, begining_timestamp) - self.plan.save_to_file(conf_dir, "plan") + self.plan.save_to_file(conf_dir, "plan") + self.statusManager.set_db_status( RebalanceStatus.IN_PROGRESS) @@ -900,7 +923,10 @@ def execute_moves(self, firstRun=True): had_error = False if self.options.end: stopTime = self.options.end - self.statusManager.set_status('EXECUTION_STARTED') + if self.options.rollback: + self.statusManager.set_status('ROLLBACK_STARTED') + else: + self.statusManager.set_status('EXECUTION_STARTED') for sequence in move_sequences: if self.shutdown_requested: break @@ -933,7 +959,6 @@ def execute_moves(self, firstRun=True): had_error = True self.logger.error(f"Could not execute role swaps:{str(e)}") break - self.statusManager.set_db_status( RebalanceStatus.IN_PROGRESS) else: @@ -978,7 +1003,10 @@ def execute_moves(self, firstRun=True): if stoppedEarly or self.shutdown_requested: self.statusManager.set_db_status( RebalanceStatus.STOPPED) - self.statusManager.set_status('EXECUTION_STOPPED') + if self.options.rollback: + self.statusManager.set_status('ROLLBACK_STOPPED') + else: + self.statusManager.set_status('EXECUTION_STOPPED') if not self.shutdown_requested: self.logger.info("Rebalance stopped due to timeout") elif had_error: @@ -986,7 +1014,10 @@ def execute_moves(self, firstRun=True): RebalanceStatus.FAILED) raise Exception("execution encountered movement erorrs") else: - self.statusManager.set_status('EXECUTION_DONE') + if self.options.rollback: + self.statusManager.set_status('ROLLBACK_DONE') + else: + self.statusManager.set_status('EXECUTION_DONE') self.statusManager.set_db_status( RebalanceStatus.COMPLETED) rmdirCmd = RemoveDirectory("remove recoverseg log dir", f"{os.path.join(os.environ.get('HOME', '.'),GPRECOVERSEG_DIR)}") @@ -997,9 +1028,12 @@ def execute_moves(self, firstRun=True): except Exception as e: - self.statusManager.set_status('EXECUTION_FAILED') + if self.options.rollback: + self.statusManager.set_status('ROLLBACK_FAILED') + else: + self.statusManager.set_status('EXECUTION_FAILED') raise - + def _execute_role_swaps(self, segids: List[SegmentId]): """Execute multiple role swaps in single gprecoverseg -r call""" if not segids: diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_status.py b/gpMgmt/bin/gprebalance_modules/rebalance_status.py index a203a843b3b2..807701916cf2 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_status.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_status.py @@ -81,6 +81,11 @@ def __init__(self, conn: dbconn.Connection, options, logger): 'EXECUTION_FAILED' : 8, 'EXECUTION_STOPPED' : 9, 'EXECUTION_DONE': 10, + 'ROLLBACK_PREPARED' : 11, + 'ROLLBACK_STARTED' : 12, + 'ROLLBACK_FAILED' : 13, + 'ROLLBACK_STOPPED' : 14, + 'ROLLBACK_DONE' : 15, } self.conf_dir = None self.target_hosts_filename = None @@ -110,7 +115,7 @@ def _read_status_file(self): 'Invalid status file. Unknown status %s' % self._status) def create_status_file(self): - """Creates a new gpexpand status file""" + """Creates a new gprebalance status file""" try: self._fp = open(self._status_filename, 'w') self._fp.write('UNINITIALIZED:None\n') @@ -135,7 +140,7 @@ def set_status(self, status, status_info=None): 'The status file is invalid and cannot be written to') if status not in self._status_values: raise InvalidStatusError( - '%s is an invalid gpexpand status' % status) + '%s is an invalid gprebalance status' % status) self._fp.write('%s:%s\n' % (status, status_info)) self._fp.flush() os.fsync(self._fp) From 0ecac16cdb7800eab7e2a519678197f1a5def471 Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Tue, 6 May 2025 12:22:50 +0300 Subject: [PATCH 07/22] Partial implementation of gprecoverseg state recovery (#1527) The changes of this patch provide the prototype for status tracking of mirror moves during rebalance. Firstly , this patch removes the usage of gpdb table for whole execution status. Secondly, the status manager is rewritten in order to track execution process with status file only. If the movement step, presented by gprecoverseg process, fails, the corresponging status (FAILED) will be written to the internal status struct first, then will be flushed to disk. The main purpose of these changes is also implementation of gprecoverseg determination. The code in analyze_gprecoverseg_states() tries to implement the SRS diagram for gprecoverseg status definition. It processes the following scenarios: 1. A mirror move failed after pg_hba conf had been updated at primary. In this case primary marks the mirror as being down. 2. A mirror move failed after gp_segment_configuration had been updated. Here our code tries to determine whether pg_basebackup was executed succesfully or not. Depending on the basebackup state, the algorithm tries to either startup the backuped mirror or rollback the configuration changes with recovering old mirror --- gpMgmt/bin/gprebalance | 34 +- gpMgmt/bin/gprebalance_modules/rebalance.py | 76 +- .../gprebalance_modules/rebalance_executor.py | 149 ++-- .../gprebalance_modules/rebalance_status.py | 824 ++++++++++++++---- 4 files changed, 748 insertions(+), 335 deletions(-) diff --git a/gpMgmt/bin/gprebalance b/gpMgmt/bin/gprebalance index 74f569eb0385..1e03b3b7d4f5 100755 --- a/gpMgmt/bin/gprebalance +++ b/gpMgmt/bin/gprebalance @@ -387,16 +387,8 @@ def main(options, args, parser): # create pid file create_pid_file(options.coordinator_data_directory) - gprebalance_db_status = None - - try: - gprebalance_db_status = GPRebalance.prepare_gpdb_state( - logger, dburl, options) - except: - pass - global gprebalance - gprebalance = GPRebalance(logger, gparray, dburl, options) + gprebalance = GPRebalance(logger, gparray, dburl, options, gpEnv) # dump existing hosts configuration if options.genconf: @@ -407,60 +399,56 @@ def main(options, args, parser): if options.clean: gprebalance.remove_status_file() - gprebalance.cleanup_schema() gprebalance.cleanup_directory() logger.info('Cleanup Finished. exiting...') sys.exit(0) from gprebalance_modules.rebalance_status import RebalanceStatus # nopep8 - if (gprebalance_db_status == RebalanceStatus.COMPLETED or - gprebalance_file_status == 'EXECUTION_DONE') and not options.rollback: + if gprebalance_file_status == RebalanceStatus.DONE and not options.rollback: logger.info('Rebalance has already completed') logger.info( 'If you want to rebalance again, run gprebalance -c to perform cleanup') - elif options.rollback and gprebalance_file_status == 'ROLLBACK_DONE': + elif options.rollback and gprebalance_file_status == RebalanceStatus.ROLLBACK_DONE: logger.info('Rollback has already completed') logger.info( 'If you want to rebalance again, run gprebalance -c to perform cleanup') - elif (gprebalance_file_status is None and gprebalance_db_status is None) or \ - gprebalance_file_status == 'UNINITIALIZED': + elif gprebalance_file_status is None or gprebalance_file_status == RebalanceStatus.UNINITIALIZED: if options.rollback: logger.error("Cannot perform rollback operation. Rebalance session status files are absent.") sys.exit(1) gprebalance.statusManager.create_status_file() possibility_checks(gprebalance) logger.info('Validation passed. Preparing rebalance...') - gprebalance.statusManager.set_status('VALIDATED', os.path.abspath( + gprebalance.statusManager.set_status(RebalanceStatus.VALIDATED, os.path.abspath( options.filename) if options.filename else None) plan = gprebalance.createPlan() gprebalance.save_plan(plan) if options.show_plan: logger.info(f"Resulting plan: \n{str(plan)}") + # should be removed when state machine will be ready + gprebalance.statusManager.set_status(RebalanceStatus.UNINITIALIZED) sys.exit(0) - gprebalance.setup_schema() gprebalance.execute_plan(plan) elif gprebalance.statusManager.conf_dir: plan = gprebalance.load_plan(gprebalance.statusManager.conf_dir, False) if options.show_plan: logger.info(f"Resulting plan: \n{str(plan)}") sys.exit(0) - if gprebalance_file_status == 'PLANNED': - gprebalance.setup_schema() + if gprebalance_file_status == RebalanceStatus.PREPARED: gprebalance.execute_plan(plan) sys.exit(0) - elif gprebalance_file_status == 'ROLLBACK_FAILED' or \ - gprebalance_file_status == 'ROLLBACK_STOPPED': + elif gprebalance_file_status == RebalanceStatus.ROLLBACK_FAILED: logger.info('Previous rollback session was interrupted due to an error. Trying to resume...') plan = gprebalance.load_plan(gprebalance.statusManager.conf_dir, True) gprebalance.resume(plan) sys.exit(0) - elif gprebalance_file_status == 'EXECUTION_FAILED': + elif gprebalance_file_status == RebalanceStatus.FAILED: if not options.rollback: logger.info('Previous rebalance session was interrupted due to an error. Trying to resume...') gprebalance.resume(plan) sys.exit(0) - elif gprebalance_file_status == 'EXECUTION_STOPPED': + elif gprebalance_file_status == RebalanceStatus.STOPPED: if not options.rollback: logger.info('Previous rebalance session was interrupted due to an timeout or user signal. Trying to resume...') gprebalance.resume(plan) diff --git a/gpMgmt/bin/gprebalance_modules/rebalance.py b/gpMgmt/bin/gprebalance_modules/rebalance.py index 66a40e38ff50..a7b2f2b93ba5 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance.py @@ -100,7 +100,7 @@ class MirrorStrategy(Enum): class GPRebalance: - def __init__(self, logger, gparray, dburl, options): + def __init__(self, logger, gparray, dburl, options, gpEnv): self.logger = logger self.dburl = dburl self.options = options @@ -155,8 +155,7 @@ def __init__(self, logger, gparray, dburl, options): self.unpreferred_segments = self.getSegmentsUnpreferredRole() self.segmentMap = {SegmentId( seg.dbid, seg.content): seg for seg in self.original_gparray.getSegmentsAsLoadedFromDb()} - self.statusManager = StatusManager( - self.conn, self.options, self.logger) + self.statusManager = StatusManager(self.options, self.logger, self.original_gparray, self.conn, gpEnv) self.executor = None @@ -228,8 +227,7 @@ def save_plan(self, plan: Plan): datadir = self.options.coordinator_data_directory + CONF_DIR os.makedirs(datadir, exist_ok=True) plan.save_to_file(datadir, "plan") - self.statusManager.set_status('PLANNED', datadir) - + def load_plan(self, datadir, rollback): filename = datadir if rollback: @@ -254,72 +252,12 @@ def execute_plan(self, plan: Plan): self.options) self.executor.execute_moves() - @staticmethod - def prepare_gpdb_state(logger, dburl, options) -> str: - gprebalance_db_status = None - - try: - gprebalance_db_status = GPRebalance.get_status_from_db( - dburl, options) - logger.debug('Expansion status returned is %s' % - gprebalance_db_status) - except Exception as e: - raise Exception( - 'Error while trying to query the gprebalance schema: %s' % e) - - return gprebalance_db_status - - @staticmethod - def get_status_from_db(dburl, options) -> RebalanceStatus: - """Gets gprebalance status from the gprebalance schema""" - status_conn = None - gprebalance_db_status = None - if get_local_db_mode(options.coordinator_data_directory) == 'NORMAL': - try: - status_conn = dbconn.connect(dburl, encoding='UTF8') - # Get the last status entry - cursor = dbconn.query( - status_conn, 'SELECT status FROM gprebalance.status ORDER BY updated DESC LIMIT 1') - if cursor.rowcount == 1: - gprebalance_db_status = cursor.fetchone()[0] - - except Exception: - # rebalance schema doesn't exists or there was a connection failure. - pass - finally: - if status_conn: - status_conn.close() - - # make sure gprebalance schema doesn't exist since it wasn't in DB provided - if not gprebalance_db_status: - conn = dbconn.connect(dburl, encoding='UTF8', utility=True) - try: - count = dbconn.querySingleton(conn, - "SELECT count(n.nspname) FROM pg_catalog.pg_namespace n WHERE n.nspname = 'gprebalance'") - if count > 0: - raise Exception( - "Existing rebalance state could not be determined, but a gprebalance schema already exists. Cannot proceed.") - finally: - conn.close() - - return RebalanceStatus(gprebalance_db_status) - def get_state_from_file(self): """Returns expansion state from status file""" - return self.statusManager.get_current_status()[0] - - def setup_schema(self): - self.statusManager.set_status('REBALANCE_PREPARE_SCHEMA_STARTED') - self.logger.info('Creating expansion schema') - self.statusManager.setup_schema() - self.statusManager.set_status('REBALANCE_PREPARE_SCHEMA_DONE') - self.statusManager.set_db_status(RebalanceStatus.INITIALIZED) - - def cleanup_schema(self): - if not self.conn: - return - self.logger.info('Dropping rebalance schema') - self.statusManager.cleanup_schema() + status = self.statusManager.get_current_status()[0] + if status: + return RebalanceStatus(status) + return None def cleanup_directory(self): self.logger.info('Dropping rebalance directory') diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_executor.py b/gpMgmt/bin/gprebalance_modules/rebalance_executor.py index 70d2efeebaf9..a36c2f738d3b 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_executor.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_executor.py @@ -6,7 +6,7 @@ import pickle from typing import List, Dict, Optional, Set, Tuple from gprebalance_modules.rebalance_plan import Move, Plan # nopep8 -from gprebalance_modules.rebalance_status import StatusManager, RebalanceStatus, MoveStatus, SqlError +from gprebalance_modules.rebalance_status import StatusManager, RebalanceStatus, MoveStatus from gprebalance_modules.rebalance import ClusterState, SegmentId, SegmentSize, Host from gppylib.gparray import GpArray, Segment from gppylib.db import dbconn @@ -107,8 +107,8 @@ def run_recovery(cmd_args: list, result_queue: multiprocessing.Queue, log_file: class SingleMoveCommand(SQLCommand): - def __init__(self, name: str, status_url: dbconn.DbURL, step_details, logger): - self.status_url = status_url + def __init__(self, name: str, step_details, logger, statusManager: StatusManager): + self.status_manager = statusManager self.logger = logger (self.segment, self.move, self.segmentSize, self.conf_dir, self.needs_switch, self.options) = step_details @@ -134,12 +134,11 @@ def write_gprecoverseg_config(self): def run(self, validateAfter=False): status_conn = None try: - status_conn = dbconn.connect(self.status_url, encoding='UTF8') + segsize = self.segmentSize.source_data_dir_usage if self.segmentSize.source_tablespace_usage: segsize += sum(self.segmentSize.source_tablespace_usage.values()) - StatusManager.record_move(status_conn, self.move, self.segment, - segsize, datetime.datetime.now()) + self.status_manager.update_move_status([self.segment.dbid], MoveStatus.IN_PROGRESS) filename = self.write_gprecoverseg_config() @@ -166,12 +165,6 @@ def run(self, validateAfter=False): ] if self.options.hba_hostnames: cmd_args.append('--hba-hostnames') - try: - StatusManager.update_record_status( - status_conn, [self.segment.dbid], MoveStatus.IN_PROGRESS) - except SqlError: - raise Exception( - f"Could not update status for move with dbid={self.segment.dbid}: ") result_queue = multiprocessing.Queue() recovery_process = multiprocessing.Process( @@ -179,8 +172,39 @@ def run(self, validateAfter=False): args=(cmd_args, result_queue, log_file) ) recovery_process.start() - result = result_queue.get() - recovery_process.join() + result = None + while recovery_process.is_alive(): + # Check if result is available without blocking + try: + if not result_queue.empty(): + result = result_queue.get(block=False) + recovery_process.join() + except Exception: + pass + time.sleep(10) + + if not result_queue.empty(): + result = result_queue.get(block=False) + + if result is None: + exit_code = recovery_process.exitcode + if exit_code < 0: + self.logger.error( + f"Could not perform mirror dbid={self.segment.dbid} " + f"move with content {self.segment.content} due to " + f"recoverseg error: Process terminated by signal {-exit_code}\n" + f"Check the gprecoverseg log file {log_file}, fix any problems, and re-run" + ) + else: + self.logger.error( + f"Could not perform mirror dbid={self.segment.dbid} " + f"move with content {self.segment.content} due to " + f"recoverseg error: Process exited with code {exit_code}\n" + f"Check the gprecoverseg log file {log_file}, fix any problems, and re-run") + self.status_manager.update_move_status([self.segment.dbid], MoveStatus.FAILED) + self.move_error = True + return + if result["status"] == "FAILED": self.logger.error( f"Could not perform mirror dbid={self.segment.dbid} " @@ -188,30 +212,15 @@ def run(self, validateAfter=False): f"recoverseg error: {result['error']}\n" f"Check the gprecoverseg log file {log_file}, fix any problems, and re-run" ) - try: - StatusManager.update_record_status( - status_conn, [self.segment.dbid], MoveStatus.FAILED) - except SqlError: - raise Exception( - f"Could not update status for move with dbid={self.segment.dbid}: ") + self.status_manager.update_move_status([self.segment.dbid], MoveStatus.FAILED) self.move_error = True - status_conn.close() return if self.needs_switch: - try: - StatusManager.update_record_status( - status_conn, [self.segment.dbid], MoveStatus.AWAITS_SWITCH) - except SqlError: - raise Exception( - f"Could not update status for move with dbid={self.segment.dbid}: ") - - try: - StatusManager.update_record_status( - status_conn, [self.segment.dbid], MoveStatus.COMPLETED) - except SqlError: - raise Exception( - f"Could not update status for move with dbid={self.segment.dbid}: ") + self.status_manager.update_move_status([self.segment.dbid], MoveStatus.AWAITS_SWITCH) + else: + self.status_manager.update_move_status([self.segment.dbid], MoveStatus.COMPLETED) + self.logger.info("Removing old segment's datadir (dbidi = %d): %s", self.segment.dbid, self.segment.datadir) @@ -225,12 +234,10 @@ def run(self, validateAfter=False): self.logger.info("Removing old segment's tablespace datadir (dbidi = %d): %s", self.segment.dbid, tblspdir) cmd.run(validateAfter=True) - status_conn.close() except Exception as ex: self.logger.error(ex.__str__().strip()) self.move_error = True - status_conn.close() return @@ -889,6 +896,23 @@ def execute_moves(self, firstRun=True): begining_timestamp = datetime.datetime.now() conf_dir = self.options.coordinator_data_directory + CONF_DIR + + #record moves in status file + detail_list = [] + for seq_no, seq in enumerate(move_sequences): + if not isinstance(seq[0], str): + for move in seq: + needs_switch = False + if move.segid.contentid in former_switches or move.segid.contentid in latter_switches: + needs_switch = True + if move.segid.contentid in former_switches and move.segid.contentid in latter_switches: + needs_switch = False + detail_list.append((seq_no, self.segmentMap[move.segid], + move, + self.segmentSizes[move.segid].source_data_dir_usage, + needs_switch)) + self.statusManager.record_moves_batch(detail_list) + hosts = set() for move in self.moves: hosts.add(move.srcHost.hostname) @@ -896,7 +920,8 @@ def execute_moves(self, firstRun=True): if firstRun: # in order to run gprecoverseg processes separately and avoid any races - # for resources gprecoverseg generates, we create separate log directories. + # for resources gprecoverseg generates (progress file), we create + # separate log directories. mkdirCmd = MakeDirectory("rebalance log dir", GPRECOVERSEG_DIR) mkdirCmd.run(validateAfter=True) for host in hosts: @@ -904,17 +929,11 @@ def execute_moves(self, firstRun=True): mkdirCmd.run(validateAfter=True) if self.options.rollback: - self.statusManager.set_status('ROLLBACK_PREPARED') + self.statusManager.set_status(RebalanceStatus.ROLLBACK_PREPARED, conf_dir) self.plan.save_to_file(conf_dir, "rollback_plan") else: - self.statusManager.set_status('EXECUTION_PREPARED') - self.statusManager.set_db_status( - RebalanceStatus.PREPARED, begining_timestamp) + self.statusManager.set_status(RebalanceStatus.PREPARED, conf_dir) self.plan.save_to_file(conf_dir, "plan") - - - self.statusManager.set_db_status( - RebalanceStatus.IN_PROGRESS) self.queue = WorkerPool(self.options.parallel) @@ -924,9 +943,9 @@ def execute_moves(self, firstRun=True): if self.options.end: stopTime = self.options.end if self.options.rollback: - self.statusManager.set_status('ROLLBACK_STARTED') + self.statusManager.set_status(RebalanceStatus.ROLLBACK_IN_PROGRESS) else: - self.statusManager.set_status('EXECUTION_STARTED') + self.statusManager.set_status(RebalanceStatus.IN_PROGRESS) for sequence in move_sequences: if self.shutdown_requested: break @@ -949,17 +968,21 @@ def execute_moves(self, firstRun=True): if self.shutdown_requested: break - self.statusManager.set_db_status( + self.statusManager.set_status( RebalanceStatus.AWAITS_SWITCH) self.logger.info( f"Executing role swaps for {len(sequence[1])} segments") try: self._execute_role_swaps(sequence[1]) + for segid in sequence[1]: + if segid in former_switches and segid not in latter_switches: + self.statusManager.update_move_status(sequence[1], MoveStatus.COMPLETED) + break except Exception as e: had_error = True self.logger.error(f"Could not execute role swaps:{str(e)}") break - self.statusManager.set_db_status( + self.statusManager.set_status( RebalanceStatus.IN_PROGRESS) else: for move in sequence: @@ -978,7 +1001,7 @@ def execute_moves(self, firstRun=True): ) cmd = SingleMoveCommand( - "name", self.dburl, step_details, self.logger) + "name", step_details, self.logger, self.statusManager) self.queue.addCommand(cmd) while not self.queue.isDone(): @@ -1001,25 +1024,23 @@ def execute_moves(self, firstRun=True): self.queue = None if stoppedEarly or self.shutdown_requested: - self.statusManager.set_db_status( - RebalanceStatus.STOPPED) - if self.options.rollback: - self.statusManager.set_status('ROLLBACK_STOPPED') - else: - self.statusManager.set_status('EXECUTION_STOPPED') + self.statusManager.set_status(RebalanceStatus.STOPPED) if not self.shutdown_requested: self.logger.info("Rebalance stopped due to timeout") elif had_error: - self.statusManager.set_db_status( + if self.options.rollback: + self.statusManager.set_status( + RebalanceStatus.ROLLBACK_FAILED) + else: + self.statusManager.set_status( RebalanceStatus.FAILED) raise Exception("execution encountered movement erorrs") else: if self.options.rollback: - self.statusManager.set_status('ROLLBACK_DONE') + self.statusManager.set_status(RebalanceStatus.ROLLBACK_DONE) else: - self.statusManager.set_status('EXECUTION_DONE') - self.statusManager.set_db_status( - RebalanceStatus.COMPLETED) + self.statusManager.set_status(RebalanceStatus.DONE) + rmdirCmd = RemoveDirectory("remove recoverseg log dir", f"{os.path.join(os.environ.get('HOME', '.'),GPRECOVERSEG_DIR)}") rmdirCmd.run() for host in hosts: @@ -1029,9 +1050,9 @@ def execute_moves(self, firstRun=True): except Exception as e: if self.options.rollback: - self.statusManager.set_status('ROLLBACK_FAILED') + self.statusManager.set_status(RebalanceStatus.ROLLBACK_FAILED) else: - self.statusManager.set_status('EXECUTION_FAILED') + self.statusManager.set_status(RebalanceStatus.FAILED) raise def _execute_role_swaps(self, segids: List[SegmentId]): @@ -1049,8 +1070,6 @@ def _execute_role_swaps(self, segids: List[SegmentId]): "content IN %s AND preferred_role = 'p'", (data,)) cur.execute("UPDATE gp_segment_configuration SET preferred_role = 'p' WHERE " "content IN %s AND preferred_role = 't'", (data,)) - cur.execute("UPDATE gprebalance.status_detail SET status = 'PENDING' WHERE " - "status = 'awaits_switch'") cur.execute("COMMIT") except Exception as e: raise Exception('could not execute SQL : %s' % str(e)) diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_status.py b/gpMgmt/bin/gprebalance_modules/rebalance_status.py index 807701916cf2..02b1c984e0b3 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_status.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_status.py @@ -1,213 +1,681 @@ +import atexit +import copy from datetime import datetime +import json import os -from gppylib.db import dbconn from enum import Enum -from typing import Dict, List +import pickle +import threading +import time +from typing import Dict, List, Tuple, Any, Optional from gprebalance_modules.rebalance_plan import Move -from gprebalance_modules.rebalance import SegmentId, Segment, SegmentSize - -create_schema_sql = "CREATE SCHEMA gprebalance" -drop_schema_sql = "DROP SCHEMA IF EXISTS gprebalance CASCADE" - -status_table_sql = """CREATE TABLE gprebalance.status - ( status varchar(255), - updated timestamp ) """ - -status_detail_table_sql = """CREATE TABLE gprebalance.status_detail - ( step_id serial, - dbid smallint, - content smallint, - role char, - preferred_role char, - source_hostname varchar(255), - source_port integer, - source_datadir varchar(255), - dest_hostname varchar(255), - target_datadir varchar(255), - target_port integer, - status varchar(255), - rebalance_started timestamp, - rebalance_finished timestamp, - source_kbytes numeric )""" +from gprebalance_modules.rebalance import SegmentId, Segment +import gppylib.operations.update_pg_hba_on_segments as hba_upd +from gppylib.operations.detect_unreachable_hosts import get_unreachable_segment_hosts, update_unreachable_flag_for_segments +from gppylib.commands import pg +from gppylib.commands.base import Command, REMOTE +from gppylib.userinput import * +from gppylib.db import dbconn +from gppylib.system import configurationInterface as configInterface +from gppylib.parseutils import * class MoveStatus(Enum): - PENDING = "pending" - IN_PROGRESS = "in_progress" - AWAITS_SWITCH = "awaits_switch" - COMPLETED = "completed" - FAILED = "failed" - STOPPED = "stopped" - REVERTED = "reverted" - + PENDING = "PENDING" + IN_PROGRESS = "IN_PROGRESS" + AWAITS_SWITCH = "AWAITS_SWITCH" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + STOPPED = "STOPPED" + REVERTED = "REVERTED" + ASK_ROLLBACK_OR_SKIP = "ASK_USER" + SKIPPED = "NEEDS_MANUAL_RECOVERY" class RebalanceStatus(Enum): - INITIALIZED = "initialized" - PREPARED = "prepared" - IN_PROGRESS = "in_progress" - COMPLETED = "completed" - FAILED = "failed" - STOPPED = "stopped" - AWAITS_SWITCH = "awaits_switch" - + UNKNOWN = "UNKNOWN" + UNINITIALIZED = "UNINITIALIZED" + VALIDATED = "VALIDATED" + PRE_EXECUTE_FAILED= "PRE_EXECUTE_NOT_PASSED" + PREPARED = "EXECUTION_PREPARED" + IN_PROGRESS = "EXECUTION_IN_PROGRESS" + AWAITS_SWITCH_APPROVE = "EXECUTION_AWAITING_SWITCHOVER_APPROVE" + AWAITS_SWITCH = "EXECUTION_AWAITING_SWITCHOVER" + STEPS_FAILED = "EXECUTION_STEPS_FAILED" + FAILED = "EXECUTION_FAILED" + DONE = "EXECUTION_DONE" + AWAITS_ROLLBACK_APPROVE = "EXECUTION_AWAITING_ROLLBACK_APPROVE" + STOPPED = "EXECUTION_STOPPED" + ROLLBACK_PREPARED = "EXECUTION_ROLLBACK_PREPARED" + ROLLBACK_IN_PROGRESS = "EXECUTION_ROLLBACK_PREPARED" + ROLLBACK_DONE = "EXECUTION_ROLLBACK_DONE" + ROLLBACK_FAILED = "EXECUTION_ROLLBACK_FAILED" class InvalidStatusError(Exception): pass +def load_plan(datadir, rollback): + filename = datadir + if rollback: + filename += "/rollback_plan.pkl" + else: + filename += "/plan.pkl" + if not os.path.exists(filename): + raise FileNotFoundError(f"No pickle file found at {filename}") + with open(filename, 'rb') as f: + plan = pickle.load(f) + return plan -class SqlError(Exception): - pass - +wait_timeout_s = 5 class StatusManager: - def __init__(self, conn: dbconn.Connection, options, logger): - - self.main_conn = conn + # Class variable for the singleton instance + _instance = None + _instance_lock = threading.Lock() + + @classmethod + def get_instance(cls, options=None, logger=None): + """ + Get or create the singleton instance of StatusManager. + + Args: + options: Configuration options (required first time) + logger: Logger object (required first time) + + Returns: + The singleton StatusManager instance + """ + with cls._instance_lock: + if cls._instance is None: + if options is None or logger is None: + raise ValueError("options and logger required for initial instance creation") + cls._instance = cls(options, logger) + return cls._instance + + def __init__(self, options, logger, gparray, conn, gpEnv): self.options = options self.logger = logger + self._status_filename = self.options.coordinator_data_directory + '/gprebalance.status.json' + self.initial_gparray= gparray + self._flush_interval = 2.0 + self._last_flush_time = 0 + self._dirty = False + self._lock = threading.RLock() + self.conn = conn + self.gpenv = gpEnv - self._status_filename = self.options.coordinator_data_directory + '/gprebalance.status' - self._fp = None - self._status = [] - self._status_info = [] - self._status_values = {'UNINITIALIZED': 1, - 'VALIDATED': 2, - 'PLANNED': 3, - 'REBALANCE_PREPARE_SCHEMA_STARTED': 4, - 'REBALANCE_PREPARE_SCHEMA_DONE': 5, - 'EXECUTION_PREPARED': 6, - 'EXECUTION_STARTED': 7, - 'EXECUTION_FAILED' : 8, - 'EXECUTION_STOPPED' : 9, - 'EXECUTION_DONE': 10, - 'ROLLBACK_PREPARED' : 11, - 'ROLLBACK_STARTED' : 12, - 'ROLLBACK_FAILED' : 13, - 'ROLLBACK_STOPPED' : 14, - 'ROLLBACK_DONE' : 15, - } + # Initialize the status data structure + self.status_data = { + "current_status": None, + "rebalance_dir" :None, + "hosts_file" : None, + "status_history": [], + "moves": [], + "last_modified": datetime.now().isoformat() + } + self.conf_dir = None self.target_hosts_filename = None + + # Load existing status file if it exists if os.path.exists(self._status_filename): self._read_status_file() + self.analyze_gprecoverseg_states() + + + atexit.register(self._ensure_flush) + self._lock_timeout = 10 # seconds to wait for lock acquisition + self._max_retries = 3 # maximum number of retries for operations + self._retry_delay = 0.5 # seconds between retries + self._stop_flush_thread = False + self._flush_thread = threading.Thread(target=self._auto_flush_worker, daemon=True) + self._flush_thread.start() + + def _auto_flush_worker(self): + """Background thread that periodically flushes dirty status to disk""" + while not self._stop_flush_thread: + if self._dirty and time.time() - self._last_flush_time >= self._flush_interval: + try: + self._flush_to_disk() + except Exception as e: + self.logger.error(f"Auto-flush failed: {str(e)}") + time.sleep(0.5) # Check every half second + + def _ensure_flush(self): + """Ensure any pending changes are flushed to disk""" + if self._dirty: + try: + self._flush_to_disk() + except Exception as e: + self.logger.error(f"Final flush failed: {str(e)}") + def _read_status_file(self): - self.logger.debug( - "Trying to read in a pre-existing gprebalance status file") + """Load status data from disk once""" + self.logger.debug(f"Loading status file: {self._status_filename}") try: - self._fp = open(self._status_filename, 'a+') - self._fp.seek(0) - - for line in self._fp: - (status, status_info) = line.rstrip().split(':') - if status == 'PLANNED': - self.conf_dir = status_info - elif status == 'VALIDATED': - self.target_hosts_filename = status_info - self._status.append(status) - self._status_info.append(status_info) - except IOError: + with open(self._status_filename, 'r') as fp: + data = json.load(fp) + + # Basic validation + required_keys = ["current_status", "status_history", "moves"] + for key in required_keys: + if key not in data: + raise InvalidStatusError(f"Status file missing required key: {key}") + + # Store the data + self.status_data = data + self.conf_dir = self.status_data["rebalance_dir"] + + self._dirty = False + self._last_flush_time = time.time() + + except json.JSONDecodeError as e: + self.logger.error(f"Invalid JSON in status file: {str(e)}") + raise InvalidStatusError(f"Status file contains invalid JSON: {str(e)}") + except Exception as e: + self.logger.error(f"Failed to load status file: {str(e)}") raise + + def _flush_to_disk(self): + """Write current status data to disk with file locking""" + with self._lock: + if not self._dirty: + return + + self.logger.debug("Flushing status data to disk") + # Update last modified timestamp + self.status_data["last_modified"] = datetime.now().isoformat() + # Create temporary file + temp_file = f"{self._status_filename}.tmp" + try: + # Write to temp file first + with open(temp_file, 'w') as fp: + json.dump(self.status_data, fp, indent=2) + fp.flush() + os.fsync(fp.fileno()) + + # Atomic rename for safer file updates + os.rename(temp_file, self._status_filename) + + self._dirty = False + self._last_flush_time = time.time() + self.logger.debug("Status data flushed successfully") + except Exception as e: + self.logger.error(f"Failed to flush status data: {str(e)}") + # Clean up temp file if it exists + if os.path.exists(temp_file): + try: + os.unlink(temp_file) + except: + pass + raise + + def create_status_file(self): + """Creates a new status file with initial state""" + with self._lock: + self.status_data = { + "current_status": "UNINITIALIZED", + "rebalance_dir" : None, + "hosts_file" : None, + "status_history": [ + { + "status": "UNINITIALIZED", + "timestamp": datetime.now().isoformat(), + "info": None + } + ], + "moves": [], + "last_modified": datetime.now().isoformat() + } + self._dirty = True + self._flush_to_disk() # Immediate flush for initial creation + + def get_current_status(self) -> Tuple[str, str]: + """Gets the current status""" + with self._lock: + if not self.status_data["status_history"]: + return (None, None) + + last_status = self.status_data["status_history"][-1] + return (last_status["status"], last_status["info"]) + + def set_status(self, statusEnum: RebalanceStatus, status_info=None): + """Set overall rebalance status""" + status = statusEnum.value + with self._lock: + status_entry = { + "status": status, + "timestamp": datetime.now().isoformat(), + "info": status_info + } + + self.status_data["current_status"] = status + self.status_data["status_history"].append(status_entry) + + # Update conf_dir or target_hosts_filename if applicable + if statusEnum == RebalanceStatus.PREPARED: + self.status_data["rebalance_dir"] = status_info + elif statusEnum == RebalanceStatus.VALIDATED: + self.status_data["hosts_file"] = status_info + + self._dirty = True + + # Consider immediate flush for important status changes + if statusEnum in [RebalanceStatus.FAILED, RebalanceStatus.PRE_EXECUTE_FAILED, RebalanceStatus.STEPS_FAILED, RebalanceStatus.ROLLBACK_FAILED]: + self._flush_to_disk() + + def record_moves_batch(self, moves_data): - if self._status[-1] not in self._status_values: - raise InvalidStatusError( - 'Invalid status file. Unknown status %s' % self._status) + with self._lock: + for seq_no, segment, move, size, needs_switch in moves_data: + move_entry = { + "dbid": segment.dbid, + "content": segment.content, + "role": segment.role, + "role_after_switch": ('p' if move.is_mirror else 'm') if needs_switch else segment.role, + "status": MoveStatus.PENDING.value, + "source_kbytes": size, + "seq_id": seq_no, + } - def create_status_file(self): - """Creates a new gprebalance status file""" + self.status_data["moves"].append(move_entry) + + if moves_data: + self._dirty = True + self._flush_to_disk() + + def update_move_status(self, dbids: List[int], status: MoveStatus): + """Update status for specified moves""" + with self._lock: + updated = False + for move in self.status_data["moves"]: + if move["dbid"] in dbids: + move["status"] = status.value + updated = True + + if updated: + self._dirty = True + + # Consider immediate flush for completion status + if status in [MoveStatus.COMPLETED, MoveStatus.FAILED]: + self._flush_to_disk() + + def get_moves_by_status(self, status: MoveStatus) -> List[Dict[str, Any]]: + """Get all moves with the specified status""" + with self._lock: + return [move.copy() for move in self.status_data["moves"] + if move["status"] == status.value] + + def get_move_by_dbid(self, dbid: int) -> Optional[Dict[str, Any]]: + """Get a specific move by dbid""" + with self._lock: + for move in self.status_data["moves"]: + if move["dbid"] == dbid: + return move.copy() + return None + + def flush(self): + """Manually flush status data to disk""" + self._flush_to_disk() + + def remove_all(self): + """Remove the status file and reset state""" + with self._lock: + self._stop_flush_thread = True + if self._flush_thread.is_alive(): + self._flush_thread.join(timeout=2.0) + + if os.path.exists(self._status_filename): + try: + os.unlink(self._status_filename) + except Exception as e: + self.logger.error(f"Failed to remove status file: {str(e)}") + + # Reset internal state + self.status_data = { + "current_status": None, + "status_history": [], + "moves": [], + "last_modified": datetime.now().isoformat() + } + self._dirty = False + self.conf_dir = None + self.target_hosts_filename = None + + def analyze_gprecoverseg_states(self): + if not self.conf_dir: + return + is_rollback = self.options.rollback + plan = load_plan(self.conf_dir, is_rollback) + failed_moves = [] + in_progress_moves = [] + not_started_moves = [] + completed_moves = [] + for move in self.status_data.get("moves", []): + move_status = move["status"] + if move_status == "FAILED": + failed_moves.append(move) + elif move_status == "IN_PROGRESS": + in_progress_moves.append(move) + elif move_status == "PENDING": + not_started_moves.append(move) + elif move_status == "COMPLETED": + completed_moves.append(move) + if failed_moves or in_progress_moves: + self.logger.info(f"Found {len(failed_moves)} failed moves and {len(in_progress_moves)} in-progress moves") + moves_to_analyze = failed_moves + in_progress_moves + self._analyze_failed_segments(moves_to_analyze, plan) + moves_to_recover = [] + for move in moves_to_analyze: + if move["status"] == "ASK_USER": + if ask_yesno('', f"Failed mirror move (dbid={move['dbid']}, content={move['content']}) cannot be rerun" + f" properly. Do you want to try to rollback it to original state?", "N"): + moves_to_recover.append(move) + else: + move["status"] = MoveStatus.SKIPPED.value + self._try_restore_config(moves_to_recover, plan) + self._dirty = True + self._flush_to_disk() + + for move in moves_to_analyze: + if move["status"] == MoveStatus.SKIPPED.value: + self.logger.info(f"Couldn't revert the move (dbid={move['dbid']}, content={move['content']})." + " Please, restore the required state and re-run") + + + def _analyze_failed_segments(self, moves, plan): + + self.logger.info(f"Analyzing {len(moves)} moves for state recovery") + segmentMap = {SegmentId( + seg.dbid, seg.content): seg for seg in self.initial_gparray.getSegmentsAsLoadedFromDb()} + hosts = set(self.initial_gparray.get_hostlist(includeCoordinator=False)) + unreachable_hosts = get_unreachable_segment_hosts(hosts, self.options.parallel) + update_unreachable_flag_for_segments(self.initial_gparray, unreachable_hosts) + plan_moves = {move.segid.dbid: move for move in plan.moves} + for move in moves: + segid = SegmentId(move["dbid"], move["content"]) + original_segment = plan.segmentMap[segid] + current_segment = segmentMap[segid] + + peer = None + is_mirror = False + for pair in self.initial_gparray.segmentPairs: + if pair.primaryDB.dbid == segid.dbid: + peer = pair.mirrorDB + break + elif pair.mirrorDB.dbid == segid.dbid: + peer = pair.primaryDB + is_mirror = True + break + + if is_mirror: + self.logger.info(f"Analyzing failed mirror move (dbid={original_segment.dbid}, content={original_segment.content}) " + f"to host {plan_moves[original_segment.dbid].dstHost.hostname}, directory {plan_moves[original_segment.dbid].target_datadir}") + if original_segment.hostname == current_segment.hostname and\ + original_segment.address == current_segment.address and\ + original_segment.dbid == current_segment.dbid and\ + (original_segment.role == current_segment.role or move["role_after_switch"] == current_segment.role) and\ + original_segment.datadir == current_segment.datadir and\ + peer.valid and not current_segment.valid: + # case 1 gprecoverseg could only have updated pg_hba_conf + # of corresponding primary before failure + # check that current_segment is running + if current_segment.unreachable: + move["status"] = MoveStatus.FAILED.value + continue + commandStr = f"pg_ctl status -D {current_segment.datadir}" + cmd_status = Command("pid", commandStr, REMOTE, current_segment.hostname) + cmd_status.run() + if cmd_status.get_return_code() != 0: + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + + #restore primary's pg_hba_conf + entries = hba_upd.create_entries(peer.hostname, original_segment.hostname, self.options.hba_hostnames) + updatecmd = hba_upd.SegUpdateHba(entries, peer.datadir, REMOTE, peer.hostname) + updatecmd.run() + if updatecmd.get_return_code() != 0: + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + host_port_tuple = (peer.hostname, peer.port) + pg.kill_existing_walsenders_on_primary([host_port_tuple]) + time.sleep(wait_timeout_s) + cmd_status = Command("pid", commandStr, REMOTE, current_segment.hostname) + cmd_status.run() + if cmd_status.get_return_code() != 0: + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + # case 2 : gp_segment_configuration has been updated by gprecoverseg + # from buildMirrorSegments.py:295 and we need to define whether + # basebackup was launched, completed or interrupred + elif original_segment.dbid == current_segment.dbid and\ + (original_segment.role == current_segment.role or\ + # case when seg is moved after 1st switch + move["role_after_switch"] == current_segment.role or + #primary only move. 2 switches, case when seg is moved after 1st switch + move["role_after_switch"] == move["role"]) and\ + (original_segment.hostname != current_segment.hostname or\ + original_segment.address != current_segment.address or\ + original_segment.datadir != current_segment.datadir): + + if current_segment.valid and current_segment.mode == 's': + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + + # check whether target directory exists + cmd = Command( + name="check_directory", + cmdStr=f"test -d {current_segment.datadir} && echo 'exists' || echo 'not_exists'", + ctxt=REMOTE, + remoteHost=current_segment.hostname) + cmd.run() + if not cmd.was_successful() or cmd.get_stdout().strip() != 'exists': + self.logger.info(f"Failed to check directory on {current_segment.hostname}") + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + + #check the port update + cmd = Command( + name="get_postgresql_conf_port", + cmdStr=f"grep -E '^port\\s*=' {current_segment.datadir}/postgresql.conf | sed -E 's/^port\\s*=\\s*([0-9]+).*/\\1/'", + ctxt=REMOTE, + remoteHost=current_segment.hostname + ) + cmd.run() + if not cmd.was_successful(): + self.logger.info(f"Failed to get port from postgresql.conf on {current_segment.hostname}: {cmd.get_stderr()}") + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + output = cmd.get_stdout().strip() + if not output or not output.isdigit(): + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + datadir_port = int(output) + if datadir_port != current_segment.port: + self.logger.info(f"Port mismatch in postgresql.conf: expected {current_segment.port}, got {datadir_port}") + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + + #check if server is running + process_running = self._check_postgres_process(current_segment.hostname, current_segment.datadir) + if not process_running: + + is_started = self._wait_for_pg_startup(current_segment.hostname, current_segment.datadir, timeout_seconds=20) + if not is_started: + attemp_start = self._attempt_start_mirror(current_segment) + if not attemp_start and not self._wait_for_pg_startup(current_segment.hostname, current_segment.datadir, timeout_seconds=20): + self.logger.info("Failed to start the mirror") + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + if not self._wait_for_config_update(current_segment.dbid, 60): + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + elif not self._wait_for_config_update(current_segment.dbid, 60): + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + + move["status"] = MoveStatus.COMPLETED.value + + + def _check_postgres_process(self, host, data_dir): try: - self._fp = open(self._status_filename, 'w') - self._fp.write('UNINITIALIZED:None\n') - self._fp.flush() - os.fsync(self._fp) - self._status.append('UNINITIALIZED') - self._status_info.append('None') - except IOError: - raise + # First check with pg_ctl status + pg_ctl_cmd = Command( + name="check_pg_ctl_status", + cmdStr=f"pg_ctl status -D {data_dir}", + ctxt=REMOTE, + remoteHost=host + ) + pg_ctl_cmd.run() + + # pg_ctl status returns 0 if server is running + if pg_ctl_cmd.get_return_code() == 0: + return True - def get_current_status(self): - """Gets the current status that has been written to the gpexpand - status file""" - if (len(self._status) > 0 and len(self._status_info) > 0): - return (self._status[-1], self._status_info[-1]) - else: - return (None, None) - - def set_status(self, status, status_info=None): - if not self._fp: - raise InvalidStatusError( - 'The status file is invalid and cannot be written to') - if status not in self._status_values: - raise InvalidStatusError( - '%s is an invalid gprebalance status' % status) - self._fp.write('%s:%s\n' % (status, status_info)) - self._fp.flush() - os.fsync(self._fp) - self._status.append(status) - self._status_info.append(status_info) - - def setup_schema(self): - dbconn.execSQL(self.main_conn, create_schema_sql) - dbconn.execSQL(self.main_conn, status_table_sql) - dbconn.execSQL(self.main_conn, status_detail_table_sql) - - def cleanup_schema(self): - dbconn.execSQL(self.main_conn, drop_schema_sql) - - def set_db_status(self, status: RebalanceStatus, stamp: datetime = None): - if not stamp: - stamp = datetime.now() - sql = "INSERT INTO gprebalance.status VALUES ('%s', '%s')" % ( - status.value, stamp) - dbconn.execSQL(self.main_conn, sql) - - @staticmethod - def record_move(conn: dbconn.Connection, move: Move, segment: Segment, source_kbytes: int, start_time: datetime): - """Record initial move details""" - - sql = """ - INSERT INTO gprebalance.status_detail ( - dbid, content, role, preferred_role, - source_hostname, source_port, source_datadir, - dest_hostname, target_datadir, target_port, status, rebalance_started, source_kbytes - ) VALUES ( - %d, %d, '%s', '%s', '%s', %d, '%s', '%s', '%s', %d, '%s', '%s', %d - ) - """ % ( - segment.dbid, - segment.content, - segment.role, - segment.preferred_role, - move.srcHost.hostname, - segment.port, - segment.datadir, - move.dstHost.hostname, - move.target_datadir, - move.target_port, - MoveStatus.PENDING.value, - str(start_time), - source_kbytes - ) - dbconn.execSQL(conn, sql) - - @staticmethod - def update_record_status(conn: dbconn.Connection, dbids: List[int], status: MoveStatus): - """Record initial move details""" - dbid_str = ','.join(str(dbid) for dbid in dbids) - if status == MoveStatus.COMPLETED: - sql = "UPDATE gprebalance.status_detail SET status = '%s' , rebalance_finished = '%s' WHERE dbid IN (%s)" % ( - status.value, datetime.now(), dbid_str + # If pg_ctl status failed, check for actual process + # Get postmaster.pid contents if it exists + pid_cmd = Command( + name="check_postmaster_pid", + cmdStr=f"test -f {data_dir}/postmaster.pid && cat {data_dir}/postmaster.pid | head -1", + ctxt=REMOTE, + remoteHost=host ) - else: - sql = "UPDATE gprebalance.status_detail SET status = '%s' WHERE dbid IN (%s)" % ( - status.value, dbid_str + pid_cmd.run() + + if not pid_cmd.was_successful() or not pid_cmd.get_stdout().strip(): + return False + + pid = pid_cmd.get_stdout().strip() + if not pid.isdigit(): + return False + + # Check if process with that PID exists and is postgres + process_cmd = Command( + name="check_postgres_process", + cmdStr=f"ps -p {pid} -o cmd= | grep postgres", + ctxt=REMOTE, + remoteHost=host ) - dbconn.execSQL(conn, sql) + process_cmd.run() - def remove_all(self): - if self._fp: - self._fp.close() - self._fp = None - if os.path.exists(self._status_filename): - os.unlink(self._status_filename) + return process_cmd.was_successful() and process_cmd.get_stdout().strip() + + except Exception as e: + self.logger.error(f"Error checking postgres process on {host}: {str(e)}") + return False + + def _wait_for_pg_startup(self, host, data_dir, timeout_seconds=120): + + self.logger.info(f"Waiting for PostgreSQL to start on {host}:{data_dir}") + + start_time = time.time() + check_interval = 5 # Check every 5 seconds + + while time.time() - start_time < timeout_seconds: + if self._check_postgres_process(host, data_dir): + elapsed = int(time.time() - start_time) + self.logger.info(f"PostgreSQL started on {host}:{data_dir} after {elapsed} seconds") + return True + + # Wait before next check + time.sleep(check_interval) + elapsed = int(time.time() - start_time) + + self.logger.info(f"Timeout waiting for PostgreSQL to start on {host}:{data_dir}") + return False + + def _wait_for_config_update(self, dbid, timeout_seconds=120): + + start_time = time.time() + check_interval = 5 + + conf_sql = """ + SELECT c.mode, c.status + FROM gp_segment_configuration AS c WHERE c.dbid = {sdbid} + """ .format(sdbid=dbid) + while time.time() - start_time < timeout_seconds: + cursor = dbconn.query(self.conn, conf_sql) + for mode, status in cursor: + if mode == 's' and status == 'u': + return True + + # Wait before next check + time.sleep(check_interval) + elapsed = int(time.time() - start_time) + + return False + + def _try_restore_config(self, moves_to_recover, plan): + from gppylib.commands.gp import GpRecoverSeg + gparray = copy.deepcopy(self.initial_gparray) + def write_gprecoverseg_config(segment, conf_dir): + filename = self.conf_dir + "_revert_" + "dbid" + str(segment.dbid) + with open(filename, 'w') as fp: + line = (f"{canonicalize_address(segment.address)}|" + f"{segment.port}|{segment.datadir}") + self.logger.info( + "About to run gprecoverseg for recovering mirror " + f"(dbid = {segment.dbid}, content = {segment.content}) {line}") + fp.write(line) + return filename + for move in moves_to_recover: + segid = SegmentId(move["dbid"], move["content"]) + original_segment = plan.segmentMap[segid] + dbids = {} + seg = None + peer = None + for pair in gparray.segmentPairs: + if pair.mirrorDB.dbid == segid.dbid: + pair.mirrorDB = copy.copy(original_segment) + pair.mirrorDB.status = 'd' + pair.mirrorDB.mode = 'n' + dbids[pair.mirrorDB.dbid] = True + pair.mirrorDB.role = 'm' + seg=pair.mirrorDB + peer = pair.primaryDB + configInterface.getConfigurationProvider().updateSystemConfig( + gparray, + "gprebalance: segment config for resync", + dbIdToForceMirrorRemoveAdd=dbids, + useUtilityMode=False, + allowPrimary=False + ) + #host_port_tuple = (peer.hostname, peer.port) + #pg.kill_existing_walsenders_on_primary([host_port_tuple]) + filename = write_gprecoverseg_config(seg, self.conf_dir) + + recoversegOptions = f"-a -l {os.path.join(os.environ.get('HOME', '.'),'gpAdminLogs/rebalance')} "\ + f"-i {filename}" + if self.options.hba_hostnames: + recoversegOptions += " --hba-hostnames" + cmd = GpRecoverSeg("Running gprecoverseg", options=recoversegOptions) + cmd.run() + recordStatus = MoveStatus.REVERTED.value + if not cmd.was_successful(): + self.logger.info(f"Failed to rollback mirror moves (dbid = {seg.dbid}, content = {seg.content}): {cmd.get_stderr()}, skipping...") + recordStatus = MoveStatus.SKIPPED.value + move["status"] = recordStatus + os.unlink(filename) + + def _attempt_start_mirror(self, segment): + from gppylib.commands.gp import SegmentStart + from gppylib.gp_era import read_era + seg = Segment(None, None, None, None, None, None, None, None, + segment.port, segment.datadir) + cmd = cmd = SegmentStart( + name="Attemping to start segment with dbid %s:" % (str(segment.dbid)) + , gpdb=seg + , numContentsInCluster=0 + , era=read_era(self.gpenv.getCoordinatorDataDir(), logger=self.logger) + , mirrormode="mirror" + , utilityMode=False + , ctxt=REMOTE + , remoteHost=segment.hostname) + self.logger.info(str(cmd)) + cmd.run() + if not cmd.was_successful(): + return False + return True \ No newline at end of file From 5b3f506b7bb0be61e8bec6c32e7b16cd764fc652 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 16 Jul 2025 01:37:46 +1000 Subject: [PATCH 08/22] Support 'ALTER TABLE REBALANCE' MVP (#1845) Problem description: There were no means to provide segments shrink feature to the 'gprebalance' tool. Fix: Add new command 'ALTER TABLE REBALANCE' (MVP level). Details: 1. 'ALTER TABLE REBALANCE' supports an optional parameter - target number of segments (ex. 'ALTER TABLE REBALANCE 2;'). 2. If the target number of segments is more than the number of segments in the table's distribution policy, rebalance command will invoke the existing functionality of 'ALTER TABLE EXPAND TABLE' (meaning that expand will always be done to the current number of segments in the cluster, even if we specified less) 3. If the target number of segments is less than the number of segments in the table's distribution policy, the table will be shrunk into the target number of segments. For hashed or randomly distributed tables, data from the excessive segments is inserted into the target segments, and then for all table types the distribution policy is updated for the target number of segments. Data from the excessive segments is not removed (we do not want to spend time on it, as most likely they will be excluded from the cluster soon anyway). 4. New GUC 'gp_target_numsegments' is added. If the target number of segments is not specified for the 'ALTER TABLE REBALANCE' command, value of 'gp_target_numsegments' is used. 5. If 'gp_target_numsegments' is set, all new tables are created using this number of segments. --- src/backend/cdb/cdbvars.c | 2 + src/backend/commands/tablecmds.c | 220 ++ src/backend/gpopt/gpdbwrappers.cpp | 11 + .../translate/CTranslatorDXLToPlStmt.cpp | 7 + src/backend/optimizer/util/pathnode.c | 9 + src/backend/parser/gram.y | 19 +- src/backend/utils/misc/guc_gp.c | 32 + src/bin/psql/tab-complete.c | 2 +- src/include/cdb/cdbvars.h | 2 + src/include/gpopt/gpdbwrappers.h | 5 +- src/include/nodes/parsenodes.h | 4 +- src/include/parser/kwlist.h | 1 + src/include/utils/sync_guc_name.h | 1 + .../alter_blocks_for_update_and_viceversa.out | 70 + .../alter_blocks_for_update_and_viceversa.sql | 37 + src/test/regress/expected/alter_rebalance.out | 3509 +++++++++++++++++ src/test/regress/greenplum_schedule | 1 + src/test/regress/sql/alter_rebalance.sql | 440 +++ 18 files changed, 4368 insertions(+), 4 deletions(-) create mode 100644 src/test/regress/expected/alter_rebalance.out create mode 100644 src/test/regress/sql/alter_rebalance.sql diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c index d29a66ab5e2a..d59727eb18c4 100644 --- a/src/backend/cdb/cdbvars.c +++ b/src/backend/cdb/cdbvars.c @@ -334,6 +334,8 @@ bool log_autostats = true; /* GUC to toggle JIT instrumentation output for EXPLAIN */ bool gp_explain_jit = true; +int gp_segment_number_for_table_shrink = 0; + /* -------------------------------------------------------------------------------------------------- * Server debugging */ diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 5e198508dac6..7a1cf6d681b5 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -78,6 +78,7 @@ #include "commands/vacuum.h" #include "executor/executor.h" #include "executor/instrument.h" +#include "executor/spi.h" #include "foreign/foreign.h" #include "miscadmin.h" #include "nodes/makefuncs.h" @@ -507,6 +508,9 @@ static void RangeVarCallbackForAlterRelation(const RangeVar *rv, Oid relid, Oid oldrelid, void *arg); static void ATExecExpandTable(List **wqueue, Relation rel, AlterTableCmd *cmd); + +static void ATExecRebalanceTable(List **wqueue, Relation rel, AlterTableCmd *cmd); + static void ATRepackTable(Relation origTable, AlteredTableInfo *tab); static void ATExecExpandPartitionTablePrepare(Relation rel); static void ATExecExpandTableCTAS(AlterTableCmd *rootCmd, Relation rel, AlterTableCmd *cmd); @@ -4800,6 +4804,7 @@ AlterTableGetLockLevel(List *cmds) case AT_ExpandTable: case AT_ExpandPartitionTablePrepare: case AT_SetDistributedBy: + case AT_Rebalance: cmd_lockmode = AccessExclusiveLock; break; case AT_RepackTable: /* GPDB: REPACK TABLE */ @@ -5293,6 +5298,59 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, ATSimpleRecursion(wqueue, rel, cmd, recurse, lockmode); pass = AT_PASS_MISC; break; + case AT_Rebalance: + ATSimplePermissions(rel, ATT_TABLE | ATT_FOREIGN_TABLE | ATT_MATVIEW); + + Assert(IsA(cmd->def, Integer)); + int targetNumSegments = intVal(cmd->def); + + if (targetNumSegments <= 0) + { + targetNumSegments = GP_POLICY_DEFAULT_NUMSEGMENTS(); + + elog(NOTICE, + "REBALANCE to current default target number of segments %d", + targetNumSegments); + + pfree(cmd->def); + cmd->def = (Node *) makeInteger(targetNumSegments); + } + + if (!recursing) + { + if (Gp_role == GP_ROLE_DISPATCH && + rel->rd_cdbpolicy->numsegments < targetNumSegments && + targetNumSegments != getgpsegmentCount()) + { + targetNumSegments = getgpsegmentCount(); + elog(NOTICE, + "REBALANCE currently supports expand only to total cluster " + "segments number, so target number of segments is forced to %d", + targetNumSegments); + } + + if (Gp_role == GP_ROLE_DISPATCH && + rel->rd_cdbpolicy->numsegments == targetNumSegments) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot rebalance table \"%s\"", + RelationGetRelationName(rel)), + errdetail("table has already been rebalanced"))); + + if (rel->rd_rel->relispartition) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot rebalance leaf or interior partition \"%s\"", + RelationGetRelationName(rel)), + errdetail("Root/leaf/interior partitions need to have same numsegments"), + errhint("Call ALTER TABLE REBALANCE on the root table instead"))); + + } + + ATSimpleRecursion(wqueue, rel, cmd, recurse, lockmode); + pass = AT_PASS_MISC; + break; + case AT_RepackTable: /* GPDB: REPACK TABLE */ ATSimplePermissions(rel, ATT_TABLE); ATSimpleRecursion(wqueue, rel, cmd, recurse, lockmode); @@ -5844,6 +5902,9 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, case AT_ExpandTable: /* EXPAND TABLE */ ATExecExpandTable(wqueue, rel, cmd); break; + case AT_Rebalance: /* REBALANCE */ + ATExecRebalanceTable(wqueue, rel, cmd); + break; case AT_RepackTable: for (int i = 0; i < AT_NUM_PASSES; ++i) { @@ -17460,6 +17521,165 @@ ATExecExpandTable(List **wqueue, Relation rel, AlterTableCmd *cmd) GpPolicyReplace(relid, newPolicy); } +/* + * Move data as needed from the excessive segments into the reduced number of + * segments. + * + * We do it only for hashed or randomly distributed tables. There is no need for + * the replicated tables. + * + * Moving is done by inserting values from the excessive segments. The plan for + * the query is forced to treat the table as randonly distributed and force + * redistribute motion for the insert action. Distribution policy for the insert + * action is adjusted at the planning stage to consider only the target number + * of segments. + */ +static void +ATExecShrinkTable(Relation rel, GpPolicy *policy) +{ + if (Gp_role == GP_ROLE_DISPATCH && GpPolicyIsPartitioned(policy)) + { + volatile bool connected = false; + StringInfoData sqlstmtInsert; + initStringInfo(&sqlstmtInsert); + char *nsp = get_namespace_name(RelationGetNamespace(rel)); + char *relname = RelationGetRelationName(rel); + + /* + * 'gp_dist_random' will cause fallback to Postgres planner, + * so no need to tweak 'optimizer' value. + */ + appendStringInfo(&sqlstmtInsert, + "insert into %s.%s select * " + "from gp_dist_random('%s.%s') " + "where gp_segment_id >= %d", + nsp, relname, + nsp, relname, + policy->numsegments); + + gp_segment_number_for_table_shrink = policy->numsegments; + + PG_TRY(); + { + if (SPI_OK_CONNECT != SPI_connect()) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("unable shrink table"), + errdetail("SPI_connect failed in %s.", __func__))); + + connected = true; + + if (SPI_execute(sqlstmtInsert.data, false, 0) <= 0) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("unable shrink table"), + errdetail("SPI_execute failed in %s.", __func__))); + + connected = false; + SPI_finish(); + } + + /* Clean up in case of error. */ + PG_CATCH(); + { + if (connected) + SPI_finish(); + + pfree(sqlstmtInsert.data); + + gp_segment_number_for_table_shrink = 0; + + /* Carry on with error handling. */ + PG_RE_THROW(); + } + PG_END_TRY(); + + pfree(sqlstmtInsert.data); + + gp_segment_number_for_table_shrink = 0; + } +} + +/* + * ALTER TABLE REBALANCE + * + * In case the target number of segments is more than the number of segments in + * the table's distribution policy, we invoke the expand functionality. + * Otherwise we shrink the table and update table's "numsegments" value to the + * target number of segments. + */ +static void +ATExecRebalanceTable(List **wqueue, Relation rel, AlterTableCmd *cmd) +{ + MemoryContext oldContext; + Oid relid = RelationGetRelid(rel); + GpPolicy *newPolicy; + GpPolicy *policy = rel->rd_cdbpolicy; + int targetNumSegments = intVal(cmd->def); + + if (Gp_role == GP_ROLE_UTILITY) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("REBALANCE not supported in utility mode"))); + + /* Permissions checks */ + if (!pg_class_ownercheck(relid, GetUserId())) + aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_TABLE, + RelationGetRelationName(rel)); + + if (IsSystemRelation(rel)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied: \"%s\" is a system catalog", + RelationGetRelationName(rel)))); + + if (targetNumSegments > policy->numsegments) + { + ATExecExpandTable(wqueue, rel, cmd); + return; + } + + oldContext = MemoryContextSwitchTo(GetMemoryChunkContext(rel)); + newPolicy = GpPolicyCopy(policy); + MemoryContextSwitchTo(oldContext); + newPolicy->numsegments = targetNumSegments; + + if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + /* + * Nothing to do on a partitioned table. But we better recurse to the + * child partitions. + */ + } + else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + { + if (rel_is_external_table(relid)) + { + ExtTableEntry *ext = GetExtTableEntry(relid); + + if (!ext->iswritable) + { + /* + * Skip expanding readable external table, since data is not + * located inside gpdb + */ + return; + } + } + else + { + /* Skip expanding foreign table, since data is not located inside gpdb */ + return; + } + } + else + { + ATExecShrinkTable(rel, newPolicy); + } + + GpPolicyReplace(relid, newPolicy); +} + /* * ALTER TABLE xxx EXPAND PARTITION PREPARE * diff --git a/src/backend/gpopt/gpdbwrappers.cpp b/src/backend/gpopt/gpdbwrappers.cpp index cc0a1c1b8040..ab2090c022df 100644 --- a/src/backend/gpopt/gpdbwrappers.cpp +++ b/src/backend/gpopt/gpdbwrappers.cpp @@ -1094,6 +1094,17 @@ gpdb::GetGPSegmentCount(void) return 0; } +int +gpdb::GetGPTargetSegmentCount(void) +{ + GP_WRAP_START; + { + return GP_POLICY_DEFAULT_NUMSEGMENTS(); + } + GP_WRAP_END; + return 0; +} + bool gpdb::HeapAttIsNull(HeapTuple tup, int attno) { diff --git a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp index e8c514060e79..a71cdad3aacf 100644 --- a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp +++ b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp @@ -6304,6 +6304,13 @@ CTranslatorDXLToPlStmt::TranslateDXLPhyCtasToDistrPolicy( num_of_distr_cols_alloc = num_of_distr_cols; } + if (gpdb::GetGPTargetSegmentCount() != gpdb::GetGPSegmentCount()) + { + // GPORCA does not support partially distributed tables yet + GPOS_RAISE(gpdxl::ExmaMD, gpdxl::ExmiDXLInvalidAttributeValue, + GPOS_WSZ_LIT("Partially Distributed Data")); + } + // always set numsegments to ALL for CTAS GpPolicy *distr_policy = gpdb::MakeGpPolicy(POLICYTYPE_PARTITIONED, num_of_distr_cols_alloc, diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 490c3679bb0c..faa9c8a228f5 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -5406,6 +5406,15 @@ adjust_modifytable_subpaths(PlannerInfo *root, CmdType operation, targetPolicy = GpPolicyFetch(rte->relid); targetPolicyType = targetPolicy->ptype; + /* + * Limit the number of segments to the target value that we want + * to achieve by the shrink operation. + */ + if (targetPolicyType == POLICYTYPE_PARTITIONED && + gp_segment_number_for_table_shrink > 0) + targetPolicy->numsegments = + Min(targetPolicy->numsegments, gp_segment_number_for_table_shrink); + numsegments = Max(targetPolicy->numsegments, numsegments); if (targetPolicyType == POLICYTYPE_PARTITIONED) diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 9e94f28411fc..3197a1c2ced0 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -837,7 +837,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ QUEUE RANDOMLY READABLE READS REJECT_P REPACK REPLICATED - RESOURCE ROOTPARTITION + RESOURCE ROOTPARTITION REBALANCE SCATTER SEGMENT SEGMENTS SPLIT SUBPARTITION @@ -3332,6 +3332,22 @@ alter_table_cmd: n->subtype = AT_ExpandPartitionTablePrepare; $$ = (Node *)n; } + /* ALTER TABLE REBALANCE*/ + | REBALANCE + { + AlterTableCmd *n = makeNode(AlterTableCmd); + n->subtype = AT_Rebalance; + n->def = (Node *) makeInteger(0); + $$ = (Node *)n; + } + /* ALTER TABLE REBALANCE */ + | REBALANCE SignedIconst + { + AlterTableCmd *n = makeNode(AlterTableCmd); + n->subtype = AT_Rebalance; + n->def = (Node *) makeInteger($2); + $$ = (Node *)n; + } /* ALTER TABLE OF */ | OF any_name { @@ -18313,6 +18329,7 @@ unreserved_keyword: | READABLE | READS | REASSIGN + | REBALANCE | RECHECK | RECURSIVE | REF diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index fd9c3969165a..18e86b1be7ec 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -95,6 +95,7 @@ static void assign_gp_default_storage_options(const char *newval, void *extra); static bool check_pljava_classpath_insecure(bool *newval, void **extra, GucSource source); static void assign_pljava_classpath_insecure(bool newval, void *extra); static bool check_gp_resource_group_bypass(bool *newval, void **extra, GucSource source); +static bool check_gp_target_numsegments(int *newval, void **extra, GucSource source); static int guc_array_compare(const void *a, const void *b); extern struct config_generic *find_option(const char *name, bool create_placeholders, int elevel); @@ -4406,6 +4407,21 @@ struct config_int ConfigureNamesInt_gp[] = }, #endif + { + {"gp_target_numsegments", PGC_SUSET, GP_ARRAY_CONFIGURATION, + gettext_noop("Target number of segments to use when a new table " + "is created or an old one is rebalanced."), + gettext_noop("Special values:\n" + "-1 - use all available segments\n" + "-2 - use random number of segments\n" + "-3 - use minimal possible number of segment\n" ), + GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE + }, + &gp_create_table_default_numsegments, + GP_DEFAULT_NUMSEGMENTS_FULL, GP_DEFAULT_NUMSEGMENTS_MINIMAL, INT_MAX, + check_gp_target_numsegments, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL @@ -5155,6 +5171,22 @@ check_gp_resource_group_bypass(bool *newval, void **extra, GucSource source) return false; } +static bool +check_gp_target_numsegments(int *newval, void **extra, GucSource source) +{ + int max_segment_number = getgpsegmentCount(); + if (*newval > max_segment_number) + { + elog(WARNING, + "Can't set value more than maximum segment number, " + "force 'gp_target_numsegments' to %d", + max_segment_number); + *newval = max_segment_number; + } + return true; +} + + static bool check_optimizer(bool *newval, void **extra, GucSource source) { diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index c8fa2e5122b9..0edcf6c1b8b5 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1883,7 +1883,7 @@ psql_completion(const char *text, int start, int end) "ENABLE", "INHERIT", "NO INHERIT", "RENAME", "RESET", "OWNER TO", "SET", "VALIDATE CONSTRAINT", "REPLICA IDENTITY", "ATTACH PARTITION", - "DETACH PARTITION"); + "DETACH PARTITION", "REBALANCE"); /* ALTER TABLE xxx ENABLE */ else if (Matches("ALTER", "TABLE", MatchAny, "ENABLE")) COMPLETE_WITH("ALWAYS", "REPLICA", "ROW LEVEL SECURITY", "RULE", diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h index 4c0a8826773d..9518af9e23d7 100644 --- a/src/include/cdb/cdbvars.h +++ b/src/include/cdb/cdbvars.h @@ -433,6 +433,8 @@ extern int gp_udpic_network_disable_ipv6; */ extern uint32 gp_interconnect_id; +extern int gp_segment_number_for_table_shrink; + /* -------------------------------------------------------------------------------------------------- * Logging */ diff --git a/src/include/gpopt/gpdbwrappers.h b/src/include/gpopt/gpdbwrappers.h index c68e0100aa75..9cd2ccfb1460 100644 --- a/src/include/gpopt/gpdbwrappers.h +++ b/src/include/gpopt/gpdbwrappers.h @@ -292,9 +292,12 @@ Oid GetTypeRelid(Oid typid); // name of the type with the given oid char *GetTypeName(Oid typid); -// number of GP segments +// total number of GP segments int GetGPSegmentCount(void); +// target number of GP segments to use +int GetGPTargetSegmentCount(void); + // heap attribute is null bool HeapAttIsNull(HeapTuple tup, int attnum); diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 04d67c653f04..1d58ed11f655 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2019,7 +2019,9 @@ typedef enum AlterTableType AT_PartTruncate, /* Truncate */ /* kept at end for ABI hygiene */ - AT_RepackTable /* REPACK TABLE */ + AT_RepackTable, /* REPACK TABLE */ + + AT_Rebalance /* REBALANCE */ } AlterTableType; typedef struct ReplicaIdentityStmt diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index cdb394701bc8..6f59aad2940a 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -372,6 +372,7 @@ PG_KEYWORD("readable", READABLE, UNRESERVED_KEYWORD) /* GPDB */ PG_KEYWORD("reads", READS, UNRESERVED_KEYWORD) PG_KEYWORD("real", REAL, COL_NAME_KEYWORD) PG_KEYWORD("reassign", REASSIGN, UNRESERVED_KEYWORD) +PG_KEYWORD("rebalance", REBALANCE, UNRESERVED_KEYWORD) /* GPDB */ PG_KEYWORD("recheck", RECHECK, UNRESERVED_KEYWORD) PG_KEYWORD("recursive", RECURSIVE, UNRESERVED_KEYWORD) PG_KEYWORD("ref", REF, UNRESERVED_KEYWORD) diff --git a/src/include/utils/sync_guc_name.h b/src/include/utils/sync_guc_name.h index 0228d8d5cf8f..c9d396025f31 100644 --- a/src/include/utils/sync_guc_name.h +++ b/src/include/utils/sync_guc_name.h @@ -179,3 +179,4 @@ "wal_sender_timeout", "work_mem", "zero_damaged_pages", + "gp_target_numsegments", diff --git a/src/test/isolation2/expected/alter_blocks_for_update_and_viceversa.out b/src/test/isolation2/expected/alter_blocks_for_update_and_viceversa.out index 7f9e1a7004a0..0a564d65930f 100644 --- a/src/test/isolation2/expected/alter_blocks_for_update_and_viceversa.out +++ b/src/test/isolation2/expected/alter_blocks_for_update_and_viceversa.out @@ -36,3 +36,73 @@ ALTER TABLE COMMIT 1<: <... completed> UPDATE 1 + +-- Check ALTER REBALANCE +1: DROP TABLE alter_block; +DROP TABLE +1: CREATE TABLE alter_block(a int, b int) DISTRIBUTED BY (a); +CREATE TABLE +1: INSERT INTO alter_block SELECT i, i FROM generate_series(1, 20)i; +INSERT 0 20 + +2: SET gp_target_numsegments = 2; +SET + +-- Validate SELECT blocks the ALTER +1: BEGIN; +BEGIN +2: BEGIN; +BEGIN +1: SELECT count(1), gp_segment_id FROM alter_block GROUP BY gp_segment_id ORDER BY gp_segment_id; + count | gp_segment_id +-------+--------------- + 8 | 0 + 4 | 1 + 8 | 2 +(3 rows) +2&: ALTER TABLE alter_block REBALANCE; +1: COMMIT; +COMMIT +2<: <... completed> +ALTER TABLE +2: ROLLBACK; +ROLLBACK + +-- Validate UPDATE blocks the ALTER +1: BEGIN; +BEGIN +2: BEGIN; +BEGIN +1: UPDATE alter_block SET b = b + 1 WHERE a = 1; +UPDATE 1 +2&: ALTER TABLE alter_block REBALANCE; +1: COMMIT; +COMMIT +2<: <... completed> +ALTER TABLE +2: ROLLBACK; +ROLLBACK + +-- Validate ALTER blocks the SELECT +1: BEGIN; +BEGIN +2: BEGIN; +BEGIN +2: ALTER TABLE alter_block REBALANCE; +ALTER TABLE +1&: SELECT count(1), gp_segment_id FROM alter_block GROUP BY gp_segment_id ORDER BY gp_segment_id; +2: COMMIT; +COMMIT +1<: <... completed> + count | gp_segment_id +-------+--------------- + 12 | 0 + 8 | 1 +(2 rows) +1: COMMIT; +COMMIT + +2: RESET gp_target_numsegments; +RESET +1: DROP TABLE alter_block; +DROP TABLE diff --git a/src/test/isolation2/sql/alter_blocks_for_update_and_viceversa.sql b/src/test/isolation2/sql/alter_blocks_for_update_and_viceversa.sql index b22000f13c30..c9ede0501bfe 100644 --- a/src/test/isolation2/sql/alter_blocks_for_update_and_viceversa.sql +++ b/src/test/isolation2/sql/alter_blocks_for_update_and_viceversa.sql @@ -17,3 +17,40 @@ 2: SELECT wait_event_type FROM pg_stat_activity where query like 'UPDATE alter_block SET %'; 2: COMMIT; 1<: + +-- Check ALTER REBALANCE +1: DROP TABLE alter_block; +1: CREATE TABLE alter_block(a int, b int) DISTRIBUTED BY (a); +1: INSERT INTO alter_block SELECT i, i FROM generate_series(1, 20)i; + +2: SET gp_target_numsegments = 2; + +-- Validate SELECT blocks the ALTER +1: BEGIN; +2: BEGIN; +1: SELECT count(1), gp_segment_id FROM alter_block GROUP BY gp_segment_id ORDER BY gp_segment_id; +2&: ALTER TABLE alter_block REBALANCE; +1: COMMIT; +2<: +2: ROLLBACK; + +-- Validate UPDATE blocks the ALTER +1: BEGIN; +2: BEGIN; +1: UPDATE alter_block SET b = b + 1 WHERE a = 1; +2&: ALTER TABLE alter_block REBALANCE; +1: COMMIT; +2<: +2: ROLLBACK; + +-- Validate ALTER blocks the SELECT +1: BEGIN; +2: BEGIN; +2: ALTER TABLE alter_block REBALANCE; +1&: SELECT count(1), gp_segment_id FROM alter_block GROUP BY gp_segment_id ORDER BY gp_segment_id; +2: COMMIT; +1<: +1: COMMIT; + +2: RESET gp_target_numsegments; +1: DROP TABLE alter_block; diff --git a/src/test/regress/expected/alter_rebalance.out b/src/test/regress/expected/alter_rebalance.out new file mode 100644 index 000000000000..1d9227b1eca8 --- /dev/null +++ b/src/test/regress/expected/alter_rebalance.out @@ -0,0 +1,3509 @@ +-- Check 'ALTER TABLE ... REBALANCE' command and 'gp_target_numsegments' GUC +-- Create hashed distributed tables +create table table_distr_hashed(a int) distributed by (a); +insert into table_distr_hashed select generate_series(1, 20); +create table table_distr_hashed_ao_row(a int) with (appendonly=true, orientation=row) distributed by (a); +insert into table_distr_hashed_ao_row select generate_series(1, 20); +create table table_distr_hashed_ao_col(a int) with (appendonly=true, orientation=column) distributed by (a); +insert into table_distr_hashed_ao_col select generate_series(1, 20); +-- Create randomly distributed tables +create table table_distr_random(a int) distributed randomly; +insert into table_distr_random select generate_series(1, 20); +create table table_distr_random_ao_row(a int) with (appendonly=true, orientation=row) distributed randomly; +insert into table_distr_random_ao_row select generate_series(1, 20); +create table table_distr_random_ao_col(a int) with (appendonly=true, orientation=column) distributed randomly; +insert into table_distr_random_ao_col select generate_series(1, 20); +-- Create replicated distributed tables +create table table_distr_replicated(a int) distributed replicated; +insert into table_distr_replicated select generate_series(1, 20); +create table table_distr_replicated_ao_row(a int) with (appendonly=true, orientation=row) distributed replicated; +insert into table_distr_replicated_ao_row select generate_series(1, 20); +create table table_distr_replicated_ao_col(a int) with (appendonly=true, orientation=column) distributed replicated; +insert into table_distr_replicated_ao_col select generate_series(1, 20); +-- Create part tables +create table part_range_table_distr_hashed (a int, b date) distributed by (a) +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into part_range_table_distr_hashed select i, '2023-01-02' from generate_series(1, 20)i; +insert into part_range_table_distr_hashed select i, '2023-05-02' from generate_series(1, 20)i; +insert into part_range_table_distr_hashed select i, '2020-05-02' from generate_series(1, 20)i; +create table part_range_table_distr_random (a int, b date) distributed randomly +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into part_range_table_distr_random select i, '2023-01-02' from generate_series(1, 20)i; +insert into part_range_table_distr_random select i, '2023-05-02' from generate_series(1, 20)i; +insert into part_range_table_distr_random select i, '2020-05-02' from generate_series(1, 20)i; +create table part_list_table_distr_hashed(a int, b text) distributed by (a) +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into part_list_table_distr_hashed select i, 'test1' from generate_series(1, 20)i; +insert into part_list_table_distr_hashed select i, 'test2' from generate_series(1, 20)i; +insert into part_list_table_distr_hashed select i, 'test3' from generate_series(1, 20)i; +create table part_list_table_distr_random(a int, b text) distributed randomly +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into part_list_table_distr_random select i, 'test1' from generate_series(1, 20)i; +insert into part_list_table_distr_random select i, 'test2' from generate_series(1, 20)i; +insert into part_list_table_distr_random select i, 'test3' from generate_series(1, 20)i; +create table multi_part_table_distr_hashed(a int, b date, c text) distributed by (a) +partition by range (b) +subpartition by list (c) subpartition template +( + subpartition subpart1 values ('test1'), + subpartition subpart2 values ('test2') +) +( + partition part1 start (date '2023-01-01'), + partition part2 start (date '2023-02-01'), + partition part3 start (date '2023-03-01') end (date '2024-01-01') +); +insert into multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(1, 20)i; +insert into multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(1, 20)i; +insert into multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(1, 20)i; +-- Now check shrink of the created tables into 2 segments +set gp_target_numsegments = 2; +alter table table_distr_hashed rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_hashed_ao_row rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_hashed_ao_col rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_random rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_random_ao_row rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_random_ao_col rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_replicated rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_replicated_ao_row rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_replicated_ao_col rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table part_range_table_distr_hashed rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table part_range_table_distr_random rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table part_list_table_distr_hashed rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table part_list_table_distr_random rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table multi_part_table_distr_hashed rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +-- Verify that data is presented only on segments #0 and #1 +select a, gp_segment_id from table_distr_hashed order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select a, gp_segment_id from table_distr_hashed_ao_row order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select a, gp_segment_id from table_distr_hashed_ao_col order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_row order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_col order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +select a from table_distr_replicated order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +(20 rows) + +select a from table_distr_replicated_ao_row order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +(20 rows) + +select a from table_distr_replicated_ao_col order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +(20 rows) + +select *, gp_segment_id from part_range_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+------------+--------------- + 1 | 05-02-2020 | 1 + 1 | 01-02-2023 | 1 + 1 | 05-02-2023 | 1 + 2 | 05-02-2020 | 0 + 2 | 01-02-2023 | 0 + 2 | 05-02-2023 | 0 + 3 | 05-02-2020 | 0 + 3 | 01-02-2023 | 0 + 3 | 05-02-2023 | 0 + 4 | 05-02-2020 | 0 + 4 | 01-02-2023 | 0 + 4 | 05-02-2023 | 0 + 5 | 05-02-2020 | 1 + 5 | 01-02-2023 | 1 + 5 | 05-02-2023 | 1 + 6 | 05-02-2020 | 0 + 6 | 01-02-2023 | 0 + 6 | 05-02-2023 | 0 + 7 | 05-02-2020 | 0 + 7 | 01-02-2023 | 0 + 7 | 05-02-2023 | 0 + 8 | 05-02-2020 | 0 + 8 | 01-02-2023 | 0 + 8 | 05-02-2023 | 0 + 9 | 05-02-2020 | 0 + 9 | 01-02-2023 | 0 + 9 | 05-02-2023 | 0 + 10 | 05-02-2020 | 0 + 10 | 01-02-2023 | 0 + 10 | 05-02-2023 | 0 + 11 | 05-02-2020 | 1 + 11 | 01-02-2023 | 1 + 11 | 05-02-2023 | 1 + 12 | 05-02-2020 | 1 + 12 | 01-02-2023 | 1 + 12 | 05-02-2023 | 1 + 13 | 05-02-2020 | 0 + 13 | 01-02-2023 | 0 + 13 | 05-02-2023 | 0 + 14 | 05-02-2020 | 1 + 14 | 01-02-2023 | 1 + 14 | 05-02-2023 | 1 + 15 | 05-02-2020 | 1 + 15 | 01-02-2023 | 1 + 15 | 05-02-2023 | 1 + 16 | 05-02-2020 | 0 + 16 | 01-02-2023 | 0 + 16 | 05-02-2023 | 0 + 17 | 05-02-2020 | 1 + 17 | 01-02-2023 | 1 + 17 | 05-02-2023 | 1 + 18 | 05-02-2020 | 0 + 18 | 01-02-2023 | 0 + 18 | 05-02-2023 | 0 + 19 | 05-02-2020 | 0 + 19 | 01-02-2023 | 0 + 19 | 05-02-2023 | 0 + 20 | 05-02-2020 | 1 + 20 | 01-02-2023 | 1 + 20 | 05-02-2023 | 1 +(60 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from part_range_table_distr_random order by a, b; + a | b | correct_segment_id +----+------------+-------------------- + 1 | 05-02-2020 | t + 1 | 01-02-2023 | t + 1 | 05-02-2023 | t + 2 | 05-02-2020 | t + 2 | 01-02-2023 | t + 2 | 05-02-2023 | t + 3 | 05-02-2020 | t + 3 | 01-02-2023 | t + 3 | 05-02-2023 | t + 4 | 05-02-2020 | t + 4 | 01-02-2023 | t + 4 | 05-02-2023 | t + 5 | 05-02-2020 | t + 5 | 01-02-2023 | t + 5 | 05-02-2023 | t + 6 | 05-02-2020 | t + 6 | 01-02-2023 | t + 6 | 05-02-2023 | t + 7 | 05-02-2020 | t + 7 | 01-02-2023 | t + 7 | 05-02-2023 | t + 8 | 05-02-2020 | t + 8 | 01-02-2023 | t + 8 | 05-02-2023 | t + 9 | 05-02-2020 | t + 9 | 01-02-2023 | t + 9 | 05-02-2023 | t + 10 | 05-02-2020 | t + 10 | 01-02-2023 | t + 10 | 05-02-2023 | t + 11 | 05-02-2020 | t + 11 | 01-02-2023 | t + 11 | 05-02-2023 | t + 12 | 05-02-2020 | t + 12 | 01-02-2023 | t + 12 | 05-02-2023 | t + 13 | 05-02-2020 | t + 13 | 01-02-2023 | t + 13 | 05-02-2023 | t + 14 | 05-02-2020 | t + 14 | 01-02-2023 | t + 14 | 05-02-2023 | t + 15 | 05-02-2020 | t + 15 | 01-02-2023 | t + 15 | 05-02-2023 | t + 16 | 05-02-2020 | t + 16 | 01-02-2023 | t + 16 | 05-02-2023 | t + 17 | 05-02-2020 | t + 17 | 01-02-2023 | t + 17 | 05-02-2023 | t + 18 | 05-02-2020 | t + 18 | 01-02-2023 | t + 18 | 05-02-2023 | t + 19 | 05-02-2020 | t + 19 | 01-02-2023 | t + 19 | 05-02-2023 | t + 20 | 05-02-2020 | t + 20 | 01-02-2023 | t + 20 | 05-02-2023 | t +(60 rows) + +select *, gp_segment_id from part_list_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+-------+--------------- + 1 | test1 | 1 + 1 | test2 | 1 + 1 | test3 | 1 + 2 | test1 | 0 + 2 | test2 | 0 + 2 | test3 | 0 + 3 | test1 | 0 + 3 | test2 | 0 + 3 | test3 | 0 + 4 | test1 | 0 + 4 | test2 | 0 + 4 | test3 | 0 + 5 | test1 | 1 + 5 | test2 | 1 + 5 | test3 | 1 + 6 | test1 | 0 + 6 | test2 | 0 + 6 | test3 | 0 + 7 | test1 | 0 + 7 | test2 | 0 + 7 | test3 | 0 + 8 | test1 | 0 + 8 | test2 | 0 + 8 | test3 | 0 + 9 | test1 | 0 + 9 | test2 | 0 + 9 | test3 | 0 + 10 | test1 | 0 + 10 | test2 | 0 + 10 | test3 | 0 + 11 | test1 | 1 + 11 | test2 | 1 + 11 | test3 | 1 + 12 | test1 | 1 + 12 | test2 | 1 + 12 | test3 | 1 + 13 | test1 | 0 + 13 | test2 | 0 + 13 | test3 | 0 + 14 | test1 | 1 + 14 | test2 | 1 + 14 | test3 | 1 + 15 | test1 | 1 + 15 | test2 | 1 + 15 | test3 | 1 + 16 | test1 | 0 + 16 | test2 | 0 + 16 | test3 | 0 + 17 | test1 | 1 + 17 | test2 | 1 + 17 | test3 | 1 + 18 | test1 | 0 + 18 | test2 | 0 + 18 | test3 | 0 + 19 | test1 | 0 + 19 | test2 | 0 + 19 | test3 | 0 + 20 | test1 | 1 + 20 | test2 | 1 + 20 | test3 | 1 +(60 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from part_list_table_distr_random order by a, b; + a | b | correct_segment_id +----+-------+-------------------- + 1 | test1 | t + 1 | test2 | t + 1 | test3 | t + 2 | test1 | t + 2 | test2 | t + 2 | test3 | t + 3 | test1 | t + 3 | test2 | t + 3 | test3 | t + 4 | test1 | t + 4 | test2 | t + 4 | test3 | t + 5 | test1 | t + 5 | test2 | t + 5 | test3 | t + 6 | test1 | t + 6 | test2 | t + 6 | test3 | t + 7 | test1 | t + 7 | test2 | t + 7 | test3 | t + 8 | test1 | t + 8 | test2 | t + 8 | test3 | t + 9 | test1 | t + 9 | test2 | t + 9 | test3 | t + 10 | test1 | t + 10 | test2 | t + 10 | test3 | t + 11 | test1 | t + 11 | test2 | t + 11 | test3 | t + 12 | test1 | t + 12 | test2 | t + 12 | test3 | t + 13 | test1 | t + 13 | test2 | t + 13 | test3 | t + 14 | test1 | t + 14 | test2 | t + 14 | test3 | t + 15 | test1 | t + 15 | test2 | t + 15 | test3 | t + 16 | test1 | t + 16 | test2 | t + 16 | test3 | t + 17 | test1 | t + 17 | test2 | t + 17 | test3 | t + 18 | test1 | t + 18 | test2 | t + 18 | test3 | t + 19 | test1 | t + 19 | test2 | t + 19 | test3 | t + 20 | test1 | t + 20 | test2 | t + 20 | test3 | t +(60 rows) + +select *, gp_segment_id from multi_part_table_distr_hashed order by a, b, c; + a | b | c | gp_segment_id +----+------------+-------+--------------- + 1 | 01-05-2023 | test1 | 1 + 1 | 02-05-2023 | test2 | 1 + 1 | 03-05-2023 | test1 | 1 + 2 | 01-05-2023 | test1 | 0 + 2 | 02-05-2023 | test2 | 0 + 2 | 03-05-2023 | test1 | 0 + 3 | 01-05-2023 | test1 | 0 + 3 | 02-05-2023 | test2 | 0 + 3 | 03-05-2023 | test1 | 0 + 4 | 01-05-2023 | test1 | 0 + 4 | 02-05-2023 | test2 | 0 + 4 | 03-05-2023 | test1 | 0 + 5 | 01-05-2023 | test1 | 1 + 5 | 02-05-2023 | test2 | 1 + 5 | 03-05-2023 | test1 | 1 + 6 | 01-05-2023 | test1 | 0 + 6 | 02-05-2023 | test2 | 0 + 6 | 03-05-2023 | test1 | 0 + 7 | 01-05-2023 | test1 | 0 + 7 | 02-05-2023 | test2 | 0 + 7 | 03-05-2023 | test1 | 0 + 8 | 01-05-2023 | test1 | 0 + 8 | 02-05-2023 | test2 | 0 + 8 | 03-05-2023 | test1 | 0 + 9 | 01-05-2023 | test1 | 0 + 9 | 02-05-2023 | test2 | 0 + 9 | 03-05-2023 | test1 | 0 + 10 | 01-05-2023 | test1 | 0 + 10 | 02-05-2023 | test2 | 0 + 10 | 03-05-2023 | test1 | 0 + 11 | 01-05-2023 | test1 | 1 + 11 | 02-05-2023 | test2 | 1 + 11 | 03-05-2023 | test1 | 1 + 12 | 01-05-2023 | test1 | 1 + 12 | 02-05-2023 | test2 | 1 + 12 | 03-05-2023 | test1 | 1 + 13 | 01-05-2023 | test1 | 0 + 13 | 02-05-2023 | test2 | 0 + 13 | 03-05-2023 | test1 | 0 + 14 | 01-05-2023 | test1 | 1 + 14 | 02-05-2023 | test2 | 1 + 14 | 03-05-2023 | test1 | 1 + 15 | 01-05-2023 | test1 | 1 + 15 | 02-05-2023 | test2 | 1 + 15 | 03-05-2023 | test1 | 1 + 16 | 01-05-2023 | test1 | 0 + 16 | 02-05-2023 | test2 | 0 + 16 | 03-05-2023 | test1 | 0 + 17 | 01-05-2023 | test1 | 1 + 17 | 02-05-2023 | test2 | 1 + 17 | 03-05-2023 | test1 | 1 + 18 | 01-05-2023 | test1 | 0 + 18 | 02-05-2023 | test2 | 0 + 18 | 03-05-2023 | test1 | 0 + 19 | 01-05-2023 | test1 | 0 + 19 | 02-05-2023 | test2 | 0 + 19 | 03-05-2023 | test1 | 0 + 20 | 01-05-2023 | test1 | 1 + 20 | 02-05-2023 | test2 | 1 + 20 | 03-05-2023 | test1 | 1 +(60 rows) + +-- Check that new data is added only to reduced set of segments +reset gp_target_numsegments; +insert into table_distr_hashed select generate_series(21, 40); +insert into table_distr_hashed_ao_row select generate_series(21, 40); +insert into table_distr_hashed_ao_col select generate_series(21, 40); +insert into table_distr_random select generate_series(21, 40); +insert into table_distr_random_ao_row select generate_series(21, 40); +insert into table_distr_random_ao_col select generate_series(21, 40); +insert into table_distr_replicated select generate_series(21, 40); +insert into table_distr_replicated_ao_row select generate_series(21, 40); +insert into table_distr_replicated_ao_col select generate_series(21, 40); +insert into part_range_table_distr_hashed select i, '2023-01-02' from generate_series(21, 40)i; +insert into part_range_table_distr_hashed select i, '2023-05-02' from generate_series(21, 40)i; +insert into part_range_table_distr_hashed select i, '2020-05-02' from generate_series(21, 40)i; +insert into part_range_table_distr_random select i, '2023-01-02' from generate_series(21, 40)i; +insert into part_range_table_distr_random select i, '2023-05-02' from generate_series(21, 40)i; +insert into part_range_table_distr_random select i, '2020-05-02' from generate_series(21, 40)i; +insert into part_list_table_distr_hashed select i, 'test1' from generate_series(21, 40)i; +insert into part_list_table_distr_hashed select i, 'test2' from generate_series(21, 40)i; +insert into part_list_table_distr_hashed select i, 'test3' from generate_series(21, 40)i; +insert into part_list_table_distr_random select i, 'test1' from generate_series(21, 40)i; +insert into part_list_table_distr_random select i, 'test2' from generate_series(21, 40)i; +insert into part_list_table_distr_random select i, 'test3' from generate_series(21, 40)i; +insert into multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(21, 40)i; +insert into multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(21, 40)i; +insert into multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(21, 40)i; +select a, gp_segment_id from table_distr_hashed order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select a, gp_segment_id from table_distr_hashed_ao_row order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select a, gp_segment_id from table_distr_hashed_ao_col order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_row order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_col order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select a from table_distr_replicated order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 + 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30 + 31 + 32 + 33 + 34 + 35 + 36 + 37 + 38 + 39 + 40 +(40 rows) + +select a from table_distr_replicated_ao_row order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 + 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30 + 31 + 32 + 33 + 34 + 35 + 36 + 37 + 38 + 39 + 40 +(40 rows) + +select a from table_distr_replicated_ao_col order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 + 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30 + 31 + 32 + 33 + 34 + 35 + 36 + 37 + 38 + 39 + 40 +(40 rows) + +select *, gp_segment_id from part_range_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+------------+--------------- + 1 | 05-02-2020 | 1 + 1 | 01-02-2023 | 1 + 1 | 05-02-2023 | 1 + 2 | 05-02-2020 | 0 + 2 | 01-02-2023 | 0 + 2 | 05-02-2023 | 0 + 3 | 05-02-2020 | 0 + 3 | 01-02-2023 | 0 + 3 | 05-02-2023 | 0 + 4 | 05-02-2020 | 0 + 4 | 01-02-2023 | 0 + 4 | 05-02-2023 | 0 + 5 | 05-02-2020 | 1 + 5 | 01-02-2023 | 1 + 5 | 05-02-2023 | 1 + 6 | 05-02-2020 | 0 + 6 | 01-02-2023 | 0 + 6 | 05-02-2023 | 0 + 7 | 05-02-2020 | 0 + 7 | 01-02-2023 | 0 + 7 | 05-02-2023 | 0 + 8 | 05-02-2020 | 0 + 8 | 01-02-2023 | 0 + 8 | 05-02-2023 | 0 + 9 | 05-02-2020 | 0 + 9 | 01-02-2023 | 0 + 9 | 05-02-2023 | 0 + 10 | 05-02-2020 | 0 + 10 | 01-02-2023 | 0 + 10 | 05-02-2023 | 0 + 11 | 05-02-2020 | 1 + 11 | 01-02-2023 | 1 + 11 | 05-02-2023 | 1 + 12 | 05-02-2020 | 1 + 12 | 01-02-2023 | 1 + 12 | 05-02-2023 | 1 + 13 | 05-02-2020 | 0 + 13 | 01-02-2023 | 0 + 13 | 05-02-2023 | 0 + 14 | 05-02-2020 | 1 + 14 | 01-02-2023 | 1 + 14 | 05-02-2023 | 1 + 15 | 05-02-2020 | 1 + 15 | 01-02-2023 | 1 + 15 | 05-02-2023 | 1 + 16 | 05-02-2020 | 0 + 16 | 01-02-2023 | 0 + 16 | 05-02-2023 | 0 + 17 | 05-02-2020 | 1 + 17 | 01-02-2023 | 1 + 17 | 05-02-2023 | 1 + 18 | 05-02-2020 | 0 + 18 | 01-02-2023 | 0 + 18 | 05-02-2023 | 0 + 19 | 05-02-2020 | 0 + 19 | 01-02-2023 | 0 + 19 | 05-02-2023 | 0 + 20 | 05-02-2020 | 1 + 20 | 01-02-2023 | 1 + 20 | 05-02-2023 | 1 + 21 | 05-02-2020 | 0 + 21 | 01-02-2023 | 0 + 21 | 05-02-2023 | 0 + 22 | 05-02-2020 | 0 + 22 | 01-02-2023 | 0 + 22 | 05-02-2023 | 0 + 23 | 05-02-2020 | 1 + 23 | 01-02-2023 | 1 + 23 | 05-02-2023 | 1 + 24 | 05-02-2020 | 0 + 24 | 01-02-2023 | 0 + 24 | 05-02-2023 | 0 + 25 | 05-02-2020 | 1 + 25 | 01-02-2023 | 1 + 25 | 05-02-2023 | 1 + 26 | 05-02-2020 | 1 + 26 | 01-02-2023 | 1 + 26 | 05-02-2023 | 1 + 27 | 05-02-2020 | 0 + 27 | 01-02-2023 | 0 + 27 | 05-02-2023 | 0 + 28 | 05-02-2020 | 0 + 28 | 01-02-2023 | 0 + 28 | 05-02-2023 | 0 + 29 | 05-02-2020 | 0 + 29 | 01-02-2023 | 0 + 29 | 05-02-2023 | 0 + 30 | 05-02-2020 | 1 + 30 | 01-02-2023 | 1 + 30 | 05-02-2023 | 1 + 31 | 05-02-2020 | 1 + 31 | 01-02-2023 | 1 + 31 | 05-02-2023 | 1 + 32 | 05-02-2020 | 0 + 32 | 01-02-2023 | 0 + 32 | 05-02-2023 | 0 + 33 | 05-02-2020 | 0 + 33 | 01-02-2023 | 0 + 33 | 05-02-2023 | 0 + 34 | 05-02-2020 | 0 + 34 | 01-02-2023 | 0 + 34 | 05-02-2023 | 0 + 35 | 05-02-2020 | 1 + 35 | 01-02-2023 | 1 + 35 | 05-02-2023 | 1 + 36 | 05-02-2020 | 1 + 36 | 01-02-2023 | 1 + 36 | 05-02-2023 | 1 + 37 | 05-02-2020 | 0 + 37 | 01-02-2023 | 0 + 37 | 05-02-2023 | 0 + 38 | 05-02-2020 | 1 + 38 | 01-02-2023 | 1 + 38 | 05-02-2023 | 1 + 39 | 05-02-2020 | 0 + 39 | 01-02-2023 | 0 + 39 | 05-02-2023 | 0 + 40 | 05-02-2020 | 1 + 40 | 01-02-2023 | 1 + 40 | 05-02-2023 | 1 +(120 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from part_range_table_distr_random order by a, b; + a | b | correct_segment_id +----+------------+-------------------- + 1 | 05-02-2020 | t + 1 | 01-02-2023 | t + 1 | 05-02-2023 | t + 2 | 05-02-2020 | t + 2 | 01-02-2023 | t + 2 | 05-02-2023 | t + 3 | 05-02-2020 | t + 3 | 01-02-2023 | t + 3 | 05-02-2023 | t + 4 | 05-02-2020 | t + 4 | 01-02-2023 | t + 4 | 05-02-2023 | t + 5 | 05-02-2020 | t + 5 | 01-02-2023 | t + 5 | 05-02-2023 | t + 6 | 05-02-2020 | t + 6 | 01-02-2023 | t + 6 | 05-02-2023 | t + 7 | 05-02-2020 | t + 7 | 01-02-2023 | t + 7 | 05-02-2023 | t + 8 | 05-02-2020 | t + 8 | 01-02-2023 | t + 8 | 05-02-2023 | t + 9 | 05-02-2020 | t + 9 | 01-02-2023 | t + 9 | 05-02-2023 | t + 10 | 05-02-2020 | t + 10 | 01-02-2023 | t + 10 | 05-02-2023 | t + 11 | 05-02-2020 | t + 11 | 01-02-2023 | t + 11 | 05-02-2023 | t + 12 | 05-02-2020 | t + 12 | 01-02-2023 | t + 12 | 05-02-2023 | t + 13 | 05-02-2020 | t + 13 | 01-02-2023 | t + 13 | 05-02-2023 | t + 14 | 05-02-2020 | t + 14 | 01-02-2023 | t + 14 | 05-02-2023 | t + 15 | 05-02-2020 | t + 15 | 01-02-2023 | t + 15 | 05-02-2023 | t + 16 | 05-02-2020 | t + 16 | 01-02-2023 | t + 16 | 05-02-2023 | t + 17 | 05-02-2020 | t + 17 | 01-02-2023 | t + 17 | 05-02-2023 | t + 18 | 05-02-2020 | t + 18 | 01-02-2023 | t + 18 | 05-02-2023 | t + 19 | 05-02-2020 | t + 19 | 01-02-2023 | t + 19 | 05-02-2023 | t + 20 | 05-02-2020 | t + 20 | 01-02-2023 | t + 20 | 05-02-2023 | t + 21 | 05-02-2020 | t + 21 | 01-02-2023 | t + 21 | 05-02-2023 | t + 22 | 05-02-2020 | t + 22 | 01-02-2023 | t + 22 | 05-02-2023 | t + 23 | 05-02-2020 | t + 23 | 01-02-2023 | t + 23 | 05-02-2023 | t + 24 | 05-02-2020 | t + 24 | 01-02-2023 | t + 24 | 05-02-2023 | t + 25 | 05-02-2020 | t + 25 | 01-02-2023 | t + 25 | 05-02-2023 | t + 26 | 05-02-2020 | t + 26 | 01-02-2023 | t + 26 | 05-02-2023 | t + 27 | 05-02-2020 | t + 27 | 01-02-2023 | t + 27 | 05-02-2023 | t + 28 | 05-02-2020 | t + 28 | 01-02-2023 | t + 28 | 05-02-2023 | t + 29 | 05-02-2020 | t + 29 | 01-02-2023 | t + 29 | 05-02-2023 | t + 30 | 05-02-2020 | t + 30 | 01-02-2023 | t + 30 | 05-02-2023 | t + 31 | 05-02-2020 | t + 31 | 01-02-2023 | t + 31 | 05-02-2023 | t + 32 | 05-02-2020 | t + 32 | 01-02-2023 | t + 32 | 05-02-2023 | t + 33 | 05-02-2020 | t + 33 | 01-02-2023 | t + 33 | 05-02-2023 | t + 34 | 05-02-2020 | t + 34 | 01-02-2023 | t + 34 | 05-02-2023 | t + 35 | 05-02-2020 | t + 35 | 01-02-2023 | t + 35 | 05-02-2023 | t + 36 | 05-02-2020 | t + 36 | 01-02-2023 | t + 36 | 05-02-2023 | t + 37 | 05-02-2020 | t + 37 | 01-02-2023 | t + 37 | 05-02-2023 | t + 38 | 05-02-2020 | t + 38 | 01-02-2023 | t + 38 | 05-02-2023 | t + 39 | 05-02-2020 | t + 39 | 01-02-2023 | t + 39 | 05-02-2023 | t + 40 | 05-02-2020 | t + 40 | 01-02-2023 | t + 40 | 05-02-2023 | t +(120 rows) + +select *, gp_segment_id from part_list_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+-------+--------------- + 1 | test1 | 1 + 1 | test2 | 1 + 1 | test3 | 1 + 2 | test1 | 0 + 2 | test2 | 0 + 2 | test3 | 0 + 3 | test1 | 0 + 3 | test2 | 0 + 3 | test3 | 0 + 4 | test1 | 0 + 4 | test2 | 0 + 4 | test3 | 0 + 5 | test1 | 1 + 5 | test2 | 1 + 5 | test3 | 1 + 6 | test1 | 0 + 6 | test2 | 0 + 6 | test3 | 0 + 7 | test1 | 0 + 7 | test2 | 0 + 7 | test3 | 0 + 8 | test1 | 0 + 8 | test2 | 0 + 8 | test3 | 0 + 9 | test1 | 0 + 9 | test2 | 0 + 9 | test3 | 0 + 10 | test1 | 0 + 10 | test2 | 0 + 10 | test3 | 0 + 11 | test1 | 1 + 11 | test2 | 1 + 11 | test3 | 1 + 12 | test1 | 1 + 12 | test2 | 1 + 12 | test3 | 1 + 13 | test1 | 0 + 13 | test2 | 0 + 13 | test3 | 0 + 14 | test1 | 1 + 14 | test2 | 1 + 14 | test3 | 1 + 15 | test1 | 1 + 15 | test2 | 1 + 15 | test3 | 1 + 16 | test1 | 0 + 16 | test2 | 0 + 16 | test3 | 0 + 17 | test1 | 1 + 17 | test2 | 1 + 17 | test3 | 1 + 18 | test1 | 0 + 18 | test2 | 0 + 18 | test3 | 0 + 19 | test1 | 0 + 19 | test2 | 0 + 19 | test3 | 0 + 20 | test1 | 1 + 20 | test2 | 1 + 20 | test3 | 1 + 21 | test1 | 0 + 21 | test2 | 0 + 21 | test3 | 0 + 22 | test1 | 0 + 22 | test2 | 0 + 22 | test3 | 0 + 23 | test1 | 1 + 23 | test2 | 1 + 23 | test3 | 1 + 24 | test1 | 0 + 24 | test2 | 0 + 24 | test3 | 0 + 25 | test1 | 1 + 25 | test2 | 1 + 25 | test3 | 1 + 26 | test1 | 1 + 26 | test2 | 1 + 26 | test3 | 1 + 27 | test1 | 0 + 27 | test2 | 0 + 27 | test3 | 0 + 28 | test1 | 0 + 28 | test2 | 0 + 28 | test3 | 0 + 29 | test1 | 0 + 29 | test2 | 0 + 29 | test3 | 0 + 30 | test1 | 1 + 30 | test2 | 1 + 30 | test3 | 1 + 31 | test1 | 1 + 31 | test2 | 1 + 31 | test3 | 1 + 32 | test1 | 0 + 32 | test2 | 0 + 32 | test3 | 0 + 33 | test1 | 0 + 33 | test2 | 0 + 33 | test3 | 0 + 34 | test1 | 0 + 34 | test2 | 0 + 34 | test3 | 0 + 35 | test1 | 1 + 35 | test2 | 1 + 35 | test3 | 1 + 36 | test1 | 1 + 36 | test2 | 1 + 36 | test3 | 1 + 37 | test1 | 0 + 37 | test2 | 0 + 37 | test3 | 0 + 38 | test1 | 1 + 38 | test2 | 1 + 38 | test3 | 1 + 39 | test1 | 0 + 39 | test2 | 0 + 39 | test3 | 0 + 40 | test1 | 1 + 40 | test2 | 1 + 40 | test3 | 1 +(120 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from part_list_table_distr_random order by a, b; + a | b | correct_segment_id +----+-------+-------------------- + 1 | test1 | t + 1 | test2 | t + 1 | test3 | t + 2 | test1 | t + 2 | test2 | t + 2 | test3 | t + 3 | test1 | t + 3 | test2 | t + 3 | test3 | t + 4 | test1 | t + 4 | test2 | t + 4 | test3 | t + 5 | test1 | t + 5 | test2 | t + 5 | test3 | t + 6 | test1 | t + 6 | test2 | t + 6 | test3 | t + 7 | test1 | t + 7 | test2 | t + 7 | test3 | t + 8 | test1 | t + 8 | test2 | t + 8 | test3 | t + 9 | test1 | t + 9 | test2 | t + 9 | test3 | t + 10 | test1 | t + 10 | test2 | t + 10 | test3 | t + 11 | test1 | t + 11 | test2 | t + 11 | test3 | t + 12 | test1 | t + 12 | test2 | t + 12 | test3 | t + 13 | test1 | t + 13 | test2 | t + 13 | test3 | t + 14 | test1 | t + 14 | test2 | t + 14 | test3 | t + 15 | test1 | t + 15 | test2 | t + 15 | test3 | t + 16 | test1 | t + 16 | test2 | t + 16 | test3 | t + 17 | test1 | t + 17 | test2 | t + 17 | test3 | t + 18 | test1 | t + 18 | test2 | t + 18 | test3 | t + 19 | test1 | t + 19 | test2 | t + 19 | test3 | t + 20 | test1 | t + 20 | test2 | t + 20 | test3 | t + 21 | test1 | t + 21 | test2 | t + 21 | test3 | t + 22 | test1 | t + 22 | test2 | t + 22 | test3 | t + 23 | test1 | t + 23 | test2 | t + 23 | test3 | t + 24 | test1 | t + 24 | test2 | t + 24 | test3 | t + 25 | test1 | t + 25 | test2 | t + 25 | test3 | t + 26 | test1 | t + 26 | test2 | t + 26 | test3 | t + 27 | test1 | t + 27 | test2 | t + 27 | test3 | t + 28 | test1 | t + 28 | test2 | t + 28 | test3 | t + 29 | test1 | t + 29 | test2 | t + 29 | test3 | t + 30 | test1 | t + 30 | test2 | t + 30 | test3 | t + 31 | test1 | t + 31 | test2 | t + 31 | test3 | t + 32 | test1 | t + 32 | test2 | t + 32 | test3 | t + 33 | test1 | t + 33 | test2 | t + 33 | test3 | t + 34 | test1 | t + 34 | test2 | t + 34 | test3 | t + 35 | test1 | t + 35 | test2 | t + 35 | test3 | t + 36 | test1 | t + 36 | test2 | t + 36 | test3 | t + 37 | test1 | t + 37 | test2 | t + 37 | test3 | t + 38 | test1 | t + 38 | test2 | t + 38 | test3 | t + 39 | test1 | t + 39 | test2 | t + 39 | test3 | t + 40 | test1 | t + 40 | test2 | t + 40 | test3 | t +(120 rows) + +select *, gp_segment_id from multi_part_table_distr_hashed order by a, b, c; + a | b | c | gp_segment_id +----+------------+-------+--------------- + 1 | 01-05-2023 | test1 | 1 + 1 | 02-05-2023 | test2 | 1 + 1 | 03-05-2023 | test1 | 1 + 2 | 01-05-2023 | test1 | 0 + 2 | 02-05-2023 | test2 | 0 + 2 | 03-05-2023 | test1 | 0 + 3 | 01-05-2023 | test1 | 0 + 3 | 02-05-2023 | test2 | 0 + 3 | 03-05-2023 | test1 | 0 + 4 | 01-05-2023 | test1 | 0 + 4 | 02-05-2023 | test2 | 0 + 4 | 03-05-2023 | test1 | 0 + 5 | 01-05-2023 | test1 | 1 + 5 | 02-05-2023 | test2 | 1 + 5 | 03-05-2023 | test1 | 1 + 6 | 01-05-2023 | test1 | 0 + 6 | 02-05-2023 | test2 | 0 + 6 | 03-05-2023 | test1 | 0 + 7 | 01-05-2023 | test1 | 0 + 7 | 02-05-2023 | test2 | 0 + 7 | 03-05-2023 | test1 | 0 + 8 | 01-05-2023 | test1 | 0 + 8 | 02-05-2023 | test2 | 0 + 8 | 03-05-2023 | test1 | 0 + 9 | 01-05-2023 | test1 | 0 + 9 | 02-05-2023 | test2 | 0 + 9 | 03-05-2023 | test1 | 0 + 10 | 01-05-2023 | test1 | 0 + 10 | 02-05-2023 | test2 | 0 + 10 | 03-05-2023 | test1 | 0 + 11 | 01-05-2023 | test1 | 1 + 11 | 02-05-2023 | test2 | 1 + 11 | 03-05-2023 | test1 | 1 + 12 | 01-05-2023 | test1 | 1 + 12 | 02-05-2023 | test2 | 1 + 12 | 03-05-2023 | test1 | 1 + 13 | 01-05-2023 | test1 | 0 + 13 | 02-05-2023 | test2 | 0 + 13 | 03-05-2023 | test1 | 0 + 14 | 01-05-2023 | test1 | 1 + 14 | 02-05-2023 | test2 | 1 + 14 | 03-05-2023 | test1 | 1 + 15 | 01-05-2023 | test1 | 1 + 15 | 02-05-2023 | test2 | 1 + 15 | 03-05-2023 | test1 | 1 + 16 | 01-05-2023 | test1 | 0 + 16 | 02-05-2023 | test2 | 0 + 16 | 03-05-2023 | test1 | 0 + 17 | 01-05-2023 | test1 | 1 + 17 | 02-05-2023 | test2 | 1 + 17 | 03-05-2023 | test1 | 1 + 18 | 01-05-2023 | test1 | 0 + 18 | 02-05-2023 | test2 | 0 + 18 | 03-05-2023 | test1 | 0 + 19 | 01-05-2023 | test1 | 0 + 19 | 02-05-2023 | test2 | 0 + 19 | 03-05-2023 | test1 | 0 + 20 | 01-05-2023 | test1 | 1 + 20 | 02-05-2023 | test2 | 1 + 20 | 03-05-2023 | test1 | 1 + 21 | 01-05-2023 | test1 | 0 + 21 | 02-05-2023 | test2 | 0 + 21 | 03-05-2023 | test1 | 0 + 22 | 01-05-2023 | test1 | 0 + 22 | 02-05-2023 | test2 | 0 + 22 | 03-05-2023 | test1 | 0 + 23 | 01-05-2023 | test1 | 1 + 23 | 02-05-2023 | test2 | 1 + 23 | 03-05-2023 | test1 | 1 + 24 | 01-05-2023 | test1 | 0 + 24 | 02-05-2023 | test2 | 0 + 24 | 03-05-2023 | test1 | 0 + 25 | 01-05-2023 | test1 | 1 + 25 | 02-05-2023 | test2 | 1 + 25 | 03-05-2023 | test1 | 1 + 26 | 01-05-2023 | test1 | 1 + 26 | 02-05-2023 | test2 | 1 + 26 | 03-05-2023 | test1 | 1 + 27 | 01-05-2023 | test1 | 0 + 27 | 02-05-2023 | test2 | 0 + 27 | 03-05-2023 | test1 | 0 + 28 | 01-05-2023 | test1 | 0 + 28 | 02-05-2023 | test2 | 0 + 28 | 03-05-2023 | test1 | 0 + 29 | 01-05-2023 | test1 | 0 + 29 | 02-05-2023 | test2 | 0 + 29 | 03-05-2023 | test1 | 0 + 30 | 01-05-2023 | test1 | 1 + 30 | 02-05-2023 | test2 | 1 + 30 | 03-05-2023 | test1 | 1 + 31 | 01-05-2023 | test1 | 1 + 31 | 02-05-2023 | test2 | 1 + 31 | 03-05-2023 | test1 | 1 + 32 | 01-05-2023 | test1 | 0 + 32 | 02-05-2023 | test2 | 0 + 32 | 03-05-2023 | test1 | 0 + 33 | 01-05-2023 | test1 | 0 + 33 | 02-05-2023 | test2 | 0 + 33 | 03-05-2023 | test1 | 0 + 34 | 01-05-2023 | test1 | 0 + 34 | 02-05-2023 | test2 | 0 + 34 | 03-05-2023 | test1 | 0 + 35 | 01-05-2023 | test1 | 1 + 35 | 02-05-2023 | test2 | 1 + 35 | 03-05-2023 | test1 | 1 + 36 | 01-05-2023 | test1 | 1 + 36 | 02-05-2023 | test2 | 1 + 36 | 03-05-2023 | test1 | 1 + 37 | 01-05-2023 | test1 | 0 + 37 | 02-05-2023 | test2 | 0 + 37 | 03-05-2023 | test1 | 0 + 38 | 01-05-2023 | test1 | 1 + 38 | 02-05-2023 | test2 | 1 + 38 | 03-05-2023 | test1 | 1 + 39 | 01-05-2023 | test1 | 0 + 39 | 02-05-2023 | test2 | 0 + 39 | 03-05-2023 | test1 | 0 + 40 | 01-05-2023 | test1 | 1 + 40 | 02-05-2023 | test2 | 1 + 40 | 03-05-2023 | test1 | 1 +(120 rows) + +-- And do some cleanup +drop table table_distr_hashed; +drop table table_distr_hashed_ao_row; +drop table table_distr_hashed_ao_col; +drop table table_distr_random; +drop table table_distr_random_ao_row; +drop table table_distr_random_ao_col; +drop table table_distr_replicated; +drop table table_distr_replicated_ao_row; +drop table table_distr_replicated_ao_col; +drop table part_range_table_distr_hashed; +drop table part_range_table_distr_random; +drop table part_list_table_distr_hashed; +drop table part_list_table_distr_random; +drop table multi_part_table_distr_hashed; +-- Check that all newly created tables have data only on segments #0 and #1 +set gp_target_numsegments = 2; +create table new_table_distr_hashed(a int) distributed by (a); +insert into new_table_distr_hashed select generate_series(1, 20); +create table new_table_distr_hashed_ao_row(a int) with (appendonly=true, orientation=row) distributed by (a); +insert into new_table_distr_hashed_ao_row select generate_series(1, 20); +create table new_table_distr_hashed_ao_col(a int) with (appendonly=true, orientation=column) distributed by (a); +insert into new_table_distr_hashed_ao_col select generate_series(1, 20); +create table new_table_distr_random(a int) distributed randomly; +insert into new_table_distr_random select generate_series(1, 20); +create table new_table_distr_random_ao_row(a int) with (appendonly=true, orientation=row) distributed randomly; +insert into new_table_distr_random_ao_row select generate_series(1, 20); +create table new_table_distr_random_ao_col(a int) with (appendonly=true, orientation=column) distributed randomly; +insert into new_table_distr_random_ao_col select generate_series(1, 20); +create table new_table_distr_replicated(a int) distributed replicated; +insert into new_table_distr_replicated select generate_series(1, 20); +create table new_table_distr_replicated_ao_row(a int) with (appendonly=true, orientation=row) distributed replicated; +insert into new_table_distr_replicated_ao_row select generate_series(1, 20); +create table new_table_distr_replicated_ao_col(a int) with (appendonly=true, orientation=column) distributed replicated; +insert into new_table_distr_replicated_ao_col select generate_series(1, 20); +create table new_part_range_table_distr_hashed (a int, b date) distributed by (a) +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into new_part_range_table_distr_hashed select i, '2023-01-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_hashed select i, '2023-05-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_hashed select i, '2020-05-02' from generate_series(1, 20)i; +create table new_part_range_table_distr_random (a int, b date) distributed randomly +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into new_part_range_table_distr_random select i, '2023-01-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_random select i, '2023-05-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_random select i, '2020-05-02' from generate_series(1, 20)i; +create table new_part_list_table_distr_hashed(a int, b text) distributed by (a) +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into new_part_list_table_distr_hashed select i, 'test1' from generate_series(1, 20)i; +insert into new_part_list_table_distr_hashed select i, 'test2' from generate_series(1, 20)i; +insert into new_part_list_table_distr_hashed select i, 'test3' from generate_series(1, 20)i; +create table new_part_list_table_distr_random(a int, b text) distributed randomly +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into new_part_list_table_distr_random select i, 'test1' from generate_series(1, 20)i; +insert into new_part_list_table_distr_random select i, 'test2' from generate_series(1, 20)i; +insert into new_part_list_table_distr_random select i, 'test3' from generate_series(1, 20)i; +create table new_multi_part_table_distr_hashed(a int, b date, c text) distributed by (a) +partition by range (b) +subpartition by list (c) subpartition template +( + subpartition subpart1 values ('test1'), + subpartition subpart2 values ('test2') +) +( + partition part1 start (date '2023-01-01'), + partition part2 start (date '2023-02-01'), + partition part3 start (date '2023-03-01') end (date '2024-01-01') +); +insert into new_multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(1, 20)i; +insert into new_multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(1, 20)i; +insert into new_multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(1, 20)i; +-- Also check CTAS statement +create table new_table_ctas as select a from generate_series(1, 20)a distributed by(a); +select * into new_table_into from generate_series(1, 20)a; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. +select a, gp_segment_id from new_table_distr_hashed order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select a, gp_segment_id from new_table_distr_hashed_ao_row order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select a, gp_segment_id from new_table_distr_hashed_ao_col order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_row order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_col order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +select a from new_table_distr_replicated order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +(20 rows) + +select a from new_table_distr_replicated_ao_row order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +(20 rows) + +select a from new_table_distr_replicated_ao_col order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +(20 rows) + +select *, gp_segment_id from new_part_range_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+------------+--------------- + 1 | 05-02-2020 | 1 + 1 | 01-02-2023 | 1 + 1 | 05-02-2023 | 1 + 2 | 05-02-2020 | 0 + 2 | 01-02-2023 | 0 + 2 | 05-02-2023 | 0 + 3 | 05-02-2020 | 0 + 3 | 01-02-2023 | 0 + 3 | 05-02-2023 | 0 + 4 | 05-02-2020 | 0 + 4 | 01-02-2023 | 0 + 4 | 05-02-2023 | 0 + 5 | 05-02-2020 | 1 + 5 | 01-02-2023 | 1 + 5 | 05-02-2023 | 1 + 6 | 05-02-2020 | 0 + 6 | 01-02-2023 | 0 + 6 | 05-02-2023 | 0 + 7 | 05-02-2020 | 0 + 7 | 01-02-2023 | 0 + 7 | 05-02-2023 | 0 + 8 | 05-02-2020 | 0 + 8 | 01-02-2023 | 0 + 8 | 05-02-2023 | 0 + 9 | 05-02-2020 | 0 + 9 | 01-02-2023 | 0 + 9 | 05-02-2023 | 0 + 10 | 05-02-2020 | 0 + 10 | 01-02-2023 | 0 + 10 | 05-02-2023 | 0 + 11 | 05-02-2020 | 1 + 11 | 01-02-2023 | 1 + 11 | 05-02-2023 | 1 + 12 | 05-02-2020 | 1 + 12 | 01-02-2023 | 1 + 12 | 05-02-2023 | 1 + 13 | 05-02-2020 | 0 + 13 | 01-02-2023 | 0 + 13 | 05-02-2023 | 0 + 14 | 05-02-2020 | 1 + 14 | 01-02-2023 | 1 + 14 | 05-02-2023 | 1 + 15 | 05-02-2020 | 1 + 15 | 01-02-2023 | 1 + 15 | 05-02-2023 | 1 + 16 | 05-02-2020 | 0 + 16 | 01-02-2023 | 0 + 16 | 05-02-2023 | 0 + 17 | 05-02-2020 | 1 + 17 | 01-02-2023 | 1 + 17 | 05-02-2023 | 1 + 18 | 05-02-2020 | 0 + 18 | 01-02-2023 | 0 + 18 | 05-02-2023 | 0 + 19 | 05-02-2020 | 0 + 19 | 01-02-2023 | 0 + 19 | 05-02-2023 | 0 + 20 | 05-02-2020 | 1 + 20 | 01-02-2023 | 1 + 20 | 05-02-2023 | 1 +(60 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from new_part_range_table_distr_random order by a, b; + a | b | correct_segment_id +----+------------+-------------------- + 1 | 05-02-2020 | t + 1 | 01-02-2023 | t + 1 | 05-02-2023 | t + 2 | 05-02-2020 | t + 2 | 01-02-2023 | t + 2 | 05-02-2023 | t + 3 | 05-02-2020 | t + 3 | 01-02-2023 | t + 3 | 05-02-2023 | t + 4 | 05-02-2020 | t + 4 | 01-02-2023 | t + 4 | 05-02-2023 | t + 5 | 05-02-2020 | t + 5 | 01-02-2023 | t + 5 | 05-02-2023 | t + 6 | 05-02-2020 | t + 6 | 01-02-2023 | t + 6 | 05-02-2023 | t + 7 | 05-02-2020 | t + 7 | 01-02-2023 | t + 7 | 05-02-2023 | t + 8 | 05-02-2020 | t + 8 | 01-02-2023 | t + 8 | 05-02-2023 | t + 9 | 05-02-2020 | t + 9 | 01-02-2023 | t + 9 | 05-02-2023 | t + 10 | 05-02-2020 | t + 10 | 01-02-2023 | t + 10 | 05-02-2023 | t + 11 | 05-02-2020 | t + 11 | 01-02-2023 | t + 11 | 05-02-2023 | t + 12 | 05-02-2020 | t + 12 | 01-02-2023 | t + 12 | 05-02-2023 | t + 13 | 05-02-2020 | t + 13 | 01-02-2023 | t + 13 | 05-02-2023 | t + 14 | 05-02-2020 | t + 14 | 01-02-2023 | t + 14 | 05-02-2023 | t + 15 | 05-02-2020 | t + 15 | 01-02-2023 | t + 15 | 05-02-2023 | t + 16 | 05-02-2020 | t + 16 | 01-02-2023 | t + 16 | 05-02-2023 | t + 17 | 05-02-2020 | t + 17 | 01-02-2023 | t + 17 | 05-02-2023 | t + 18 | 05-02-2020 | t + 18 | 01-02-2023 | t + 18 | 05-02-2023 | t + 19 | 05-02-2020 | t + 19 | 01-02-2023 | t + 19 | 05-02-2023 | t + 20 | 05-02-2020 | t + 20 | 01-02-2023 | t + 20 | 05-02-2023 | t +(60 rows) + +select *, gp_segment_id from new_part_list_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+-------+--------------- + 1 | test1 | 1 + 1 | test2 | 1 + 1 | test3 | 1 + 2 | test1 | 0 + 2 | test2 | 0 + 2 | test3 | 0 + 3 | test1 | 0 + 3 | test2 | 0 + 3 | test3 | 0 + 4 | test1 | 0 + 4 | test2 | 0 + 4 | test3 | 0 + 5 | test1 | 1 + 5 | test2 | 1 + 5 | test3 | 1 + 6 | test1 | 0 + 6 | test2 | 0 + 6 | test3 | 0 + 7 | test1 | 0 + 7 | test2 | 0 + 7 | test3 | 0 + 8 | test1 | 0 + 8 | test2 | 0 + 8 | test3 | 0 + 9 | test1 | 0 + 9 | test2 | 0 + 9 | test3 | 0 + 10 | test1 | 0 + 10 | test2 | 0 + 10 | test3 | 0 + 11 | test1 | 1 + 11 | test2 | 1 + 11 | test3 | 1 + 12 | test1 | 1 + 12 | test2 | 1 + 12 | test3 | 1 + 13 | test1 | 0 + 13 | test2 | 0 + 13 | test3 | 0 + 14 | test1 | 1 + 14 | test2 | 1 + 14 | test3 | 1 + 15 | test1 | 1 + 15 | test2 | 1 + 15 | test3 | 1 + 16 | test1 | 0 + 16 | test2 | 0 + 16 | test3 | 0 + 17 | test1 | 1 + 17 | test2 | 1 + 17 | test3 | 1 + 18 | test1 | 0 + 18 | test2 | 0 + 18 | test3 | 0 + 19 | test1 | 0 + 19 | test2 | 0 + 19 | test3 | 0 + 20 | test1 | 1 + 20 | test2 | 1 + 20 | test3 | 1 +(60 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from new_part_list_table_distr_random order by a, b; + a | b | correct_segment_id +----+-------+-------------------- + 1 | test1 | t + 1 | test2 | t + 1 | test3 | t + 2 | test1 | t + 2 | test2 | t + 2 | test3 | t + 3 | test1 | t + 3 | test2 | t + 3 | test3 | t + 4 | test1 | t + 4 | test2 | t + 4 | test3 | t + 5 | test1 | t + 5 | test2 | t + 5 | test3 | t + 6 | test1 | t + 6 | test2 | t + 6 | test3 | t + 7 | test1 | t + 7 | test2 | t + 7 | test3 | t + 8 | test1 | t + 8 | test2 | t + 8 | test3 | t + 9 | test1 | t + 9 | test2 | t + 9 | test3 | t + 10 | test1 | t + 10 | test2 | t + 10 | test3 | t + 11 | test1 | t + 11 | test2 | t + 11 | test3 | t + 12 | test1 | t + 12 | test2 | t + 12 | test3 | t + 13 | test1 | t + 13 | test2 | t + 13 | test3 | t + 14 | test1 | t + 14 | test2 | t + 14 | test3 | t + 15 | test1 | t + 15 | test2 | t + 15 | test3 | t + 16 | test1 | t + 16 | test2 | t + 16 | test3 | t + 17 | test1 | t + 17 | test2 | t + 17 | test3 | t + 18 | test1 | t + 18 | test2 | t + 18 | test3 | t + 19 | test1 | t + 19 | test2 | t + 19 | test3 | t + 20 | test1 | t + 20 | test2 | t + 20 | test3 | t +(60 rows) + +select *, gp_segment_id from new_multi_part_table_distr_hashed order by a, b, c; + a | b | c | gp_segment_id +----+------------+-------+--------------- + 1 | 01-05-2023 | test1 | 1 + 1 | 02-05-2023 | test2 | 1 + 1 | 03-05-2023 | test1 | 1 + 2 | 01-05-2023 | test1 | 0 + 2 | 02-05-2023 | test2 | 0 + 2 | 03-05-2023 | test1 | 0 + 3 | 01-05-2023 | test1 | 0 + 3 | 02-05-2023 | test2 | 0 + 3 | 03-05-2023 | test1 | 0 + 4 | 01-05-2023 | test1 | 0 + 4 | 02-05-2023 | test2 | 0 + 4 | 03-05-2023 | test1 | 0 + 5 | 01-05-2023 | test1 | 1 + 5 | 02-05-2023 | test2 | 1 + 5 | 03-05-2023 | test1 | 1 + 6 | 01-05-2023 | test1 | 0 + 6 | 02-05-2023 | test2 | 0 + 6 | 03-05-2023 | test1 | 0 + 7 | 01-05-2023 | test1 | 0 + 7 | 02-05-2023 | test2 | 0 + 7 | 03-05-2023 | test1 | 0 + 8 | 01-05-2023 | test1 | 0 + 8 | 02-05-2023 | test2 | 0 + 8 | 03-05-2023 | test1 | 0 + 9 | 01-05-2023 | test1 | 0 + 9 | 02-05-2023 | test2 | 0 + 9 | 03-05-2023 | test1 | 0 + 10 | 01-05-2023 | test1 | 0 + 10 | 02-05-2023 | test2 | 0 + 10 | 03-05-2023 | test1 | 0 + 11 | 01-05-2023 | test1 | 1 + 11 | 02-05-2023 | test2 | 1 + 11 | 03-05-2023 | test1 | 1 + 12 | 01-05-2023 | test1 | 1 + 12 | 02-05-2023 | test2 | 1 + 12 | 03-05-2023 | test1 | 1 + 13 | 01-05-2023 | test1 | 0 + 13 | 02-05-2023 | test2 | 0 + 13 | 03-05-2023 | test1 | 0 + 14 | 01-05-2023 | test1 | 1 + 14 | 02-05-2023 | test2 | 1 + 14 | 03-05-2023 | test1 | 1 + 15 | 01-05-2023 | test1 | 1 + 15 | 02-05-2023 | test2 | 1 + 15 | 03-05-2023 | test1 | 1 + 16 | 01-05-2023 | test1 | 0 + 16 | 02-05-2023 | test2 | 0 + 16 | 03-05-2023 | test1 | 0 + 17 | 01-05-2023 | test1 | 1 + 17 | 02-05-2023 | test2 | 1 + 17 | 03-05-2023 | test1 | 1 + 18 | 01-05-2023 | test1 | 0 + 18 | 02-05-2023 | test2 | 0 + 18 | 03-05-2023 | test1 | 0 + 19 | 01-05-2023 | test1 | 0 + 19 | 02-05-2023 | test2 | 0 + 19 | 03-05-2023 | test1 | 0 + 20 | 01-05-2023 | test1 | 1 + 20 | 02-05-2023 | test2 | 1 + 20 | 03-05-2023 | test1 | 1 +(60 rows) + +select a, gp_segment_id from new_table_ctas order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from new_table_into order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +-- Validate the insertion works fine with the new tables +-- after 'gp_target_numsegments' reset +reset gp_target_numsegments; +insert into new_table_ctas select generate_series(21, 40); +insert into new_table_into select generate_series(21, 40); +insert into new_table_distr_hashed select generate_series(21, 40); +insert into new_table_distr_hashed_ao_row select generate_series(21, 40); +insert into new_table_distr_hashed_ao_col select generate_series(21, 40); +insert into new_table_distr_random select generate_series(21, 40); +insert into new_table_distr_random_ao_row select generate_series(21, 40); +insert into new_table_distr_random_ao_col select generate_series(21, 40); +insert into new_part_range_table_distr_hashed select i, '2023-01-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_hashed select i, '2023-05-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_hashed select i, '2020-05-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_random select i, '2023-01-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_random select i, '2023-05-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_random select i, '2020-05-02' from generate_series(21, 40)i; +insert into new_part_list_table_distr_hashed select i, 'test1' from generate_series(21, 40)i; +insert into new_part_list_table_distr_hashed select i, 'test2' from generate_series(21, 40)i; +insert into new_part_list_table_distr_hashed select i, 'test3' from generate_series(21, 40)i; +insert into new_part_list_table_distr_random select i, 'test1' from generate_series(21, 40)i; +insert into new_part_list_table_distr_random select i, 'test2' from generate_series(21, 40)i; +insert into new_part_list_table_distr_random select i, 'test3' from generate_series(21, 40)i; +insert into new_multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(21, 40)i; +insert into new_multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(21, 40)i; +insert into new_multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(21, 40)i; +select a, gp_segment_id from new_table_ctas order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from new_table_into order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select a, gp_segment_id from new_table_distr_hashed order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select a, gp_segment_id from new_table_distr_hashed_ao_row order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select a, gp_segment_id from new_table_distr_hashed_ao_col order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_row order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_col order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select *, gp_segment_id from new_part_range_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+------------+--------------- + 1 | 05-02-2020 | 1 + 1 | 01-02-2023 | 1 + 1 | 05-02-2023 | 1 + 2 | 05-02-2020 | 0 + 2 | 01-02-2023 | 0 + 2 | 05-02-2023 | 0 + 3 | 05-02-2020 | 0 + 3 | 01-02-2023 | 0 + 3 | 05-02-2023 | 0 + 4 | 05-02-2020 | 0 + 4 | 01-02-2023 | 0 + 4 | 05-02-2023 | 0 + 5 | 05-02-2020 | 1 + 5 | 01-02-2023 | 1 + 5 | 05-02-2023 | 1 + 6 | 05-02-2020 | 0 + 6 | 01-02-2023 | 0 + 6 | 05-02-2023 | 0 + 7 | 05-02-2020 | 0 + 7 | 01-02-2023 | 0 + 7 | 05-02-2023 | 0 + 8 | 05-02-2020 | 0 + 8 | 01-02-2023 | 0 + 8 | 05-02-2023 | 0 + 9 | 05-02-2020 | 0 + 9 | 01-02-2023 | 0 + 9 | 05-02-2023 | 0 + 10 | 05-02-2020 | 0 + 10 | 01-02-2023 | 0 + 10 | 05-02-2023 | 0 + 11 | 05-02-2020 | 1 + 11 | 01-02-2023 | 1 + 11 | 05-02-2023 | 1 + 12 | 05-02-2020 | 1 + 12 | 01-02-2023 | 1 + 12 | 05-02-2023 | 1 + 13 | 05-02-2020 | 0 + 13 | 01-02-2023 | 0 + 13 | 05-02-2023 | 0 + 14 | 05-02-2020 | 1 + 14 | 01-02-2023 | 1 + 14 | 05-02-2023 | 1 + 15 | 05-02-2020 | 1 + 15 | 01-02-2023 | 1 + 15 | 05-02-2023 | 1 + 16 | 05-02-2020 | 0 + 16 | 01-02-2023 | 0 + 16 | 05-02-2023 | 0 + 17 | 05-02-2020 | 1 + 17 | 01-02-2023 | 1 + 17 | 05-02-2023 | 1 + 18 | 05-02-2020 | 0 + 18 | 01-02-2023 | 0 + 18 | 05-02-2023 | 0 + 19 | 05-02-2020 | 0 + 19 | 01-02-2023 | 0 + 19 | 05-02-2023 | 0 + 20 | 05-02-2020 | 1 + 20 | 01-02-2023 | 1 + 20 | 05-02-2023 | 1 + 21 | 05-02-2020 | 0 + 21 | 01-02-2023 | 0 + 21 | 05-02-2023 | 0 + 22 | 05-02-2020 | 0 + 22 | 01-02-2023 | 0 + 22 | 05-02-2023 | 0 + 23 | 05-02-2020 | 1 + 23 | 01-02-2023 | 1 + 23 | 05-02-2023 | 1 + 24 | 05-02-2020 | 0 + 24 | 01-02-2023 | 0 + 24 | 05-02-2023 | 0 + 25 | 05-02-2020 | 1 + 25 | 01-02-2023 | 1 + 25 | 05-02-2023 | 1 + 26 | 05-02-2020 | 1 + 26 | 01-02-2023 | 1 + 26 | 05-02-2023 | 1 + 27 | 05-02-2020 | 0 + 27 | 01-02-2023 | 0 + 27 | 05-02-2023 | 0 + 28 | 05-02-2020 | 0 + 28 | 01-02-2023 | 0 + 28 | 05-02-2023 | 0 + 29 | 05-02-2020 | 0 + 29 | 01-02-2023 | 0 + 29 | 05-02-2023 | 0 + 30 | 05-02-2020 | 1 + 30 | 01-02-2023 | 1 + 30 | 05-02-2023 | 1 + 31 | 05-02-2020 | 1 + 31 | 01-02-2023 | 1 + 31 | 05-02-2023 | 1 + 32 | 05-02-2020 | 0 + 32 | 01-02-2023 | 0 + 32 | 05-02-2023 | 0 + 33 | 05-02-2020 | 0 + 33 | 01-02-2023 | 0 + 33 | 05-02-2023 | 0 + 34 | 05-02-2020 | 0 + 34 | 01-02-2023 | 0 + 34 | 05-02-2023 | 0 + 35 | 05-02-2020 | 1 + 35 | 01-02-2023 | 1 + 35 | 05-02-2023 | 1 + 36 | 05-02-2020 | 1 + 36 | 01-02-2023 | 1 + 36 | 05-02-2023 | 1 + 37 | 05-02-2020 | 0 + 37 | 01-02-2023 | 0 + 37 | 05-02-2023 | 0 + 38 | 05-02-2020 | 1 + 38 | 01-02-2023 | 1 + 38 | 05-02-2023 | 1 + 39 | 05-02-2020 | 0 + 39 | 01-02-2023 | 0 + 39 | 05-02-2023 | 0 + 40 | 05-02-2020 | 1 + 40 | 01-02-2023 | 1 + 40 | 05-02-2023 | 1 +(120 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from new_part_range_table_distr_random order by a, b; + a | b | correct_segment_id +----+------------+-------------------- + 1 | 05-02-2020 | t + 1 | 01-02-2023 | t + 1 | 05-02-2023 | t + 2 | 05-02-2020 | t + 2 | 01-02-2023 | t + 2 | 05-02-2023 | t + 3 | 05-02-2020 | t + 3 | 01-02-2023 | t + 3 | 05-02-2023 | t + 4 | 05-02-2020 | t + 4 | 01-02-2023 | t + 4 | 05-02-2023 | t + 5 | 05-02-2020 | t + 5 | 01-02-2023 | t + 5 | 05-02-2023 | t + 6 | 05-02-2020 | t + 6 | 01-02-2023 | t + 6 | 05-02-2023 | t + 7 | 05-02-2020 | t + 7 | 01-02-2023 | t + 7 | 05-02-2023 | t + 8 | 05-02-2020 | t + 8 | 01-02-2023 | t + 8 | 05-02-2023 | t + 9 | 05-02-2020 | t + 9 | 01-02-2023 | t + 9 | 05-02-2023 | t + 10 | 05-02-2020 | t + 10 | 01-02-2023 | t + 10 | 05-02-2023 | t + 11 | 05-02-2020 | t + 11 | 01-02-2023 | t + 11 | 05-02-2023 | t + 12 | 05-02-2020 | t + 12 | 01-02-2023 | t + 12 | 05-02-2023 | t + 13 | 05-02-2020 | t + 13 | 01-02-2023 | t + 13 | 05-02-2023 | t + 14 | 05-02-2020 | t + 14 | 01-02-2023 | t + 14 | 05-02-2023 | t + 15 | 05-02-2020 | t + 15 | 01-02-2023 | t + 15 | 05-02-2023 | t + 16 | 05-02-2020 | t + 16 | 01-02-2023 | t + 16 | 05-02-2023 | t + 17 | 05-02-2020 | t + 17 | 01-02-2023 | t + 17 | 05-02-2023 | t + 18 | 05-02-2020 | t + 18 | 01-02-2023 | t + 18 | 05-02-2023 | t + 19 | 05-02-2020 | t + 19 | 01-02-2023 | t + 19 | 05-02-2023 | t + 20 | 05-02-2020 | t + 20 | 01-02-2023 | t + 20 | 05-02-2023 | t + 21 | 05-02-2020 | t + 21 | 01-02-2023 | t + 21 | 05-02-2023 | t + 22 | 05-02-2020 | t + 22 | 01-02-2023 | t + 22 | 05-02-2023 | t + 23 | 05-02-2020 | t + 23 | 01-02-2023 | t + 23 | 05-02-2023 | t + 24 | 05-02-2020 | t + 24 | 01-02-2023 | t + 24 | 05-02-2023 | t + 25 | 05-02-2020 | t + 25 | 01-02-2023 | t + 25 | 05-02-2023 | t + 26 | 05-02-2020 | t + 26 | 01-02-2023 | t + 26 | 05-02-2023 | t + 27 | 05-02-2020 | t + 27 | 01-02-2023 | t + 27 | 05-02-2023 | t + 28 | 05-02-2020 | t + 28 | 01-02-2023 | t + 28 | 05-02-2023 | t + 29 | 05-02-2020 | t + 29 | 01-02-2023 | t + 29 | 05-02-2023 | t + 30 | 05-02-2020 | t + 30 | 01-02-2023 | t + 30 | 05-02-2023 | t + 31 | 05-02-2020 | t + 31 | 01-02-2023 | t + 31 | 05-02-2023 | t + 32 | 05-02-2020 | t + 32 | 01-02-2023 | t + 32 | 05-02-2023 | t + 33 | 05-02-2020 | t + 33 | 01-02-2023 | t + 33 | 05-02-2023 | t + 34 | 05-02-2020 | t + 34 | 01-02-2023 | t + 34 | 05-02-2023 | t + 35 | 05-02-2020 | t + 35 | 01-02-2023 | t + 35 | 05-02-2023 | t + 36 | 05-02-2020 | t + 36 | 01-02-2023 | t + 36 | 05-02-2023 | t + 37 | 05-02-2020 | t + 37 | 01-02-2023 | t + 37 | 05-02-2023 | t + 38 | 05-02-2020 | t + 38 | 01-02-2023 | t + 38 | 05-02-2023 | t + 39 | 05-02-2020 | t + 39 | 01-02-2023 | t + 39 | 05-02-2023 | t + 40 | 05-02-2020 | t + 40 | 01-02-2023 | t + 40 | 05-02-2023 | t +(120 rows) + +select *, gp_segment_id from new_part_list_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+-------+--------------- + 1 | test1 | 1 + 1 | test2 | 1 + 1 | test3 | 1 + 2 | test1 | 0 + 2 | test2 | 0 + 2 | test3 | 0 + 3 | test1 | 0 + 3 | test2 | 0 + 3 | test3 | 0 + 4 | test1 | 0 + 4 | test2 | 0 + 4 | test3 | 0 + 5 | test1 | 1 + 5 | test2 | 1 + 5 | test3 | 1 + 6 | test1 | 0 + 6 | test2 | 0 + 6 | test3 | 0 + 7 | test1 | 0 + 7 | test2 | 0 + 7 | test3 | 0 + 8 | test1 | 0 + 8 | test2 | 0 + 8 | test3 | 0 + 9 | test1 | 0 + 9 | test2 | 0 + 9 | test3 | 0 + 10 | test1 | 0 + 10 | test2 | 0 + 10 | test3 | 0 + 11 | test1 | 1 + 11 | test2 | 1 + 11 | test3 | 1 + 12 | test1 | 1 + 12 | test2 | 1 + 12 | test3 | 1 + 13 | test1 | 0 + 13 | test2 | 0 + 13 | test3 | 0 + 14 | test1 | 1 + 14 | test2 | 1 + 14 | test3 | 1 + 15 | test1 | 1 + 15 | test2 | 1 + 15 | test3 | 1 + 16 | test1 | 0 + 16 | test2 | 0 + 16 | test3 | 0 + 17 | test1 | 1 + 17 | test2 | 1 + 17 | test3 | 1 + 18 | test1 | 0 + 18 | test2 | 0 + 18 | test3 | 0 + 19 | test1 | 0 + 19 | test2 | 0 + 19 | test3 | 0 + 20 | test1 | 1 + 20 | test2 | 1 + 20 | test3 | 1 + 21 | test1 | 0 + 21 | test2 | 0 + 21 | test3 | 0 + 22 | test1 | 0 + 22 | test2 | 0 + 22 | test3 | 0 + 23 | test1 | 1 + 23 | test2 | 1 + 23 | test3 | 1 + 24 | test1 | 0 + 24 | test2 | 0 + 24 | test3 | 0 + 25 | test1 | 1 + 25 | test2 | 1 + 25 | test3 | 1 + 26 | test1 | 1 + 26 | test2 | 1 + 26 | test3 | 1 + 27 | test1 | 0 + 27 | test2 | 0 + 27 | test3 | 0 + 28 | test1 | 0 + 28 | test2 | 0 + 28 | test3 | 0 + 29 | test1 | 0 + 29 | test2 | 0 + 29 | test3 | 0 + 30 | test1 | 1 + 30 | test2 | 1 + 30 | test3 | 1 + 31 | test1 | 1 + 31 | test2 | 1 + 31 | test3 | 1 + 32 | test1 | 0 + 32 | test2 | 0 + 32 | test3 | 0 + 33 | test1 | 0 + 33 | test2 | 0 + 33 | test3 | 0 + 34 | test1 | 0 + 34 | test2 | 0 + 34 | test3 | 0 + 35 | test1 | 1 + 35 | test2 | 1 + 35 | test3 | 1 + 36 | test1 | 1 + 36 | test2 | 1 + 36 | test3 | 1 + 37 | test1 | 0 + 37 | test2 | 0 + 37 | test3 | 0 + 38 | test1 | 1 + 38 | test2 | 1 + 38 | test3 | 1 + 39 | test1 | 0 + 39 | test2 | 0 + 39 | test3 | 0 + 40 | test1 | 1 + 40 | test2 | 1 + 40 | test3 | 1 +(120 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from new_part_list_table_distr_random order by a, b; + a | b | correct_segment_id +----+-------+-------------------- + 1 | test1 | t + 1 | test2 | t + 1 | test3 | t + 2 | test1 | t + 2 | test2 | t + 2 | test3 | t + 3 | test1 | t + 3 | test2 | t + 3 | test3 | t + 4 | test1 | t + 4 | test2 | t + 4 | test3 | t + 5 | test1 | t + 5 | test2 | t + 5 | test3 | t + 6 | test1 | t + 6 | test2 | t + 6 | test3 | t + 7 | test1 | t + 7 | test2 | t + 7 | test3 | t + 8 | test1 | t + 8 | test2 | t + 8 | test3 | t + 9 | test1 | t + 9 | test2 | t + 9 | test3 | t + 10 | test1 | t + 10 | test2 | t + 10 | test3 | t + 11 | test1 | t + 11 | test2 | t + 11 | test3 | t + 12 | test1 | t + 12 | test2 | t + 12 | test3 | t + 13 | test1 | t + 13 | test2 | t + 13 | test3 | t + 14 | test1 | t + 14 | test2 | t + 14 | test3 | t + 15 | test1 | t + 15 | test2 | t + 15 | test3 | t + 16 | test1 | t + 16 | test2 | t + 16 | test3 | t + 17 | test1 | t + 17 | test2 | t + 17 | test3 | t + 18 | test1 | t + 18 | test2 | t + 18 | test3 | t + 19 | test1 | t + 19 | test2 | t + 19 | test3 | t + 20 | test1 | t + 20 | test2 | t + 20 | test3 | t + 21 | test1 | t + 21 | test2 | t + 21 | test3 | t + 22 | test1 | t + 22 | test2 | t + 22 | test3 | t + 23 | test1 | t + 23 | test2 | t + 23 | test3 | t + 24 | test1 | t + 24 | test2 | t + 24 | test3 | t + 25 | test1 | t + 25 | test2 | t + 25 | test3 | t + 26 | test1 | t + 26 | test2 | t + 26 | test3 | t + 27 | test1 | t + 27 | test2 | t + 27 | test3 | t + 28 | test1 | t + 28 | test2 | t + 28 | test3 | t + 29 | test1 | t + 29 | test2 | t + 29 | test3 | t + 30 | test1 | t + 30 | test2 | t + 30 | test3 | t + 31 | test1 | t + 31 | test2 | t + 31 | test3 | t + 32 | test1 | t + 32 | test2 | t + 32 | test3 | t + 33 | test1 | t + 33 | test2 | t + 33 | test3 | t + 34 | test1 | t + 34 | test2 | t + 34 | test3 | t + 35 | test1 | t + 35 | test2 | t + 35 | test3 | t + 36 | test1 | t + 36 | test2 | t + 36 | test3 | t + 37 | test1 | t + 37 | test2 | t + 37 | test3 | t + 38 | test1 | t + 38 | test2 | t + 38 | test3 | t + 39 | test1 | t + 39 | test2 | t + 39 | test3 | t + 40 | test1 | t + 40 | test2 | t + 40 | test3 | t +(120 rows) + +select *, gp_segment_id from new_multi_part_table_distr_hashed order by a, b, c; + a | b | c | gp_segment_id +----+------------+-------+--------------- + 1 | 01-05-2023 | test1 | 1 + 1 | 02-05-2023 | test2 | 1 + 1 | 03-05-2023 | test1 | 1 + 2 | 01-05-2023 | test1 | 0 + 2 | 02-05-2023 | test2 | 0 + 2 | 03-05-2023 | test1 | 0 + 3 | 01-05-2023 | test1 | 0 + 3 | 02-05-2023 | test2 | 0 + 3 | 03-05-2023 | test1 | 0 + 4 | 01-05-2023 | test1 | 0 + 4 | 02-05-2023 | test2 | 0 + 4 | 03-05-2023 | test1 | 0 + 5 | 01-05-2023 | test1 | 1 + 5 | 02-05-2023 | test2 | 1 + 5 | 03-05-2023 | test1 | 1 + 6 | 01-05-2023 | test1 | 0 + 6 | 02-05-2023 | test2 | 0 + 6 | 03-05-2023 | test1 | 0 + 7 | 01-05-2023 | test1 | 0 + 7 | 02-05-2023 | test2 | 0 + 7 | 03-05-2023 | test1 | 0 + 8 | 01-05-2023 | test1 | 0 + 8 | 02-05-2023 | test2 | 0 + 8 | 03-05-2023 | test1 | 0 + 9 | 01-05-2023 | test1 | 0 + 9 | 02-05-2023 | test2 | 0 + 9 | 03-05-2023 | test1 | 0 + 10 | 01-05-2023 | test1 | 0 + 10 | 02-05-2023 | test2 | 0 + 10 | 03-05-2023 | test1 | 0 + 11 | 01-05-2023 | test1 | 1 + 11 | 02-05-2023 | test2 | 1 + 11 | 03-05-2023 | test1 | 1 + 12 | 01-05-2023 | test1 | 1 + 12 | 02-05-2023 | test2 | 1 + 12 | 03-05-2023 | test1 | 1 + 13 | 01-05-2023 | test1 | 0 + 13 | 02-05-2023 | test2 | 0 + 13 | 03-05-2023 | test1 | 0 + 14 | 01-05-2023 | test1 | 1 + 14 | 02-05-2023 | test2 | 1 + 14 | 03-05-2023 | test1 | 1 + 15 | 01-05-2023 | test1 | 1 + 15 | 02-05-2023 | test2 | 1 + 15 | 03-05-2023 | test1 | 1 + 16 | 01-05-2023 | test1 | 0 + 16 | 02-05-2023 | test2 | 0 + 16 | 03-05-2023 | test1 | 0 + 17 | 01-05-2023 | test1 | 1 + 17 | 02-05-2023 | test2 | 1 + 17 | 03-05-2023 | test1 | 1 + 18 | 01-05-2023 | test1 | 0 + 18 | 02-05-2023 | test2 | 0 + 18 | 03-05-2023 | test1 | 0 + 19 | 01-05-2023 | test1 | 0 + 19 | 02-05-2023 | test2 | 0 + 19 | 03-05-2023 | test1 | 0 + 20 | 01-05-2023 | test1 | 1 + 20 | 02-05-2023 | test2 | 1 + 20 | 03-05-2023 | test1 | 1 + 21 | 01-05-2023 | test1 | 0 + 21 | 02-05-2023 | test2 | 0 + 21 | 03-05-2023 | test1 | 0 + 22 | 01-05-2023 | test1 | 0 + 22 | 02-05-2023 | test2 | 0 + 22 | 03-05-2023 | test1 | 0 + 23 | 01-05-2023 | test1 | 1 + 23 | 02-05-2023 | test2 | 1 + 23 | 03-05-2023 | test1 | 1 + 24 | 01-05-2023 | test1 | 0 + 24 | 02-05-2023 | test2 | 0 + 24 | 03-05-2023 | test1 | 0 + 25 | 01-05-2023 | test1 | 1 + 25 | 02-05-2023 | test2 | 1 + 25 | 03-05-2023 | test1 | 1 + 26 | 01-05-2023 | test1 | 1 + 26 | 02-05-2023 | test2 | 1 + 26 | 03-05-2023 | test1 | 1 + 27 | 01-05-2023 | test1 | 0 + 27 | 02-05-2023 | test2 | 0 + 27 | 03-05-2023 | test1 | 0 + 28 | 01-05-2023 | test1 | 0 + 28 | 02-05-2023 | test2 | 0 + 28 | 03-05-2023 | test1 | 0 + 29 | 01-05-2023 | test1 | 0 + 29 | 02-05-2023 | test2 | 0 + 29 | 03-05-2023 | test1 | 0 + 30 | 01-05-2023 | test1 | 1 + 30 | 02-05-2023 | test2 | 1 + 30 | 03-05-2023 | test1 | 1 + 31 | 01-05-2023 | test1 | 1 + 31 | 02-05-2023 | test2 | 1 + 31 | 03-05-2023 | test1 | 1 + 32 | 01-05-2023 | test1 | 0 + 32 | 02-05-2023 | test2 | 0 + 32 | 03-05-2023 | test1 | 0 + 33 | 01-05-2023 | test1 | 0 + 33 | 02-05-2023 | test2 | 0 + 33 | 03-05-2023 | test1 | 0 + 34 | 01-05-2023 | test1 | 0 + 34 | 02-05-2023 | test2 | 0 + 34 | 03-05-2023 | test1 | 0 + 35 | 01-05-2023 | test1 | 1 + 35 | 02-05-2023 | test2 | 1 + 35 | 03-05-2023 | test1 | 1 + 36 | 01-05-2023 | test1 | 1 + 36 | 02-05-2023 | test2 | 1 + 36 | 03-05-2023 | test1 | 1 + 37 | 01-05-2023 | test1 | 0 + 37 | 02-05-2023 | test2 | 0 + 37 | 03-05-2023 | test1 | 0 + 38 | 01-05-2023 | test1 | 1 + 38 | 02-05-2023 | test2 | 1 + 38 | 03-05-2023 | test1 | 1 + 39 | 01-05-2023 | test1 | 0 + 39 | 02-05-2023 | test2 | 0 + 39 | 03-05-2023 | test1 | 0 + 40 | 01-05-2023 | test1 | 1 + 40 | 02-05-2023 | test2 | 1 + 40 | 03-05-2023 | test1 | 1 +(120 rows) + +-- And do some cleanup +drop table new_table_distr_hashed; +drop table new_table_distr_hashed_ao_row; +drop table new_table_distr_hashed_ao_col; +drop table new_table_distr_random; +drop table new_table_distr_random_ao_row; +drop table new_table_distr_random_ao_col; +drop table new_table_distr_replicated; +drop table new_table_distr_replicated_ao_row; +drop table new_table_distr_replicated_ao_col; +drop table new_part_range_table_distr_hashed; +drop table new_part_range_table_distr_random; +drop table new_part_list_table_distr_hashed; +drop table new_part_list_table_distr_random; +drop table new_multi_part_table_distr_hashed; +drop table new_table_ctas; +drop table new_table_into; +-- Check rollback of alter rebalance operation +create table table_distr_hashed(a int) distributed by (a); +insert into table_distr_hashed select generate_series(1, 20); +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 8 | 0 + 4 | 1 + 8 | 2 +(3 rows) + +begin; +set gp_target_numsegments = 2; +alter table table_distr_hashed rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 12 | 0 + 8 | 1 +(2 rows) + +rollback; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 8 | 0 + 4 | 1 + 8 | 2 +(3 rows) + +reset gp_target_numsegments; +drop table table_distr_hashed; +-- Check rebalance with parameter +create table table_distr_hashed(a int) distributed by (a); +insert into table_distr_hashed select generate_series(1, 20); +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 8 | 0 + 4 | 1 + 8 | 2 +(3 rows) + +-- Shrink to 2 segments +alter table table_distr_hashed rebalance 2; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 12 | 0 + 8 | 1 +(2 rows) + +-- Shrink to 1 segment +alter table table_distr_hashed rebalance 1; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 20 | 0 +(1 row) + +-- Expand back to 3 segments +alter table table_distr_hashed rebalance 3; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 8 | 0 + 4 | 1 + 8 | 2 +(3 rows) + +-- Try to expand to 4 segments - should do nothing +alter table table_distr_hashed rebalance 3; +ERROR: cannot rebalance table "table_distr_hashed" +DETAIL: table has already been rebalanced +drop table table_distr_hashed; diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index 41df7af9f115..bf5f1b59229f 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -363,4 +363,5 @@ test: hba_conf test: gp_query_id +test: alter_rebalance # end of tests diff --git a/src/test/regress/sql/alter_rebalance.sql b/src/test/regress/sql/alter_rebalance.sql new file mode 100644 index 000000000000..4dec8b1d159c --- /dev/null +++ b/src/test/regress/sql/alter_rebalance.sql @@ -0,0 +1,440 @@ +-- Check 'ALTER TABLE ... REBALANCE' command and 'gp_target_numsegments' GUC + +-- Create hashed distributed tables +create table table_distr_hashed(a int) distributed by (a); +insert into table_distr_hashed select generate_series(1, 20); + +create table table_distr_hashed_ao_row(a int) with (appendonly=true, orientation=row) distributed by (a); +insert into table_distr_hashed_ao_row select generate_series(1, 20); + +create table table_distr_hashed_ao_col(a int) with (appendonly=true, orientation=column) distributed by (a); +insert into table_distr_hashed_ao_col select generate_series(1, 20); + +-- Create randomly distributed tables +create table table_distr_random(a int) distributed randomly; +insert into table_distr_random select generate_series(1, 20); + +create table table_distr_random_ao_row(a int) with (appendonly=true, orientation=row) distributed randomly; +insert into table_distr_random_ao_row select generate_series(1, 20); + +create table table_distr_random_ao_col(a int) with (appendonly=true, orientation=column) distributed randomly; +insert into table_distr_random_ao_col select generate_series(1, 20); + +-- Create replicated distributed tables +create table table_distr_replicated(a int) distributed replicated; +insert into table_distr_replicated select generate_series(1, 20); + +create table table_distr_replicated_ao_row(a int) with (appendonly=true, orientation=row) distributed replicated; +insert into table_distr_replicated_ao_row select generate_series(1, 20); + +create table table_distr_replicated_ao_col(a int) with (appendonly=true, orientation=column) distributed replicated; +insert into table_distr_replicated_ao_col select generate_series(1, 20); + +-- Create part tables +create table part_range_table_distr_hashed (a int, b date) distributed by (a) +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into part_range_table_distr_hashed select i, '2023-01-02' from generate_series(1, 20)i; +insert into part_range_table_distr_hashed select i, '2023-05-02' from generate_series(1, 20)i; +insert into part_range_table_distr_hashed select i, '2020-05-02' from generate_series(1, 20)i; + +create table part_range_table_distr_random (a int, b date) distributed randomly +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into part_range_table_distr_random select i, '2023-01-02' from generate_series(1, 20)i; +insert into part_range_table_distr_random select i, '2023-05-02' from generate_series(1, 20)i; +insert into part_range_table_distr_random select i, '2020-05-02' from generate_series(1, 20)i; + +create table part_list_table_distr_hashed(a int, b text) distributed by (a) +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into part_list_table_distr_hashed select i, 'test1' from generate_series(1, 20)i; +insert into part_list_table_distr_hashed select i, 'test2' from generate_series(1, 20)i; +insert into part_list_table_distr_hashed select i, 'test3' from generate_series(1, 20)i; + +create table part_list_table_distr_random(a int, b text) distributed randomly +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into part_list_table_distr_random select i, 'test1' from generate_series(1, 20)i; +insert into part_list_table_distr_random select i, 'test2' from generate_series(1, 20)i; +insert into part_list_table_distr_random select i, 'test3' from generate_series(1, 20)i; + +create table multi_part_table_distr_hashed(a int, b date, c text) distributed by (a) +partition by range (b) +subpartition by list (c) subpartition template +( + subpartition subpart1 values ('test1'), + subpartition subpart2 values ('test2') +) +( + partition part1 start (date '2023-01-01'), + partition part2 start (date '2023-02-01'), + partition part3 start (date '2023-03-01') end (date '2024-01-01') +); +insert into multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(1, 20)i; +insert into multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(1, 20)i; +insert into multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(1, 20)i; + +-- Now check shrink of the created tables into 2 segments +set gp_target_numsegments = 2; + +alter table table_distr_hashed rebalance; +alter table table_distr_hashed_ao_row rebalance; +alter table table_distr_hashed_ao_col rebalance; + +alter table table_distr_random rebalance; +alter table table_distr_random_ao_row rebalance; +alter table table_distr_random_ao_col rebalance; + +alter table table_distr_replicated rebalance; +alter table table_distr_replicated_ao_row rebalance; +alter table table_distr_replicated_ao_col rebalance; + +alter table part_range_table_distr_hashed rebalance; +alter table part_range_table_distr_random rebalance; + +alter table part_list_table_distr_hashed rebalance; +alter table part_list_table_distr_random rebalance; + +alter table multi_part_table_distr_hashed rebalance; + +-- Verify that data is presented only on segments #0 and #1 +select a, gp_segment_id from table_distr_hashed order by a; +select a, gp_segment_id from table_distr_hashed_ao_row order by a; +select a, gp_segment_id from table_distr_hashed_ao_col order by a; + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random order by a; +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_row order by a; +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_col order by a; + +select a from table_distr_replicated order by a; +select a from table_distr_replicated_ao_row order by a; +select a from table_distr_replicated_ao_col order by a; + +select *, gp_segment_id from part_range_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from part_range_table_distr_random order by a, b; + +select *, gp_segment_id from part_list_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from part_list_table_distr_random order by a, b; + +select *, gp_segment_id from multi_part_table_distr_hashed order by a, b, c; + +-- Check that new data is added only to reduced set of segments +reset gp_target_numsegments; + +insert into table_distr_hashed select generate_series(21, 40); +insert into table_distr_hashed_ao_row select generate_series(21, 40); +insert into table_distr_hashed_ao_col select generate_series(21, 40); + +insert into table_distr_random select generate_series(21, 40); +insert into table_distr_random_ao_row select generate_series(21, 40); +insert into table_distr_random_ao_col select generate_series(21, 40); + +insert into table_distr_replicated select generate_series(21, 40); +insert into table_distr_replicated_ao_row select generate_series(21, 40); +insert into table_distr_replicated_ao_col select generate_series(21, 40); + +insert into part_range_table_distr_hashed select i, '2023-01-02' from generate_series(21, 40)i; +insert into part_range_table_distr_hashed select i, '2023-05-02' from generate_series(21, 40)i; +insert into part_range_table_distr_hashed select i, '2020-05-02' from generate_series(21, 40)i; + +insert into part_range_table_distr_random select i, '2023-01-02' from generate_series(21, 40)i; +insert into part_range_table_distr_random select i, '2023-05-02' from generate_series(21, 40)i; +insert into part_range_table_distr_random select i, '2020-05-02' from generate_series(21, 40)i; + +insert into part_list_table_distr_hashed select i, 'test1' from generate_series(21, 40)i; +insert into part_list_table_distr_hashed select i, 'test2' from generate_series(21, 40)i; +insert into part_list_table_distr_hashed select i, 'test3' from generate_series(21, 40)i; + +insert into part_list_table_distr_random select i, 'test1' from generate_series(21, 40)i; +insert into part_list_table_distr_random select i, 'test2' from generate_series(21, 40)i; +insert into part_list_table_distr_random select i, 'test3' from generate_series(21, 40)i; + +insert into multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(21, 40)i; +insert into multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(21, 40)i; +insert into multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(21, 40)i; + +select a, gp_segment_id from table_distr_hashed order by a; +select a, gp_segment_id from table_distr_hashed_ao_row order by a; +select a, gp_segment_id from table_distr_hashed_ao_col order by a; + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random order by a; +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_row order by a; +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_col order by a; + +select a from table_distr_replicated order by a; +select a from table_distr_replicated_ao_row order by a; +select a from table_distr_replicated_ao_col order by a; + +select *, gp_segment_id from part_range_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from part_range_table_distr_random order by a, b; + +select *, gp_segment_id from part_list_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from part_list_table_distr_random order by a, b; + +select *, gp_segment_id from multi_part_table_distr_hashed order by a, b, c; + +-- And do some cleanup +drop table table_distr_hashed; +drop table table_distr_hashed_ao_row; +drop table table_distr_hashed_ao_col; + +drop table table_distr_random; +drop table table_distr_random_ao_row; +drop table table_distr_random_ao_col; + +drop table table_distr_replicated; +drop table table_distr_replicated_ao_row; +drop table table_distr_replicated_ao_col; + +drop table part_range_table_distr_hashed; +drop table part_range_table_distr_random; + +drop table part_list_table_distr_hashed; +drop table part_list_table_distr_random; + +drop table multi_part_table_distr_hashed; + +-- Check that all newly created tables have data only on segments #0 and #1 +set gp_target_numsegments = 2; +create table new_table_distr_hashed(a int) distributed by (a); +insert into new_table_distr_hashed select generate_series(1, 20); + +create table new_table_distr_hashed_ao_row(a int) with (appendonly=true, orientation=row) distributed by (a); +insert into new_table_distr_hashed_ao_row select generate_series(1, 20); + +create table new_table_distr_hashed_ao_col(a int) with (appendonly=true, orientation=column) distributed by (a); +insert into new_table_distr_hashed_ao_col select generate_series(1, 20); + +create table new_table_distr_random(a int) distributed randomly; +insert into new_table_distr_random select generate_series(1, 20); + +create table new_table_distr_random_ao_row(a int) with (appendonly=true, orientation=row) distributed randomly; +insert into new_table_distr_random_ao_row select generate_series(1, 20); + +create table new_table_distr_random_ao_col(a int) with (appendonly=true, orientation=column) distributed randomly; +insert into new_table_distr_random_ao_col select generate_series(1, 20); + +create table new_table_distr_replicated(a int) distributed replicated; +insert into new_table_distr_replicated select generate_series(1, 20); + +create table new_table_distr_replicated_ao_row(a int) with (appendonly=true, orientation=row) distributed replicated; +insert into new_table_distr_replicated_ao_row select generate_series(1, 20); + +create table new_table_distr_replicated_ao_col(a int) with (appendonly=true, orientation=column) distributed replicated; +insert into new_table_distr_replicated_ao_col select generate_series(1, 20); + +create table new_part_range_table_distr_hashed (a int, b date) distributed by (a) +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into new_part_range_table_distr_hashed select i, '2023-01-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_hashed select i, '2023-05-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_hashed select i, '2020-05-02' from generate_series(1, 20)i; + +create table new_part_range_table_distr_random (a int, b date) distributed randomly +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into new_part_range_table_distr_random select i, '2023-01-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_random select i, '2023-05-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_random select i, '2020-05-02' from generate_series(1, 20)i; + +create table new_part_list_table_distr_hashed(a int, b text) distributed by (a) +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into new_part_list_table_distr_hashed select i, 'test1' from generate_series(1, 20)i; +insert into new_part_list_table_distr_hashed select i, 'test2' from generate_series(1, 20)i; +insert into new_part_list_table_distr_hashed select i, 'test3' from generate_series(1, 20)i; + +create table new_part_list_table_distr_random(a int, b text) distributed randomly +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into new_part_list_table_distr_random select i, 'test1' from generate_series(1, 20)i; +insert into new_part_list_table_distr_random select i, 'test2' from generate_series(1, 20)i; +insert into new_part_list_table_distr_random select i, 'test3' from generate_series(1, 20)i; + +create table new_multi_part_table_distr_hashed(a int, b date, c text) distributed by (a) +partition by range (b) +subpartition by list (c) subpartition template +( + subpartition subpart1 values ('test1'), + subpartition subpart2 values ('test2') +) +( + partition part1 start (date '2023-01-01'), + partition part2 start (date '2023-02-01'), + partition part3 start (date '2023-03-01') end (date '2024-01-01') +); +insert into new_multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(1, 20)i; +insert into new_multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(1, 20)i; +insert into new_multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(1, 20)i; + +-- Also check CTAS statement +create table new_table_ctas as select a from generate_series(1, 20)a distributed by(a); +select * into new_table_into from generate_series(1, 20)a; + +select a, gp_segment_id from new_table_distr_hashed order by a; +select a, gp_segment_id from new_table_distr_hashed_ao_row order by a; +select a, gp_segment_id from new_table_distr_hashed_ao_col order by a; + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random order by a; +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_row order by a; +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_col order by a; + +select a from new_table_distr_replicated order by a; +select a from new_table_distr_replicated_ao_row order by a; +select a from new_table_distr_replicated_ao_col order by a; + +select *, gp_segment_id from new_part_range_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from new_part_range_table_distr_random order by a, b; + +select *, gp_segment_id from new_part_list_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from new_part_list_table_distr_random order by a, b; + +select *, gp_segment_id from new_multi_part_table_distr_hashed order by a, b, c; + +select a, gp_segment_id from new_table_ctas order by a; +select *, (gp_segment_id < 2) as correct_segment_id from new_table_into order by a; + +-- Validate the insertion works fine with the new tables +-- after 'gp_target_numsegments' reset +reset gp_target_numsegments; + +insert into new_table_ctas select generate_series(21, 40); +insert into new_table_into select generate_series(21, 40); + +insert into new_table_distr_hashed select generate_series(21, 40); +insert into new_table_distr_hashed_ao_row select generate_series(21, 40); +insert into new_table_distr_hashed_ao_col select generate_series(21, 40); + +insert into new_table_distr_random select generate_series(21, 40); +insert into new_table_distr_random_ao_row select generate_series(21, 40); +insert into new_table_distr_random_ao_col select generate_series(21, 40); + +insert into new_part_range_table_distr_hashed select i, '2023-01-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_hashed select i, '2023-05-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_hashed select i, '2020-05-02' from generate_series(21, 40)i; + +insert into new_part_range_table_distr_random select i, '2023-01-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_random select i, '2023-05-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_random select i, '2020-05-02' from generate_series(21, 40)i; + +insert into new_part_list_table_distr_hashed select i, 'test1' from generate_series(21, 40)i; +insert into new_part_list_table_distr_hashed select i, 'test2' from generate_series(21, 40)i; +insert into new_part_list_table_distr_hashed select i, 'test3' from generate_series(21, 40)i; + +insert into new_part_list_table_distr_random select i, 'test1' from generate_series(21, 40)i; +insert into new_part_list_table_distr_random select i, 'test2' from generate_series(21, 40)i; +insert into new_part_list_table_distr_random select i, 'test3' from generate_series(21, 40)i; + +insert into new_multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(21, 40)i; +insert into new_multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(21, 40)i; +insert into new_multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(21, 40)i; + +select a, gp_segment_id from new_table_ctas order by a; +select *, (gp_segment_id < 2) as correct_segment_id from new_table_into order by a; + +select a, gp_segment_id from new_table_distr_hashed order by a; +select a, gp_segment_id from new_table_distr_hashed_ao_row order by a; +select a, gp_segment_id from new_table_distr_hashed_ao_col order by a; + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random order by a; +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_row order by a; +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_col order by a; + +select *, gp_segment_id from new_part_range_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from new_part_range_table_distr_random order by a, b; + +select *, gp_segment_id from new_part_list_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from new_part_list_table_distr_random order by a, b; + +select *, gp_segment_id from new_multi_part_table_distr_hashed order by a, b, c; + +-- And do some cleanup +drop table new_table_distr_hashed; +drop table new_table_distr_hashed_ao_row; +drop table new_table_distr_hashed_ao_col; + +drop table new_table_distr_random; +drop table new_table_distr_random_ao_row; +drop table new_table_distr_random_ao_col; + +drop table new_table_distr_replicated; +drop table new_table_distr_replicated_ao_row; +drop table new_table_distr_replicated_ao_col; + +drop table new_part_range_table_distr_hashed; +drop table new_part_range_table_distr_random; + +drop table new_part_list_table_distr_hashed; +drop table new_part_list_table_distr_random; + +drop table new_multi_part_table_distr_hashed; + +drop table new_table_ctas; +drop table new_table_into; + +-- Check rollback of alter rebalance operation +create table table_distr_hashed(a int) distributed by (a); +insert into table_distr_hashed select generate_series(1, 20); + +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + +begin; +set gp_target_numsegments = 2; +alter table table_distr_hashed rebalance; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; +rollback; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + +reset gp_target_numsegments; +drop table table_distr_hashed; + +-- Check rebalance with parameter +create table table_distr_hashed(a int) distributed by (a); +insert into table_distr_hashed select generate_series(1, 20); +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + +-- Shrink to 2 segments +alter table table_distr_hashed rebalance 2; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + +-- Shrink to 1 segment +alter table table_distr_hashed rebalance 1; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + +-- Expand back to 3 segments +alter table table_distr_hashed rebalance 3; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + +-- Try to expand to 4 segments - should do nothing +alter table table_distr_hashed rebalance 3; + +drop table table_distr_hashed; From c4ebcffd1545e5071860e1f530252be0749cb29e Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Fri, 8 Aug 2025 13:01:50 +0300 Subject: [PATCH 09/22] Use shared variable for default number of segments (#1941) Commit 5b3f506b7bb0be61e8bec6c32e7b16cd764fc652 introduced new command ALTER TABLE REBALANCE with shrink support. The target number of segments (if not specified in ALTER command) is taken from GP_POLICY_DEFAULT_NUMSEGMENTS() macro. Therefore, we need somehow to set and maintain the creation number across all backends. This patch introduces a mechanism for managing the default number of segments used in table creation during a rebalance operation in GPDB. A new shared variable gp_create_table_rebalance_numsegments is introduced in gpexpand.h to track the number of segments to use during table creation while a rebalancing operation is in progress. The shared variable is initialized in shared memory with appropriate size and get functionality. Corresponding SQL functions are created in gp_toolkit extension. The system now checks if a rebalancing operation is active by verifying locks before allowing modifications to the number of segments. If a lock is not already acquired in current transaction (indicating that no rebalancing is underway), an appropriate error message is returned. Tests from 5b3f506b7bb0be61e8bec6c32e7b16cd764fc652 are updated to support the new functonality gp_debug_numsegments extension preserves its behaviour. But we disallow to modify local numsegments value when gp_create_table_rebalance_numsegments is set. --- .../gp_debug_numsegments.c | 16 ++++ gpcontrib/gp_toolkit/Makefile | 4 +- .../gp_toolkit/gp_rebalance_numsegments.c | 87 +++++++++++++++++++ gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql | 27 ++++++ gpcontrib/gp_toolkit/gp_toolkit.control | 2 +- src/backend/storage/ipc/ipci.c | 5 ++ src/backend/utils/misc/gpexpand.c | 26 ++++++ src/backend/utils/misc/guc_gp.c | 32 ------- src/include/catalog/gp_distribution_policy.h | 9 +- src/include/utils/gpexpand.h | 4 + src/include/utils/sync_guc_name.h | 1 - .../alter_blocks_for_update_and_viceversa.out | 10 +-- .../expected/rebalance_numsegments.out | 84 ++++++++++++++++++ src/test/isolation2/isolation2_schedule | 1 + .../alter_blocks_for_update_and_viceversa.sql | 8 +- .../isolation2/sql/rebalance_numsegments.sql | 29 +++++++ src/test/regress/expected/alter_rebalance.out | 66 ++++++++++++-- src/test/regress/sql/alter_rebalance.sql | 25 ++++-- 18 files changed, 371 insertions(+), 65 deletions(-) create mode 100644 gpcontrib/gp_toolkit/gp_rebalance_numsegments.c create mode 100644 gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql create mode 100644 src/test/isolation2/expected/rebalance_numsegments.out create mode 100644 src/test/isolation2/sql/rebalance_numsegments.sql diff --git a/gpcontrib/gp_debug_numsegments/gp_debug_numsegments.c b/gpcontrib/gp_debug_numsegments/gp_debug_numsegments.c index 736101ac53b0..98b4830b272e 100644 --- a/gpcontrib/gp_debug_numsegments/gp_debug_numsegments.c +++ b/gpcontrib/gp_debug_numsegments/gp_debug_numsegments.c @@ -16,6 +16,8 @@ #include "utils/builtins.h" #include "cdb/cdbutil.h" #include "catalog/pg_type.h" +#include "port/atomics.h" +#include "utils/gpexpand.h" #ifdef PG_MODULE_MAGIC @@ -53,6 +55,13 @@ gp_debug_set_create_table_default_numsegments(PG_FUNCTION_ARGS) Assert(1 == PG_NARGS()); + uint32 gp_rebalance_numsegs = pg_atomic_read_u32(gp_create_table_rebalance_numsegments); + if (gp_rebalance_numsegs != GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("gprebalance in progress, all relations will be created at %u segments", + gp_rebalance_numsegs))); + argtypeoid = get_fn_expr_argtype(fcinfo->flinfo, 0); /* Accept argument in integer / text format */ @@ -138,6 +147,13 @@ gp_debug_get_create_table_default_numsegments(PG_FUNCTION_ARGS) { char buf[64]; const char *result = NULL; + uint32 gp_rebalance_numsegs = pg_atomic_read_u32(gp_create_table_rebalance_numsegments); + + if (gp_rebalance_numsegs != GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("gprebalance in progress, all relations will be created at %u segments", + gp_rebalance_numsegs))); switch (gp_create_table_default_numsegments) { diff --git a/gpcontrib/gp_toolkit/Makefile b/gpcontrib/gp_toolkit/Makefile index 82fc7f7dee99..852b3d93ba82 100644 --- a/gpcontrib/gp_toolkit/Makefile +++ b/gpcontrib/gp_toolkit/Makefile @@ -1,10 +1,10 @@ EXTENSION = gp_toolkit DATA = gp_toolkit--1.1--1.2.sql gp_toolkit--1.0--1.1.sql gp_toolkit--1.0.sql \ gp_toolkit--1.2--1.3.sql gp_toolkit--1.3.sql gp_toolkit--1.3--1.4.sql \ - gp_toolkit--1.4--1.5.sql gp_toolkit--1.5--1.6.sql + gp_toolkit--1.4--1.5.sql gp_toolkit--1.5--1.6.sql gp_toolkit--1.6--1.7.sql MODULE_big = gp_toolkit ifeq ($(shell uname -s), Linux) -OBJS = resgroup.o gp_partition_maint.o +OBJS = resgroup.o gp_partition_maint.o gp_rebalance_numsegments.o else OBJS = resgroup-dummy.o gp_partition_maint.o endif diff --git a/gpcontrib/gp_toolkit/gp_rebalance_numsegments.c b/gpcontrib/gp_toolkit/gp_rebalance_numsegments.c new file mode 100644 index 000000000000..2ae7d74ee109 --- /dev/null +++ b/gpcontrib/gp_toolkit/gp_rebalance_numsegments.c @@ -0,0 +1,87 @@ +#include "postgres.h" + +#include "fmgr.h" +#include "utils/builtins.h" +#include "cdb/cdbutil.h" +#include "catalog/pg_type.h" +#include "storage/lock.h" +#include "utils/gpexpand.h" + +static void +errorOutOnLock(void) +{ + LOCKTAG gp_expand_locktag = + { + /* FIXME: how to fill the locktag? */ + .locktag_field1 = 0xdead, + .locktag_field2 = 0xdead, + .locktag_field3 = 0xdead, + .locktag_field4 = 0xdd, + .locktag_type = LOCKTAG_USERLOCK, + .locktag_lockmethodid = USER_LOCKMETHOD, + }; + LockAcquireResult acquired = LockAcquire(&gp_expand_locktag, AccessExclusiveLock, false, true); + + if (acquired != LOCKACQUIRE_ALREADY_HELD) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("gprebalance not in progress, function can be used only under catalog lock"))); +} + +/* + * Get the rebalance numsegments when creating a table. + */ +PG_FUNCTION_INFO_V1(gp_get_rebalance_numsegments); +Datum +gp_get_rebalance_numsegments(PG_FUNCTION_ARGS) +{ + errorOutOnLock(); + + uint32 gp_rebalance_numsegs = pg_atomic_read_u32(gp_create_table_rebalance_numsegments); + + PG_RETURN_UINT32(gp_rebalance_numsegs); +} + +/* + * Set the numsegments when creating tables during gprebalance run. + * + * For integer argument the valid range is [1, gp_num_contents_in_cluster]. + */ +PG_FUNCTION_INFO_V1(gp_set_rebalance_numsegments); +Datum +gp_set_rebalance_numsegments(PG_FUNCTION_ARGS) +{ + int numsegments = -1; + + Assert(1 == PG_NARGS()); + + errorOutOnLock(); + + numsegments = PG_GETARG_INT32(0); + + if (numsegments < 1 || numsegments > getgpsegmentCount()) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("invalid integer value for default numsegments: %d", + numsegments), + errhint("Valid range: [1, %d (gp_num_contents_in_cluster)]", + getgpsegmentCount()))); + + pg_atomic_write_u32(gp_create_table_rebalance_numsegments, numsegments); + + return gp_get_rebalance_numsegments(fcinfo); +} + +/* + * Reset the rebalance numsegments when creating a table. + */ +PG_FUNCTION_INFO_V1(gp_reset_rebalance_numsegments); +Datum +gp_reset_rebalance_numsegments(PG_FUNCTION_ARGS) +{ + errorOutOnLock(); + + pg_atomic_write_u32(gp_create_table_rebalance_numsegments, GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET); + + PG_RETURN_VOID(); +} diff --git a/gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql b/gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql new file mode 100644 index 000000000000..cb62c578ea1d --- /dev/null +++ b/gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql @@ -0,0 +1,27 @@ +/* gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION gp_toolkit UPDATE TO '1.7" to load this file. \quit + +-- This function set the numsegments when creating tables during rebalance operation. +-- This form accepts an integer argument: [1, gp_num_contents_in_cluster]. +CREATE FUNCTION gp_toolkit.gp_set_rebalance_numsegments(integer) RETURNS int +AS '$libdir/gp_toolkit','gp_set_rebalance_numsegments' +LANGUAGE C STRICT; + +REVOKE ALL ON FUNCTION gp_toolkit.gp_set_rebalance_numsegments(integer) FROM public; + +-- This function reset the default numsegments. +-- This function resets numsegments directly to default value INT_MAX +CREATE FUNCTION gp_toolkit.gp_reset_rebalance_numsegments() RETURNS void +AS '$libdir/gp_toolkit','gp_reset_rebalance_numsegments' +LANGUAGE C STRICT; + +REVOKE ALL ON FUNCTION gp_toolkit.gp_reset_rebalance_numsegments() FROM public; + +-- This function get the default numsegments when creating tables. +CREATE FUNCTION gp_toolkit.gp_get_rebalance_numsegments() RETURNS int +AS '$libdir/gp_toolkit','gp_get_rebalance_numsegments' +LANGUAGE C STRICT; + +REVOKE ALL ON FUNCTION gp_toolkit.gp_get_rebalance_numsegments() FROM public; diff --git a/gpcontrib/gp_toolkit/gp_toolkit.control b/gpcontrib/gp_toolkit/gp_toolkit.control index 0d9b47e7e6d6..c8c0a4bacad4 100644 --- a/gpcontrib/gp_toolkit/gp_toolkit.control +++ b/gpcontrib/gp_toolkit/gp_toolkit.control @@ -1,5 +1,5 @@ # gp_toolkit extension comment = 'various GPDB administrative views/functions' -default_version = '1.6' +default_version = '1.7' schema = gp_toolkit diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index bd71b5df69ca..8e95fc556e34 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -227,6 +227,9 @@ CreateSharedMemoryAndSemaphores(int port) /* size of expand version */ size = add_size(size, GpExpandVersionShmemSize()); + /* size of rebalance numsegments variable */ + size = add_size(size, GpRebalanceNumsegmentsShmemSize()); + /* size of token and endpoint shared memory */ size = add_size(size, EndpointShmemSize()); @@ -385,6 +388,8 @@ CreateSharedMemoryAndSemaphores(int port) GpExpandVersionShmemInit(); + GpRebalanceNumsegmentsShmemInit(); + #ifdef EXEC_BACKEND /* diff --git a/src/backend/utils/misc/gpexpand.c b/src/backend/utils/misc/gpexpand.c index 2cd826516087..c937c2551274 100644 --- a/src/backend/utils/misc/gpexpand.c +++ b/src/backend/utils/misc/gpexpand.c @@ -29,9 +29,13 @@ #include "utils/rel.h" #include "utils/relcache.h" #include "utils/gpexpand.h" +#include "port/atomics.h" +#include "catalog/gp_distribution_policy.h" static volatile int *gp_expand_version; +volatile pg_atomic_uint32 *gp_create_table_rebalance_numsegments; + /* * Catalog lock. */ @@ -157,3 +161,25 @@ gp_expand_protect_catalog_changes(Relation relation) "catalog changes are disallowed", oldVersion, newVersion))); } +int +GpRebalanceNumsegmentsShmemSize(void) +{ + return sizeof(*gp_create_table_rebalance_numsegments); +} + +void +GpRebalanceNumsegmentsShmemInit(void) +{ + if (IsUnderPostmaster) + return; + + /* only postmaster initialize it */ + gp_create_table_rebalance_numsegments = (volatile pg_atomic_uint32 *)ShmemAlloc(GpRebalanceNumsegmentsShmemSize()); + pg_atomic_init_u32(gp_create_table_rebalance_numsegments, GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET); +} + +uint32 +getRebalanceNumsegments(void) +{ + return pg_atomic_read_u32(gp_create_table_rebalance_numsegments); +} diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index 18e86b1be7ec..fd9c3969165a 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -95,7 +95,6 @@ static void assign_gp_default_storage_options(const char *newval, void *extra); static bool check_pljava_classpath_insecure(bool *newval, void **extra, GucSource source); static void assign_pljava_classpath_insecure(bool newval, void *extra); static bool check_gp_resource_group_bypass(bool *newval, void **extra, GucSource source); -static bool check_gp_target_numsegments(int *newval, void **extra, GucSource source); static int guc_array_compare(const void *a, const void *b); extern struct config_generic *find_option(const char *name, bool create_placeholders, int elevel); @@ -4407,21 +4406,6 @@ struct config_int ConfigureNamesInt_gp[] = }, #endif - { - {"gp_target_numsegments", PGC_SUSET, GP_ARRAY_CONFIGURATION, - gettext_noop("Target number of segments to use when a new table " - "is created or an old one is rebalanced."), - gettext_noop("Special values:\n" - "-1 - use all available segments\n" - "-2 - use random number of segments\n" - "-3 - use minimal possible number of segment\n" ), - GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE - }, - &gp_create_table_default_numsegments, - GP_DEFAULT_NUMSEGMENTS_FULL, GP_DEFAULT_NUMSEGMENTS_MINIMAL, INT_MAX, - check_gp_target_numsegments, NULL, NULL - }, - /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL @@ -5171,22 +5155,6 @@ check_gp_resource_group_bypass(bool *newval, void **extra, GucSource source) return false; } -static bool -check_gp_target_numsegments(int *newval, void **extra, GucSource source) -{ - int max_segment_number = getgpsegmentCount(); - if (*newval > max_segment_number) - { - elog(WARNING, - "Can't set value more than maximum segment number, " - "force 'gp_target_numsegments' to %d", - max_segment_number); - *newval = max_segment_number; - } - return true; -} - - static bool check_optimizer(bool *newval, void **extra, GucSource source) { diff --git a/src/include/catalog/gp_distribution_policy.h b/src/include/catalog/gp_distribution_policy.h index fdb864e68137..76ba7839935b 100644 --- a/src/include/catalog/gp_distribution_policy.h +++ b/src/include/catalog/gp_distribution_policy.h @@ -59,6 +59,10 @@ typedef FormData_gp_distribution_policy *Form_gp_distribution_policy; * gp_create_table_default_numsegments. */ #define GP_POLICY_DEFAULT_NUMSEGMENTS() \ +( (getRebalanceNumsegments() != GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET) ? \ + getRebalanceNumsegments() : GP_POLICY_ORIGINAL_BEHAVIOR() ) + +#define GP_POLICY_ORIGINAL_BEHAVIOR() \ ( gp_create_table_default_numsegments == GP_DEFAULT_NUMSEGMENTS_FULL ? getgpsegmentCount() \ : gp_create_table_default_numsegments == GP_DEFAULT_NUMSEGMENTS_RANDOM ? (1 + random() % getgpsegmentCount()) \ : gp_create_table_default_numsegments == GP_DEFAULT_NUMSEGMENTS_MINIMAL ? 1 \ @@ -67,12 +71,14 @@ typedef FormData_gp_distribution_policy *Form_gp_distribution_policy; /* * The the default numsegments policies when creating a table. * + * - UNSET: shared value used during rebalance is not set, default variable is used * - FULL: all the segments; * - RANDOM: pick a random set of segments each time; * - MINIMAL: the minimal set of segments; */ enum { + GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET = 0x7fffffff, GP_DEFAULT_NUMSEGMENTS_FULL = -1, GP_DEFAULT_NUMSEGMENTS_RANDOM = -2, GP_DEFAULT_NUMSEGMENTS_MINIMAL = -3, @@ -113,7 +119,6 @@ typedef struct GpPolicy * Global Variables */ extern int gp_create_table_default_numsegments; - /* * GpPolicyCopy -- Return a copy of a GpPolicy object. * @@ -167,4 +172,6 @@ extern GpPolicy *createHashPartitionedPolicy(List *keys, List *opclasses, int nu extern bool IsReplicatedTable(Oid relid); +extern uint32 getRebalanceNumsegments(void); + #endif /* GP_DISTRIBUTION_POLICY_H */ diff --git a/src/include/utils/gpexpand.h b/src/include/utils/gpexpand.h index 8614c3434077..cd8f8d4bb44b 100644 --- a/src/include/utils/gpexpand.h +++ b/src/include/utils/gpexpand.h @@ -24,6 +24,10 @@ extern void gp_expand_protect_catalog_changes(Relation relation); extern Datum gp_expand_bump_version(PG_FUNCTION_ARGS); +extern volatile pg_atomic_uint32 *gp_create_table_rebalance_numsegments; + +extern void GpRebalanceNumsegmentsShmemInit(void); +extern int GpRebalanceNumsegmentsShmemSize(void); #endif /* GPEXPAND_H */ diff --git a/src/include/utils/sync_guc_name.h b/src/include/utils/sync_guc_name.h index c9d396025f31..0228d8d5cf8f 100644 --- a/src/include/utils/sync_guc_name.h +++ b/src/include/utils/sync_guc_name.h @@ -179,4 +179,3 @@ "wal_sender_timeout", "work_mem", "zero_damaged_pages", - "gp_target_numsegments", diff --git a/src/test/isolation2/expected/alter_blocks_for_update_and_viceversa.out b/src/test/isolation2/expected/alter_blocks_for_update_and_viceversa.out index 0a564d65930f..0c9abaef9e62 100644 --- a/src/test/isolation2/expected/alter_blocks_for_update_and_viceversa.out +++ b/src/test/isolation2/expected/alter_blocks_for_update_and_viceversa.out @@ -45,8 +45,6 @@ CREATE TABLE 1: INSERT INTO alter_block SELECT i, i FROM generate_series(1, 20)i; INSERT 0 20 -2: SET gp_target_numsegments = 2; -SET -- Validate SELECT blocks the ALTER 1: BEGIN; @@ -60,7 +58,7 @@ BEGIN 4 | 1 8 | 2 (3 rows) -2&: ALTER TABLE alter_block REBALANCE; +2&: ALTER TABLE alter_block REBALANCE 2; 1: COMMIT; COMMIT 2<: <... completed> @@ -75,7 +73,7 @@ BEGIN BEGIN 1: UPDATE alter_block SET b = b + 1 WHERE a = 1; UPDATE 1 -2&: ALTER TABLE alter_block REBALANCE; +2&: ALTER TABLE alter_block REBALANCE 2; 1: COMMIT; COMMIT 2<: <... completed> @@ -88,7 +86,7 @@ ROLLBACK BEGIN 2: BEGIN; BEGIN -2: ALTER TABLE alter_block REBALANCE; +2: ALTER TABLE alter_block REBALANCE 2; ALTER TABLE 1&: SELECT count(1), gp_segment_id FROM alter_block GROUP BY gp_segment_id ORDER BY gp_segment_id; 2: COMMIT; @@ -102,7 +100,5 @@ COMMIT 1: COMMIT; COMMIT -2: RESET gp_target_numsegments; -RESET 1: DROP TABLE alter_block; DROP TABLE diff --git a/src/test/isolation2/expected/rebalance_numsegments.out b/src/test/isolation2/expected/rebalance_numsegments.out new file mode 100644 index 000000000000..db00c1a2115b --- /dev/null +++ b/src/test/isolation2/expected/rebalance_numsegments.out @@ -0,0 +1,84 @@ +-- Test setting numsegments value for table creation during rebalance operation +-- Test error throwing when catalog is not locked by gprebalance +select gp_toolkit.gp_set_rebalance_numsegments(2); +ERROR: gprebalance not in progress, function can be used only under catalog lock +select gp_toolkit.gp_get_rebalance_numsegments(); +ERROR: gprebalance not in progress, function can be used only under catalog lock +select gp_toolkit.gp_reset_rebalance_numsegments(); +ERROR: gprebalance not in progress, function can be used only under catalog lock +-- Test the numsegment value setting mechanics +1: begin; +BEGIN +1: select gp_expand_lock_catalog(); + gp_expand_lock_catalog +------------------------ + +(1 row) +2: create table t_reb (i int) distributed by (i); +ERROR: gpexpand in progress, catalog changes are disallowed. +1: select gp_toolkit.gp_get_rebalance_numsegments(); + gp_get_rebalance_numsegments +------------------------------ + 2147483647 +(1 row) +1: select gp_toolkit.gp_set_rebalance_numsegments(2); + gp_set_rebalance_numsegments +------------------------------ + 2 +(1 row) +1: select gp_toolkit.gp_get_rebalance_numsegments(); + gp_get_rebalance_numsegments +------------------------------ + 2 +(1 row) +1: select gp_toolkit.gp_reset_rebalance_numsegments(); + gp_reset_rebalance_numsegments +-------------------------------- + +(1 row) +1: select gp_toolkit.gp_get_rebalance_numsegments(); + gp_get_rebalance_numsegments +------------------------------ + 2147483647 +(1 row) +1: select gp_toolkit.gp_set_rebalance_numsegments(2); + gp_set_rebalance_numsegments +------------------------------ + 2 +(1 row) +1: commit; +COMMIT +-- Test that newly created tables are using updated number of segmetns +1: create table t_reb (i int) distributed by (i); +CREATE TABLE +1: select numsegments from gp_distribution_policy where localoid = 't_reb'::regclass; + numsegments +------------- + 2 +(1 row) +1: drop table t_reb; +DROP TABLE + +1: begin; +BEGIN +1: select gp_expand_lock_catalog(); + gp_expand_lock_catalog +------------------------ + +(1 row) +1: select gp_toolkit.gp_reset_rebalance_numsegments(); + gp_reset_rebalance_numsegments +-------------------------------- + +(1 row) +1: create table t_reb (i int) distributed by (i); +CREATE TABLE +1: select numsegments from gp_distribution_policy where localoid = 't_reb'::regclass; + numsegments +------------- + 3 +(1 row) +1: rollback; +ROLLBACK +1q: ... +2q: ... diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 45afebf07d92..3ebfb5eab1b1 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -373,3 +373,4 @@ test: brin_heap # Intensive tests for BRIN test: uao/brin_chain_row uao/brin_chain_column +test: rebalance_numsegments diff --git a/src/test/isolation2/sql/alter_blocks_for_update_and_viceversa.sql b/src/test/isolation2/sql/alter_blocks_for_update_and_viceversa.sql index c9ede0501bfe..f89a077abb28 100644 --- a/src/test/isolation2/sql/alter_blocks_for_update_and_viceversa.sql +++ b/src/test/isolation2/sql/alter_blocks_for_update_and_viceversa.sql @@ -23,13 +23,12 @@ 1: CREATE TABLE alter_block(a int, b int) DISTRIBUTED BY (a); 1: INSERT INTO alter_block SELECT i, i FROM generate_series(1, 20)i; -2: SET gp_target_numsegments = 2; -- Validate SELECT blocks the ALTER 1: BEGIN; 2: BEGIN; 1: SELECT count(1), gp_segment_id FROM alter_block GROUP BY gp_segment_id ORDER BY gp_segment_id; -2&: ALTER TABLE alter_block REBALANCE; +2&: ALTER TABLE alter_block REBALANCE 2; 1: COMMIT; 2<: 2: ROLLBACK; @@ -38,7 +37,7 @@ 1: BEGIN; 2: BEGIN; 1: UPDATE alter_block SET b = b + 1 WHERE a = 1; -2&: ALTER TABLE alter_block REBALANCE; +2&: ALTER TABLE alter_block REBALANCE 2; 1: COMMIT; 2<: 2: ROLLBACK; @@ -46,11 +45,10 @@ -- Validate ALTER blocks the SELECT 1: BEGIN; 2: BEGIN; -2: ALTER TABLE alter_block REBALANCE; +2: ALTER TABLE alter_block REBALANCE 2; 1&: SELECT count(1), gp_segment_id FROM alter_block GROUP BY gp_segment_id ORDER BY gp_segment_id; 2: COMMIT; 1<: 1: COMMIT; -2: RESET gp_target_numsegments; 1: DROP TABLE alter_block; diff --git a/src/test/isolation2/sql/rebalance_numsegments.sql b/src/test/isolation2/sql/rebalance_numsegments.sql new file mode 100644 index 000000000000..fb8c75a13d36 --- /dev/null +++ b/src/test/isolation2/sql/rebalance_numsegments.sql @@ -0,0 +1,29 @@ +-- Test setting numsegments value for table creation during rebalance operation +-- Test error throwing when catalog is not locked by gprebalance +select gp_toolkit.gp_set_rebalance_numsegments(2); +select gp_toolkit.gp_get_rebalance_numsegments(); +select gp_toolkit.gp_reset_rebalance_numsegments(); +-- Test the numsegment value setting mechanics +1: begin; +1: select gp_expand_lock_catalog(); +2: create table t_reb (i int) distributed by (i); +1: select gp_toolkit.gp_get_rebalance_numsegments(); +1: select gp_toolkit.gp_set_rebalance_numsegments(2); +1: select gp_toolkit.gp_get_rebalance_numsegments(); +1: select gp_toolkit.gp_reset_rebalance_numsegments(); +1: select gp_toolkit.gp_get_rebalance_numsegments(); +1: select gp_toolkit.gp_set_rebalance_numsegments(2); +1: commit; +-- Test that newly created tables are using updated number of segmetns +1: create table t_reb (i int) distributed by (i); +1: select numsegments from gp_distribution_policy where localoid = 't_reb'::regclass; +1: drop table t_reb; + +1: begin; +1: select gp_expand_lock_catalog(); +1: select gp_toolkit.gp_reset_rebalance_numsegments(); +1: create table t_reb (i int) distributed by (i); +1: select numsegments from gp_distribution_policy where localoid = 't_reb'::regclass; +1: rollback; +1q: +2q: diff --git a/src/test/regress/expected/alter_rebalance.out b/src/test/regress/expected/alter_rebalance.out index 1d9227b1eca8..1c2b65ae8703 100644 --- a/src/test/regress/expected/alter_rebalance.out +++ b/src/test/regress/expected/alter_rebalance.out @@ -75,7 +75,20 @@ insert into multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from g insert into multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(1, 20)i; insert into multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(1, 20)i; -- Now check shrink of the created tables into 2 segments -set gp_target_numsegments = 2; +begin; +select gp_expand_lock_catalog(); + gp_expand_lock_catalog +------------------------ + +(1 row) + +select gp_toolkit.gp_set_rebalance_numsegments(2); + gp_set_rebalance_numsegments +------------------------------ + 2 +(1 row) + +end; alter table table_distr_hashed rebalance; NOTICE: REBALANCE to current default target number of segments 2 alter table table_distr_hashed_ao_row rebalance; @@ -656,7 +669,20 @@ select *, gp_segment_id from multi_part_table_distr_hashed order by a, b, c; (60 rows) -- Check that new data is added only to reduced set of segments -reset gp_target_numsegments; +begin; +select gp_expand_lock_catalog(); + gp_expand_lock_catalog +------------------------ + +(1 row) + +select gp_toolkit.gp_reset_rebalance_numsegments(); + gp_reset_rebalance_numsegments +-------------------------------- + +(1 row) + +end; insert into table_distr_hashed select generate_series(21, 40); insert into table_distr_hashed_ao_row select generate_series(21, 40); insert into table_distr_hashed_ao_col select generate_series(21, 40); @@ -1727,7 +1753,20 @@ drop table part_list_table_distr_hashed; drop table part_list_table_distr_random; drop table multi_part_table_distr_hashed; -- Check that all newly created tables have data only on segments #0 and #1 -set gp_target_numsegments = 2; +begin; +select gp_expand_lock_catalog(); + gp_expand_lock_catalog +------------------------ + +(1 row) + +select gp_toolkit.gp_set_rebalance_numsegments(2); + gp_set_rebalance_numsegments +------------------------------ + 2 +(1 row) + +end; create table new_table_distr_hashed(a int) distributed by (a); insert into new_table_distr_hashed select generate_series(1, 20); create table new_table_distr_hashed_ao_row(a int) with (appendonly=true, orientation=row) distributed by (a); @@ -1802,7 +1841,6 @@ insert into new_multi_part_table_distr_hashed select i, '2023-03-05', 'test1' fr -- Also check CTAS statement create table new_table_ctas as select a from generate_series(1, 20)a distributed by(a); select * into new_table_into from generate_series(1, 20)a; -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. select a, gp_segment_id from new_table_distr_hashed order by a; a | gp_segment_id ----+--------------- @@ -2405,7 +2443,20 @@ select *, (gp_segment_id < 2) as correct_segment_id from new_table_into order by -- Validate the insertion works fine with the new tables -- after 'gp_target_numsegments' reset -reset gp_target_numsegments; +begin; +select gp_expand_lock_catalog(); + gp_expand_lock_catalog +------------------------ + +(1 row) + +select gp_toolkit.gp_reset_rebalance_numsegments(); + gp_reset_rebalance_numsegments +-------------------------------- + +(1 row) + +end; insert into new_table_ctas select generate_series(21, 40); insert into new_table_into select generate_series(21, 40); insert into new_table_distr_hashed select generate_series(21, 40); @@ -3443,9 +3494,7 @@ select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id or (3 rows) begin; -set gp_target_numsegments = 2; -alter table table_distr_hashed rebalance; -NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_hashed rebalance 2; select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; count | gp_segment_id -------+--------------- @@ -3462,7 +3511,6 @@ select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id or 8 | 2 (3 rows) -reset gp_target_numsegments; drop table table_distr_hashed; -- Check rebalance with parameter create table table_distr_hashed(a int) distributed by (a); diff --git a/src/test/regress/sql/alter_rebalance.sql b/src/test/regress/sql/alter_rebalance.sql index 4dec8b1d159c..accacc6db888 100644 --- a/src/test/regress/sql/alter_rebalance.sql +++ b/src/test/regress/sql/alter_rebalance.sql @@ -90,7 +90,10 @@ insert into multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from g insert into multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(1, 20)i; -- Now check shrink of the created tables into 2 segments -set gp_target_numsegments = 2; +begin; +select gp_expand_lock_catalog(); +select gp_toolkit.gp_set_rebalance_numsegments(2); +end; alter table table_distr_hashed rebalance; alter table table_distr_hashed_ao_row rebalance; @@ -134,7 +137,10 @@ select *, (gp_segment_id < 2) as correct_segment_id from part_list_table_distr_r select *, gp_segment_id from multi_part_table_distr_hashed order by a, b, c; -- Check that new data is added only to reduced set of segments -reset gp_target_numsegments; +begin; +select gp_expand_lock_catalog(); +select gp_toolkit.gp_reset_rebalance_numsegments(); +end; insert into table_distr_hashed select generate_series(21, 40); insert into table_distr_hashed_ao_row select generate_series(21, 40); @@ -210,7 +216,11 @@ drop table part_list_table_distr_random; drop table multi_part_table_distr_hashed; -- Check that all newly created tables have data only on segments #0 and #1 -set gp_target_numsegments = 2; +begin; +select gp_expand_lock_catalog(); +select gp_toolkit.gp_set_rebalance_numsegments(2); +end; + create table new_table_distr_hashed(a int) distributed by (a); insert into new_table_distr_hashed select generate_series(1, 20); @@ -325,7 +335,10 @@ select *, (gp_segment_id < 2) as correct_segment_id from new_table_into order by -- Validate the insertion works fine with the new tables -- after 'gp_target_numsegments' reset -reset gp_target_numsegments; +begin; +select gp_expand_lock_catalog(); +select gp_toolkit.gp_reset_rebalance_numsegments(); +end; insert into new_table_ctas select generate_series(21, 40); insert into new_table_into select generate_series(21, 40); @@ -408,13 +421,11 @@ insert into table_distr_hashed select generate_series(1, 20); select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; begin; -set gp_target_numsegments = 2; -alter table table_distr_hashed rebalance; +alter table table_distr_hashed rebalance 2; select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; rollback; select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; -reset gp_target_numsegments; drop table table_distr_hashed; -- Check rebalance with parameter From 69fb3e98d4f3e1dcc69adeb54fc4ff358ce2d688 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Tue, 23 Sep 2025 20:09:57 +1000 Subject: [PATCH 10/22] Implement skeleton for basic shrink scenario (#1971) This patch implements a state machine skeleton for a basic shrink scenario based on 'transitions' library. It consists of a new 'ggrebalance' tool, which will be a single entry point for shrink, expand, and cluster rebalance functionality, and 'shrink.py', which contains the state machine itself with the shrink logic. The main purpose of this half-MVP is to evaluate the state machine pattern suitability. Therefore it implements only a limited set of requirements for the shrink, which allows you to support basic shrink workflow. --- README.Ubuntu.bash | 1 + gpMgmt/bin/Makefile | 2 +- gpMgmt/bin/ggrebalance | 221 ++++++++++ gpMgmt/bin/gprebalance_modules/Makefile | 2 +- gpMgmt/bin/gprebalance_modules/shrink.py | 508 +++++++++++++++++++++++ 5 files changed, 732 insertions(+), 2 deletions(-) create mode 100755 gpMgmt/bin/ggrebalance create mode 100644 gpMgmt/bin/gprebalance_modules/shrink.py diff --git a/README.Ubuntu.bash b/README.Ubuntu.bash index 2a383b9d8d31..543af4a6536d 100755 --- a/README.Ubuntu.bash +++ b/README.Ubuntu.bash @@ -43,6 +43,7 @@ apt-get install -y \ python3-psycopg2 \ python3-psutil \ python3-yaml \ + python3-transitions \ zlib1g-dev tee -a /etc/sysctl.conf << EOF diff --git a/gpMgmt/bin/Makefile b/gpMgmt/bin/Makefile index f65048d56cab..c5ea3abd2e0e 100644 --- a/gpMgmt/bin/Makefile +++ b/gpMgmt/bin/Makefile @@ -17,7 +17,7 @@ PROGRAMS= analyzedb gpactivatestandby gpaddmirrors gpcheckcat gpcheckperf \ gpinitsystem gpload gpload.py gplogfilter gpmovemirrors \ gprecoverseg gpreload gpsync gpsd gpssh gpssh-exkeys gpstart \ gpstate gpstop minirepro gpmemwatcher gpmemreport gpcheckresgroupv2impl \ - gprebalance + gprebalance ggrebalance installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)/lib' diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance new file mode 100755 index 000000000000..70f797f734a8 --- /dev/null +++ b/gpMgmt/bin/ggrebalance @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 + +from gppylib.mainUtils import getProgramName + +from psycopg2 import DatabaseError +try: + from gppylib.commands.unix import * + from gppylib.commands.gp import * + from gppylib.gpparseopts import OptParser, OptChecker + from gppylib.gplog import * + from gppylib.gparray import GpArray + from gppylib.db import dbconn + from gppylib.userinput import * + from gppylib.system.environment import GpCoordinatorEnvironment + from gppylib.system import configurationInterface, configurationImplGpdb + from gprebalance_modules.shrink import GGShrink, DBNAME +except ImportError as e: + sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) + + +_description = (""" + TBD. Curretnly only minimum shrink implementation is implemented. +""") + +_help = [""" +TBD +"""] + +_usage = """ + TBD +""" + +EXECNAME = os.path.split(__file__)[-1] +MAX_BATCH_SIZE = 128 +MAX_PARALLEL_WORKERS = 96 + + +def parseargs(): + parser = OptParser(option_class=OptChecker, + description=' '.join(_description.split()), + version='%prog version $Revision$') + parser.setHelp(_help) + parser.set_usage('%prog ' + _usage) + parser.remove_option('-h') + + parser.add_option('-x', '--target-segment-count', type='int', dest='target_segment_count', metavar='', + help='Target segments number of the whole cluster') + parser.add_option('-B', '--batch-size', type='int', default=4, dest='batch_size', metavar='', + help='Max worker number in WorkerPool instance. Valid values are 1-%d.' % MAX_BATCH_SIZE) + parser.add_option('-n', '--parallel', type="int", default=4, dest="parallel", metavar='', + help='Max number of workers at a time. Valid values are 1-%d.' % MAX_PARALLEL_WORKERS) + parser.add_option('-c', '--clean', action='store_true', dest="clean_required", + help='remove the rebalance schema.') + parser.add_option('-h', '-?', '--help', action='help', + help='show this help message and exit.') + + parser.set_defaults(verbose=False) + + # Parse the command line arguments + options, args = parser.parse_args() + return options, args, parser + + +def validate_options(options, args, parser): + if len(args) > 0: + logger.error(f'Unknown argument {args[0]}') + parser.exit(1) + + try: + options.coordinator_data_directory = get_coordinatordatadir() + options.gphome = get_gphome() + except GpError as msg: + logger.error(msg) + parser.exit(1) + + if not os.path.exists(options.coordinator_data_directory): + logger.error('Coordinator data directory %s does not exist.' % + options.coordinator_data_directory) + parser.exit(1) + + if options.batch_size < 1 or options.batch_size > MAX_BATCH_SIZE: + logger.error( + 'Invalid argument. batch-size value must be >= 1 and <= %d' % MAX_BATCH_SIZE) + parser.exit(1) + + if options.parallel < 1 or options.parallel > MAX_PARALLEL_WORKERS: + logger.error( + 'Invalid argument. parallel value must be >= 1 and <= %d' % MAX_PARALLEL_WORKERS) + parser.exit(1) + + return options, args + + +def check_running_gputils(dburl: dbconn.DbURL, coordinator_data_directory: str): + """ + Checks if there are any running instances of + gprebalance/gpbackup/gpexpand/gpshrink/gpresize + """ + for util in ('ggrebalance', 'gprebalance', 'gpexpand', 'gpshrink', 'gpresize'): + try: + with open(f'{coordinator_data_directory}/{util}.pid', 'r') as fp: + pid = int(fp.readline().strip()) + if check_pid(pid): + logger.error(f'{util} is already running.') + sys.exit(1) + except IOError: + pass + + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + cursor = dbconn.query(conn, '''SELECT datid + FROM pg_stat_activity + WHERE application_name LIKE 'gpbackup%' OR + application_name LIKE 'gprestore%' ''') + if cursor.rowcount > 0: + logger.error( + '''gpbackup/gprestore utility is already running.''') + sys.exit(1) + + if is_gprecoverseg_running(): + logger.error( + '''gprecoverseg is already running.''') + sys.exit(1) + + +def create_pid_file(coordinator_data_directory: str): + """ + Creates ggrebalance pid file + """ + with open(coordinator_data_directory + '/ggrebalance.pid', 'w') as fp: + fp.write(str(os.getpid())) + + +def remove_pid_file(coordinator_data_directory: str): + """ + Removes gprebalance pid file + """ + try: + os.unlink(coordinator_data_directory + '/ggrebalance.pid') + except IOError: + pass + +gg_shrink = None + +def sig_handler(sig, arg): + if gg_shrink is not None: + gg_shrink.shutdown() + else: + sys.exit(1) + +def main(options, args, parser): + try: + signal.signal(signal.SIGTERM, sig_handler) + signal.signal(signal.SIGHUP, sig_handler) + signal.signal(signal.SIGINT, sig_handler) + + logger = get_default_logger() + setup_tool_logging(EXECNAME, getLocalHostname(), getUserName()) + + options, args = validate_options(options, args, parser) + + if options.verbose: + enable_verbose_logging() + + gpenv = GpCoordinatorEnvironment( + options.coordinator_data_directory, True) + + # can we remove it? + configurationInterface.registerConfigurationProvider( + configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) + + configurationInterface.getConfigurationProvider( + ).initializeProvider(gpenv.getCoordinatorPort()) + + dburl = dbconn.DbURL(dbname=DBNAME, port=gpenv.getCoordinatorPort()) + + check_running_gputils(dburl, options.coordinator_data_directory) + + create_pid_file(options.coordinator_data_directory) + + gparray_dump_filename = options.coordinator_data_directory + '/gparraydump' + + if os.path.exists(gparray_dump_filename): + logger.info('Init gparray from file %s' % gparray_dump_filename) + gparray = GpArray.initFromFile(gparray_dump_filename) + else: + logger.info('Init gparray from catalog') + try: + gparray = GpArray.initFromCatalog(dburl, utility=True) + except DatabaseError as ex: + logger.error("Failed to connect to database. Make sure the" + " Greengage instance you wish to expand is running" + " and that your environment is correct, then rerun" + " gprebalance" + ' '.join(sys.argv[1:])) + sys.exit(1) + except ConnectionError as ex: + logger.error(f"{str(ex)}") + sys.exit(1) + + if options.target_segment_count != None and gparray.get_segment_count() <= options.target_segment_count: + logger.error('Target segment count (%s) >= current segment count (%s).\n' + 'Currently only shrink is supported (target segment count < current segment count).' + % (options.target_segment_count, gparray.get_segment_count())) + sys.exit(1) + else: + global gg_shrink + gg_shrink = GGShrink(logger, dburl, options, gpenv, gparray, gparray_dump_filename) + gg_shrink.run() + + sys.exit(0) + except Exception as e: + if options and options.verbose: + logger.exception('ggrebalance failed. exiting...') + else: + logger.error(f'ggrebalance failed: {e} \n\nExiting...') + sys.exit(1) + finally: + remove_pid_file(get_coordinatordatadir()) + +if __name__ == '__main__': + options, args, parser = parseargs() + main(options, args, parser) diff --git a/gpMgmt/bin/gprebalance_modules/Makefile b/gpMgmt/bin/gprebalance_modules/Makefile index 692d727c7073..d71eb8f3d35d 100644 --- a/gpMgmt/bin/gprebalance_modules/Makefile +++ b/gpMgmt/bin/gprebalance_modules/Makefile @@ -3,7 +3,7 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -PROGRAMS= __init__.py rebalance_validator.py rebalance.py rebalance_plan.py rebalance_executor.py rebalance_status.py +PROGRAMS= __init__.py rebalance_validator.py rebalance.py rebalance_plan.py rebalance_executor.py rebalance_status.py shrink.py installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)/gprebalance_modules' diff --git a/gpMgmt/bin/gprebalance_modules/shrink.py b/gpMgmt/bin/gprebalance_modules/shrink.py new file mode 100644 index 000000000000..ccc236e339e1 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/shrink.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 + +from transitions import Machine +from contextlib import closing +from typing import Any + +try: + from gppylib.commands.unix import * + from gppylib.commands.gp import * + from gppylib.gplog import * + from gppylib.db import dbconn + from gppylib.userinput import * + from gppylib.commands import base + from gppylib.commands.gp import SEGMENT_STOP_TIMEOUT_DEFAULT, SegmentStop + from gppylib.system.environment import * +except ImportError as e: + sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) + + +DBNAME = 'postgres' + +def print_progress(pool: WorkerPool, interval: int = 10) -> None: + """ + Waits for a WorkerPool to complete, printing a progress percentage marker + once at the beginning of the call, and thereafter at the provided interval + (default ten seconds). A final 100% marker is printed upon completion. + """ + def print_completed_percentage() -> bool: + # pool.completed can change asynchronously; save its value. + completed = pool.completed + + pct = 0 + if pool.assigned: + pct = float(completed) / pool.assigned + + pool.logger.info(f'{pct:.2%} of jobs completed') + return completed >= pool.assigned + + # print_completed_percentage() returns True if we're done. + while not print_completed_percentage(): + if pool.join(interval): + return + +class GGShrink: + timeout = SEGMENT_STOP_TIMEOUT_DEFAULT + stop_mode = 'fast' + + rebalance_schema_name = 'ggrebalance' + rebalance_status = 'rebalance_status' + table_rebalance_status_detail = 'table_rebalance_status_detail' + + states = [ + 'STATE_START', + 'STATE_OPTIONS_VALIDATION', + 'STATE_CHECK_PREVIOUS_RUN', + 'STATE_END', + 'STATE_CLEANUP', + 'STATE_ERROR', + 'STATE_ROLLBACK' + ] + + # Note: order of states in the list below is important, + # as we rely on it when recover from an interrupted state. + # These states are separated from 'states' above, because + # these states exist after the shrink schema is created, + # so they are reflected in the status table inside the schema, + # and we can re-enter the interrupted state. + states_main_shrink_flow = [ + 'STATE_SETUP_SHRINK_SCHEMA_STARTED', + 'STATE_SETUP_SHRINK_SCHEMA_DONE', + 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED', + 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE', + 'STATE_PREPARE_SHRINK_SCHEMA_STARTED', + 'STATE_PREPARE_SHRINK_SCHEMA_DONE', + 'STATE_SHRINK_TABLES_STARTED', + 'STATE_SHRINK_TABLES_DONE', + 'STATE_SHRINK_CATALOG_STARTED', + 'STATE_SHRINK_CATALOG_DONE', + 'STATE_SHRINK_SEGMENTS_STOP_STARTED', + 'STATE_SHRINK_SEGMENTS_STOP_DONE', + 'STATE_SHRINK_DONE' + ] + + transitions = [ + { + 'trigger': 'start', + 'source': 'STATE_START', + 'dest': 'STATE_OPTIONS_VALIDATION' + }, + { + 'trigger': 'move_to_STATE_CLEANUP', + 'source': 'STATE_OPTIONS_VALIDATION', + 'dest': 'STATE_CLEANUP' + }, + { + 'trigger': 'move_to_STATE_CHECK_PREVIOUS_RUN', + 'source': 'STATE_OPTIONS_VALIDATION', + 'dest': 'STATE_CHECK_PREVIOUS_RUN' + }, + { + 'trigger': 'move_to_STATE_SETUP_SHRINK_SCHEMA_STARTED', + 'source': 'STATE_CHECK_PREVIOUS_RUN', + 'dest': 'STATE_SETUP_SHRINK_SCHEMA_STARTED' + }, + { + 'trigger': 'move_to_STATE_SETUP_SHRINK_SCHEMA_DONE', + 'source': 'STATE_SETUP_SHRINK_SCHEMA_STARTED', + 'dest': 'STATE_SETUP_SHRINK_SCHEMA_DONE' + }, + { + 'trigger': 'move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED', + 'source': 'STATE_SETUP_SHRINK_SCHEMA_DONE', + 'dest': 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED' + }, + { + 'trigger': 'move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE', + 'source': 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED', + 'dest': 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE' + }, + { + 'trigger': 'move_to_STATE_PREPARE_SHRINK_SCHEMA_STARTED', + 'source': 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE', + 'dest': 'STATE_PREPARE_SHRINK_SCHEMA_STARTED' + }, + { + 'trigger': 'move_to_STATE_PREPARE_SHRINK_SCHEMA_DONE', + 'source': 'STATE_PREPARE_SHRINK_SCHEMA_STARTED', + 'dest': 'STATE_PREPARE_SHRINK_SCHEMA_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_TABLES_STARTED', + 'source': 'STATE_PREPARE_SHRINK_SCHEMA_DONE', + 'dest': 'STATE_SHRINK_TABLES_STARTED' + }, + { + 'trigger': 'move_to_STATE_SHRINK_TABLES_DONE', + 'source': 'STATE_SHRINK_TABLES_STARTED', + 'dest': 'STATE_SHRINK_TABLES_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_CATALOG_STARTED', + 'source': 'STATE_SHRINK_TABLES_DONE', + 'dest': 'STATE_SHRINK_CATALOG_STARTED' + }, + { + 'trigger': 'move_to_STATE_SHRINK_CATALOG_DONE', + 'source': 'STATE_SHRINK_CATALOG_STARTED', + 'dest': 'STATE_SHRINK_CATALOG_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_SEGMENTS_STOP_STARTED', + 'source': 'STATE_SHRINK_CATALOG_DONE', + 'dest': 'STATE_SHRINK_SEGMENTS_STOP_STARTED' + }, + { + 'trigger': 'move_to_STATE_SHRINK_SEGMENTS_STOP_DONE', + 'source': 'STATE_SHRINK_SEGMENTS_STOP_STARTED', + 'dest': 'STATE_SHRINK_SEGMENTS_STOP_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_DONE', + 'source': 'STATE_SHRINK_SEGMENTS_STOP_DONE', + 'dest': 'STATE_SHRINK_DONE' + }, + { + 'trigger': 'move_to_STATE_END', + 'source': ['STATE_SHRINK_DONE', 'STATE_CHECK_PREVIOUS_RUN', 'STATE_CLEANUP'], + 'dest': 'STATE_END' + }, + { + 'trigger': 'move_to_STATE_ERROR', + 'source': '*', + 'dest': 'STATE_ERROR' + } + ] + + def __init__(self, logger: Any, dburl: dbconn.DbURL, options: Any, gpEnv: GpCoordinatorEnvironment, gpArray: gparray.GpArray, gpArrayDumpFilename: str) -> None: + self.logger = logger + self.dburl = dburl + self.options = options + self.gpEnv = gpEnv + self.conn = dbconn.connect( + self.dburl, encoding='UTF8', allowSystemTableMods=True) + self.shutdown_requested = False + self.workers_for_tables_rebalance = None + self.workers_for_segment_stop = None + self.gparray = gpArray + self.gparray_dump_file = gpArrayDumpFilename + + self.machine = Machine(model = self, + queued=True, + states = self.states + self.states_main_shrink_flow, + transitions = self.transitions, + initial = 'STATE_START', + before_state_change = 'on_every_state') + + def run(self) -> None: + self.trigger('start') + + def ggrebalance_schema_exists(self) -> bool: + row = dbconn.queryRow(self.conn, f"SELECT COUNT(1) FROM pg_namespace WHERE nspname = '{self.rebalance_schema_name}'") + return int(row[0]) == 1 + + def get_state_from_previous_run(self) -> str: + cursor = dbconn.query(self.conn, f'SELECT status FROM {self.rebalance_schema_name}.{self.rebalance_status} ORDER BY updated DESC LIMIT 1') + if cursor.rowcount > 0: + return str(cursor.fetchone()[0]) + return 'not defined' + + def on_every_state(self) -> None: + if self.shutdown_requested: + self.logger.info('Shrink was interrupted') + sys.exit(1) + + assert self.state in self.states + self.states_main_shrink_flow + + # insert status if the schema already exists + if self.state in self.states_main_shrink_flow: + if self.ggrebalance_schema_exists(): + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.rebalance_schema_name}.{self.rebalance_status} + VALUES ('{self.state}', NOW())''') + + # state callbacks start here + def on_enter_STATE_OPTIONS_VALIDATION(self) -> None: + if self.options.clean_required: + self.trigger('move_to_STATE_CLEANUP') + else: + self.trigger('move_to_STATE_CHECK_PREVIOUS_RUN') + + def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: + # check if rebalance schema exists + # and whether we can get the state where we stopped in previous run + # in order to proceed from the same point + if self.ggrebalance_schema_exists(): + self.logger.info('Rebalance schema already exists') + state_from_prev_run = self.get_state_from_previous_run() + # check maybe the state is the final one + if state_from_prev_run == self.states_main_shrink_flow[-1]: + self.logger.info('Previous run was completed successfully. Please execute cleanup before a new run.') + self.trigger('move_to_STATE_END') + else: + self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") + try: + next_state = self.states_main_shrink_flow[ self.states_main_shrink_flow.index(state_from_prev_run) + 1 ] + except: + self.logger.error("Can't determine next state") + self.trigger('move_to_STATE_ERROR') + return + # use auto to_«state» method to recover + self.trigger(f'to_{next_state}') + else: + self.trigger('move_to_STATE_SETUP_SHRINK_SCHEMA_STARTED') + + def on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED(self) -> None: + # Create schema and status tables + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, f'DROP SCHEMA IF EXISTS {self.rebalance_schema_name} CASCADE') + dbconn.execSQL(self.conn, f'CREATE SCHEMA {self.rebalance_schema_name}') + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.rebalance_schema_name}.{self.rebalance_status} + (status TEXT, updated TIMESTAMP WITH TIME ZONE) + DISTRIBUTED REPLICATED''') + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.rebalance_schema_name}.{self.table_rebalance_status_detail} + (db_name TEXT, schema_name TEXT, rel_name TEXT, status TEXT, + CONSTRAINT unique_fqn UNIQUE (db_name, schema_name, rel_name)) + DISTRIBUTED REPLICATED''') + dbconn.execSQL(self.conn, 'COMMIT') + + self.trigger('move_to_STATE_SETUP_SHRINK_SCHEMA_DONE') + + def on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE(self) -> None: + self.logger.info(f'Created shrink schema {self.rebalance_schema_name}') + self.trigger('move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED') + + def get_table_distr_segment_count(self, schema_name, table_name) -> int: + row = dbconn.queryRow(self.conn, + f'''SELECT p.numsegments + FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid + JOIN gp_distribution_policy p ON c.oid = p.localoid + WHERE n.nspname='{schema_name}' AND c.relname='{table_name}';''') + return int(row[0]) + + def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED(self) -> None: + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + dbconn.execSQL(self.conn, 'CHECKPOINT') + dbconn.execSQL(self.conn, f'SELECT gp_toolkit.gp_set_rebalance_numsegments({self.options.target_segment_count})') + + self.gparray.dumpToFile(self.gparray_dump_file) + + # Rebalance the status tables we've created previously right here before we start to rebalance all other tables. + # Before that check if the tables are already rebalanced + # (in case we re-enter after interruption that happened after COMMIT but before new state) + if self.get_table_distr_segment_count(self.rebalance_schema_name, + self.rebalance_status) > self.options.target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.rebalance_schema_name}"."{self.rebalance_status}" + REBALANCE {self.options.target_segment_count}''') + + if self.get_table_distr_segment_count(self.rebalance_schema_name, + self.table_rebalance_status_detail) > self.options.target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.rebalance_schema_name}"."{self.table_rebalance_status_detail}" + REBALANCE {self.options.target_segment_count}''') + + dbconn.execSQL(self.conn, 'COMMIT') + + self.trigger('move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE') + + def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE(self) -> None: + self.logger.info(f'Updated target segment count to {self.options.target_segment_count}') + self.trigger('move_to_STATE_PREPARE_SHRINK_SCHEMA_STARTED') + + def on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED(self) -> None: + # collect databases and tables that require 'ALTER TABLE REBALANCE' + # and store in 'table_rebalance_status_detail' table + + dbconn.execSQL(self.conn, 'BEGIN') + + # cleanup table_rebalance_status_detail for the case we re-enter this state after we were interrupted right after it + dbconn.execSQL(self.conn, f'TRUNCATE {self.rebalance_schema_name}.{self.table_rebalance_status_detail}') + + cursor = dbconn.query(self.conn, 'SELECT datname FROM pg_database') + databases_to_process = [] + for record in cursor: + database_name = record[0] + if database_name != 'template0': + databases_to_process.append(database_name) + + for db in databases_to_process: + dburl = dbconn.DbURL(dbname=db, port=self.gpEnv.getCoordinatorPort()) + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + cursor = dbconn.query(conn, + f'''SELECT n.nspname, c.relname + FROM pg_class c + JOIN pg_namespace n ON c.relnamespace = n.oid + JOIN gp_distribution_policy p ON c.oid = p.localoid + WHERE c.relkind IN ('r', 'p') AND c.relispartition = FALSE AND + p.numsegments > {self.options.target_segment_count} AND + n.nspname NOT IN ('pg_catalog', 'information_schema', '{self.rebalance_schema_name}')''') + for schema_name, rel_name in cursor: + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.rebalance_schema_name}.{self.table_rebalance_status_detail} + VALUES ('{db}', '{schema_name}', '{rel_name}', 'none')''') + + dbconn.execSQL(self.conn, 'COMMIT') + + self.trigger('move_to_STATE_PREPARE_SHRINK_SCHEMA_DONE') + + def on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE(self) -> None: + self.logger.info(f'Initiated {self.rebalance_schema_name}.{self.table_rebalance_status_detail}') + self.trigger('move_to_STATE_SHRINK_TABLES_STARTED') + + class TableRebalanceTask(SQLCommand): + def __init__(self, + shrink: 'GGShrink', + db_name: str, + schema_name: str, + rel_name: str) -> None: + self.shrink = shrink + self.db_name = db_name + self.schema_name = schema_name + self.rel_name = rel_name + SQLCommand.__init__(self, f'task rebalance for {self.db_name}.{self.schema_name}.{self.rel_name}') + + def run(self) -> None: + dburl = dbconn.DbURL(dbname=self.db_name, port=self.shrink.gpEnv.getCoordinatorPort()) + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + dbconn.execSQL(conn, 'BEGIN') + dbconn.execSQL(conn, + f'''ALTER TABLE "{self.schema_name}"."{self.rel_name}" + REBALANCE {self.shrink.options.target_segment_count}''') + dbconn.execSQL(self.shrink.conn, + f'''UPDATE {self.shrink.rebalance_schema_name}.{self.shrink.table_rebalance_status_detail} SET status = 'done' + WHERE db_name = '{self.db_name}' AND schema_name = '{self.schema_name}' AND rel_name = '{self.rel_name}';''') + dbconn.execSQL(conn, 'COMMIT') + self.set_results(CommandResult(0, b'', b'', True, False)) + + def on_enter_STATE_SHRINK_TABLES_STARTED(self) -> None: + self.logger.info('Start tables rebalance') + + # perform 'ALTER TABLE REBALANCE' for all not yet processed tables + cursor = dbconn.query(self.conn, + f"SELECT db_name, schema_name, rel_name FROM {self.rebalance_schema_name}.{self.table_rebalance_status_detail} WHERE status = 'none'") + + self.logger.info(f'Tables to process {cursor.rowcount}') + + if cursor.rowcount > 0: + self.workers_for_tables_rebalance = WorkerPool(numWorkers=min(cursor.rowcount, self.options.parallel)) + + for db_name, schema_name, rel_name in cursor: + task = self.TableRebalanceTask(self, + db_name, + schema_name, + rel_name) + self.workers_for_tables_rebalance.addCommand(task) + + print_progress(self.workers_for_tables_rebalance, interval=1) + + self.workers_for_tables_rebalance.haltWork() + self.workers_for_tables_rebalance.joinWorkers() + + for task in self.workers_for_tables_rebalance.getCompletedItems(): + if not task.was_successful(): + raise Exception(f'Failed to do ALTER REBALANCE: {task.get_results().stderr}') + + self.workers_for_tables_rebalance = None + + self.trigger('move_to_STATE_SHRINK_TABLES_DONE') + + def on_enter_STATE_SHRINK_TABLES_DONE(self) -> None: + self.logger.info('Tables rebalance complete') + self.trigger('move_to_STATE_SHRINK_CATALOG_STARTED') + + def on_enter_STATE_SHRINK_CATALOG_STARTED(self) -> None: + self.logger.info('Start catalog shrink') + + ## Shrink catalog + dbconn.execSQL(self.conn, 'BEGIN') + cursor = dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + dbconn.execSQL(self.conn, f'DELETE FROM gp_segment_configuration WHERE content >= {self.options.target_segment_count}') + dbconn.execSQL(self.conn, 'CHECKPOINT') + dbconn.execSQL(self.conn, 'SELECT gp_expand_bump_version()') + cursor = dbconn.query(self.conn, 'SELECT gp_toolkit.gp_reset_rebalance_numsegments()') + dbconn.execSQL(self.conn, 'COMMIT') + + self.trigger('move_to_STATE_SHRINK_CATALOG_DONE') + + def on_enter_STATE_SHRINK_CATALOG_DONE(self) -> None: + self.logger.info('Catalog shrink complete') + self.trigger('move_to_STATE_SHRINK_SEGMENTS_STOP_STARTED') + + def on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED(self) -> None: + self.logger.info('Stopping shrinked segments...') + + segments_to_stop = self.gparray.get_segment_count() - self.options.target_segment_count + segments_to_stop = segments_to_stop * 2 # consider mirrors + self.workers_for_segment_stop = WorkerPool(numWorkers=min(segments_to_stop, self.options.batch_size)) + + for seg_pair in self.gparray.getSegmentList(): + primary_seg = seg_pair.primaryDB + mirror_seq = seg_pair.mirrorDB + if primary_seg.getSegmentContentId() >= self.options.target_segment_count: + if primary_seg.isSegmentUp(): + cmd = SegmentStop(f'stop primary (content {primary_seg.getSegmentContentId()}, dbid {primary_seg.getSegmentDbId()})', + primary_seg.getSegmentDataDirectory(), + mode=self.stop_mode, + timeout=self.timeout, + ctxt=base.REMOTE, + remoteHost=primary_seg.getSegmentHostName()) + self.workers_for_segment_stop.addCommand(cmd) + + if mirror_seq != None and mirror_seq.isSegmentUp(): + cmd = SegmentStop(f'stop mirror (content {mirror_seq.getSegmentContentId()}, dbid {mirror_seq.getSegmentDbId()})', + mirror_seq.getSegmentDataDirectory(), + mode=self.stop_mode, + timeout=self.timeout, + ctxt=base.REMOTE, + remoteHost=mirror_seq.getSegmentHostName()) + self.workers_for_segment_stop.addCommand(cmd) + + print_progress(self.workers_for_segment_stop, interval=1) + + self.workers_for_segment_stop.haltWork() + self.workers_for_segment_stop.joinWorkers() + + for task in self.workers_for_segment_stop.getCompletedItems(): + if not task.was_successful(): + raise Exception('Failed to stop segments') + + self.workers_for_segment_stop = None + + self.trigger('move_to_STATE_SHRINK_SEGMENTS_STOP_DONE') + + def on_enter_STATE_SHRINK_SEGMENTS_STOP_DONE(self) -> None: + self.logger.info('Shrinked segments were stopped') + self.trigger('move_to_STATE_SHRINK_DONE') + + def on_enter_STATE_SHRINK_DONE(self) -> None: + os.remove(self.gparray_dump_file) + self.logger.info('Shrink is complete') + self.trigger('move_to_STATE_END') + + def on_enter_STATE_CLEANUP(self) -> None: + dbconn.execSQL(self.conn, f'DROP SCHEMA {self.rebalance_schema_name} CASCADE') + self.logger.info('Cleanup is complete') + self.trigger('move_to_STATE_END') + + def on_enter_STATE_END(self) -> None: + self.conn.close() + + def on_enter_STATE_ERROR(self) -> None: + sys.exit(1) + + # state callbacks end here + + def shutdown(self) -> None: + if self.workers_for_tables_rebalance != None: + self.workers_for_tables_rebalance.haltWork() + self.workers_for_tables_rebalance.joinWorkers() + + if self.workers_for_segment_stop != None: + self.workers_for_segment_stop.haltWork() + self.workers_for_segment_stop.joinWorkers() + + self.shutdown_requested = True From 1134de6887ffcd8883fdbcc825b6fc485a008519 Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Thu, 16 Oct 2025 20:14:02 +0300 Subject: [PATCH 11/22] Add gp_rebalance_numsegments reset check (#1977) This patch adds a check for probable scenario when during interruption of ggrebalance the cluster could be restarted. In this case the shared variable gp_rebalance_numsegments is unset, and new table may be created at old segment count. Thus, during recovering of shrink process the STATE_CHECK_PREVIOUS_RUN callback calls get_state_after_interrupt() function, which checks the mentioned situation. If cluster is restarted the state machine executes transition to STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED state. The interface for gp_rebalance_numsegments variable is updated via gp_rebalance_numsegments_is_set() SQL function in order to provide convenient way to monitor variable status. Before that, the comparison with INT_MAX value was required. Additionally, fault injection interface was returned to behave tests to cause workflow interruptions. The behave tests utility code was also adjusted to support some of the shrink scenarios. The code related to table population is fixed to make it follow declared semantics. gpaddmirrors test is updated as well. Co-Authored-By: Roman Eskin r.eskin@arenadata.io --- gpMgmt/bin/ggrebalance | 35 +++++----- gpMgmt/bin/gppylib/Makefile | 1 + gpMgmt/bin/gppylib/fault_injection.py | 17 +++++ gpMgmt/bin/gprebalance_modules/shrink.py | 64 +++++++++++++++++-- .../behave/mgmt_utils/ggrebalance.feature | 36 +++++++++++ .../behave/mgmt_utils/gpaddmirrors.feature | 6 +- .../behave/mgmt_utils/steps/mgmt_utils.py | 26 ++++++++ gpMgmt/test/behave_utils/utils.py | 4 +- .../gp_toolkit/gp_rebalance_numsegments.c | 12 ++++ gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql | 6 ++ 10 files changed, 177 insertions(+), 30 deletions(-) create mode 100755 gpMgmt/bin/gppylib/fault_injection.py create mode 100755 gpMgmt/test/behave/mgmt_utils/ggrebalance.feature diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index 70f797f734a8..f62cdc72ca27 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -179,25 +179,22 @@ def main(options, args, parser): gparray_dump_filename = options.coordinator_data_directory + '/gparraydump' - if os.path.exists(gparray_dump_filename): - logger.info('Init gparray from file %s' % gparray_dump_filename) - gparray = GpArray.initFromFile(gparray_dump_filename) - else: - logger.info('Init gparray from catalog') - try: - gparray = GpArray.initFromCatalog(dburl, utility=True) - except DatabaseError as ex: - logger.error("Failed to connect to database. Make sure the" - " Greengage instance you wish to expand is running" - " and that your environment is correct, then rerun" - " gprebalance" + ' '.join(sys.argv[1:])) - sys.exit(1) - except ConnectionError as ex: - logger.error(f"{str(ex)}") - sys.exit(1) - - if options.target_segment_count != None and gparray.get_segment_count() <= options.target_segment_count: - logger.error('Target segment count (%s) >= current segment count (%s).\n' + logger.info('Init gparray from catalog') + try: + gparray = GpArray.initFromCatalog(dburl, utility=True) + except DatabaseError as ex: + logger.error("Failed to connect to database. Make sure the" + " Greengage instance you wish to expand is running" + " and that your environment is correct, then rerun" + " gprebalance" + ' '.join(sys.argv[1:])) + sys.exit(1) + except ConnectionError as ex: + logger.error(f"{str(ex)}") + sys.exit(1) + + # the equality case is considered by shrink internally + if options.target_segment_count != None and gparray.get_segment_count() < options.target_segment_count: + logger.error('Target segment count (%s) > current segment count (%s).\n' 'Currently only shrink is supported (target segment count < current segment count).' % (options.target_segment_count, gparray.get_segment_count())) sys.exit(1) diff --git a/gpMgmt/bin/gppylib/Makefile b/gpMgmt/bin/gppylib/Makefile index 04972f5345fe..cfe4ef59f9fe 100644 --- a/gpMgmt/bin/gppylib/Makefile +++ b/gpMgmt/bin/gppylib/Makefile @@ -10,6 +10,7 @@ $(recurse) PROGRAMS= gparray.py gpunit unit2 DATA= __init__.py \ + fault_injection.py \ datetimeutils.py \ gp_era.py \ gpcatalog.py \ diff --git a/gpMgmt/bin/gppylib/fault_injection.py b/gpMgmt/bin/gppylib/fault_injection.py new file mode 100755 index 000000000000..43f437cc7671 --- /dev/null +++ b/gpMgmt/bin/gppylib/fault_injection.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +import os + +GPMGMT_FAULT_POINT = 'GPMGMT_FAULT_POINT' + +def inject_fault(fault_point): + if GPMGMT_FAULT_POINT in os.environ and fault_point == os.environ[GPMGMT_FAULT_POINT]: + raise Exception('Fault Injection %s' % os.environ[GPMGMT_FAULT_POINT]) + +# decorator for test purposes +def wrap_state_func_with_faults(func): + def func_with_faults(*args): + inject_fault(f'{func.__name__}_begin') + func(*args) + inject_fault(f'{func.__name__}_end') + return func_with_faults diff --git a/gpMgmt/bin/gprebalance_modules/shrink.py b/gpMgmt/bin/gprebalance_modules/shrink.py index ccc236e339e1..fbd077754692 100644 --- a/gpMgmt/bin/gprebalance_modules/shrink.py +++ b/gpMgmt/bin/gprebalance_modules/shrink.py @@ -9,6 +9,7 @@ from gppylib.commands.gp import * from gppylib.gplog import * from gppylib.db import dbconn + from gppylib.fault_injection import * from gppylib.userinput import * from gppylib.commands import base from gppylib.commands.gp import SEGMENT_STOP_TIMEOUT_DEFAULT, SegmentStop @@ -186,6 +187,8 @@ def __init__(self, logger: Any, dburl: dbconn.DbURL, options: Any, gpEnv: GpCoor self.workers_for_segment_stop = None self.gparray = gpArray self.gparray_dump_file = gpArrayDumpFilename + self.needs_repopulate = False + self.dumped_gparray = gparray.GpArray.initFromFile(self.gparray_dump_file) if os.path.exists(self.gparray_dump_file) else None self.machine = Machine(model = self, queued=True, @@ -206,6 +209,27 @@ def get_state_from_previous_run(self) -> str: if cursor.rowcount > 0: return str(cursor.fetchone()[0]) return 'not defined' + + def get_state_after_interrupt(self, prev_state) -> str: + prev_idx = self.states_main_shrink_flow.index(prev_state) + lower = self.states_main_shrink_flow.index('STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED') + upper = self.states_main_shrink_flow.index('STATE_SHRINK_TABLES_DONE') + if prev_idx >= lower and prev_idx <= upper: + + #if shrink is interrupted after catalog update and before the state is logged + if prev_state == 'STATE_SHRINK_TABLES_DONE' and \ + self.dumped_gparray is not None \ + and self.gparray.get_segment_count() + self.options.target_segment_count == self.dumped_gparray.get_segment_count(): + return 'STATE_SHRINK_CATALOG_DONE' + + row = dbconn.queryRow(self.conn, 'SELECT gp_toolkit.gp_rebalance_numsegments_is_set();') + # means that target rebalance numsegments is reset, and new tables are created at old segment count + if bool(row[0]) is False: + self.logger.info("Cluster restarted after previous run, trying to repopulate the relation queue") + self.needs_repopulate = True + return 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED' + + return self.states_main_shrink_flow[prev_idx + 1] def on_every_state(self) -> None: if self.shutdown_requested: @@ -222,12 +246,20 @@ def on_every_state(self) -> None: VALUES ('{self.state}', NOW())''') # state callbacks start here + @wrap_state_func_with_faults def on_enter_STATE_OPTIONS_VALIDATION(self) -> None: if self.options.clean_required: self.trigger('move_to_STATE_CLEANUP') + elif not self.ggrebalance_schema_exists() and \ + self.gparray.get_segment_count() == self.options.target_segment_count: + self.logger.error('Target segment count (%s) = current segment count (%s).\n' + 'Currently only shrink is supported (target segment count < current segment count).' + % (self.options.target_segment_count, self.gparray.get_segment_count())) + raise ValueError("Wrong target segment count") else: self.trigger('move_to_STATE_CHECK_PREVIOUS_RUN') + @wrap_state_func_with_faults def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: # check if rebalance schema exists # and whether we can get the state where we stopped in previous run @@ -242,7 +274,7 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: else: self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") try: - next_state = self.states_main_shrink_flow[ self.states_main_shrink_flow.index(state_from_prev_run) + 1 ] + next_state = self.get_state_after_interrupt(state_from_prev_run) except: self.logger.error("Can't determine next state") self.trigger('move_to_STATE_ERROR') @@ -252,6 +284,7 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: else: self.trigger('move_to_STATE_SETUP_SHRINK_SCHEMA_STARTED') + @wrap_state_func_with_faults def on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED(self) -> None: # Create schema and status tables dbconn.execSQL(self.conn, 'BEGIN') @@ -270,6 +303,7 @@ def on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED(self) -> None: self.trigger('move_to_STATE_SETUP_SHRINK_SCHEMA_DONE') + @wrap_state_func_with_faults def on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE(self) -> None: self.logger.info(f'Created shrink schema {self.rebalance_schema_name}') self.trigger('move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED') @@ -282,6 +316,7 @@ def get_table_distr_segment_count(self, schema_name, table_name) -> int: WHERE n.nspname='{schema_name}' AND c.relname='{table_name}';''') return int(row[0]) + @wrap_state_func_with_faults def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED(self) -> None: dbconn.execSQL(self.conn, 'BEGIN') dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') @@ -309,18 +344,22 @@ def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED(self) self.trigger('move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE') + @wrap_state_func_with_faults def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE(self) -> None: self.logger.info(f'Updated target segment count to {self.options.target_segment_count}') self.trigger('move_to_STATE_PREPARE_SHRINK_SCHEMA_STARTED') + @wrap_state_func_with_faults def on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED(self) -> None: # collect databases and tables that require 'ALTER TABLE REBALANCE' # and store in 'table_rebalance_status_detail' table dbconn.execSQL(self.conn, 'BEGIN') - # cleanup table_rebalance_status_detail for the case we re-enter this state after we were interrupted right after it - dbconn.execSQL(self.conn, f'TRUNCATE {self.rebalance_schema_name}.{self.table_rebalance_status_detail}') + # cleanup table_rebalance_status_detail for the case we re-enter this state after we were interrupted + dbconn.execSQL(self.conn, + f'''DELETE FROM {self.rebalance_schema_name}.{self.table_rebalance_status_detail} + WHERE (status <> 'done')''') cursor = dbconn.query(self.conn, 'SELECT datname FROM pg_database') databases_to_process = [] @@ -349,6 +388,7 @@ def on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED(self) -> None: self.trigger('move_to_STATE_PREPARE_SHRINK_SCHEMA_DONE') + @wrap_state_func_with_faults def on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE(self) -> None: self.logger.info(f'Initiated {self.rebalance_schema_name}.{self.table_rebalance_status_detail}') self.trigger('move_to_STATE_SHRINK_TABLES_STARTED') @@ -378,6 +418,7 @@ def run(self) -> None: dbconn.execSQL(conn, 'COMMIT') self.set_results(CommandResult(0, b'', b'', True, False)) + @wrap_state_func_with_faults def on_enter_STATE_SHRINK_TABLES_STARTED(self) -> None: self.logger.info('Start tables rebalance') @@ -410,10 +451,12 @@ def on_enter_STATE_SHRINK_TABLES_STARTED(self) -> None: self.trigger('move_to_STATE_SHRINK_TABLES_DONE') + @wrap_state_func_with_faults def on_enter_STATE_SHRINK_TABLES_DONE(self) -> None: self.logger.info('Tables rebalance complete') self.trigger('move_to_STATE_SHRINK_CATALOG_STARTED') + @wrap_state_func_with_faults def on_enter_STATE_SHRINK_CATALOG_STARTED(self) -> None: self.logger.info('Start catalog shrink') @@ -428,18 +471,25 @@ def on_enter_STATE_SHRINK_CATALOG_STARTED(self) -> None: self.trigger('move_to_STATE_SHRINK_CATALOG_DONE') + @wrap_state_func_with_faults def on_enter_STATE_SHRINK_CATALOG_DONE(self) -> None: self.logger.info('Catalog shrink complete') self.trigger('move_to_STATE_SHRINK_SEGMENTS_STOP_STARTED') + @wrap_state_func_with_faults def on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED(self) -> None: self.logger.info('Stopping shrinked segments...') - segments_to_stop = self.gparray.get_segment_count() - self.options.target_segment_count + gp_array = self.dumped_gparray + + if gp_array is None: + gp_array = self.gparray + + segments_to_stop = gp_array.get_segment_count() - self.options.target_segment_count segments_to_stop = segments_to_stop * 2 # consider mirrors self.workers_for_segment_stop = WorkerPool(numWorkers=min(segments_to_stop, self.options.batch_size)) - for seg_pair in self.gparray.getSegmentList(): + for seg_pair in gp_array.getSegmentList(): primary_seg = seg_pair.primaryDB mirror_seq = seg_pair.mirrorDB if primary_seg.getSegmentContentId() >= self.options.target_segment_count: @@ -474,20 +524,24 @@ def on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED(self) -> None: self.trigger('move_to_STATE_SHRINK_SEGMENTS_STOP_DONE') + @wrap_state_func_with_faults def on_enter_STATE_SHRINK_SEGMENTS_STOP_DONE(self) -> None: self.logger.info('Shrinked segments were stopped') self.trigger('move_to_STATE_SHRINK_DONE') + @wrap_state_func_with_faults def on_enter_STATE_SHRINK_DONE(self) -> None: os.remove(self.gparray_dump_file) self.logger.info('Shrink is complete') self.trigger('move_to_STATE_END') + @wrap_state_func_with_faults def on_enter_STATE_CLEANUP(self) -> None: dbconn.execSQL(self.conn, f'DROP SCHEMA {self.rebalance_schema_name} CASCADE') self.logger.info('Cleanup is complete') self.trigger('move_to_STATE_END') + @wrap_state_func_with_faults def on_enter_STATE_END(self) -> None: self.conn.close() diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature new file mode 100755 index 000000000000..c1f7da7518f8 --- /dev/null +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature @@ -0,0 +1,36 @@ +@ggrebalance +Feature: ggrebalance behave tests + + Scenario Outline: test shrink continue after cluster restart + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When set fault inject "" + And the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "gpstop -arf" + Then gpstart should return a return code of 0 + When there is a "heap" table "test_schema_2.test_table_3" in "test_db_2" with data + And the user runs "ggrebalance -x 1" + Then ggrebalance should print "Cluster restarted after previous run, trying to repopulate the relation queue" to logfile + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_3" with data in "test_db_2" is equal to segment count = 1, row count = 1094 + And ggrebalance should return a return code of 0 + Examples: + | fault_name | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_begin | diff --git a/gpMgmt/test/behave/mgmt_utils/gpaddmirrors.feature b/gpMgmt/test/behave/mgmt_utils/gpaddmirrors.feature index 9817183b5792..e446aedb6769 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpaddmirrors.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpaddmirrors.feature @@ -493,9 +493,9 @@ Feature: Tests for gpaddmirrors And the segments are synchronized When user stops all primary processes And user can start transactions - Then verify that there is a "heap" table "public.heap_table" in "gptest" with "202" rows - Then verify that there is a "ao" table "public.ao_table" in "gptest" with "202" rows - Then verify that there is a "co" table "public.co_table" in "gptest" with "202" rows + Then verify that there is a "heap" table "public.heap_table" in "gptest" with "100" rows + Then verify that there is a "ao" table "public.ao_table" in "gptest" with "100" rows + Then verify that there is a "co" table "public.co_table" in "gptest" with "100" rows And the user runs "gpstop -aqM fast" @concourse_cluster diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 022d51ebb69e..27748082efec 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -3481,6 +3481,20 @@ def impl(context, table, dbname): context.pre_redistribution_row_count = _get_row_count_per_segment(table, dbname) context.pre_redistribution_dist_policy = _get_dist_policy_per_partition(table, dbname) +@then('distribution information from table "{table}" with data in "{dbname}" is equal to segment count = {seg_cnt}, row count = {row_cnt}') +def impl(context, table, dbname, seg_cnt, row_cnt): + with closing(dbconn.connect(dbconn.DbURL(dbname=dbname), unsetSearchPath=False)) as conn: + query = "SELECT count(1) FROM (SELECT gp_segment_id FROM %s GROUP BY gp_segment_id) t" % table + cursor = dbconn.query(conn, query) + table_segments = cursor.fetchone()[0] + query = "SELECT count(1) FROM %s" % table + cursor = dbconn.query(conn, query) + table_rows = cursor.fetchone()[0] + if int(table_segments) != int(seg_cnt): + raise Exception("Expected table %s in db %s to be distributed on %s segments, but it is distributed on %s segments" % (table, dbname, seg_cnt, table_segments)) + if int(table_rows) != int(row_cnt): + raise Exception("Expected table %s in db %s to have %s rows, but got %s rows" % (table, dbname, row_cnt, table_rows)) + @then('distribution information from table "{table}" with data in "{dbname}" is verified against saved data') def impl(context, table, dbname): pre_distribution_row_count = context.pre_redistribution_row_count @@ -4409,3 +4423,15 @@ def step_impl(context): @given(u'the cluster is running in IC proxy mode with new proxy address {address}') def step_impl(context, address): set_ic_proxy_and_address(context, address) + +@given('set fault inject "{fault}"') +@then('set fault inject "{fault}"') +@when('set fault inject "{fault}"') +def impl(context, fault): + os.environ['GPMGMT_FAULT_POINT'] = fault + +@given('unset fault inject') +@then('unset fault inject') +@when('unset fault inject') +def impl(context): + os.environ['GPMGMT_FAULT_POINT'] = "" diff --git a/gpMgmt/test/behave_utils/utils.py b/gpMgmt/test/behave_utils/utils.py index 5a94dcc75eb2..976d1a87be72 100644 --- a/gpMgmt/test/behave_utils/utils.py +++ b/gpMgmt/test/behave_utils/utils.py @@ -472,9 +472,7 @@ def create_partition(context, tablename, storage_type, dbname, compression_type= # same data size as populate partition, but different values def populate_partition(tablename, start_date, dbname, data_offset, rowcount=1094, host=None, port=0, user=None): - insert_sql_str = "insert into %s select i+%d, 'backup', i + date '%s' from generate_series(0,%d) as i" % ( - tablename, data_offset, start_date, rowcount) - insert_sql_str += "; insert into %s select i+%d, 'restore', i + date '%s' from generate_series(0,%d) as i" % ( + insert_sql_str = "insert into %s select i+%d, 'backup', i + date '%s' from generate_series(1,%d) as i" % ( tablename, data_offset, start_date, rowcount) with closing(dbconn.connect(dbconn.DbURL(hostname=host, port=port, username=user, dbname=dbname), unsetSearchPath=False)) as conn: diff --git a/gpcontrib/gp_toolkit/gp_rebalance_numsegments.c b/gpcontrib/gp_toolkit/gp_rebalance_numsegments.c index 2ae7d74ee109..96dba865886a 100644 --- a/gpcontrib/gp_toolkit/gp_rebalance_numsegments.c +++ b/gpcontrib/gp_toolkit/gp_rebalance_numsegments.c @@ -85,3 +85,15 @@ gp_reset_rebalance_numsegments(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } + +/* + * Check if the rebalance numsegments is set. + */ +PG_FUNCTION_INFO_V1(gp_rebalance_numsegments_is_set); +Datum +gp_rebalance_numsegments_is_set(PG_FUNCTION_ARGS) +{ + PG_RETURN_BOOL( + pg_atomic_read_u32(gp_create_table_rebalance_numsegments) + != GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET); +} diff --git a/gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql b/gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql index cb62c578ea1d..3e0c21b413fa 100644 --- a/gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql +++ b/gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql @@ -25,3 +25,9 @@ AS '$libdir/gp_toolkit','gp_get_rebalance_numsegments' LANGUAGE C STRICT; REVOKE ALL ON FUNCTION gp_toolkit.gp_get_rebalance_numsegments() FROM public; + +CREATE FUNCTION gp_toolkit.gp_rebalance_numsegments_is_set() RETURNS int +AS '$libdir/gp_toolkit','gp_rebalance_numsegments_is_set' +LANGUAGE C STRICT; + +REVOKE ALL ON FUNCTION gp_toolkit.gp_rebalance_numsegments_is_set() FROM public; From 14f4669f0f744530d1ce0b5ad1fbfc319a134ffe Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 24 Oct 2025 17:35:36 +1000 Subject: [PATCH 12/22] Implement cleanup and rollback options for the cluster shrink (#1979) In this patch: 1. The new option '--clean' is added for the cluster shrink by the ggrebalance tool. 2. The new option '--rollback' is added for the cluster shrink by the ggrebalance tool. 3. The new option '--non-interactive-mode' is added for the ggrebalance tool. It is essential to allow auto testing of some cleanup scenarios that would expect user confirmation without such an option. 4. As the existing 'main' and the new 'rollback' shrink workflows use similar functionality, the shrink code is reorganized to reduce code duplication: a. New functions that are used in both 'main' and 'rollback' workflows are introduced (like 'prepare_shrink_schema()', 'rebalance_tables()'). b. All logic related to the ggrebalance schema handling is moved to a separate class named 'RebalanceSchema' in 'rebalance_commons.py'. 5. A new entity, 'Plan,' is added. It is used to pass information about required shrink configuration of the target cluster to the shrink engine. We store it in the rebalance schema and used for the 'rollback' workflow, and when we recover from an interrupted shrink state. It is added due to the following reasons: a. As already stated above, we need it during rollback. When the user starts the rollback operation, he doesn't specify the target segment count that was used at the preceding shrink operation. Thus we need to store this information at shrink for the later usage. b. When the user tries to re-enter the shrink procedure from an interrupted state, we need to re-start with the same target segment count that was specified originally. Otherwise we may get the cluster in some invalid configuration where tables are shrunk to different segment counts. Giving the user the ability to specify target segment count for the re-enter launch opens the way for such error prone scenarios. So we just forbid specifying segment count configuration if we re-enter the interrupted state or start the rollback, and use the saved plan information that we got at the very first operation start. c. According to the current design, at the later phase we'll introduce a Planner entity, that will perform planning for all shrink/expand/rebalance operations. And its output Plan will be the input to the shrink engine. So this change is aligned with the overall design. 6. New behave test cases are added. The test cases cover not only the 'cleanup' and 'rollback' flows, but also the existing 'main' shrink flow, as we can't guarantee the correctness of rollback without proving the 'main' flow works Ok. The existing test case is renamed to 'test 2.4' and moved to be near the new tests that cover similar functionality. 7. New steps are added to mgmt_utils.py, that are used to verify that the shrinked segments are actually down. Also a small change in 'SegmentIsShutDown' is done - it is required to check that the mirror is down. 8. In order to recover properly, if we are interrupted in the middle of stopping shrinked segments, a new class 'SegmentStopAfterShrink' is introduced. It wraps the 'SegmentStop' with the checking whether the segment is actually still running. Without it, if shrink was re-entered and some segments were already shut down by the preceding interrupted launch, we got an error when trying to shut down such segments. --- gpMgmt/bin/ggrebalance | 39 +- gpMgmt/bin/gppylib/commands/gp.py | 2 +- gpMgmt/bin/gprebalance_modules/Makefile | 2 +- gpMgmt/bin/gprebalance_modules/planner.py | 19 + .../gprebalance_modules/rebalance_commons.py | 121 ++++ gpMgmt/bin/gprebalance_modules/shrink.py | 613 +++++++++++------ .../behave/mgmt_utils/ggrebalance.feature | 620 +++++++++++++++++- .../behave/mgmt_utils/steps/mgmt_utils.py | 25 + 8 files changed, 1206 insertions(+), 235 deletions(-) create mode 100755 gpMgmt/bin/gprebalance_modules/planner.py create mode 100755 gpMgmt/bin/gprebalance_modules/rebalance_commons.py diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index f62cdc72ca27..79a99a423825 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -14,6 +14,7 @@ try: from gppylib.system.environment import GpCoordinatorEnvironment from gppylib.system import configurationInterface, configurationImplGpdb from gprebalance_modules.shrink import GGShrink, DBNAME + from gprebalance_modules.planner import * except ImportError as e: sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) @@ -51,8 +52,12 @@ def parseargs(): help='Max number of workers at a time. Valid values are 1-%d.' % MAX_PARALLEL_WORKERS) parser.add_option('-c', '--clean', action='store_true', dest="clean_required", help='remove the rebalance schema.') + parser.add_option('-r', '--rollback', action='store_true', dest="rollback_required", + help='performs rebalance rollback if possible.') parser.add_option('-h', '-?', '--help', action='help', help='show this help message and exit.') + parser.add_option('-y', '--non-interactive-mode', dest="interactive", action='store_false', default=True, + help="non-interactive mode, do not require user input for confirmations, use defaults.") parser.set_defaults(verbose=False) @@ -88,6 +93,15 @@ def validate_options(options, args, parser): 'Invalid argument. parallel value must be >= 1 and <= %d' % MAX_PARALLEL_WORKERS) parser.exit(1) + if options.clean_required and options.rollback_required: + logger.error("Can't use together options '--clean' and '--rollback'") + parser.exit(1) + + if ((options.clean_required or options.rollback_required) and + options.target_segment_count != None): + logger.error("Can't use '--target-segment-count' together option '--clean' or '--rollback'") + parser.exit(1) + return options, args @@ -192,17 +206,20 @@ def main(options, args, parser): logger.error(f"{str(ex)}") sys.exit(1) - # the equality case is considered by shrink internally - if options.target_segment_count != None and gparray.get_segment_count() < options.target_segment_count: - logger.error('Target segment count (%s) > current segment count (%s).\n' - 'Currently only shrink is supported (target segment count < current segment count).' - % (options.target_segment_count, gparray.get_segment_count())) - sys.exit(1) - else: - global gg_shrink - gg_shrink = GGShrink(logger, dburl, options, gpenv, gparray, gparray_dump_filename) - gg_shrink.run() - + global gg_shrink + gg_shrink = GGShrink(logger, dburl, options, gpenv, gparray, gparray_dump_filename) + + # Note: the plan for a shrink later will be provided by the planner component. + # But for now we simply create a Plan object from the options directly. + # We'll keep this stub till planner is ready. + # If shrink_plan is None, we assume that we need to continue previous operation + shrink_plan = None + if options.target_segment_count != None: + shrink_plan = Plan() + shrink_plan.setTargetSegmentCount(options.target_segment_count) + + gg_shrink.run(shrink_plan) + sys.exit(0) except Exception as e: if options and options.verbose: diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index 034e694f2af2..214206adaf33 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -453,7 +453,7 @@ def __init__(self,name,directory,ctxt=LOCAL,remoteHost=None): def is_shutdown(self): for key, value in self.results.split_stdout(): if key == 'Database cluster state': - return value.strip() == 'shut down' + return value.strip() == 'shut down' or value.strip() == 'shut down in recovery' return False @staticmethod diff --git a/gpMgmt/bin/gprebalance_modules/Makefile b/gpMgmt/bin/gprebalance_modules/Makefile index d71eb8f3d35d..71e2ca3d8ea3 100644 --- a/gpMgmt/bin/gprebalance_modules/Makefile +++ b/gpMgmt/bin/gprebalance_modules/Makefile @@ -3,7 +3,7 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -PROGRAMS= __init__.py rebalance_validator.py rebalance.py rebalance_plan.py rebalance_executor.py rebalance_status.py shrink.py +PROGRAMS= __init__.py rebalance_validator.py rebalance.py rebalance_plan.py rebalance_executor.py rebalance_status.py shrink.py planner.py rebalance_commons.py installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)/gprebalance_modules' diff --git a/gpMgmt/bin/gprebalance_modules/planner.py b/gpMgmt/bin/gprebalance_modules/planner.py new file mode 100755 index 000000000000..f248caf4f9a5 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/planner.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +import pickle + +class Plan: + def __init__(self): + self.target_segment_count = 0 + + def getTargetSegmentCount(self) -> int: + return self.target_segment_count + + def setTargetSegmentCount(self, target_segment_count: int) -> None: + self.target_segment_count = target_segment_count + + def serializePlan(self) -> bytes: + return pickle.dumps(self) + +def deserializePlan(input: bytes) -> Plan: + return pickle.loads(input) diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_commons.py b/gpMgmt/bin/gprebalance_modules/rebalance_commons.py new file mode 100755 index 000000000000..0775b94160bd --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_commons.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 + +import psycopg2 +from psycopg2.extensions import cursor +from gppylib.db import dbconn +from gprebalance_modules import planner + +def get_table_distr_segment_count(conn: dbconn.Connection, schema_name: str, table_name: str) -> int: + row = dbconn.queryRow(conn, + f'''SELECT p.numsegments + FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid + JOIN gp_distribution_policy p ON c.oid = p.localoid + WHERE n.nspname='{schema_name}' AND c.relname='{table_name}';''') + return int(row[0]) + +class RebalanceSchema: + def __init__(self, conn: dbconn.Connection): + self.schema_name = 'ggrebalance' + self.rebalance_status = 'rebalance_status' + self.table_rebalance_status_detail = 'table_rebalance_status_detail' + self.saved_plan = 'saved_plan' + self.conn = conn + + def createSchema(self, plan: planner.Plan) -> None: + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, f'CREATE SCHEMA {self.schema_name}') + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.rebalance_status} + (status TEXT, updated TIMESTAMP WITH TIME ZONE) + DISTRIBUTED REPLICATED''') + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.table_rebalance_status_detail} + (db_name TEXT, schema_name TEXT, rel_name TEXT, status TEXT, + CONSTRAINT unique_fqn UNIQUE (db_name, schema_name, rel_name)) + DISTRIBUTED REPLICATED''') + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.saved_plan} + (plan BYTEA) + DISTRIBUTED REPLICATED''') + + self.savePlan(plan) + + dbconn.execSQL(self.conn, 'COMMIT') + + def dropSchema(self) -> None: + dbconn.execSQL(self.conn, f'DROP SCHEMA {self.schema_name} CASCADE') + + def getSchemaName(self) -> str: + return self.schema_name + + def savePlan(self, plan: planner.Plan) -> None: + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.schema_name}.{self.saved_plan} + VALUES ('\\x{plan.serializePlan().hex()}')''') + + def retrieveSavedPlan(self) -> planner.Plan: + if not self.schemaExists(): + return None + + row = dbconn.queryRow(self.conn, f'SELECT count(1) FROM {self.schema_name}.{self.saved_plan}') + plan_count = int(row[0]) + if plan_count > 1: + raise Exception(f'Number of saved plans ({plan_count}) is > 1') + if plan_count == 0: + return None + + row = dbconn.queryRow(self.conn, f'SELECT plan FROM {self.schema_name}.{self.saved_plan} LIMIT 1') + return planner.deserializePlan(row[0]) + + def schemaExists(self) -> bool: + row = dbconn.queryRow(self.conn, f"SELECT COUNT(1) FROM pg_namespace WHERE nspname = '{self.schema_name}'") + return int(row[0]) == 1 + + def getStateFromPreviousRun(self) -> str: + cursor = dbconn.query(self.conn, f'SELECT status FROM {self.schema_name}.{self.rebalance_status} ORDER BY updated DESC LIMIT 1') + if cursor.rowcount > 0: + return str(cursor.fetchone()[0]) + return 'not defined' + + def rebalanceSchema(self, target_segment_count: int) -> None: + # Before rebalancing check if the tables are already rebalanced + # (in case we re-enter after interruption that happened after COMMIT but before new state) + if get_table_distr_segment_count(self.conn, self.schema_name, self.rebalance_status) > target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.schema_name}"."{self.rebalance_status}" + REBALANCE {target_segment_count}''') + + if get_table_distr_segment_count(self.conn, self.schema_name, self.table_rebalance_status_detail) > target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.schema_name}"."{self.table_rebalance_status_detail}" + REBALANCE {target_segment_count}''') + + if get_table_distr_segment_count(self.conn, self.schema_name, self.saved_plan) > target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.schema_name}"."{self.saved_plan}" + REBALANCE {target_segment_count}''') + + def storeState(self, state: str) -> None: + if self.schemaExists(): + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.schema_name}.{self.rebalance_status} + VALUES ('{state}', NOW())''') + + def clearTablesToRebalanceWithStatus(self, status: str) -> None: + dbconn.execSQL(self.conn, + f'''DELETE FROM {self.schema_name}.{self.table_rebalance_status_detail} + WHERE (status = '{status}')''') + + def addTableToRebalance(self, db: str, schema_name: str, rel_name: str, status: str) -> None: + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.schema_name}.{self.table_rebalance_status_detail} + VALUES ('{db}', '{schema_name}', '{rel_name}', '{status}')''') + + def setStatusForTableToRebalance(self, db: str, schema_name: str, rel_name: str, status: str) -> None: + dbconn.execSQL(self.conn, + f'''UPDATE {self.schema_name}.{self.table_rebalance_status_detail} SET status = '{status}' + WHERE db_name = '{db}' AND schema_name = '{schema_name}' AND rel_name = '{rel_name}';''') + + def getTablesToRebalanceWithStatus(self, status: str) -> cursor: + return dbconn.query(self.conn, f"""SELECT db_name, schema_name, rel_name FROM + {self.schema_name}.{self.table_rebalance_status_detail} WHERE status = '{status}'""") diff --git a/gpMgmt/bin/gprebalance_modules/shrink.py b/gpMgmt/bin/gprebalance_modules/shrink.py index fbd077754692..92f421963e51 100644 --- a/gpMgmt/bin/gprebalance_modules/shrink.py +++ b/gpMgmt/bin/gprebalance_modules/shrink.py @@ -9,11 +9,15 @@ from gppylib.commands.gp import * from gppylib.gplog import * from gppylib.db import dbconn + from gppylib import userinput + from gppylib.gparray import GpArray, Segment from gppylib.fault_injection import * from gppylib.userinput import * from gppylib.commands import base from gppylib.commands.gp import SEGMENT_STOP_TIMEOUT_DEFAULT, SegmentStop from gppylib.system.environment import * + from gprebalance_modules.planner import * + from gprebalance_modules.rebalance_commons import RebalanceSchema except ImportError as e: sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) @@ -46,10 +50,6 @@ class GGShrink: timeout = SEGMENT_STOP_TIMEOUT_DEFAULT stop_mode = 'fast' - rebalance_schema_name = 'ggrebalance' - rebalance_status = 'rebalance_status' - table_rebalance_status_detail = 'table_rebalance_status_detail' - states = [ 'STATE_START', 'STATE_OPTIONS_VALIDATION', @@ -57,7 +57,8 @@ class GGShrink: 'STATE_END', 'STATE_CLEANUP', 'STATE_ERROR', - 'STATE_ROLLBACK' + 'STATE_ROLLBACK', + 'STATE_END_FROM_ROLLBACK' ] # Note: order of states in the list below is important, @@ -82,6 +83,21 @@ class GGShrink: 'STATE_SHRINK_DONE' ] + # Note: order of states in the list below is important, + # as we rely on it when recover from an interrupted state. + # These states are separated from 'states' and 'states_main_shrink_flow' + # above in order to support re-enter the interrupted state of rollback. + states_rollback_flow = [ + 'STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START', + 'STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE', + 'STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START', + 'STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE', + 'STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START', + 'STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE', + 'STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START', + 'STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE' + ] + transitions = [ { 'trigger': 'start', @@ -94,8 +110,13 @@ class GGShrink: 'dest': 'STATE_CLEANUP' }, { - 'trigger': 'move_to_STATE_CHECK_PREVIOUS_RUN', + 'trigger': 'move_to_STATE_ROLLBACK', 'source': 'STATE_OPTIONS_VALIDATION', + 'dest': 'STATE_ROLLBACK' + }, + { + 'trigger': 'move_to_STATE_CHECK_PREVIOUS_RUN', + 'source': ['STATE_OPTIONS_VALIDATION', 'STATE_ROLLBACK'], 'dest': 'STATE_CHECK_PREVIOUS_RUN' }, { @@ -165,9 +186,54 @@ class GGShrink: }, { 'trigger': 'move_to_STATE_END', - 'source': ['STATE_SHRINK_DONE', 'STATE_CHECK_PREVIOUS_RUN', 'STATE_CLEANUP'], + 'source': ['STATE_SHRINK_DONE', 'STATE_CHECK_PREVIOUS_RUN', 'STATE_CLEANUP', 'STATE_END_FROM_ROLLBACK'], 'dest': 'STATE_END' }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START', + 'source': 'STATE_CHECK_PREVIOUS_RUN', + 'dest': 'STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE', + 'source': 'STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START', + 'dest': 'STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START', + 'source': 'STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE', + 'dest': 'STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE', + 'source': 'STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START', + 'dest': 'STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START', + 'source': 'STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE', + 'dest': 'STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE', + 'source': 'STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START', + 'dest': 'STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START', + 'source': 'STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE', + 'dest': 'STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE', + 'source': 'STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START', + 'dest': 'STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE' + }, + { + 'trigger': 'move_to_STATE_END_FROM_ROLLBACK', + 'source': ['STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE', 'STATE_ROLLBACK', 'STATE_CHECK_PREVIOUS_RUN'], + 'dest': 'STATE_END_FROM_ROLLBACK' + }, { 'trigger': 'move_to_STATE_ERROR', 'source': '*', @@ -187,29 +253,22 @@ def __init__(self, logger: Any, dburl: dbconn.DbURL, options: Any, gpEnv: GpCoor self.workers_for_segment_stop = None self.gparray = gpArray self.gparray_dump_file = gpArrayDumpFilename + self.rebalance_schema = RebalanceSchema(self.conn) + self.shrink_plan = None self.needs_repopulate = False self.dumped_gparray = gparray.GpArray.initFromFile(self.gparray_dump_file) if os.path.exists(self.gparray_dump_file) else None self.machine = Machine(model = self, queued=True, - states = self.states + self.states_main_shrink_flow, + states = self.states + self.states_main_shrink_flow + self.states_rollback_flow, transitions = self.transitions, initial = 'STATE_START', before_state_change = 'on_every_state') - def run(self) -> None: + def run(self, shrinkPlan: Plan) -> None: + self.shrink_plan = shrinkPlan self.trigger('start') - def ggrebalance_schema_exists(self) -> bool: - row = dbconn.queryRow(self.conn, f"SELECT COUNT(1) FROM pg_namespace WHERE nspname = '{self.rebalance_schema_name}'") - return int(row[0]) == 1 - - def get_state_from_previous_run(self) -> str: - cursor = dbconn.query(self.conn, f'SELECT status FROM {self.rebalance_schema_name}.{self.rebalance_status} ORDER BY updated DESC LIMIT 1') - if cursor.rowcount > 0: - return str(cursor.fetchone()[0]) - return 'not defined' - def get_state_after_interrupt(self, prev_state) -> str: prev_idx = self.states_main_shrink_flow.index(prev_state) lower = self.states_main_shrink_flow.index('STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED') @@ -219,7 +278,7 @@ def get_state_after_interrupt(self, prev_state) -> str: #if shrink is interrupted after catalog update and before the state is logged if prev_state == 'STATE_SHRINK_TABLES_DONE' and \ self.dumped_gparray is not None \ - and self.gparray.get_segment_count() + self.options.target_segment_count == self.dumped_gparray.get_segment_count(): + and self.gparray.get_segment_count() + self.shrink_plan.target_segment_count == self.dumped_gparray.get_segment_count(): return 'STATE_SHRINK_CATALOG_DONE' row = dbconn.queryRow(self.conn, 'SELECT gp_toolkit.gp_rebalance_numsegments_is_set();') @@ -234,28 +293,20 @@ def get_state_after_interrupt(self, prev_state) -> str: def on_every_state(self) -> None: if self.shutdown_requested: self.logger.info('Shrink was interrupted') - sys.exit(1) + raise Exception('Shrink was interrupted') - assert self.state in self.states + self.states_main_shrink_flow + assert self.state in self.states + self.states_main_shrink_flow + self.states_rollback_flow - # insert status if the schema already exists - if self.state in self.states_main_shrink_flow: - if self.ggrebalance_schema_exists(): - dbconn.execSQL(self.conn, - f'''INSERT INTO {self.rebalance_schema_name}.{self.rebalance_status} - VALUES ('{self.state}', NOW())''') + if self.state in self.states_main_shrink_flow + self.states_rollback_flow: + self.rebalance_schema.storeState(self.state) # state callbacks start here @wrap_state_func_with_faults def on_enter_STATE_OPTIONS_VALIDATION(self) -> None: if self.options.clean_required: self.trigger('move_to_STATE_CLEANUP') - elif not self.ggrebalance_schema_exists() and \ - self.gparray.get_segment_count() == self.options.target_segment_count: - self.logger.error('Target segment count (%s) = current segment count (%s).\n' - 'Currently only shrink is supported (target segment count < current segment count).' - % (self.options.target_segment_count, self.gparray.get_segment_count())) - raise ValueError("Wrong target segment count") + elif self.options.rollback_required: + self.trigger('move_to_STATE_ROLLBACK') else: self.trigger('move_to_STATE_CHECK_PREVIOUS_RUN') @@ -264,81 +315,95 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: # check if rebalance schema exists # and whether we can get the state where we stopped in previous run # in order to proceed from the same point - if self.ggrebalance_schema_exists(): - self.logger.info('Rebalance schema already exists') - state_from_prev_run = self.get_state_from_previous_run() + if self.rebalance_schema.schemaExists(): + self.logger.info('Rebalance schema exists') + state_from_prev_run = self.rebalance_schema.getStateFromPreviousRun() # check maybe the state is the final one if state_from_prev_run == self.states_main_shrink_flow[-1]: - self.logger.info('Previous run was completed successfully. Please execute cleanup before a new run.') - self.trigger('move_to_STATE_END') + if self.options.rollback_required: + self.logger.info(f"Previous run was completed successfully. Can't perform rollback.") + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + else: + self.logger.error('Previous run was completed successfully. Please execute cleanup before a new run.') + self.trigger('move_to_STATE_END') + return + + elif self.shrink_plan != None: + self.logger.error("Can't start a new operation, because the previous one was interrupted. " + "Please try to launch again without a plan to continue from the interrupted state, " + "or use '--rollback' or '--cleanup' options.") + self.trigger('move_to_STATE_ERROR') + return + else: - self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") - try: - next_state = self.get_state_after_interrupt(state_from_prev_run) - except: - self.logger.error("Can't determine next state") + self.shrink_plan = self.rebalance_schema.retrieveSavedPlan() + if self.shrink_plan == None: + self.logger.error('No saved plan found. Try to execute cleanup.') self.trigger('move_to_STATE_ERROR') return + + if state_from_prev_run in self.states_rollback_flow: + self.logger.info('Continue interrupted rollback operation...') + self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") + try: + next_state = self.states_rollback_flow[ self.states_rollback_flow.index(state_from_prev_run) + 1 ] + except: + self.logger.error("Can't determine next rollback state") + self.trigger('move_to_STATE_ERROR') + return + else: + if self.options.rollback_required: + self.trigger('move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START') + return + + self.logger.info('Continue interrupted operation...') + self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") + try: + next_state = self.get_state_after_interrupt(state_from_prev_run) + except: + self.logger.error("Can't determine next state. Try to execute cleanup.") + self.trigger('move_to_STATE_ERROR') + return + # use auto to_«state» method to recover self.trigger(f'to_{next_state}') else: + if self.shrink_plan == None: + self.logger.error("Rebalance schema doesn't exists and no shrink plan is supplied. Please specify shrink plan.") + self.trigger('move_to_STATE_ERROR') + return + if self.gparray.get_segment_count() <= self.shrink_plan.target_segment_count: + logger.error('Target segment count (%s) >= current segment count (%s).\n' + 'Currently only shrink is supported (target segment count < current segment count).' + % (self.shrink_plan.target_segment_count, self.gparray.get_segment_count())) + self.trigger('move_to_STATE_ERROR') + return + self.trigger('move_to_STATE_SETUP_SHRINK_SCHEMA_STARTED') @wrap_state_func_with_faults def on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED(self) -> None: - # Create schema and status tables - dbconn.execSQL(self.conn, 'BEGIN') - dbconn.execSQL(self.conn, f'DROP SCHEMA IF EXISTS {self.rebalance_schema_name} CASCADE') - dbconn.execSQL(self.conn, f'CREATE SCHEMA {self.rebalance_schema_name}') - dbconn.execSQL(self.conn, - f'''CREATE TABLE {self.rebalance_schema_name}.{self.rebalance_status} - (status TEXT, updated TIMESTAMP WITH TIME ZONE) - DISTRIBUTED REPLICATED''') - dbconn.execSQL(self.conn, - f'''CREATE TABLE {self.rebalance_schema_name}.{self.table_rebalance_status_detail} - (db_name TEXT, schema_name TEXT, rel_name TEXT, status TEXT, - CONSTRAINT unique_fqn UNIQUE (db_name, schema_name, rel_name)) - DISTRIBUTED REPLICATED''') - dbconn.execSQL(self.conn, 'COMMIT') - + # Create schema and status tables. + # It will also save plan in order to use it for recovering after interruption + self.rebalance_schema.createSchema(self.shrink_plan) self.trigger('move_to_STATE_SETUP_SHRINK_SCHEMA_DONE') @wrap_state_func_with_faults def on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE(self) -> None: - self.logger.info(f'Created shrink schema {self.rebalance_schema_name}') + self.logger.info(f'Created "{self.rebalance_schema.getSchemaName()}" schema') self.trigger('move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED') - def get_table_distr_segment_count(self, schema_name, table_name) -> int: - row = dbconn.queryRow(self.conn, - f'''SELECT p.numsegments - FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid - JOIN gp_distribution_policy p ON c.oid = p.localoid - WHERE n.nspname='{schema_name}' AND c.relname='{table_name}';''') - return int(row[0]) - @wrap_state_func_with_faults def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED(self) -> None: dbconn.execSQL(self.conn, 'BEGIN') dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') dbconn.execSQL(self.conn, 'CHECKPOINT') - dbconn.execSQL(self.conn, f'SELECT gp_toolkit.gp_set_rebalance_numsegments({self.options.target_segment_count})') + dbconn.execSQL(self.conn, f'SELECT gp_toolkit.gp_set_rebalance_numsegments({self.shrink_plan.getTargetSegmentCount()})') self.gparray.dumpToFile(self.gparray_dump_file) # Rebalance the status tables we've created previously right here before we start to rebalance all other tables. - # Before that check if the tables are already rebalanced - # (in case we re-enter after interruption that happened after COMMIT but before new state) - if self.get_table_distr_segment_count(self.rebalance_schema_name, - self.rebalance_status) > self.options.target_segment_count: - dbconn.execSQL(self.conn, - f'''ALTER TABLE "{self.rebalance_schema_name}"."{self.rebalance_status}" - REBALANCE {self.options.target_segment_count}''') - - if self.get_table_distr_segment_count(self.rebalance_schema_name, - self.table_rebalance_status_detail) > self.options.target_segment_count: - dbconn.execSQL(self.conn, - f'''ALTER TABLE "{self.rebalance_schema_name}"."{self.table_rebalance_status_detail}" - REBALANCE {self.options.target_segment_count}''') + self.rebalance_schema.rebalanceSchema(self.shrink_plan.getTargetSegmentCount()) dbconn.execSQL(self.conn, 'COMMIT') @@ -346,109 +411,24 @@ def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED(self) @wrap_state_func_with_faults def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE(self) -> None: - self.logger.info(f'Updated target segment count to {self.options.target_segment_count}') + self.logger.info(f'Updated target segment count to {self.shrink_plan.getTargetSegmentCount()}') self.trigger('move_to_STATE_PREPARE_SHRINK_SCHEMA_STARTED') @wrap_state_func_with_faults def on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED(self) -> None: - # collect databases and tables that require 'ALTER TABLE REBALANCE' - # and store in 'table_rebalance_status_detail' table - - dbconn.execSQL(self.conn, 'BEGIN') - - # cleanup table_rebalance_status_detail for the case we re-enter this state after we were interrupted - dbconn.execSQL(self.conn, - f'''DELETE FROM {self.rebalance_schema_name}.{self.table_rebalance_status_detail} - WHERE (status <> 'done')''') - - cursor = dbconn.query(self.conn, 'SELECT datname FROM pg_database') - databases_to_process = [] - for record in cursor: - database_name = record[0] - if database_name != 'template0': - databases_to_process.append(database_name) - - for db in databases_to_process: - dburl = dbconn.DbURL(dbname=db, port=self.gpEnv.getCoordinatorPort()) - with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: - cursor = dbconn.query(conn, - f'''SELECT n.nspname, c.relname - FROM pg_class c - JOIN pg_namespace n ON c.relnamespace = n.oid - JOIN gp_distribution_policy p ON c.oid = p.localoid - WHERE c.relkind IN ('r', 'p') AND c.relispartition = FALSE AND - p.numsegments > {self.options.target_segment_count} AND - n.nspname NOT IN ('pg_catalog', 'information_schema', '{self.rebalance_schema_name}')''') - for schema_name, rel_name in cursor: - dbconn.execSQL(self.conn, - f'''INSERT INTO {self.rebalance_schema_name}.{self.table_rebalance_status_detail} - VALUES ('{db}', '{schema_name}', '{rel_name}', 'none')''') - - dbconn.execSQL(self.conn, 'COMMIT') - + self.prepare_shrink_schema(False) self.trigger('move_to_STATE_PREPARE_SHRINK_SCHEMA_DONE') @wrap_state_func_with_faults def on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE(self) -> None: - self.logger.info(f'Initiated {self.rebalance_schema_name}.{self.table_rebalance_status_detail}') + self.logger.info(f'Initiated list of tables to rebalance') self.trigger('move_to_STATE_SHRINK_TABLES_STARTED') - class TableRebalanceTask(SQLCommand): - def __init__(self, - shrink: 'GGShrink', - db_name: str, - schema_name: str, - rel_name: str) -> None: - self.shrink = shrink - self.db_name = db_name - self.schema_name = schema_name - self.rel_name = rel_name - SQLCommand.__init__(self, f'task rebalance for {self.db_name}.{self.schema_name}.{self.rel_name}') - - def run(self) -> None: - dburl = dbconn.DbURL(dbname=self.db_name, port=self.shrink.gpEnv.getCoordinatorPort()) - with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: - dbconn.execSQL(conn, 'BEGIN') - dbconn.execSQL(conn, - f'''ALTER TABLE "{self.schema_name}"."{self.rel_name}" - REBALANCE {self.shrink.options.target_segment_count}''') - dbconn.execSQL(self.shrink.conn, - f'''UPDATE {self.shrink.rebalance_schema_name}.{self.shrink.table_rebalance_status_detail} SET status = 'done' - WHERE db_name = '{self.db_name}' AND schema_name = '{self.schema_name}' AND rel_name = '{self.rel_name}';''') - dbconn.execSQL(conn, 'COMMIT') - self.set_results(CommandResult(0, b'', b'', True, False)) - @wrap_state_func_with_faults def on_enter_STATE_SHRINK_TABLES_STARTED(self) -> None: - self.logger.info('Start tables rebalance') - + self.logger.info('Start tables rebalance for shrink') # perform 'ALTER TABLE REBALANCE' for all not yet processed tables - cursor = dbconn.query(self.conn, - f"SELECT db_name, schema_name, rel_name FROM {self.rebalance_schema_name}.{self.table_rebalance_status_detail} WHERE status = 'none'") - - self.logger.info(f'Tables to process {cursor.rowcount}') - - if cursor.rowcount > 0: - self.workers_for_tables_rebalance = WorkerPool(numWorkers=min(cursor.rowcount, self.options.parallel)) - - for db_name, schema_name, rel_name in cursor: - task = self.TableRebalanceTask(self, - db_name, - schema_name, - rel_name) - self.workers_for_tables_rebalance.addCommand(task) - - print_progress(self.workers_for_tables_rebalance, interval=1) - - self.workers_for_tables_rebalance.haltWork() - self.workers_for_tables_rebalance.joinWorkers() - - for task in self.workers_for_tables_rebalance.getCompletedItems(): - if not task.was_successful(): - raise Exception(f'Failed to do ALTER REBALANCE: {task.get_results().stderr}') - - self.workers_for_tables_rebalance = None - + self.rebalance_tables('none', 'done', self.shrink_plan.getTargetSegmentCount()) self.trigger('move_to_STATE_SHRINK_TABLES_DONE') @wrap_state_func_with_faults @@ -462,11 +442,11 @@ def on_enter_STATE_SHRINK_CATALOG_STARTED(self) -> None: ## Shrink catalog dbconn.execSQL(self.conn, 'BEGIN') - cursor = dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') - dbconn.execSQL(self.conn, f'DELETE FROM gp_segment_configuration WHERE content >= {self.options.target_segment_count}') + dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + dbconn.execSQL(self.conn, f'DELETE FROM gp_segment_configuration WHERE content >= {self.shrink_plan.getTargetSegmentCount()}') dbconn.execSQL(self.conn, 'CHECKPOINT') dbconn.execSQL(self.conn, 'SELECT gp_expand_bump_version()') - cursor = dbconn.query(self.conn, 'SELECT gp_toolkit.gp_reset_rebalance_numsegments()') + dbconn.execSQL(self.conn, 'SELECT gp_toolkit.gp_reset_rebalance_numsegments()') dbconn.execSQL(self.conn, 'COMMIT') self.trigger('move_to_STATE_SHRINK_CATALOG_DONE') @@ -485,30 +465,20 @@ def on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED(self) -> None: if gp_array is None: gp_array = self.gparray - segments_to_stop = gp_array.get_segment_count() - self.options.target_segment_count + segments_to_stop = gp_array.get_segment_count() - self.shrink_plan.getTargetSegmentCount() segments_to_stop = segments_to_stop * 2 # consider mirrors self.workers_for_segment_stop = WorkerPool(numWorkers=min(segments_to_stop, self.options.batch_size)) for seg_pair in gp_array.getSegmentList(): primary_seg = seg_pair.primaryDB - mirror_seq = seg_pair.mirrorDB - if primary_seg.getSegmentContentId() >= self.options.target_segment_count: + mirror_seg = seg_pair.mirrorDB + if primary_seg.getSegmentContentId() >= self.shrink_plan.getTargetSegmentCount(): if primary_seg.isSegmentUp(): - cmd = SegmentStop(f'stop primary (content {primary_seg.getSegmentContentId()}, dbid {primary_seg.getSegmentDbId()})', - primary_seg.getSegmentDataDirectory(), - mode=self.stop_mode, - timeout=self.timeout, - ctxt=base.REMOTE, - remoteHost=primary_seg.getSegmentHostName()) + cmd = self.SegmentStopAfterShrink(self, primary_seg) self.workers_for_segment_stop.addCommand(cmd) - if mirror_seq != None and mirror_seq.isSegmentUp(): - cmd = SegmentStop(f'stop mirror (content {mirror_seq.getSegmentContentId()}, dbid {mirror_seq.getSegmentDbId()})', - mirror_seq.getSegmentDataDirectory(), - mode=self.stop_mode, - timeout=self.timeout, - ctxt=base.REMOTE, - remoteHost=mirror_seq.getSegmentHostName()) + if mirror_seg != None and mirror_seg.isSegmentUp(): + cmd = self.SegmentStopAfterShrink(self, mirror_seg) self.workers_for_segment_stop.addCommand(cmd) print_progress(self.workers_for_segment_stop, interval=1) @@ -537,19 +507,274 @@ def on_enter_STATE_SHRINK_DONE(self) -> None: @wrap_state_func_with_faults def on_enter_STATE_CLEANUP(self) -> None: - dbconn.execSQL(self.conn, f'DROP SCHEMA {self.rebalance_schema_name} CASCADE') - self.logger.info('Cleanup is complete') + if not self.rebalance_schema.schemaExists(): + self.logger.info(f"Rebalance schema doesn't exist. Cleanup is not required.") + else: + state_from_prev_run = self.rebalance_schema.getStateFromPreviousRun() + if state_from_prev_run != self.states_main_shrink_flow[-1]: + self.logger.warning("ggrebalance hasn't finished shrink process properly. Previous run was interrupted. " + "Some unbalanced tables can still exist.") + + # get default num segments + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + row = dbconn.queryRow(self.conn, 'SELECT gp_toolkit.gp_rebalance_numsegments_is_set()') + numsegments_is_set = bool(row[0]) + dbconn.execSQL(self.conn, 'END') + + if numsegments_is_set: + self.logger.warning('Current numsegments is not equal to default value.') + self.logger.info('Suggestion: explicitly reset the value before cleanup. Note: cluster restart will implicitly reset the value.') + + if (self.options.interactive and + not userinput.ask_yesno(None, "\nContinue with cleanup?", 'Y')): + self.logger.info('Cleanup was interrupted...') + self.trigger('move_to_STATE_END') + return + + if (numsegments_is_set and + (not self.options.interactive or userinput.ask_yesno(None, "\nReset numsegments to default?", 'Y'))): + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + dbconn.execSQL(self.conn, 'SELECT gp_toolkit.gp_reset_rebalance_numsegments()') + dbconn.execSQL(self.conn, 'COMMIT') + self.logger.info('Reset numsegments to default is done.') + + if os.path.exists(self.gparray_dump_file): + os.remove(self.gparray_dump_file) + self.rebalance_schema.dropSchema() + self.logger.info('Cleanup is complete') + self.trigger('move_to_STATE_END') + + @wrap_state_func_with_faults + def on_enter_STATE_ROLLBACK(self) -> None: + if not self.rebalance_schema.schemaExists(): + self.logger.info("Rebalance schema doesn't exist. Can't perform rollback.") + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + return + else: + state_from_prev_run = self.rebalance_schema.getStateFromPreviousRun() + if state_from_prev_run != 'not defined': + # check maybe the state is the final one + if state_from_prev_run == self.states_main_shrink_flow[-1]: + self.logger.info("Previous run was completed successfully. Can't perform rollback.") + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + return + + if not self.state_can_rollback(state_from_prev_run) or self.is_gp_segment_configuration_shrinked(): + self.logger.info("Can't perform rollback as the catalog is already updated") + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + return + + self.trigger('move_to_STATE_CHECK_PREVIOUS_RUN') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START(self) -> None: + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + dbconn.execSQL(self.conn, 'SELECT gp_toolkit.gp_reset_rebalance_numsegments()') + # Store state here in case we fail before we enter 'on_every_state()' + # because after COMMIT we are on a one-way road of rollback. + self.rebalance_schema.storeState(self.state) + dbconn.execSQL(self.conn, 'COMMIT') + + self.trigger('move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE(self) -> None: + self.trigger('move_to_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START(self) -> None: + self.prepare_shrink_schema(True) + self.trigger('move_to_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE(self) -> None: + self.trigger('move_to_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START(self) -> None: + self.logger.info('Start tables rebalance for rollback') + # perform 'ALTER TABLE REBALANCE' for all not yet processed tables + self.rebalance_tables('done', 'none', self.gparray.get_segment_count()) + self.trigger('move_to_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE(self) -> None: + self.trigger('move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START(self) -> None: + if os.path.exists(self.gparray_dump_file): + os.remove(self.gparray_dump_file) + self.rebalance_schema.dropSchema() + self.trigger('move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE(self) -> None: + self.logger.info('Rollback is complete.') + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + + @wrap_state_func_with_faults + def on_enter_STATE_END_FROM_ROLLBACK(self) -> None: self.trigger('move_to_STATE_END') @wrap_state_func_with_faults def on_enter_STATE_END(self) -> None: self.conn.close() + @wrap_state_func_with_faults def on_enter_STATE_ERROR(self) -> None: - sys.exit(1) + raise Exception('Shrink entered STATE_ERROR') # state callbacks end here + class SegmentStopAfterShrink(SegmentStop): + def __init__(self, shrink: 'GGShrink', segment: Segment) -> None: + self.shrink = shrink + self.segment = segment + if self.segment.isSegmentPrimary(): + name = f'stop primary (content {self.segment.getSegmentContentId()}, dbid {self.segment.getSegmentDbId()})' + else: + name = f'stop mirror (content {self.segment.getSegmentContentId()}, dbid {self.segment.getSegmentDbId()})' + self.checkRunningSegment = SegmentIsShutDown(name, self.segment.getSegmentDataDirectory(), base.REMOTE, self.segment.getSegmentHostName()) + SegmentStop.__init__(self, + name, + self.segment.getSegmentDataDirectory(), + self.shrink.stop_mode, + False, + base.REMOTE, + self.segment.getSegmentHostName(), + self.shrink.timeout) + + # decorator to inject a fault before running SegmentStopAfterShrink for a specific dbid + def wrap_segment_stop_with_faults(fun): + def func_with_faults(self): + inject_fault(f'fault_segment_stop_dbid_{self.segment.getSegmentDbId()}') + fun(self) + return func_with_faults + + @wrap_segment_stop_with_faults + def run(self) -> None: + self.shrink.logger.info(f'Stopping shrinked segment dbid {self.segment.getSegmentDbId()} @ host={self.remoteHost}, datadir={self.segment.getSegmentDataDirectory()}') + self.checkRunningSegment.run() + if self.checkRunningSegment.is_shutdown(): + self.shrink.logger.info(f'Segment dbid {self.segment.getSegmentDbId()} is already down @ host={self.remoteHost}, datadir={self.segment.getSegmentDataDirectory()} ') + self.set_results(CommandResult(0, b'', b'', True, False)) + else: + SegmentStop.run(self) + self.shrink.logger.info(f'Stopped shrinked segment dbid {self.segment.getSegmentDbId()} @ host={self.remoteHost}, datadir={self.segment.getSegmentDataDirectory()}') + + class TableRebalanceTask(SQLCommand): + def __init__(self, + shrink: 'GGShrink', + db_name: str, + schema_name: str, + rel_name: str, + target_segment_count: int, + table_status_after_rebalance: str) -> None: + self.shrink = shrink + self.db_name = db_name + self.schema_name = schema_name + self.rel_name = rel_name + self.target_segment_count = target_segment_count + self.table_status_after_rebalance = table_status_after_rebalance + SQLCommand.__init__(self, f'task rebalance for {self.db_name}.{self.schema_name}.{self.rel_name}') + + # decorator to inject a fault before running TableRebalanceTask for a specific {db_name, schema_name, rel_name} + def wrap_table_rebalance_with_faults(fun): + def func_with_faults(self): + inject_fault(f'fault_rebalance_table_{self.db_name}.{self.schema_name}.{self.rel_name}') + fun(self) + return func_with_faults + + @wrap_table_rebalance_with_faults + def run(self) -> None: + self.shrink.logger.info(f'Start table rebalance for "{self.db_name}"."{self.schema_name}"."{self.rel_name}" to {self.target_segment_count} segments') + dburl = dbconn.DbURL(dbname=self.db_name, port=self.shrink.gpEnv.getCoordinatorPort()) + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + dbconn.execSQL(conn, 'BEGIN') + dbconn.execSQL(conn, + f'''ALTER TABLE "{self.schema_name}"."{self.rel_name}" + REBALANCE {self.target_segment_count}''') + self.shrink.rebalance_schema.setStatusForTableToRebalance(self.db_name, self.schema_name, self.rel_name, self.table_status_after_rebalance) + dbconn.execSQL(conn, 'COMMIT') + self.shrink.logger.info(f'Complete table rebalance for "{self.db_name}"."{self.schema_name}"."{self.rel_name}"') + self.set_results(CommandResult(0, b'', b'', True, False)) + + def prepare_shrink_schema(self, is_rollback: bool) -> None: + status = 'done' if is_rollback else 'none' + cmp = '<=' if is_rollback else '>' + + dbconn.execSQL(self.conn, 'BEGIN') + + # cleanup list of tables that require rebalance + # for the case we re-enter this state after we were interrupted right after it + self.rebalance_schema.clearTablesToRebalanceWithStatus(status) + + cursor = dbconn.query(self.conn, 'SELECT datname FROM pg_database') + databases_to_process = [] + for record in cursor: + database_name = record[0] + if database_name != 'template0': + databases_to_process.append(database_name) + + for db in databases_to_process: + dburl = dbconn.DbURL(dbname=db, port=self.gpEnv.getCoordinatorPort()) + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + cursor = dbconn.query(conn, + f'''SELECT n.nspname, c.relname + FROM pg_class c + JOIN pg_namespace n ON c.relnamespace = n.oid + JOIN gp_distribution_policy p ON c.oid = p.localoid + WHERE c.relkind IN ('r', 'p') AND c.relispartition = FALSE AND + p.numsegments {cmp} {self.shrink_plan.getTargetSegmentCount()} AND + n.nspname NOT IN ('pg_catalog', 'information_schema', '{self.rebalance_schema.getSchemaName()}')''') + for schema_name, rel_name in cursor: + self.rebalance_schema.addTableToRebalance(db, schema_name, rel_name, status) + + dbconn.execSQL(self.conn, 'COMMIT') + + def rebalance_tables(self, original_status: str, target_status: str, target_segment_count: int) -> None: + cursor = self.rebalance_schema.getTablesToRebalanceWithStatus(original_status) + + self.logger.info(f'Tables to process {cursor.rowcount}') + + if cursor.rowcount > 0: + self.workers_for_tables_rebalance = WorkerPool(numWorkers=min(cursor.rowcount, self.options.parallel)) + + for db_name, schema_name, rel_name in cursor: + task = self.TableRebalanceTask(self, + db_name, + schema_name, + rel_name, + target_segment_count, + target_status) + self.workers_for_tables_rebalance.addCommand(task) + + print_progress(self.workers_for_tables_rebalance, interval=1) + + self.workers_for_tables_rebalance.haltWork() + self.workers_for_tables_rebalance.joinWorkers() + + for task in self.workers_for_tables_rebalance.getCompletedItems(): + if not task.was_successful(): + raise Exception(f'Failed to do ALTER REBALANCE: {task.get_results().stderr}') + + self.workers_for_tables_rebalance = None + + def state_can_rollback(self, state: str) -> bool: + if (state in self.states_main_shrink_flow): + if self.states_main_shrink_flow.index(state) <= self.states_main_shrink_flow.index('STATE_SHRINK_TABLES_DONE'): + return True + return False + + def is_gp_segment_configuration_shrinked(self) -> bool: + if self.dumped_gparray is None: + return False + return self.dumped_gparray.get_segment_count() != self.gparray.get_segment_count() + def shutdown(self) -> None: if self.workers_for_tables_rebalance != None: self.workers_for_tables_rebalance.haltWork() diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature index c1f7da7518f8..c7a07c4f9d3c 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature @@ -1,36 +1,600 @@ @ggrebalance Feature: ggrebalance behave tests - Scenario Outline: test shrink continue after cluster restart - Given the database is not running - And a working directory of the test as '/data/gpdata/ggrebalance' - And a cluster is created with mirrors on "cdw" and "sdw1" - And all files in gpAdminLogs directory are deleted - And database "test_db_1" exists - And schema "test_schema_1" exists in "test_db_1" - And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows - And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows - And database "test_db_2" exists - And schema "test_schema_2" exists in "test_db_2" - And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows - And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When set fault inject "" - And the user runs "ggrebalance -x 1" - Then ggrebalance should return a return code of 1 - And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp - And unset fault inject - When the user runs "gpstop -arf" - Then gpstart should return a return code of 0 - When there is a "heap" table "test_schema_2.test_table_3" in "test_db_2" with data - And the user runs "ggrebalance -x 1" - Then ggrebalance should print "Cluster restarted after previous run, trying to repopulate the relation queue" to logfile - And ggrebalance should print "Shrink is complete" to logfile with latest timestamp - And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 - And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 - And distribution information from table "test_schema_2.test_table_3" with data in "test_db_2" is equal to segment count = 1, row count = 1094 - And ggrebalance should return a return code of 0 + Scenario: test 1.1 ggrebalance simple scenarios + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2" + And segment information for content 1 is saved in context + And segment information for content 2 is saved in context + And segment information for content 3 is saved in context + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 4" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Target segment count (4) >= current segment count (4)." to logfile with latest timestamp + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance schema doesn't exist. Cleanup is not required." to logfile with latest timestamp + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance schema doesn't exist. Cleanup is not required." to logfile with latest timestamp + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance schema doesn't exist. Can't perform rollback." to logfile with latest timestamp + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance schema doesn't exist. Can't perform rollback." to logfile with latest timestamp + When the user runs "ggrebalance -x 2" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Previous run was completed successfully. Please execute cleanup before a new run." to logfile with latest timestamp + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Previous run was completed successfully. Can't perform rollback." to logfile with latest timestamp + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Cleanup is complete" to logfile with latest timestamp + + Scenario: test 1.2. check cleanup after the target segment count was updated + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end" + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -c -y" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Reset numsegments to default is done." to logfile with latest timestamp + And ggrebalance should print "Cleanup is complete" to logfile with latest timestamp + + Scenario: test 2.1. shrink - check continue after interrupted state, if interruption is done before the rebalance schema creation + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED_begin" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Rebalance schema doesn't exists and no shrink plan is supplied. Please specify shrink plan." to logfile with latest timestamp + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 1, row count = 100 + + Scenario: test 2.2. shrink - check continue after interrupted state, if interruption is done after the rebalance schema creation, but before any state is saved there + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED_end" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Can't determine next state. Try to execute cleanup." to logfile with latest timestamp + When the user runs "ggrebalance -c -y" + Then ggrebalance should return a return code of 0 + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 1, row count = 100 + + Scenario Outline: test 2.3. shrink - check continue after interrupted state + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Can't start a new operation, because the previous one was interrupted" to logfile with latest timestamp + When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Can't start a new operation, because the previous one was interrupted" to logfile with latest timestamp + When the user runs "ggrebalance --parallel 1 --batch-size 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 1, row count = 100 + + Examples: + | fault_name | + | on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE_begin | + | on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE_end | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_begin | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_end | + | on_enter_STATE_SHRINK_TABLES_STARTED_begin | + | on_enter_STATE_SHRINK_TABLES_STARTED_end | + | on_enter_STATE_SHRINK_TABLES_DONE_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_end | + | on_enter_STATE_SHRINK_CATALOG_STARTED_begin | + | on_enter_STATE_SHRINK_CATALOG_STARTED_end | + | on_enter_STATE_SHRINK_CATALOG_DONE_begin | + | on_enter_STATE_SHRINK_CATALOG_DONE_end | + | on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED_begin | + | on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | + | fault_segment_stop_dbid_3 | + + Scenario Outline: test 2.4. test shrink continue after cluster restart + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When set fault inject "" + And the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "gpstop -arf" + Then gpstart should return a return code of 0 + When there is a "heap" table "test_schema_2.test_table_3" in "test_db_2" with data + And the user runs "ggrebalance" + Then ggrebalance should print "Cluster restarted after previous run, trying to repopulate the relation queue" to logfile + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_3" with data in "test_db_2" is equal to segment count = 1, row count = 1094 + And ggrebalance should return a return code of 0 + Examples: + | fault_name | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_begin | + + Scenario: test 3.1. shrink - check rollback after interrupted state, if interruption is done before the rebalance schema creation + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED_begin" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance schema doesn't exist. Can't perform rollback." to logfile with latest timestamp + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 2, row count = 100 + + Scenario Outline: test 3.2. shrink - check rollback after interrupted state + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is complete" to logfile with latest timestamp + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 2, row count = 100 + + Examples: + | fault_name | + | on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED_end | + | on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE_begin | + | on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE_end | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_begin | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_end | + | on_enter_STATE_SHRINK_TABLES_STARTED_begin | + | on_enter_STATE_SHRINK_TABLES_STARTED_end | + | on_enter_STATE_SHRINK_TABLES_DONE_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_end | + | on_enter_STATE_SHRINK_CATALOG_STARTED_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | + + Scenario Outline: test 3.3. shrink - check rollback after interrupted state (interruption is done after the point of no return). Rollback fails. So just continue shrink. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Can't perform rollback as the catalog is already updated" to logfile with latest timestamp + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 1, row count = 100 + Examples: | fault_name | + | on_enter_STATE_SHRINK_CATALOG_STARTED_end | + | on_enter_STATE_SHRINK_CATALOG_DONE_begin | + | on_enter_STATE_SHRINK_CATALOG_DONE_end | + | on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED_begin | + | on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED_end | + | fault_segment_stop_dbid_3 | + + Scenario Outline: test 3.4. shrink - check rollback after interrupted state and cluster restart + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When set fault inject "" + And the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And there is a "heap" table "test_schema_2.test_table_3" in "test_db_2" with "200" rows + When the user runs "gpstop -arf" + Then gpstart should return a return code of 0 + When there is a "heap" table "test_schema_2.test_table_4" in "test_db_2" with "300" rows + And the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is complete" to logfile with latest timestamp + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_3" with data in "test_db_2" is equal to segment count = 2, row count = 200 + And distribution information from table "test_schema_2.test_table_4" with data in "test_db_2" is equal to segment count = 2, row count = 300 + Examples: + | fault_name | + | on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED_end | + | on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE_begin | + | on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE_end | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_begin | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end | | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_end | + | on_enter_STATE_SHRINK_TABLES_STARTED_begin | + | on_enter_STATE_SHRINK_TABLES_STARTED_end | | on_enter_STATE_SHRINK_TABLES_DONE_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_end | + | on_enter_STATE_SHRINK_CATALOG_STARTED_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | + + Scenario Outline: test 4.1. shrink - check continue after interrupted rollback state. In this case we fail in rollback too early, and normal shrink will be complete. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And set fault inject "" + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 1, row count = 100 + + Examples: + | fault_name_shrink | fault_name_rollback | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_begin | + + Scenario Outline: test 4.2. shrink - check continue after interrupted rollback state + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And set fault inject "" + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is complete" to logfile with latest timestamp + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 2, row count = 100 + + Examples: + | fault_name_shrink | fault_name_rollback | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_begin | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | + + Scenario Outline: test 4.3. shrink - check continue after interrupted rollback state (interruption is done after rebalance schema is dropped) + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And set fault inject "" + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Rebalance schema doesn't exists and no shrink plan is supplied. Please specify shrink plan." to logfile with latest timestamp + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 2, row count = 100 + + Examples: + | fault_name_shrink | fault_name_rollback | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE_end | + + Scenario Outline: test 4.4. shrink - check continue after interrupted rollback state and cluster restart + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And set fault inject "" + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And there is a "heap" table "test_schema_2.test_table_3" in "test_db_2" with "200" rows + When the user runs "gpstop -arf" + Then gpstart should return a return code of 0 + When there is a "heap" table "test_schema_2.test_table_4" in "test_db_2" with "300" rows + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is complete" to logfile with latest timestamp + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_3" with data in "test_db_2" is equal to segment count = 2, row count = 200 + And distribution information from table "test_schema_2.test_table_4" with data in "test_db_2" is equal to segment count = 2, row count = 300 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 2, row count = 100 + + Examples: + | fault_name_shrink | fault_name_rollback | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_begin | + diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 27748082efec..21b9f0784597 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -2283,6 +2283,8 @@ def impl(context): ''') @given('there is a "{tabletype}" table "{tablename}" in "{dbname}" with "{numrows}" rows') +@then('there is a "{tabletype}" table "{tablename}" in "{dbname}" with "{numrows}" rows') +@when('there is a "{tabletype}" table "{tablename}" in "{dbname}" with "{numrows}" rows') def impl(context, tabletype, tablename, dbname, numrows): populate_regular_table_data(context, tabletype, tablename, dbname, compression_type=None, with_data=True, rowcount=int(numrows)) @@ -4158,6 +4160,29 @@ def impl(context): raise Exception("Postgres process {0} not killed on {1}.".format(pid, host)) +@given('segment information for content {content} is saved in context') +@when('segment information for content {content} is saved in context') +@then('segment information for content {content} is saved in context') +def impl(context, content): + segment_information = [] + segs = GpArray.initFromCatalog(dbconn.DbURL()).getDbList() + for seg in segs: + if int(content) == int(seg.getSegmentContentId()): + segment_information.append(seg) + + context.segment_information = segment_information + + +@given('verify no segment running for saved segment information') +@when('verify no segment running for saved segment information') +@then('verify no segment running for saved segment information') +def impl(context): + for seg in context.segment_information: + segment_check = gp.SegmentIsShutDown('', seg.getSegmentDataDirectory(), REMOTE, seg.getSegmentHostName()) + segment_check.run() + if not segment_check.is_shutdown(): + raise Exception(f'Segment is up (dbid {seg.getSegmentDbId()})') + @then('the database segments are in execute mode') def impl(context): # Get all primary segments details From 62c583a93c68c6513c3da7fa29faf22a38fbfc02 Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Tue, 2 Dec 2025 12:42:29 +0300 Subject: [PATCH 13/22] Add basic shrink/rebalance algorithm (#2002) This patch adds foundations of shrink/rebalance planner. Some extra planning details and proper integration of planning stage into the ggrebalance state machine are going to be considered in separate tickets. The main feature of provided code is an abstract balancing algorithm, which represents manual primary/mirror host assignment following greedy strategy. In short, algorithm structure consists of several phases: 1) Primary assignment. Sort segments by relocation priority: firstly, must-move segments - those lying at decomissioned hosts, encoded in initial_primary as indexes >= n_target_hosts. Then move from overloaded to underloaded hosts. Assign each segment to least-loaded host, preferring original placement when possible. 2) Mirror assignment. Is built according to simple logic: prefer original mirror hosts, use least-loaded mirror hosts. 3) Optional improvement. Using adaptive large neighborhood search, where we try build near solutions by destroying and reassigning parts of the initial one. Quite volatile, but in some cases can bring better solution. Proposed to use in the ggrebalance utility. Reentrancy could be achieved by saving first plan into the database. Unit tests are moved from gppylib into gprebalance_modules in order to achieve better tests granularity and possibility to import separate modules. --- gpMgmt/bin/Makefile | 3 +- gpMgmt/bin/ggrebalance | 84 +- .../data/gprebalance/balanced_grouped.array | 16 - .../gprebalance/balanced_no_mirrors.array | 11 - .../data/gprebalance/balanced_spread.array | 22 - .../unit/data/gprebalance/segments_down.array | 12 - .../data/gprebalance/unbalanced_neg.array | 9 - .../data/gprebalance/unbalanced_pos.array | 8 - .../unbalanced_to_grouped_pos.array | 12 - .../unbalanced_to_spread_neg.array | 29 - .../unbalanced_to_spread_pos.array | 17 - .../gprebalance/unbalanced_unpreferred.array | 12 - .../test/unit/test_unit_gprebalance.py | 191 -- gpMgmt/bin/gprebalance_modules/Makefile | 10 +- gpMgmt/bin/gprebalance_modules/planner.py | 453 ++++ .../gprebalance_modules/rebalance_commons.py | 179 +- .../gprebalance_modules/rebalance_schema.py | 120 + gpMgmt/bin/gprebalance_modules/shrink.py | 8 +- gpMgmt/bin/gprebalance_modules/solver.py | 1526 ++++++++++++ .../bin/gprebalance_modules/test/__init__.py | 0 gpMgmt/bin/gprebalance_modules/test/config.py | 34 + .../test/data/1000_50_balanced_grouped.array | 2052 +++++++++++++++++ .../test/data/1000_50_balanced_spread.array | 2052 +++++++++++++++++ .../data/1000_50_unbalanced_grouped.array | 2052 +++++++++++++++++ .../test/data/1000_50_unbalanced_spread.array | 2052 +++++++++++++++++ .../test/data/120_20_balanced_grouped.array | 262 +++ .../test/data/120_20_balanced_spread.array | 262 +++ .../test/data/120_20_unbalanced_grouped.array | 262 +++ .../test/data/120_20_unbalanced_spread.array | 262 +++ .../test/data/35_7_balanced_grouped.array | 79 + .../test/data/35_7_balanced_spread.array | 79 + .../test/data/40_5_unbalanced_grouped.array | 87 + .../test/data/40_5_unbalanced_spread.array | 87 + .../test/data/40_8_unbalanced_spread.array | 90 + .../test/data/balanced_grouped_6.array | 16 + .../test/data/balanced_spread_24.array | 54 + .../test/data/config_generator.py | 616 +++++ .../test/data/role_mismatch.array | 12 + .../test/data/seg_down.array | 12 + .../test/test_unit_ggrebalance.py | 123 + .../test/test_unit_solver.py | 502 ++++ gpMgmt/test/behave/mgmt_utils/environment.py | 6 +- .../behave/mgmt_utils/ggrebalance.feature | 44 +- 43 files changed, 13321 insertions(+), 498 deletions(-) delete mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_grouped.array delete mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_no_mirrors.array delete mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_spread.array delete mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/segments_down.array delete mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_neg.array delete mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_pos.array delete mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_grouped_pos.array delete mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_neg.array delete mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_pos.array delete mode 100644 gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_unpreferred.array delete mode 100644 gpMgmt/bin/gppylib/test/unit/test_unit_gprebalance.py create mode 100644 gpMgmt/bin/gprebalance_modules/rebalance_schema.py create mode 100644 gpMgmt/bin/gprebalance_modules/solver.py create mode 100644 gpMgmt/bin/gprebalance_modules/test/__init__.py create mode 100644 gpMgmt/bin/gprebalance_modules/test/config.py create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_grouped.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_spread.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_grouped.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_spread.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_grouped.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_spread.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_grouped.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_spread.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_grouped.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_spread.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_grouped.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_spread.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/40_8_unbalanced_spread.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/balanced_grouped_6.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/balanced_spread_24.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/config_generator.py create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/role_mismatch.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/seg_down.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py create mode 100644 gpMgmt/bin/gprebalance_modules/test/test_unit_solver.py diff --git a/gpMgmt/bin/Makefile b/gpMgmt/bin/Makefile index c5ea3abd2e0e..9d2dbc1d7065 100644 --- a/gpMgmt/bin/Makefile +++ b/gpMgmt/bin/Makefile @@ -110,7 +110,8 @@ unitdevel: @echo "Running pure unit tests..." PYTHONPATH=$(SERVER_SRC):$(SERVER_SBIN):$(PYTHONPATH):$(PYTHONSRC_INSTALL_PYTHON_PATH):$(SRC)/ext:$(SBIN_DIR):$(LIB_DIR):$(PYLIB_DIR)/mock-1.0.1 \ python3 -m unittest discover --verbose -s $(SRC)/gppylib -p "test_unit*.py" - + PYTHONPATH=$(SERVER_SRC):$(SERVER_SBIN):$(PYTHONPATH):$(PYTHONSRC_INSTALL_PYTHON_PATH):$(SRC)/ext:$(SBIN_DIR):$(LIB_DIR):$(PYLIB_DIR)/mock-1.0.1 \ + python3 -m unittest discover --verbose -s $(SRC)/gprebalance_modules -t $(SRC) .PHONY: installcheck-bash installcheck-bash: diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index 79a99a423825..9df9b07e644d 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -58,6 +58,20 @@ def parseargs(): help='show this help message and exit.') parser.add_option('-y', '--non-interactive-mode', dest="interactive", action='store_false', default=True, help="non-interactive mode, do not require user input for confirmations, use defaults.") + parser.add_option('-H', '--target-hosts', dest='target_hosts', metavar='', + help='Set of hostnames where rebalance will distribute segments to') + parser.add_option('-A', '--add-hosts', dest='add_hosts', metavar='', + help='Hosts extending the configuration') + parser.add_option('-R', '--remove-hosts', dest='remove_hosts', metavar='', + help='Decomissioned hosts') + parser.add_option('-m', '--mirror-mode', dest='mirror_mode', metavar='', + help='Desirable mirroring strategy after rebalance.') + parser.add_option('--skip-rebalance', dest='skip_rebalance', metavar='', action='store_true', default=False, + help='Perform shrink or expand only.') + parser.add_option('-p', '--show-plan', dest='show_plan', metavar='', action='store_true', default=False, + help='Show rebalance plan') + parser.add_option('-d', '--target-datadirs', dest='target_datadirs', metavar='', + help='segment datadirs on new hosts') parser.set_defaults(verbose=False) @@ -101,6 +115,57 @@ def validate_options(options, args, parser): options.target_segment_count != None): logger.error("Can't use '--target-segment-count' together option '--clean' or '--rollback'") parser.exit(1) + + if options.clean_required or options.rollback_required: + if options.target_hosts: + logger.error("Can't use '--target-hosts' with '--clean' or '--rollback'") + parser.exit(1) + + if options.add_hosts: + logger.error("Can't use '--add-hosts' with '--clean' or '--rollback'") + parser.exit(1) + + if options.remove_hosts: + logger.error("Can't use '--remove-hosts' with '--clean' or '--rollback'") + parser.exit(1) + + if options.mirror_mode: + logger.error("Can't use '--mirror-mode' with '--clean' or '--rollback'") + parser.exit(1) + + if options.skip_rebalance: + logger.error("Can't use '--skip-rebalance' with '--clean' or '--rollback'") + parser.exit(1) + + if options.target_datadirs: + logger.error("Can't use '--target-datadirs' with '--clean' or '--rollback'") + parser.exit(1) + + if options.target_hosts: + if options.add_hosts: + logger.error("Can't use together options '--target-hosts' and '--add-hosts'") + parser.exit(1) + elif options.remove_hosts: + logger.error("Can't use together options '--target-hosts' and '--remove-hosts'") + parser.exit(1) + + if (options.target_hosts or options.add_hosts)\ + and not options.target_datadirs: + logger.error("-target-datadirs options is not provided") + parser.exit(1) + + if not (options.target_hosts or options.add_hosts)\ + and options.target_datadirs: + logger.error("--target-hosts option is not provided") + parser.exit(1) + + if options.mirror_mode and options.mirror_mode not in ('grouped', 'spread'): + logger.error('Mirroring strategy %s is not supported' % + options.mirroring) + parser.exit(1) + elif options.mirror_mode and options.skip_rebalance: + logger.error("Can't use together options '--skip-rebalance' and '--mirror-mode'") + parser.exit(1) return options, args @@ -213,19 +278,20 @@ def main(options, args, parser): # But for now we simply create a Plan object from the options directly. # We'll keep this stub till planner is ready. # If shrink_plan is None, we assume that we need to continue previous operation - shrink_plan = None + plan = None if options.target_segment_count != None: - shrink_plan = Plan() - shrink_plan.setTargetSegmentCount(options.target_segment_count) - - gg_shrink.run(shrink_plan) + plan = Planner(logger, dburl, gparray, options).plan() + + if options.target_segment_count != None and options.show_plan: + logger.info(f"Final plan:\n{plan}") + sys.exit(0) + + if plan is None or isinstance(plan, ShrinkPlan): + gg_shrink.run(plan) sys.exit(0) except Exception as e: - if options and options.verbose: - logger.exception('ggrebalance failed. exiting...') - else: - logger.error(f'ggrebalance failed: {e} \n\nExiting...') + logger.error(f'ggrebalance failed: {e} \n\nExiting...') sys.exit(1) finally: remove_pid_file(get_coordinatordatadir()) diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_grouped.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_grouped.array deleted file mode 100644 index 605e0a6afd16..000000000000 --- a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_grouped.array +++ /dev/null @@ -1,16 +0,0 @@ -# Coordinator -1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator -# SDW1 -2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 -3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 -4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 -5|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 -6|4|m|m|s|u|sdw1|sdw1|50004|/data/mirror4 -7|5|m|m|s|u|sdw1|sdw1|50005|/data/mirror5 -# SDW2 -8|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 -9|4|p|p|s|u|sdw2|sdw2|40004|/data/primary4 -10|5|p|p|s|u|sdw2|sdw2|40005|/data/primary5 -11|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 -12|1|m|m|s|u|sdw2|sdw2|50001|/data/mirror1 -13|2|m|m|s|u|sdw2|sdw2|50002|/data/mirror2 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_no_mirrors.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_no_mirrors.array deleted file mode 100644 index 23b1219c8b16..000000000000 --- a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_no_mirrors.array +++ /dev/null @@ -1,11 +0,0 @@ -1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator -# SDW1 -2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 -3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 -4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 -5|3|p|p|s|u|sdw1|sdw1|40003|/data/primary3 -# SDW2 -6|4|p|p|s|u|sdw2|sdw2|40004|/data/primary4 -7|5|p|p|s|u|sdw2|sdw2|40005|/data/primary5 -8|6|p|p|s|u|sdw2|sdw2|40006|/data/primary6 -9|7|p|p|s|u|sdw2|sdw2|40007|/data/primary7 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_spread.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_spread.array deleted file mode 100644 index 61bb5538c581..000000000000 --- a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/balanced_spread.array +++ /dev/null @@ -1,22 +0,0 @@ -# Coordinator -1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator -# SDW1 -2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 -3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 -4|5|m|m|s|u|sdw1|sdw1|50005|/data/mirror5 -5|6|m|m|s|u|sdw1|sdw1|50006|/data/mirror6 -# SDW2 -6|2|p|p|s|u|sdw2|sdw2|40002|/data/primary2 -7|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 -8|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 -9|7|m|m|s|u|sdw2|sdw2|50007|/data/mirror7 -# SDW3 -10|4|p|p|s|u|sdw3|sdw3|40004|/data/primary4 -11|5|p|p|s|u|sdw3|sdw3|40005|/data/primary5 -12|1|m|m|s|u|sdw3|sdw3|50001|/data/mirror1 -13|2|m|m|s|u|sdw3|sdw3|50002|/data/mirror2 -# SDW4 -14|6|p|p|s|u|sdw4|sdw4|40006|/data/primary6 -15|7|p|p|s|u|sdw4|sdw4|40007|/data/primary7 -16|3|m|m|s|u|sdw4|sdw4|50003|/data/mirror3 -17|4|m|m|s|u|sdw4|sdw4|50004|/data/mirror4 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/segments_down.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/segments_down.array deleted file mode 100644 index 6faa871528ad..000000000000 --- a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/segments_down.array +++ /dev/null @@ -1,12 +0,0 @@ -# Coordinator -1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator -# SDW1 -2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 -3|1|p|p|s|d|sdw1|sdw1|40001|/data/primary1 -4|2|m|m|s|u|sdw1|sdw1|50002|/data/mirror2 -5|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 -# SDW2 -6|2|p|p|s|u|sdw2|sdw2|40002|/data/primary2 -7|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 -8|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 -9|1|m|m|s|u|sdw2|sdw2|50001|/data/mirror1 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_neg.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_neg.array deleted file mode 100644 index 0ef9b91a4e8f..000000000000 --- a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_neg.array +++ /dev/null @@ -1,9 +0,0 @@ -# Coordinator -1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator -# SDW1 (3 primaries) -2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 -3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 -4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 -# SDW2 (2 primary) -5|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 -6|4|p|p|s|u|sdw2|sdw2|40004|/data/primary4 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_pos.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_pos.array deleted file mode 100644 index b6fc095d2984..000000000000 --- a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_pos.array +++ /dev/null @@ -1,8 +0,0 @@ -# Coordinator -1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator -# SDW1 (3 primaries) -2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 -3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 -4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 -# SDW2 (1 primary) -5|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_grouped_pos.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_grouped_pos.array deleted file mode 100644 index 0afdfa70aee7..000000000000 --- a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_grouped_pos.array +++ /dev/null @@ -1,12 +0,0 @@ -# Coordinator -1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator -# SDW1 (3 primaries) -2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 -3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 -4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 -5|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 -# SDW2 (1 primary) -6|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 -7|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 -8|1|m|m|s|u|sdw2|sdw2|50001|/data/mirror1 -9|2|m|m|s|u|sdw2|sdw2|50002|/data/mirror2 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_neg.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_neg.array deleted file mode 100644 index 9710b89543a1..000000000000 --- a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_neg.array +++ /dev/null @@ -1,29 +0,0 @@ -# Coordinator -1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator -# SDW0 (5 primaries + 3 mirrors) -2|0|p|p|s|u|sdw0|sdw0|40000|/data/primary0 -3|1|p|p|s|u|sdw0|sdw0|40001|/data/primary1 -4|2|p|p|s|u|sdw0|sdw0|40002|/data/primary2 -5|3|p|p|s|u|sdw0|sdw0|40003|/data/primary3 -6|4|p|p|s|u|sdw0|sdw0|40004|/data/primary4 -7|8|m|m|s|u|sdw0|sdw0|50008|/data/mirror8 -8|9|m|m|s|u|sdw0|sdw0|50009|/data/mirror9 -9|10|m|m|s|u|sdw0|sdw0|50010|/data/mirror10 -# SDW1 (4 primaries + 5 mirrors) -10|5|p|p|s|u|sdw1|sdw1|40005|/data/primary5 -11|6|p|p|s|u|sdw1|sdw1|40006|/data/primary6 -12|7|p|p|s|u|sdw1|sdw1|40007|/data/primary7 -13|8|p|p|s|u|sdw1|sdw1|40008|/data/primary8 -14|0|m|m|s|u|sdw1|sdw1|50000|/data/mirror0 -15|1|m|m|s|u|sdw1|sdw1|50001|/data/mirror1 -16|2|m|m|s|u|sdw1|sdw1|50002|/data/mirror2 -17|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 -18|4|m|m|s|u|sdw1|sdw1|50004|/data/mirror4 -# SDW2 (3 primaries + 4 mirrors) -19|9|p|p|s|u|sdw2|sdw2|40009|/data/primary9 -20|10|p|p|s|u|sdw2|sdw2|40010|/data/primary10 -21|11|p|p|s|u|sdw2|sdw2|40011|/data/primary11 -22|5|m|m|s|u|sdw2|sdw2|50005|/data/mirror5 -23|6|m|m|s|u|sdw2|sdw2|50006|/data/mirror6 -24|7|m|m|s|u|sdw2|sdw2|50007|/data/mirror7 -25|11|m|m|s|u|sdw2|sdw2|50011|/data/mirror11 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_pos.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_pos.array deleted file mode 100644 index 8edd92d5c9fd..000000000000 --- a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_to_spread_pos.array +++ /dev/null @@ -1,17 +0,0 @@ -# Coordinator -1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator -# SDW0 -2|0|p|p|s|u|sdw0|sdw0|40000|/data/primary0 -3|1|p|p|s|u|sdw0|sdw0|40001|/data/primary1 -4|2|p|p|s|u|sdw0|sdw0|40002|/data/primary2 -5|3|p|p|s|u|sdw0|sdw0|40003|/data/primary3 -6|4|m|m|s|u|sdw0|sdw0|50004|/data/mirror4 -7|5|m|m|s|u|sdw0|sdw0|50005|/data/mirror5 -# SDW1 -8|4|p|p|s|u|sdw1|sdw1|40004|/data/primary4 -9|0|m|m|s|u|sdw1|sdw1|50000|/data/mirror0 -10|1|m|m|s|u|sdw1|sdw1|50001|/data/mirror1 -# SDW2 -11|5|p|p|s|u|sdw2|sdw2|40005|/data/primary5 -12|2|m|m|s|u|sdw2|sdw2|50002|/data/mirror2 -13|3|m|m|s|u|sdw2|sdw2|50003|/data/mirror3 diff --git a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_unpreferred.array b/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_unpreferred.array deleted file mode 100644 index c34ab72f1bbe..000000000000 --- a/gpMgmt/bin/gppylib/test/unit/data/gprebalance/unbalanced_unpreferred.array +++ /dev/null @@ -1,12 +0,0 @@ -# Coordinator -1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator -# SDW1 -2|0|m|p|s|u|sdw1|sdw1|40000|/data/primary0 -3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 -4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 -5|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 -# SDW2 -8|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 -11|0|p|m|s|u|sdw2|sdw2|50000|/data/mirror0 -12|1|m|m|s|u|sdw2|sdw2|50001|/data/mirror1 -13|2|m|m|s|u|sdw2|sdw2|50002|/data/mirror2 diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gprebalance.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gprebalance.py deleted file mode 100644 index 9ef921dbd42f..000000000000 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gprebalance.py +++ /dev/null @@ -1,191 +0,0 @@ -import os -import imp - -from .gp_unittest import * -from mock import * -from gppylib.gparray import Segment, GpArray -from gppylib.db.dbconn import DbURL -from gppylib.db import catalog -from gppylib.gplog import * -from gppylib.system.configurationInterface import GpConfigurationProvider -from gppylib.system.environment import GpCoordinatorEnvironment -from gppylib.db import dbconn -import io -import sys - - -def initGparrayFromFile(basename): - filename = os.path.dirname(__file__) + \ - "/data/gprebalance/" + basename + ".array" - segdbs = [] - with open(filename, 'r') as fp: - for line in fp: - if not line.lstrip().startswith('#'): - segdbs.append(Segment.initFromString(line)) - return GpArray(segdbs, segdbs) - - -class GpTestRebalance(GpTestCase): - def setUp(self): - gprebalance_file = os.path.abspath( - os.path.dirname(__file__) + "/../../../gprebalance") - self.subject = imp.load_source('gprebalance', gprebalance_file) - self.old_sys_argv = sys.argv - sys.argv = [] - self.options, self.args, self.parser = self.subject.parseargs() - self.options.mirroring = 'grouped' - - self.subject.logger = Mock( - spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal', 'exception']) - - self.subject.check_running_gputils = Mock(return_value=False) - - self.apply_patches([ - patch('builtins.open', mock_open(), create=True), - patch('builtins.input'), - patch('gprebalance.dbconn.DbURL', return_value=Mock(), spec=DbURL), - patch('gprebalance.dbconn.connect', return_value=Mock()), - patch('gprebalance.GpCoordinatorEnvironment', - return_value=Mock(), spec=GpCoordinatorEnvironment), - patch('gprebalance.configurationInterface.getConfigurationProvider'), - patch('os.path.exists', return_value=Mock()), - patch('gprebalance.get_default_logger', - return_value=self.subject.logger), - ]) - self.input_mock = self.get_mock_from_apply_patch("input") - self.getConfigProviderFunctionMock = self.get_mock_from_apply_patch( - "getConfigurationProvider") - self.gpCoordinatorEnvironmentMock = self.get_mock_from_apply_patch( - "GpCoordinatorEnvironment") - self.previous_coordinator_data_directory = os.getenv( - 'COORDINATOR_DATA_DIRECTORY', '') - os.environ["COORDINATOR_DATA_DIRECTORY"] = '/tmp/dirdoesnotexist' - configProviderMock = Mock(spec=GpConfigurationProvider) - self.getConfigProviderFunctionMock.return_value = configProviderMock - configProviderMock.initializeProvider.return_value = configProviderMock - self.gpCoordinatorEnvironmentMock.return_value.getCoordinatorPort.return_value = 123456 - - def tearDown(self): - os.environ['COORDINATOR_DATA_DIRECTORY'] = self.previous_coordinator_data_directory - sys.argv = self.old_sys_argv - super(GpTestRebalance, self).tearDown() - - @patch('gprebalance.GpArray.initFromCatalog', - return_value=initGparrayFromFile("balanced_grouped")) - def test_already_balanced_grouped(self, mock1): - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.info.assert_any_call( - "Cluster is already balanced") - - @patch('gprebalance.GpArray.initFromCatalog', - return_value=initGparrayFromFile("balanced_spread")) - def test_already_balanced_spread(self, mock1): - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.info.assert_any_call( - "Cluster is already balanced") - - @patch('gprebalance.GpArray.initFromCatalog', - return_value=initGparrayFromFile("balanced_no_mirrors")) - def test_no_mirrors(self, mock1): - self.input_mock.return_value = "N" - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.error.assert_any_call( - "User Aborted. Exiting...") - self.input_mock.return_value = "Y" - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.info.assert_any_call( - "Cluster is already balanced") - - @patch('gprebalance.GpArray.initFromCatalog', - return_value=initGparrayFromFile("balanced_no_mirrors")) - def test_no_mirrors_allow_mirrorless(self, mock1): - saved = self.options.allow_mirrorless - self.options.allow_mirrorless = True - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.info.assert_any_call( - "Cluster is already balanced") - self.options.allow_mirrorless = saved - - @patch('gprebalance.GpArray.initFromCatalog', - return_value=initGparrayFromFile("segments_down")) - def test_segments_down(self, mock1): - self.input_mock.return_value = "N" - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.error.assert_any_call( - "User Aborted. Exiting...") - self.input_mock.return_value = "Y" - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.info.assert_any_call( - "Cluster is already balanced") - - @patch('gprebalance.GpArray.initFromCatalog', - return_value=initGparrayFromFile("unbalanced_pos")) - def test_unbalanced_pos(self, mock1): - self.input_mock.return_value = "Y" - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.info.assert_any_call( - "Validation passed. Preparing rebalance...") - - @patch('gprebalance.GpArray.initFromCatalog', - return_value=initGparrayFromFile("unbalanced_neg")) - def test_unbalanced_neg(self, mock1): - self.input_mock.return_value = "Y" - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.error.assert_called_with("gprebalance failed: Cannot evenly distribute 5 segments across 2 hosts. " - "\n\nExiting...") - - @patch('gprebalance.GpArray.initFromCatalog', - return_value=initGparrayFromFile("unbalanced_to_grouped_pos")) - def test_unbalanced_to_grouped_pos(self, mock1): - saved = self.options.mirroring - self.options.mirroring = 'grouped' - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.info.assert_any_call( - "Validation passed. Preparing rebalance...") - self.options.mirroring = saved - - @patch('gprebalance.GpArray.initFromCatalog', - return_value=initGparrayFromFile("unbalanced_to_spread_pos")) - def test_unbalanced_to_spread_pos(self, mock1): - saved = self.options.mirroring - self.options.mirroring = 'spread' - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.info.assert_any_call( - "Validation passed. Preparing rebalance...") - self.options.mirroring = saved - - @patch('gprebalance.GpArray.initFromCatalog', - return_value=initGparrayFromFile("unbalanced_to_spread_neg")) - def test_unbalanced_to_spread_neg(self, mock1): - saved = self.options.mirroring - self.options.mirroring = 'spread' - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.error.assert_called_with( - "gprebalance failed: Cannot support spread mirroring strategy on given configuration." - " Use cluster utilities like gpresize or gpexpand to get desired cluster configuration \n\nExiting...") - self.options.mirroring = saved - - @patch('gprebalance.GpArray.initFromCatalog', - return_value=initGparrayFromFile("unbalanced_unpreferred")) - def test_unbalanced_unpreferred(self, mock1): - self.input_mock.return_value = "Y" - with self.assertRaises(SystemExit): - self.subject.main(self.options, self.args, self.parser) - self.subject.logger.info.assert_any_call( - "Validation passed. Preparing rebalance...") - - -if __name__ == '__main__': - run_tests() diff --git a/gpMgmt/bin/gprebalance_modules/Makefile b/gpMgmt/bin/gprebalance_modules/Makefile index 71e2ca3d8ea3..ef363a6ab844 100644 --- a/gpMgmt/bin/gprebalance_modules/Makefile +++ b/gpMgmt/bin/gprebalance_modules/Makefile @@ -3,7 +3,15 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -PROGRAMS= __init__.py rebalance_validator.py rebalance.py rebalance_plan.py rebalance_executor.py rebalance_status.py shrink.py planner.py rebalance_commons.py +PROGRAMS= __init__.py rebalance_validator.py \ +rebalance.py \ +rebalance_plan.py \ +rebalance_executor.py \ +rebalance_status.py \ +shrink.py planner.py \ +rebalance_commons.py \ +rebalance_schema.py \ +solver.py installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)/gprebalance_modules' diff --git a/gpMgmt/bin/gprebalance_modules/planner.py b/gpMgmt/bin/gprebalance_modules/planner.py index f248caf4f9a5..929907996862 100755 --- a/gpMgmt/bin/gprebalance_modules/planner.py +++ b/gpMgmt/bin/gprebalance_modules/planner.py @@ -1,19 +1,472 @@ #!/usr/bin/env python3 +from collections import defaultdict +import os import pickle +import re +from typing import Any, Set, Tuple, List, Dict +from dataclasses import dataclass +from copy import deepcopy + +from gppylib.db import dbconn +import gppylib.gparray as gparray +from gprebalance_modules.rebalance_commons import Host, HostStatus, CandidateSegment +from gprebalance_modules.solver import GreedySolver, HostId, SolverConfig + +class ValidationError(Exception): + pass +class PlanningError(Exception): + pass + +PRIMARY_PATH = '/data/primary/gpseg' +MIRROR_PATH = '/data/mirror/gpseg' + +GROUPED = 'grouped' +SPREAD = 'spread' + +HostMapping = Dict[Host, HostId] + +@dataclass +class LogicalMove: + """ + Move operation assumed by rebalance + + Attributes: + seg: segment info + dst: destination host + target_datadir: target data directory + target_port: target port + """ + seg: CandidateSegment + dstHost: Host + target_datadir: str + target_port: int + + def __str__(self): + """Pretty print logical move""" + seg_info = self.seg.info + + # Source information + src_host = self.seg.host.hostname + src_datadir = seg_info.datadir + src_port = seg_info.port + + # Destination information + dst_host = self.dstHost.hostname + dst_datadir = self.target_datadir + dst_port = self.target_port + + # Size information (if available) + size_str = "" + if self.seg.size: + size_str = str(self.seg.size) + + return ( + f"Move Segment(content={seg_info.content}, dbid={seg_info.dbid}, " + f"role={seg_info.role}){size_str}\n" + f" From: {src_host}:{src_port} → {src_datadir}\n" + f" To: {dst_host}:{dst_port} → {dst_datadir}" + ) + class Plan: def __init__(self): self.target_segment_count = 0 + self.moves = None def getTargetSegmentCount(self) -> int: return self.target_segment_count def setTargetSegmentCount(self, target_segment_count: int) -> None: self.target_segment_count = target_segment_count + + def getMoves(self)-> List[LogicalMove]: + return self.moves + + def setMoves(self, moves: List[LogicalMove]): + self.moves = moves def serializePlan(self) -> bytes: return pickle.dumps(self) + + def __str__(self): + lines = [] + + lines.append(f"\n{'BALANCE MOVES':-^80}") + if self.moves: + lines.append(f"Total moves planned: {len(self.moves)}") + lines.append("") + + for idx, move in enumerate(self.moves, 1): + lines.append(f" [{idx}] {move}") + if idx < len(self.moves): + lines.append("") + else: + lines.append(" No data migration needed.") + + + return "\n".join(lines) def deserializePlan(input: bytes) -> Plan: return pickle.loads(input) + +class ShrinkPlan(Plan): + """ + Shrink plan + """ + def __init__(self, shrinked_segs: List[gparray.SegmentPair]): + Plan.__init__(self) + self.shrinked_segments = shrinked_segs + + def __str__(self): + """ + Pretty print shrink plan with segments to remove and moves to execute + """ + lines = [] + lines.append("=" * 80) + lines.append("SHRINK PLAN".center(80)) + lines.append("=" * 80) + + # Target segment count + lines.append(f"\nTarget Segment Count: {self.target_segment_count}") + + # Shrinked segments section + lines.append(f"\n{'SEGMENTS TO REMOVE':-^80}") + if self.shrinked_segments: + lines.append(f"Total segments to shrink: {len(self.shrinked_segments)}") + lines.append("") + + for idx, seg_pair in enumerate(self.shrinked_segments, 1): + lines.append(f" [{idx}] Segment Pair:") + + # Primary segment + if seg_pair.primaryDB: + lines.append(f" Primary:") + lines.append(f" Content: {seg_pair.primaryDB.content}") + lines.append(f" DbId: {seg_pair.primaryDB.dbid}") + lines.append(f" Host: {seg_pair.primaryDB.hostname}") + lines.append(f" Datadir: {seg_pair.primaryDB.datadir}") + lines.append(f" Port: {seg_pair.primaryDB.port}") + + # Mirror segment + if seg_pair.mirrorDB: + lines.append(f" Mirror:") + lines.append(f" Content: {seg_pair.mirrorDB.content}") + lines.append(f" DbId: {seg_pair.mirrorDB.dbid}") + lines.append(f" Host: {seg_pair.mirrorDB.hostname}") + lines.append(f" Datadir: {seg_pair.mirrorDB.datadir}") + lines.append(f" Port: {seg_pair.mirrorDB.port}") + + if idx < len(self.shrinked_segments): + lines.append("") + else: + lines.append(" No segments to remove.") + + # Moves section + lines.append(super().__str__()) + + lines.append("\n" + "=" * 80) + + return "\n".join(lines) + +class ConfigurationEncoder: + + @staticmethod + def encode_configuration(gparray: gparray.GpArray, + target_hosts: List[Host], + strategy: str) -> Tuple[SolverConfig, HostMapping]: + """ + The rebalance solvers work with abstract input segment configuration. + Each host must have an id in [0, n-1] interval, where n is a size + of full hosts set, including decommissioned and new hosts. We encode + segment placement as a vecor (h0, h1, ..., hj, ... hk), k - number of segments, + hj - host id, j - segment contentid. E.x. H0 = {p0, p1, p2, m3, m4, m5} + H1= {p3, p4, p5. m0, m1, m2}. We want to decommission H1 and add 2 new hosts H2 + and H3. Thus, the configuration will be encoded as: + primaries (0, 0, 0, 3, 3, 3) mirrors (3, 3, 3, 0, 0, 0). Host id equal 3 corresponds + to H1, since we want the decommissioned hosts be out of interval [0, nt-1] nt <= n, + where nt - number of target hosts, which the rebalance should be performed on. + The initial host load can encoded as (6, 0, 0, 6). n = 4, nt = 3. + H0 H2 H3 H1 + After rebalance we want to get the configuration such that the load (4, 4, 4, 0) + and mirroring strategy are achieved. + """ + host_mapping = {} + sorted_hosts = sorted(target_hosts, key=lambda h: h.status) + for i, h in enumerate(sorted_hosts): + host_mapping[h] = i + primary_plcmnt = [0] * gparray.get_primary_count() + mirror_plcmnt = [0] * gparray.get_primary_count() + for pair in gparray.segmentPairs: + primary_plcmnt[pair.primaryDB.content] = host_mapping[Host(hostname=pair.primaryDB.hostname, + address=pair.primaryDB.address)] + mirror_plcmnt[pair.mirrorDB.content] = host_mapping[Host(hostname=pair.mirrorDB.hostname, + address=pair.mirrorDB.address)] + n_initial = len(target_hosts) + n_target = sum([1 for h in target_hosts if h.status != HostStatus.DECOMMISSIONED]) + conf = SolverConfig(gparray.get_primary_count(), + n_target, + n_initial, + primary_plcmnt, + mirror_plcmnt, + strategy) + return (conf, host_mapping) + +class Planner: + """ + Performs the main planning procedure. Decides the operations to be performed + on given configuration. + """ + def __init__(self, logger: Any, dburl: dbconn.DbURL, gpArray: gparray.GpArray, options: Any): + self.dburl = dburl + self.gparray = gpArray + self.options = options + self.logger = logger + self.virtual_gparray = deepcopy(self.gparray) + if options.target_datadirs: + self.dir_template_p, self.dir_template_m = self.get_datadirs() + else: + self.dir_template_p, self.dir_template_m = PRIMARY_PATH, MIRROR_PATH + self.target_hosts, self.host_set_changed = Planner.get_target_hosts(self.virtual_gparray, + options, + self.dir_template_p, + self.dir_template_m) + + def plan(self) -> Plan: + # TODO Remove with future development. Temp code before state machine is implemented. + from gprebalance_modules.rebalance_schema import RebalanceSchema + conn = dbconn.connect(self.dburl, encoding='UTF8') + rebalance_schema = RebalanceSchema(conn) + if rebalance_schema.schemaExists(): + state_from_prev_run = rebalance_schema.getStateFromPreviousRun() + if state_from_prev_run in ['STATE_SHRINK_TABLES_DONE', + 'STATE_SHRINK_CATALOG_STARTED', + 'STATE_SHRINK_CATALOG_DONE', + 'STATE_SHRINK_SEGMENTS_STOP_STARTED', + 'STATE_SHRINK_SEGMENTS_STOP_DONE', + 'STATE_SHRINK_DONE']: + plan = ShrinkPlan([]) + plan.setTargetSegmentCount(self.options.target_segment_count) + conn.close() + return plan + conn.close() + + # Planning starts here + plan = Plan() + + self.validate_segment_status() + if self.options.target_segment_count < self.gparray.get_segment_count(): + plan = self.plan_shrink() + + elif self.options.target_segment_count > self.gparray.get_segment_count(): + raise PlanningError("Expand is not supported yet") + + if self.options.skip_rebalance: + self.logger.info("Skipping rebalance") + return plan + + rebalance_moves = self.form_moves() + + if rebalance_moves is None: + self.logger.info("Cluster is already balanced, no segment moves will be held.") + + plan.setMoves(rebalance_moves) + plan.setTargetSegmentCount(self.options.target_segment_count) + return plan + + def validate_segment_status(self): + if len([1 for pair in self.gparray.segmentPairs\ + if not pair.primaryDB.valid or not pair.mirrorDB.valid]): + raise ValidationError("Some segments in 'down' status. ggrebalance can't proceed further") + + + def plan_shrink(self) -> ShrinkPlan: + self.logger.info("Planning shrink") + shrinkedSegs = [] + for seg_pair in self.gparray.getSegmentList(): + primary_seg = seg_pair.primaryDB + if primary_seg.getSegmentContentId() >= self.options.target_segment_count: + shrinkedSegs.append(seg_pair) + self.remove_segpair_from_array(seg_pair) + plan = ShrinkPlan(shrinkedSegs) + plan.setTargetSegmentCount(self.options.target_segment_count) + return plan + + def remove_segpair_from_array(self, segPair: gparray.SegmentPair): + """ + Removes the segment pair from gparray. + """ + self.virtual_gparray.numPrimarySegments -= 1 + self.virtual_gparray.segmentPairs = list(filter(lambda x: x.primaryDB.dbid != segPair.primaryDB.dbid, + self.virtual_gparray.segmentPairs)) + segs_list = self.virtual_gparray.getSegmentsAsLoadedFromDb() + if segs_list: + segs_list = list(filter(lambda x: x.dbid != segPair.primaryDB.dbid, + segs_list)) + if segPair.mirrorDB: + segs_list = list(filter(lambda x: x.dbid != segPair.mirrorDB.dbid, + segs_list)) + self.virtual_gparray.setSegmentsAsLoadedFromDb(segs_list) + + @staticmethod + def get_target_hosts(array: gparray.GpArray, + options: Any, + dir_template_p: str = PRIMARY_PATH, + dir_template_m: str = MIRROR_PATH) -> Tuple[List[Host], bool]: + """ + Form set of hosts where we need to rebalance segments on + """ + hosts = {} + host_set_changed = False + for seg in array.segmentPairs: + if seg.primaryDB.content >= 0: + if seg.primaryDB.hostname not in hosts: + hosts[seg.primaryDB.hostname] = Host(hostname=seg.primaryDB.hostname,\ + address=seg.primaryDB.address,\ + primary_datadirs={dir_template_p},\ + mirror_datadirs={dir_template_m},\ + status = HostStatus.ACTIVE) + if seg.mirrorDB.hostname not in hosts: + hosts[seg.mirrorDB.hostname] = Host(hostname=seg.mirrorDB.hostname,\ + address=seg.mirrorDB.address,\ + primary_datadirs={dir_template_p},\ + mirror_datadirs={dir_template_m},\ + status = HostStatus.ACTIVE) + for pair in array.segmentPairs: + primary = pair.primaryDB + mirror = pair.mirrorDB + hosts[primary.hostname].primary_datadirs.add( + re.sub(r'\d+$', '',primary.datadir)) + if mirror: + hosts[mirror.hostname].mirror_datadirs.add( + re.sub(r'\d+$', '',mirror.datadir)) + if options.target_hosts: + hl = list(map(str.strip, options.target_hosts.split(','))) + for host in hosts.keys(): + if host not in hl: + hosts[host].status = HostStatus.DECOMMISSIONED + host_set_changed = True + for host in hl: + if host not in hosts: + hosts[host] = Host(hostname=host,\ + address=host,\ + primary_datadirs={dir_template_p},\ + mirror_datadirs={dir_template_m},\ + status = HostStatus.NEW) + host_set_changed = True + if options.add_hosts: + hl = list(map(str.strip, options.add_hosts.split(','))) + for host in hl: + if host not in hosts: + hosts[host] = Host(hostname=host,\ + address=host,\ + primary_datadirs={dir_template_p},\ + mirror_datadirs={dir_template_m},\ + status = HostStatus.NEW) + host_set_changed = True + if options.remove_hosts: + hl = list(map(str.strip, options.remove_hosts.split(','))) + for host in hosts.keys(): + if host in hl: + hosts[host].status = HostStatus.DECOMMISSIONED + host_set_changed = True + + return hosts.values(), host_set_changed + + def get_datadirs(self) -> Tuple[str, str]: + parts = self.options.target_datadirs.split(',') + if len(parts) != 2: + raise ValidationError('--target_datadirs options should have format like ' + '"/data/primary/gpseg{content} , /data/mirror/gpseg{content}"') + # Remove {content} placeholder from each part + cleaned_parts = [] + for part in parts: + # Strip whitespace + part = part.strip() + match = re.search(r'\{.*\}$', part) + if not match: + raise ValidationError('--target_datadirs options should have format like ' + '"/data/primary/gpseg{content} , /data/mirror/gpseg{content}"') + # Remove trailing {...} + part = re.sub(r'\{.*\}$', '', part) + cleaned_parts.append(part) + + return tuple(cleaned_parts) + + def form_moves(self) -> List[LogicalMove]: + self.logger.info("Validation of rebalance possibility") + + if not self.virtual_gparray.hasMirrors: + raise ValidationError("Cluster has mirroring disabled. Can't proceed with rebalance") + + for pair in self.virtual_gparray.segmentPairs: + prim = pair.primaryDB + mir = pair.mirrorDB + if prim.role != prim.preferred_role and mir.role != mir.preferred_role: + raise ValidationError("Current role does not match preferred role for several segments.") + + total_primaries = self.virtual_gparray.get_primary_count() + total_hosts = len([h for h in self.target_hosts\ + if h.status == HostStatus.NEW or h.status == HostStatus.ACTIVE]) + if total_hosts < 2: + raise ValidationError("Cannot perform rebalance at 1 host") + if total_primaries % total_hosts != 0: + raise ValidationError(f"Cannot evenly distribute {total_primaries}" + f" segments across {total_hosts} hosts.") + + expected_per_host = total_primaries // total_hosts + strat = self.options.mirror_mode + + if not strat: + strat = GROUPED + + if strat == SPREAD and expected_per_host > total_hosts - 1: + raise ValidationError("Cannot provide spread mirroring. Specify other " + "mirroring strategy via -m option") + + if not self.host_set_changed\ + and self.already_balanced(expected_per_host): + return None + + # dry movements are planned here + config, host_mapping = ConfigurationEncoder.encode_configuration(self.virtual_gparray, self.target_hosts, strat) + id_to_host = {v: k for k, v in host_mapping.items()} + self.logger.info("Planning rebalance moves. Can take up to 60s.") + planning_seed_value = int.from_bytes(os.urandom(16) , 'big') + self.logger.info(f"Running randomized plan improvement with seed:{planning_seed_value}") + solution, cost = GreedySolver(config, seed=planning_seed_value).solve() + moves = [] + for pair in self.virtual_gparray.segmentPairs: + prim = pair.primaryDB + mir = pair.mirrorDB + plcmnt = solution[prim.content] + # TODO - resource estimation, ports, directories, size planning + if host_mapping[Host(prim.hostname, prim.address)] != plcmnt[0]: + cseg = CandidateSegment(prim, [h for h in self.target_hosts if h == Host(prim.hostname, prim.address)][0], None) + moves.append(LogicalMove(cseg, id_to_host[plcmnt[0]], list(id_to_host[plcmnt[0]].primary_datadirs)[-1] + str(prim.content), 7002)) + if host_mapping[Host(mir.hostname, mir.address)] != plcmnt[1]: + cseg = CandidateSegment(mir, [h for h in self.target_hosts if h == Host(mir.hostname, mir.address)][0], None) + moves.append(LogicalMove(cseg, id_to_host[plcmnt[1]], list(id_to_host[plcmnt[1]].mirror_datadirs)[-1] + str(mir.content), 7003)) + + if len(moves) == 0: + return None + + return moves + + def already_balanced(self, load: int) -> bool: + primaries_by_host = defaultdict(int) + mirrors_by_host = defaultdict(int) + for pair in self.virtual_gparray.segmentPairs: + primaries_by_host[pair.primaryDB.hostname] += 1 + if pair.mirrorDB: + mirrors_by_host[pair.mirrorDB.hostname] += 1 + # This is mirroring strategy violation + if pair.mirrorDB.hostname == pair.primaryDB.hostname: + return False + for n in primaries_by_host.values(): + if n != load: + return False + for n in mirrors_by_host.values(): + if n != load: + return False + return True diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_commons.py b/gpMgmt/bin/gprebalance_modules/rebalance_commons.py index 0775b94160bd..912ff434edcb 100755 --- a/gpMgmt/bin/gprebalance_modules/rebalance_commons.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_commons.py @@ -1,121 +1,62 @@ #!/usr/bin/env python3 -import psycopg2 -from psycopg2.extensions import cursor -from gppylib.db import dbconn -from gprebalance_modules import planner - -def get_table_distr_segment_count(conn: dbconn.Connection, schema_name: str, table_name: str) -> int: - row = dbconn.queryRow(conn, - f'''SELECT p.numsegments - FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid - JOIN gp_distribution_policy p ON c.oid = p.localoid - WHERE n.nspname='{schema_name}' AND c.relname='{table_name}';''') - return int(row[0]) - -class RebalanceSchema: - def __init__(self, conn: dbconn.Connection): - self.schema_name = 'ggrebalance' - self.rebalance_status = 'rebalance_status' - self.table_rebalance_status_detail = 'table_rebalance_status_detail' - self.saved_plan = 'saved_plan' - self.conn = conn - - def createSchema(self, plan: planner.Plan) -> None: - dbconn.execSQL(self.conn, 'BEGIN') - dbconn.execSQL(self.conn, f'CREATE SCHEMA {self.schema_name}') - dbconn.execSQL(self.conn, - f'''CREATE TABLE {self.schema_name}.{self.rebalance_status} - (status TEXT, updated TIMESTAMP WITH TIME ZONE) - DISTRIBUTED REPLICATED''') - dbconn.execSQL(self.conn, - f'''CREATE TABLE {self.schema_name}.{self.table_rebalance_status_detail} - (db_name TEXT, schema_name TEXT, rel_name TEXT, status TEXT, - CONSTRAINT unique_fqn UNIQUE (db_name, schema_name, rel_name)) - DISTRIBUTED REPLICATED''') - dbconn.execSQL(self.conn, - f'''CREATE TABLE {self.schema_name}.{self.saved_plan} - (plan BYTEA) - DISTRIBUTED REPLICATED''') - - self.savePlan(plan) - - dbconn.execSQL(self.conn, 'COMMIT') - - def dropSchema(self) -> None: - dbconn.execSQL(self.conn, f'DROP SCHEMA {self.schema_name} CASCADE') - - def getSchemaName(self) -> str: - return self.schema_name - - def savePlan(self, plan: planner.Plan) -> None: - dbconn.execSQL(self.conn, - f'''INSERT INTO {self.schema_name}.{self.saved_plan} - VALUES ('\\x{plan.serializePlan().hex()}')''') - - def retrieveSavedPlan(self) -> planner.Plan: - if not self.schemaExists(): - return None - - row = dbconn.queryRow(self.conn, f'SELECT count(1) FROM {self.schema_name}.{self.saved_plan}') - plan_count = int(row[0]) - if plan_count > 1: - raise Exception(f'Number of saved plans ({plan_count}) is > 1') - if plan_count == 0: - return None - - row = dbconn.queryRow(self.conn, f'SELECT plan FROM {self.schema_name}.{self.saved_plan} LIMIT 1') - return planner.deserializePlan(row[0]) - - def schemaExists(self) -> bool: - row = dbconn.queryRow(self.conn, f"SELECT COUNT(1) FROM pg_namespace WHERE nspname = '{self.schema_name}'") - return int(row[0]) == 1 - - def getStateFromPreviousRun(self) -> str: - cursor = dbconn.query(self.conn, f'SELECT status FROM {self.schema_name}.{self.rebalance_status} ORDER BY updated DESC LIMIT 1') - if cursor.rowcount > 0: - return str(cursor.fetchone()[0]) - return 'not defined' - - def rebalanceSchema(self, target_segment_count: int) -> None: - # Before rebalancing check if the tables are already rebalanced - # (in case we re-enter after interruption that happened after COMMIT but before new state) - if get_table_distr_segment_count(self.conn, self.schema_name, self.rebalance_status) > target_segment_count: - dbconn.execSQL(self.conn, - f'''ALTER TABLE "{self.schema_name}"."{self.rebalance_status}" - REBALANCE {target_segment_count}''') - - if get_table_distr_segment_count(self.conn, self.schema_name, self.table_rebalance_status_detail) > target_segment_count: - dbconn.execSQL(self.conn, - f'''ALTER TABLE "{self.schema_name}"."{self.table_rebalance_status_detail}" - REBALANCE {target_segment_count}''') - - if get_table_distr_segment_count(self.conn, self.schema_name, self.saved_plan) > target_segment_count: - dbconn.execSQL(self.conn, - f'''ALTER TABLE "{self.schema_name}"."{self.saved_plan}" - REBALANCE {target_segment_count}''') - - def storeState(self, state: str) -> None: - if self.schemaExists(): - dbconn.execSQL(self.conn, - f'''INSERT INTO {self.schema_name}.{self.rebalance_status} - VALUES ('{state}', NOW())''') - - def clearTablesToRebalanceWithStatus(self, status: str) -> None: - dbconn.execSQL(self.conn, - f'''DELETE FROM {self.schema_name}.{self.table_rebalance_status_detail} - WHERE (status = '{status}')''') - - def addTableToRebalance(self, db: str, schema_name: str, rel_name: str, status: str) -> None: - dbconn.execSQL(self.conn, - f'''INSERT INTO {self.schema_name}.{self.table_rebalance_status_detail} - VALUES ('{db}', '{schema_name}', '{rel_name}', '{status}')''') - - def setStatusForTableToRebalance(self, db: str, schema_name: str, rel_name: str, status: str) -> None: - dbconn.execSQL(self.conn, - f'''UPDATE {self.schema_name}.{self.table_rebalance_status_detail} SET status = '{status}' - WHERE db_name = '{db}' AND schema_name = '{schema_name}' AND rel_name = '{rel_name}';''') - - def getTablesToRebalanceWithStatus(self, status: str) -> cursor: - return dbconn.query(self.conn, f"""SELECT db_name, schema_name, rel_name FROM - {self.schema_name}.{self.table_rebalance_status_detail} WHERE status = '{status}'""") +from dataclasses import dataclass +from typing import Set, Optional +from enum import IntEnum +from gppylib.gparray import Segment + +class HostStatus(IntEnum): + ACTIVE = 1 + NEW = 2 + DECOMMISSIONED = 3 + + +@dataclass +class Host: + """ + Segment host representaion + + Attributes: + hostname: hostname from gp_segment_configuration + address: address from gp_segment_configuration + primary_datadirs: set of datadirs which primary catalogs belong to + mirror_datadirs: set of datadirs which mirror catalogs belong to + status: intendend host usage + """ + hostname: str + address: str + primary_datadirs: Set[str] = None + mirror_datadirs: Set[str] = None + status: HostStatus = None + + def __hash__(self): + return hash((self.hostname, self.address)) + + def __eq__(self, other): + if not isinstance(other, Host): + return NotImplemented + return self.hostname == other.hostname and self.address == other.address + + def __str__(self): + pass + +class SegmentSize: + """ + TODO. Should contain storage size of + segment instance with all tablespace info. + """ + pass + +@dataclass +class CandidateSegment: + """ + Accumulates the info of rebalance segments + + Attributes: + info: segment info from gparray + size: size estimation before execution + host: placement host + """ + info: Segment + host: Host + size: Optional[SegmentSize] \ No newline at end of file diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py new file mode 100644 index 000000000000..000ce689dc75 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +from psycopg2.extensions import cursor +from gppylib.db import dbconn +from gprebalance_modules.planner import Plan, deserializePlan + +def get_table_distr_segment_count(conn: dbconn.Connection, schema_name: str, table_name: str) -> int: + row = dbconn.queryRow(conn, + f'''SELECT p.numsegments + FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid + JOIN gp_distribution_policy p ON c.oid = p.localoid + WHERE n.nspname='{schema_name}' AND c.relname='{table_name}';''') + return int(row[0]) + +class RebalanceSchema: + def __init__(self, conn: dbconn.Connection): + self.schema_name = 'ggrebalance' + self.rebalance_status = 'rebalance_status' + self.table_rebalance_status_detail = 'table_rebalance_status_detail' + self.saved_plan = 'saved_plan' + self.conn = conn + + def createSchema(self, plan: Plan) -> None: + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, f'CREATE SCHEMA {self.schema_name}') + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.rebalance_status} + (status TEXT, updated TIMESTAMP WITH TIME ZONE) + DISTRIBUTED REPLICATED''') + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.table_rebalance_status_detail} + (db_name TEXT, schema_name TEXT, rel_name TEXT, status TEXT, + CONSTRAINT unique_fqn UNIQUE (db_name, schema_name, rel_name)) + DISTRIBUTED REPLICATED''') + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.saved_plan} + (plan BYTEA) + DISTRIBUTED REPLICATED''') + + self.savePlan(plan) + + dbconn.execSQL(self.conn, 'COMMIT') + + def dropSchema(self) -> None: + dbconn.execSQL(self.conn, f'DROP SCHEMA {self.schema_name} CASCADE') + + def getSchemaName(self) -> str: + return self.schema_name + + def savePlan(self, plan: Plan) -> None: + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.schema_name}.{self.saved_plan} + VALUES ('\\x{plan.serializePlan().hex()}')''') + + def retrieveSavedPlan(self) -> Plan: + if not self.schemaExists(): + return None + + row = dbconn.queryRow(self.conn, f'SELECT count(1) FROM {self.schema_name}.{self.saved_plan}') + plan_count = int(row[0]) + if plan_count > 1: + raise Exception(f'Number of saved plans ({plan_count}) is > 1') + if plan_count == 0: + return None + + row = dbconn.queryRow(self.conn, f'SELECT plan FROM {self.schema_name}.{self.saved_plan} LIMIT 1') + return deserializePlan(row[0]) + + def schemaExists(self) -> bool: + row = dbconn.queryRow(self.conn, f"SELECT COUNT(1) FROM pg_namespace WHERE nspname = '{self.schema_name}'") + return int(row[0]) == 1 + + def getStateFromPreviousRun(self) -> str: + cursor = dbconn.query(self.conn, f'SELECT status FROM {self.schema_name}.{self.rebalance_status} ORDER BY updated DESC LIMIT 1') + if cursor.rowcount > 0: + return str(cursor.fetchone()[0]) + return 'not defined' + + def rebalanceSchema(self, target_segment_count: int) -> None: + # Before rebalancing check if the tables are already rebalanced + # (in case we re-enter after interruption that happened after COMMIT but before new state) + if get_table_distr_segment_count(self.conn, self.schema_name, self.rebalance_status) > target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.schema_name}"."{self.rebalance_status}" + REBALANCE {target_segment_count}''') + + if get_table_distr_segment_count(self.conn, self.schema_name, self.table_rebalance_status_detail) > target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.schema_name}"."{self.table_rebalance_status_detail}" + REBALANCE {target_segment_count}''') + + if get_table_distr_segment_count(self.conn, self.schema_name, self.saved_plan) > target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.schema_name}"."{self.saved_plan}" + REBALANCE {target_segment_count}''') + + def storeState(self, state: str) -> None: + if self.schemaExists(): + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.schema_name}.{self.rebalance_status} + VALUES ('{state}', NOW())''') + + def clearTablesToRebalanceWithStatus(self, status: str) -> None: + dbconn.execSQL(self.conn, + f'''DELETE FROM {self.schema_name}.{self.table_rebalance_status_detail} + WHERE (status = '{status}')''') + + def addTableToRebalance(self, db: str, schema_name: str, rel_name: str, status: str) -> None: + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.schema_name}.{self.table_rebalance_status_detail} + VALUES ('{db}', '{schema_name}', '{rel_name}', '{status}')''') + + def setStatusForTableToRebalance(self, db: str, schema_name: str, rel_name: str, status: str) -> None: + dbconn.execSQL(self.conn, + f'''UPDATE {self.schema_name}.{self.table_rebalance_status_detail} SET status = '{status}' + WHERE db_name = '{db}' AND schema_name = '{schema_name}' AND rel_name = '{rel_name}';''') + + def getTablesToRebalanceWithStatus(self, status: str) -> cursor: + return dbconn.query(self.conn, f"""SELECT db_name, schema_name, rel_name FROM + {self.schema_name}.{self.table_rebalance_status_detail} WHERE status = '{status}'""") diff --git a/gpMgmt/bin/gprebalance_modules/shrink.py b/gpMgmt/bin/gprebalance_modules/shrink.py index 92f421963e51..da73f5849929 100644 --- a/gpMgmt/bin/gprebalance_modules/shrink.py +++ b/gpMgmt/bin/gprebalance_modules/shrink.py @@ -17,7 +17,7 @@ from gppylib.commands.gp import SEGMENT_STOP_TIMEOUT_DEFAULT, SegmentStop from gppylib.system.environment import * from gprebalance_modules.planner import * - from gprebalance_modules.rebalance_commons import RebalanceSchema + from gprebalance_modules.rebalance_schema import RebalanceSchema except ImportError as e: sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) @@ -265,7 +265,7 @@ def __init__(self, logger: Any, dburl: dbconn.DbURL, options: Any, gpEnv: GpCoor initial = 'STATE_START', before_state_change = 'on_every_state') - def run(self, shrinkPlan: Plan) -> None: + def run(self, shrinkPlan: ShrinkPlan) -> None: self.shrink_plan = shrinkPlan self.trigger('start') @@ -372,8 +372,8 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: self.logger.error("Rebalance schema doesn't exists and no shrink plan is supplied. Please specify shrink plan.") self.trigger('move_to_STATE_ERROR') return - if self.gparray.get_segment_count() <= self.shrink_plan.target_segment_count: - logger.error('Target segment count (%s) >= current segment count (%s).\n' + if self.gparray.get_segment_count() < self.shrink_plan.target_segment_count: + logger.error('Target segment count (%s) > current segment count (%s).\n' 'Currently only shrink is supported (target segment count < current segment count).' % (self.shrink_plan.target_segment_count, self.gparray.get_segment_count())) self.trigger('move_to_STATE_ERROR') diff --git a/gpMgmt/bin/gprebalance_modules/solver.py b/gpMgmt/bin/gprebalance_modules/solver.py new file mode 100644 index 000000000000..f74c93c83b9c --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/solver.py @@ -0,0 +1,1526 @@ +from dataclasses import dataclass +from enum import Enum +import math +import random +from typing import NewType, Optional, Set, Dict, List, Tuple, Union +from collections import defaultdict +import time + +# 0 <= hostid <= n_hosts_initial +HostId = NewType('HostId', int) +# 0 <= contentid <= n_segments +ContentId = NewType('ContentId', int) +# number of moves +Cost = NewType('Cost', int) +# load +Load = NewType('Load', int) +# {'contendid' - > (primary host, mirror host)} +Solution = Dict[ContentId, Tuple[HostId, HostId]] + +@dataclass +class SolverConfig: + """ + Configuration for the balancing problem + Attributes: + n_segments: planned numsegments after shrink/expand + n_hosts_target: target number of hosts + n_hosts_initial: initial number of hosts + initial_primary_mapping: initial_primary_mapping[contentid] = hostid + initial_mirror_mapping: initial_mirror_mapping[contentid] = hostid, 0 <= hostid <= n_hosts_initial + strategy: mirroring strategy + """ + n_segments: int + n_hosts_target: int + n_hosts_initial: int + initial_primary_mapping: List[HostId] + initial_mirror_mapping: List[HostId] + strategy: str + +class GreedySolver: + """ + Greedy construction of a balanced primary/mirror assignment + with an optional ALNS improvement phase. + + Segments: + 0 .. n_segments-1 + + Hosts: + initial hosts: 0 .. n_hosts_initial-1 + target hosts: 0 .. n_hosts_target-1 (subset of initial hosts) + + Constraints: + - Each segment has exactly one primary host and one mirror host. + - Primary host != Mirror host for every segment. + - Each target host has: + target_primary_load primaries + target_primary_load mirrors + => total 2 * target_primary_load segments (primary+mirror) per host. + - `strategy`: + 'grouped': all segments with same primary host share a single mirror host + 'spread' : all segments with same primary host have distinct mirror hosts + """ + def __init__(self, + config: SolverConfig, + run_improve: bool = True, + printing: bool = False, + seed: int = None): + + self.n_segments = config.n_segments + self.n_hosts_target = config.n_hosts_target + self.n_hosts_initial = config.n_hosts_initial + self.initial_primary_mapping = config.initial_primary_mapping + self.initial_mirror_mapping = config.initial_mirror_mapping + self.strategy = config.strategy + + self.run_improve = run_improve + self.printing = printing + self.target_primary_load = self.n_segments // self.n_hosts_target + self.target_load = 2 * self.n_segments // self.n_hosts_target + + if self.n_hosts_target < 2: + raise ValueError("Cannot balance to single host") + + if self.n_segments % self.n_hosts_target != 0: + raise ValueError(f"Cannot evenly distribute {self.n_segments}" + f"segments across {self.n_hosts_target} hosts") + + if self.strategy == 'spread': + if self.target_primary_load > self.n_hosts_target - 1: + raise ValueError("Cannot follow spread mirroring strategy") + + # Best known solution + self.best_primary_mapping: List[HostId] | None = None + self.best_mirror_mapping: List[HostId] | None = None + self.best_cost = None + + # seed to use in ALNS + self.seed = seed + + def solve(self) -> Tuple[Solution, Cost]: + """ + Build an initial greedy solution and optionally improve it with ALNS. + + Returns: + (solution, cost) where solution is a mapping from ContentId to + (primary_host, mirror_host), and cost is number of moves from + the initial placement. + """ + # Phase 1: Balance primaries + primary_mapping = self._balance_primaries() + + # Phase 2: Assign mirrors + mirror_mapping = self._assign_mirror_hosts(primary_mapping) + + # Optional improvement by ALNS. + if self.run_improve: + # at least 10 iterations per segment + config = ALNSConfig(max_iterations=10 * self.n_segments) + alns = ALNS(self, config) + if self.seed: + random.seed(self.seed) + primary_mapping, mirror_mapping = alns.optimize(primary_mapping, mirror_mapping) + + solution: Solution = { + ContentId(i): (HostId(primary_mapping[i]), HostId(mirror_mapping[i])) + for i in range(self.n_segments) + } + + cost = self._calculate_cost(primary_mapping, mirror_mapping) + + assert(self._validate_solution(solution)) + + self.best_primary_mapping = primary_mapping + self.best_mirror_mapping = mirror_mapping + + return solution, cost + + # --------------------------------------------------------------------- # + # Phase 1: Primaries + # --------------------------------------------------------------------- # + def _balance_primaries(self) -> List[HostId]: + """ + Assign primaries to target hosts, keeping: + - all host loads equal to target_primary_load, + - as many primaries as possible on their original host, + - moving out primaries from: + * hosts not in target count, + * overloaded hosts. + """ + primary_mapping = [-1] * self.n_segments + + # Count initial load only on target hosts. + initial_load = [0] * self.n_hosts_initial + for p in self.initial_primary_mapping: + if not self.is_decomissioned(p): + initial_load[p] += 1 + + # Segment processing order: + # must_move first (out-of-target or overloaded), + # then by descending load on their original host (move from heavier first). + segment_order = [] + for i in range(self.n_segments): + orig_host = self.initial_primary_mapping[i] + must_move = ( + self.is_decomissioned(orig_host) + or initial_load[orig_host] > self.target_primary_load + ) + segment_order.append((must_move, initial_load[orig_host], i)) + + segment_order.sort(reverse=True, key=lambda x: (x[0], x[1])) + + # Deficit + # Calculate how many segments each host needs to receive + deficit = [] + for h in range(self.n_hosts_target): + if h < self.n_hosts_initial: + deficit.append(self.target_primary_load - initial_load[h]) + else: + deficit.append(self.target_primary_load) + + # Current primary load for each target host. + current_load = [0] * self.n_hosts_target + + for _, _, seg_id in segment_order: + orig_host = self.initial_primary_mapping[seg_id] + + # Try to keep on original host if possible + if (not self.is_decomissioned(orig_host) and + current_load[orig_host] < self.target_primary_load): + host = orig_host + else: + # Find host with largest remaining deficit + host = max(range(self.n_hosts_target), key=lambda h: deficit[h]) + + primary_mapping[seg_id] = host + current_load[host] += 1 + deficit[host] -= 1 + + return primary_mapping + + def is_decomissioned(self, host: HostId): + return host >= self.n_hosts_target + + # --------------------------------------------------------------------- # + # Phase 2: Mirrors + # --------------------------------------------------------------------- # + def _assign_mirror_hosts(self, primary_mapping: List[HostId]) -> List[HostId]: + """ + Assign mirror hosts for all segments, respecting: + - load balance + - primary host != mirror host + - chosen mirroring strategy (grouped/spread) + - minimizing deviation from initial_mirror_mapping where possible. + """ + mirror_mapping = [-1] * self.n_segments + mirror_load = [0] * self.n_hosts_target + + # Group segments by primary host + groups: Dict[HostId, List[ContentId]] = defaultdict(list) + for seg in range(self.n_segments): + groups[primary_mapping[seg]].append(seg) + + if self.strategy == 'grouped': + self._assign_mirrors_grouped(primary_mapping, mirror_mapping, mirror_load, groups) + elif self.strategy == 'spread': + self._assign_mirrors_spread(primary_mapping, mirror_mapping, mirror_load) + + return mirror_mapping + + def _assign_mirrors_grouped(self, + primary_mapping: List[HostId], + mirror_mapping: List[HostId], + mirror_load: List[Load], + groups: Dict[HostId, List[int]]): + """ + Grouped strategy: + + All segments that share a primary host p_host must share the same + mirror host m_host (per-group mirroring). + """ + # Track mirror choice for each primary host. + phost_to_mhost: Dict[HostId, HostId] = {} + + preferences = self._compute_mirror_preferences(groups) + + sorted_p_hosts = sorted(groups.keys(), + key=lambda p: -max(preferences[p].values(), default=0)) + + for p_host in sorted_p_hosts: + segments = groups[p_host] + best_mirror_host = self._select_group_mirror( + p_host=p_host, + mirror_mapping=mirror_mapping, + mirror_load=mirror_load, + phost_to_mhost=phost_to_mhost, + groups=groups, + preferences=preferences + ) + # Fallback to straightforward assignment (shouldn't happen) + if best_mirror_host is None: + self._fill_naive_grouped(primary_mapping, mirror_mapping) + return + # Assign the group to chosen mirror. + for seg in segments: + mirror_mapping[seg] = best_mirror_host + mirror_load[best_mirror_host] += 1 + + phost_to_mhost[p_host] = best_mirror_host + + def _fill_naive_grouped(self, + primary_mapping: List[HostId], + mirror_mapping: List[HostId]): + """ + GROUPED: Mirrors from same primary host lie at the same host + """ + for content, primary_host in enumerate(primary_mapping): + mirror_mapping[content] = (primary_host + 1) % self.n_hosts_target + + def _fill_naive_spread(self, + primary_mapping: List[HostId], + mirror_mapping: List[HostId]): + """ + SPREAD: Mirrors from same primary host spread across different hosts + """ + segments_per_primary = defaultdict(int) + + for content_id, primary_host in enumerate(primary_mapping): + # Calculate local index within primary host's segments + local_idx = segments_per_primary[primary_host] + segments_per_primary[primary_host] += 1 + + # Formula: mirror = (primary + local_idx + 1) % n_hosts + mirror_host = (primary_host + 1 + local_idx) % self.n_hosts_target + + # Ensure no colocation + if mirror_host == primary_host: + mirror_host = (mirror_host + 1) % self.n_hosts_target + + mirror_mapping[content_id] = mirror_host + + def _compute_mirror_preferences(self, + groups: Dict[HostId, List[ContentId]]) -> Dict[HostId, Dict[HostId, int]]: + """ + Precompute uses (preferences) for each primary host -> possible mirror host. + Returns: {p_host: {m_host: use_count}} + """ + preferences: Dict[HostId, Dict[HostId, int]] = defaultdict(lambda: defaultdict(int)) + for p_host, segments in groups.items(): + for seg in segments: + orig_mirror_host = self.initial_mirror_mapping[seg] + if not self.is_decomissioned(orig_mirror_host) and orig_mirror_host != p_host: + preferences[p_host][orig_mirror_host] += 1 + return preferences + + def _select_group_mirror(self, + p_host: HostId, + mirror_mapping: List[HostId], + mirror_load: List[Load], + phost_to_mhost: Dict[HostId, HostId], + groups: Dict[HostId, List[ContentId]], + preferences: Dict[HostId, Dict[HostId, int]]) -> HostId: + """ + Pick a mirror host for a primary group (grouped strategy), + with the following priority: + 1. Most-used original mirror host. + 2. Least loaded available host with least contention by other primaries. + 3. Swap another already-assigned group with current in + case of conflict when the only mirror host is primary one. + """ + + mirror_uses = preferences[p_host] + segs = groups[p_host] + group_size = len(segs) + assigned_p_hosts = set(phost_to_mhost.keys()) + + best_mirror_host: HostId | None = None + + # Priority 1: Most used original mirror host (if has capacity) + if mirror_uses: + candidates_with_capacity = [] + for h, uses in mirror_uses.items(): + if mirror_load[h] + group_size <= self.target_primary_load: + # Contention: sum of uses from unassigned groups for this host + contention = sum(preferences[other_p].get(h, 0) + for other_p in preferences + if other_p != p_host and other_p not in assigned_p_hosts) + candidates_with_capacity.append((uses, contention, mirror_load[h], h)) + if candidates_with_capacity: + candidates_with_capacity.sort(key=lambda x: (-x[0], x[1], x[2], x[3])) + best_mirror_host = candidates_with_capacity[0][3] + + # Priority 2: Least loaded host + if best_mirror_host is None: + available_hosts = [] + for h in range(self.n_hosts_target): + if h != p_host and mirror_load[h] + group_size <= self.target_primary_load: + contention = sum(preferences[other_p].get(h, 0) + for other_p in preferences + if other_p != p_host and other_p not in assigned_p_hosts) + available_hosts.append((contention, mirror_load[h], h)) + if available_hosts: + best_mirror_host = min(available_hosts)[2] + + # Priority 3: DEADLOCK - Try swapping with already assigned group + if best_mirror_host is None: + best_mirror_host = self._swap_to_resolve_deadlock( + blocked_p_host=p_host, + mirror_load=mirror_load, + phost_to_mhost=phost_to_mhost, + groups=groups, + mirror_mapping=mirror_mapping) + + return best_mirror_host + + def _swap_to_resolve_deadlock(self, + blocked_p_host: HostId, + mirror_load: List[Load], + phost_to_mhost: Dict[HostId, HostId], + groups: Dict[HostId, List[ContentId]], + mirror_mapping: List[HostId]): + """ + Resolve a deadlock in grouped mirroring by moving another + mirror group to a different mirror. + + Idea: + - Look at already assigned primary hosts (other_p_host). + - For each, see if we can move that group to some alternative mirror. + - If that frees enough capacity on its current mirror for blocked_p_host, + we do the move and return the freed mirror as the new mirror for + blocked_p_host. + """ + + for other_p_host, current_mirror_host in phost_to_mhost.items(): + if other_p_host == blocked_p_host: + continue + + other_size = len(groups[other_p_host]) + + # Find alternative mirror where to move other_p_host's mirror group + alternative_mirror_host = next( + (h for h in range(self.n_hosts_target) + if h != other_p_host and # Can't be other's primary + mirror_load[h] + other_size <= self.target_primary_load), + None + ) + + if alternative_mirror_host is None: + continue + + space_after_move = mirror_load[current_mirror_host] - other_size + if space_after_move + len(groups[blocked_p_host]) > self.target_primary_load: + continue + + # Swap: move other_p_host to alternative_mirror + for seg in groups[other_p_host]: + mirror_mapping[seg] = alternative_mirror_host + + mirror_load[current_mirror_host] -= other_size + mirror_load[alternative_mirror_host] += other_size + phost_to_mhost[other_p_host] = alternative_mirror_host + + return current_mirror_host + + return None + + def _assign_mirrors_spread(self, + primary_mapping: List[HostId], + mirror_mapping: List[HostId], + mirror_load: List[Load]): + """ + Spread strategy: + + For each primary host p_host, the mirrors of all segments in that group + must be distinct and not equal to p_host. + """ + + used_in_group: Dict[HostId, Set[HostId]] = defaultdict(set) + + # Phase 1: Try to assign segments to their original mirrors + unassigned: List[ContentId] = [] + + for seg in range(self.n_segments): + p_host = primary_mapping[seg] + orig_mirror_host = self.initial_mirror_mapping[seg] + + # Check if original mirror is valid and available + can_use_original = ( + not self.is_decomissioned(orig_mirror_host) and + orig_mirror_host != p_host and + orig_mirror_host not in used_in_group[p_host] and + mirror_load[orig_mirror_host] < self.target_primary_load + ) + + if can_use_original: + mirror_mapping[seg] = orig_mirror_host + mirror_load[orig_mirror_host] += 1 + used_in_group[p_host].add(orig_mirror_host) + else: + unassigned.append(seg) + + # Phase 2: Assign remaining segments with load balancing + for seg in unassigned: + p_host = primary_mapping[seg] + + available = [ + h for h in range(self.n_hosts_target) + if (h != p_host and + h not in used_in_group[p_host] and + mirror_load[h] < self.target_primary_load) + ] + + if available: + best_host = min(available, key=lambda h: mirror_load[h]) + mirror_mapping[seg] = best_host + mirror_load[best_host] += 1 + used_in_group[p_host].add(best_host) + else: + # Priority 3: DEADLOCK - try swap + best_host = self._resolve_spread_deadlock_for_segment(seg, + primary_mapping, + mirror_mapping, + mirror_load, + used_in_group) + + if best_host is None: + self._fill_naive_spread(primary_mapping, mirror_mapping) + return + + mirror_mapping[seg] = best_host + mirror_load[best_host] += 1 + used_in_group[p_host].add(best_host) + + def _resolve_spread_deadlock_for_segment(self, + seg: ContentId, + primary_mapping: List[HostId], + mirror_mapping: List[HostId], + mirror_load: List[Load], + used_in_group: Dict[HostId, Set[HostId]], + ) -> Optional[HostId]: + """ + Attempt to resolve a deadlock for one segment in the spread strategy + by relocating another segment's mirror to a host with capacity. + """ + p_host = primary_mapping[seg] + + hosts_with_capacity = [ + h for h in range(self.n_hosts_target) + if mirror_load[h] < self.target_primary_load] + + if not hosts_with_capacity: + return None + + # Find hosts we could use (not in our group, at any load level) + candidate_hosts = [ + h for h in range(self.n_hosts_target) + if h != p_host and h not in used_in_group[p_host] + ] + # Try to find a segment using one of these hosts that can move mirror to p_host + for candidate_host in candidate_hosts: + # Find segments currently using candidate_host as mirror + for other_seg in range(self.n_segments): + if mirror_mapping[other_seg] != candidate_host: + continue + + other_p_host = primary_mapping[other_seg] + # Check if other_seg can use p_host as mirror + for dest_host in hosts_with_capacity: + # Check if other_seg can move to dest_host + if dest_host == other_p_host: + continue + if dest_host in used_in_group[other_p_host]: + continue + + # Perform the swap + # Remove other_seg from candidate_host + used_in_group[other_p_host].remove(candidate_host) + mirror_load[candidate_host] -= 1 + + # Move other_seg to dest_host + mirror_mapping[other_seg] = dest_host + used_in_group[other_p_host].add(dest_host) + mirror_load[dest_host] += 1 + + return candidate_host + + return None + + def _calculate_cost(self, primary: List[HostId], mirror: List[HostId]) -> int: + """ + Calculate movement cost + """ + return sum( + (1 if primary[i] != self.initial_primary_mapping[i] else 0) + + (1 if mirror[i] != self.initial_mirror_mapping[i] else 0) + for i in range(self.n_segments) + ) + + def _validate_solution(self, solution: Solution) -> bool: + """ + Validate that solution satisfies all constraints + """ + + # Check 1: All segments assigned + if len(solution) != self.n_segments: + return False + + # Check 2: Primary host != Mirror host + for i, (p, m) in solution.items(): + if p == m: + return False + + # Check 3: Load balance + load = [0] * (self.n_hosts_target) + for i, (p, m) in solution.items(): + load[p] += 1 + load[m] += 1 + + for h in range(self.n_hosts_target): + if load[h] != self.target_primary_load * 2 : + return False + + # Check 4: Strategy constraints + segments_by_host = defaultdict(list) + for i, (p, m) in solution.items(): + segments_by_host[p].append((i, m)) + + for _, segs in segments_by_host.items(): + if len(segs) < 2: + continue + + mirror_hsts = [r for (i, r) in segs] + + if self.strategy == 'grouped': + if len(set(mirror_hsts)) > 1: + return False + elif self.strategy == 'spread': + if len(set(mirror_hsts)) != len(mirror_hsts): + return False + + return True + + +@dataclass +class ALNSConfig: + """ + ALNS algorithm configuration. + """ + max_iterations: int = 1000 + timeout: float = 60.0 + temperature_decay: float = 0.95 + min_temperature: float = 0.01 + local_search_frequency: int = 5 + stagnation_threshold: int = 30 + restart_threshold: int = 20 + + +@dataclass +class SearchState: + """ + Current search state and statistics. + """ + current_primary_mapping: List[HostId] + current_mirror_mapping: List[HostId] + current_cost: Cost + best_primary_mapping: List[HostId] + best_mirror_mapping: List[HostId] + best_cost: Cost + iteration: int = 0 + stagnation_count: int = 0 + last_improvement_iter: int = 0 + + def get_temperature(self, config: ALNSConfig) -> float: + """ + Calculate current temperature. + """ + temp = 1.0 * (config.temperature_decay ** self.iteration) + return max(config.min_temperature, temp) + + def get_destroy_size(self, config: ALNSConfig) -> float: + """ + Calculate adaptive destroy size based on progress. + """ + if self.iteration - self.last_improvement_iter > config.restart_threshold: + return random.uniform(0.20, 0.40) # Stuck: larger neighborhoods + elif self.iteration < config.max_iterations * 0.3: + return random.uniform(0.15, 0.30) # Early phase: explore + else: + return random.uniform(0.10, 0.20) # Late phase: intensify + + def update_best(self, primary_mapping: List[HostId], mirror_mapping: List[HostId], cost: Cost) -> bool: + """ + Update best solution found. + """ + if cost < self.best_cost: + self.best_primary_mapping = primary_mapping[:] + self.best_mirror_mapping = mirror_mapping[:] + self.best_cost = cost + self.last_improvement_iter = self.iteration + return True + return False + + def should_restart(self, config: ALNSConfig) -> bool: + """ + Check if search should restart from best. + """ + return self.stagnation_count > config.stagnation_threshold + + def restart_from_best(self): + """ + Restart search from best solution. + """ + self.current_primary_mapping = self.best_primary_mapping[:] + self.current_mirror_mapping = self.best_mirror_mapping[:] + self.current_cost = self.best_cost + self.stagnation_count = 0 + + +class ALNSDestroyMethod(Enum): + GROUP_DESTROY = 'group_destroy' + BAD_SEGMENTS = 'bad_segments' + SHAW_REMOVAL = 'shaw_removal' + RANDOM_SEGMENTS = 'random_segments' + + +class ALNSDestroyOperators: + """ + Collection of destroy operators for ALNS. + """ + def __init__(self, n_segments: int, strategy: str, + initial_primary: List[HostId], + initial_mirror: List[HostId]): + self.n_segments = n_segments + self.strategy = strategy + self.initial_primary = initial_primary + self.initial_mirror = initial_mirror + + def select_method(self, stagnation: int) -> ALNSDestroyMethod: + """ + Select destroy method based on search state. + """ + if stagnation > 15: + # Stuck: use aggressive methods + return random.choice([ALNSDestroyMethod.GROUP_DESTROY, ALNSDestroyMethod.RANDOM_SEGMENTS]) + + if self.strategy == 'grouped': + # For grouped, favor group-based destroy + return random.choices( + [ALNSDestroyMethod.GROUP_DESTROY, + ALNSDestroyMethod.BAD_SEGMENTS, + ALNSDestroyMethod.SHAW_REMOVAL, + ALNSDestroyMethod.RANDOM_SEGMENTS], + weights=[0.4, 0.3, 0.2, 0.1] + )[0] + else: + # For spread, favor segment-based methods + return random.choices( + [ALNSDestroyMethod.BAD_SEGMENTS, + ALNSDestroyMethod.SHAW_REMOVAL, + ALNSDestroyMethod.RANDOM_SEGMENTS], + weights=[0.4, 0.4, 0.2] + )[0] + + def destroy_random(self, destroy_size: float) -> Set[ContentId]: + """ + Destroy random segments. + """ + n_destroy = max(1, int(self.n_segments * destroy_size)) + return set(random.sample(range(self.n_segments), n_destroy)) + + def destroy_primary_groups(self, primary: List[HostId], mirror: List[HostId], + destroy_size: float) -> Set[ContentId]: + """ + Destroy complete primary groups (for grouped strategy). + """ + groups = defaultdict(list) + for seg in range(self.n_segments): + groups[primary[seg]].append(seg) + + # Score groups by badness + primary_badness = self._calculate_group_badness(groups, primary, mirror) + + # Select groups probabilistically + return self._select_groups_by_badness(groups, primary_badness, destroy_size) + + def destroy_bad_segments(self, primary: List[HostId], mirror: List[HostId], + destroy_size: float) -> Set[ContentId]: + """ + Destroy segments that differ from initial placement. + """ + n_destroy = max(1, int(self.n_segments * destroy_size)) + + # Find segments that moved from original position + bad_segments = [] + for seg in range(self.n_segments): + badness = 0 + if primary[seg] != self.initial_primary[seg]: + badness += 1 + if mirror[seg] != self.initial_mirror[seg]: + badness += 1 + + if badness > 0: + bad_segments.append((badness, seg)) + + if not bad_segments: + return self.destroy_random(destroy_size) + + return self._select_bad_segments_with_relatedness(bad_segments, primary, mirror, n_destroy) + + def shaw_removal(self, primary: List[HostId], mirror: List[HostId], + destroy_size: float) -> Set[ContentId]: + """ + Remove related segments (same primary or mirror host). + """ + n_destroy = max(1, int(self.n_segments * destroy_size)) + + seed_seg = random.randint(0, self.n_segments - 1) + destroyed = {seed_seg} + + # Calculate relatedness scores + relatedness = [] + for seg in range(self.n_segments): + if seg == seed_seg: + continue + + score = 0 + if primary[seg] == primary[seed_seg]: + score += 2 + if mirror[seg] == mirror[seed_seg]: + score += 1 + + relatedness.append((score, seg)) + + # Sort by relatedness and take most related + relatedness.sort(reverse=True) + for _, seg in relatedness[:n_destroy-1]: + destroyed.add(seg) + + return destroyed + + def _calculate_group_badness(self, groups: Dict[HostId, List[ContentId]], + primary: List[HostId], mirror: List[HostId]) -> Dict[HostId, float]: + """ + Calculate badness score for each primary group. + (how many segments deviate from original) + """ + badness = {} + for p_host, segments in groups.items(): + moved_mirrors = sum(1 for seg in segments + if mirror[seg] != self.initial_mirror[seg]) + moved_primaries = sum(1 for seg in segments + if primary[seg] != self.initial_primary[seg]) + + total_moved = moved_mirrors + moved_primaries + badness[p_host] = total_moved / len(segments) if segments else 0 + + return badness + + def _select_groups_by_badness(self, groups: Dict[HostId, List[ContentId]], + badness: Dict[HostId, float], + destroy_size: float) -> Set[ContentId]: + """ + Select primary groups weighted by badness. + """ + n_destroy = max(1, int(self.n_segments * destroy_size)) + primaries = list(groups.keys()) + weights = [badness.get(p, 0.0) + 0.1 for p in primaries] # Add exploration constant 0.1 + + destroyed = set() + available_primaries = primaries[:] + available_weights = weights[:] + + attempts = 0 + while len(destroyed) < n_destroy and attempts < 20 and available_primaries: + p_host = random.choices(available_primaries, weights=available_weights)[0] + destroyed.update(groups[p_host]) + + # Remove from further selection + idx = available_primaries.index(p_host) + available_primaries.pop(idx) + available_weights.pop(idx) + attempts += 1 + + # Trim to size if necessary + if len(destroyed) > n_destroy: + destroyed = set(random.sample(list(destroyed), n_destroy)) + + return destroyed + + def _select_bad_segments_with_relatedness(self, bad_segments: List[Tuple[int, ContentId]], + primary: List[HostId], mirror: List[HostId], + n_destroy: int) -> Set[ContentId]: + """ + Select bad segments and add related ones. + """ + bad_segments.sort(reverse=True) + destroyed = set() + + # Start with worst segments + for _, seg in bad_segments[:min(n_destroy, len(bad_segments))]: + destroyed.add(seg) + + # Add related segments to reach quota + if len(destroyed) < n_destroy: + seed_segments = list(destroyed) + for seed_seg in seed_segments: + if len(destroyed) >= n_destroy: + break + + for seg in range(self.n_segments): + if seg in destroyed or len(destroyed) >= n_destroy: + continue + + if (primary[seg] == primary[seed_seg] or + mirror[seg] == mirror[seed_seg]): + destroyed.add(seg) + + return destroyed + + +class ALNSRepairMethod(Enum): + GREEDY_REPAIR = 'repair_greedy' + CONSTRAINT_REPAIR = 'repair_constrained' + REGRET_REPAIR = "repair_regret" + + +class ALNSRepairOperators: + """Repair operators for ALNS.""" + + def __init__(self, n_segments: int, + n_hosts: int, strategy: str, + initial_primary_mapping: List[HostId], + initial_mirror_mapping: List[HostId], + target_load: Load): + self.n_segments = n_segments + self.n_hosts = n_hosts + self.strategy = strategy + self.initial_primary_mapping = initial_primary_mapping + self.initial_mirror_mapping = initial_mirror_mapping + self.target_load = target_load + self._host_set = set(range(self.n_hosts)) + + def select_method(self, iteration: int, max_iterations: int) -> ALNSRepairMethod: + """ + Select destroy method based on search state. + """ + # Phase 1: Early exploration (first 20% of iterations) + if iteration < max_iterations * 0.2: + return random.choices( + [ALNSRepairMethod.GREEDY_REPAIR, + ALNSRepairMethod.CONSTRAINT_REPAIR], + weights=[0.9, 0.1] + )[0] + + # Phase 2: Quality optimization (middle 60%) + elif iteration < max_iterations * 0.8: + if self.strategy == 'spread': + # Spread is harder - more Most Constrained + return random.choices( + [ALNSRepairMethod.CONSTRAINT_REPAIR, + ALNSRepairMethod.GREEDY_REPAIR, + ALNSRepairMethod.REGRET_REPAIR], + weights=[0.3, 0.5, 0.2] + )[0] + else: + # Grouped is easier - more Greedy/Regret + return random.choices( + [ALNSRepairMethod.GREEDY_REPAIR, + ALNSRepairMethod.REGRET_REPAIR, + ALNSRepairMethod.CONSTRAINT_REPAIR], + weights=[0.4, 0.4, 0.2] + )[0] + + # Phase 3: Final intensification (last 20%) + else: + return random.choices( + [ALNSRepairMethod.REGRET_REPAIR, + ALNSRepairMethod.GREEDY_REPAIR, + ALNSRepairMethod.CONSTRAINT_REPAIR], + weights=[0.5, 0.3, 0.2] + )[0] + + def repair_greedy(self, primary_mapping: List[HostId], mirror_mapping: List[HostId], + destroyed: Set[ContentId]) -> Tuple[List[HostId], List[HostId]]: + """ + Greedy: Assign each segment to cheapest valid option immediately. + O(n * h²) complexity. Fast, reliable. + """ + new_primary_mapping = primary_mapping[:] + new_mirror_mapping = mirror_mapping[:] + + # Clear destroyed segments + for seg in destroyed: + new_primary_mapping[seg] = -1 + new_mirror_mapping[seg] = -1 + + # Initialize load tracker + primary_capacity = [self.target_load] * self.n_hosts + mirror_capacity = [self.target_load] * self.n_hosts + + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_capacity[new_primary_mapping[seg]] -= 1 + mirror_capacity[new_mirror_mapping[seg]] -= 1 + + # Build constraint cache + if self.strategy == 'grouped': + primary_to_mirror = {} + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_to_mirror[new_primary_mapping[seg]] = new_mirror_mapping[seg] + elif self.strategy == 'spread': + primary_to_used_mirrors = defaultdict(set) + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_to_used_mirrors[new_primary_mapping[seg]].add(new_mirror_mapping[seg]) + + # Shuffle to avoid bias + destroyed_list = list(destroyed) + random.shuffle(destroyed_list) + + for seg in destroyed_list: + orig_p = self.initial_primary_mapping[seg] + orig_m = self.initial_mirror_mapping[seg] + best_cost = float('inf') + best_p, best_m = -1, -1 + # Try each primary + for p_host in range(self.n_hosts): + if primary_capacity[p_host] <= 0: + continue + + # Get valid mirrors for this primary + valid_mirrors = self._get_valid_mirrors( + p_host, mirror_capacity, + primary_to_mirror if self.strategy == 'grouped' else primary_to_used_mirrors + ) + if not valid_mirrors: + continue + + # Calculate primary cost + p_cost = 0 if p_host == orig_p else 1 + # Try each valid mirror + for m_host in valid_mirrors: + m_cost = 0 if m_host == orig_m else 1 + total_cost = p_cost + m_cost + if total_cost < best_cost: + best_cost = total_cost + best_p, best_m = p_host, m_host + # Early exit on perfect match + if total_cost == 0: + break + + if best_cost == 0: + break + + # Assign if found + if best_p != -1: + new_primary_mapping[seg] = best_p + new_mirror_mapping[seg] = best_m + primary_capacity[best_p] -= 1 + mirror_capacity[best_m] -= 1 + # Update cache + if self.strategy == 'grouped' and best_p not in primary_to_mirror: + primary_to_mirror[best_p] = best_m + elif self.strategy == 'spread': + primary_to_used_mirrors[best_p].add(best_m) + + return new_primary_mapping, new_mirror_mapping + + def _get_valid_mirrors(self, + primary_host: HostId, + mirror_capacity: List[int], + constraints_cache: Union[Dict[HostId, HostId], Dict[HostId, Set[HostId]]]) -> Set[HostId]: + """ + Fast lookup of valid mirror hosts for a given primary host. + + Args: + primary_host: The primary host we want to assign mirror hostto + mirror_capacity: Remaining capacity on each mirror host + constraints_cache: Either: + - For grouped: Dict[primary -> mirror] mapping + - For spread: Dict[primary -> Set[used_mirrors]] + + Returns: + Set of valid mirror host ids that can be used + """ + + if self.strategy == 'grouped': + # GROUPED STRATEGY: Each primary must use exactly ONE mirror for all segments + + if primary_host in constraints_cache: + # This primary already has an assigned mirror + existing_mirror = constraints_cache[primary_host] + + # Can only use this existing mirror if it has capacity + if mirror_capacity[existing_mirror] > 0: + return {existing_mirror} + else: + return set() # No valid mirrors (existing one is full) + + else: + # New primary: can use any mirror with capacity (except itself) + valid = set() + for m_host in range(self.n_hosts): + if m_host != primary_host and mirror_capacity[m_host] > 0: + valid.add(m_host) + return valid + + elif self.strategy == 'spread': + # SPREAD STRATEGY: Each primary must use DIFFERENT mirrors for each segment + + used_mirrors = constraints_cache.get(primary_host, set()) + + valid = set() + for m_host in range(self.n_hosts): + if (m_host != primary_host and + m_host not in used_mirrors and + mirror_capacity[m_host] > 0): + valid.add(m_host) + + return valid + + def repair_most_constrained(self, primary_mapping: List[HostId], mirror_mapping: List[HostId], + destroyed: Set[ContentId]) -> Tuple[List[HostId], List[HostId]]: + """ + Most Constrained: Assign segments with fewest valid options first. + Useful for sread strategy. + O(n**2 * h**2) complexity. + """ + new_primary_mapping = primary_mapping[:] + new_mirror_mapping = mirror_mapping[:] + + for seg in destroyed: + new_primary_mapping[seg] = -1 + new_mirror_mapping[seg] = -1 + + primary_capacity = [self.target_load] * self.n_hosts + mirror_capacity = [self.target_load] * self.n_hosts + + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_capacity[new_primary_mapping[seg]] -= 1 + mirror_capacity[new_mirror_mapping[seg]] -= 1 + + # Build constraint cache + if self.strategy == 'grouped': + primary_to_mirror = {} + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_to_mirror[new_primary_mapping[seg]] = new_mirror_mapping[seg] + elif self.strategy == 'spread': + primary_to_used_mirrors = defaultdict(set) + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_to_used_mirrors[new_primary_mapping[seg]].add(new_mirror_mapping[seg]) + + unassigned = set(destroyed) + + while unassigned: + # Evaluate ALL unassigned segments + candidates = [] + + for seg in unassigned: + orig_p = self.initial_primary_mapping[seg] + orig_m = self.initial_mirror_mapping[seg] + + # Find ALL valid (primary, mirror) pairs for this segment + valid_options = [] + + for p_host in range(self.n_hosts): + if primary_capacity[p_host] <= 0: + continue + + # Get valid mirrors dynamically + valid_mirrors = self._get_valid_mirrors( + p_host, mirror_capacity, + primary_to_mirror if self.strategy == 'grouped' else primary_to_used_mirrors + ) + + for m_host in valid_mirrors: + # Calculate cost for this option + cost = (0 if p_host == orig_p else 1) + (0 if m_host == orig_m else 1) + valid_options.append((cost, p_host, m_host)) + + if not valid_options: + # No valid options - this segment is problematic + # Add with infinity to handle at end (might fail gracefully) + candidates.append((float('inf'), float('inf'), seg, None)) + else: + # Sort by cost and pick best + valid_options.sort() + best_cost, best_p, best_m = valid_options[0] + option_count = len(valid_options) + + # Store: (option_count, best_cost, segment, best_assignment) + candidates.append((option_count, best_cost, seg, (best_p, best_m))) + + if not candidates: + break + + # Sort: by option_count (ascending), by cost (ascending) + # This ensures we assign the hardest segment first + candidates.sort(key=lambda x: (x[0], x[1])) + + # Process most constrained segment + option_count, cost, seg, assignment = candidates[0] + + if assignment is None: + # No valid assignment possible - skip or handle + unassigned.remove(seg) + continue + + best_p, best_m = assignment + + # Assign + new_primary_mapping[seg] = best_p + new_mirror_mapping[seg] = best_m + primary_capacity[best_p] -= 1 + mirror_capacity[best_m] -= 1 + + # Update cache + if self.strategy == 'grouped' and best_p not in primary_to_mirror: + primary_to_mirror[best_p] = best_m + elif self.strategy == 'spread': + primary_to_used_mirrors[best_p].add(best_m) + + unassigned.remove(seg) + + return new_primary_mapping, new_mirror_mapping + + def repair_regret(self, primary_mapping: List[HostId], mirror_mapping: List[HostId], + destroyed: Set[ContentId]) -> Tuple[List[HostId], List[HostId]]: + """ + Regret: Assign segment with highest regret score (best - 2nd_best) first. + O(n**2 * h**2) complexity. + """ + new_primary_mapping = primary_mapping[:] + new_mirror_mapping = mirror_mapping[:] + + for seg in destroyed: + new_primary_mapping[seg] = -1 + new_mirror_mapping[seg] = -1 + + primary_capacity = [self.target_load] * self.n_hosts + mirror_capacity = [self.target_load] * self.n_hosts + + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_capacity[new_primary_mapping[seg]] -= 1 + mirror_capacity[new_mirror_mapping[seg]] -= 1 + + # Build constraint cache + if self.strategy == 'grouped': + primary_to_mirror = {} + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_to_mirror[new_primary_mapping[seg]] = new_mirror_mapping[seg] + elif self.strategy == 'spread': + primary_to_used_mirrors = defaultdict(set) + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_to_used_mirrors[new_primary_mapping[seg]].add(new_mirror_mapping[seg]) + + unassigned = list(destroyed) + + while unassigned: + best_regret = -1 + best_seg = None + best_assignment = None + + # Calculate regret for each unassigned segment + for seg in unassigned: + orig_p = self.initial_primary_mapping[seg] + orig_m = self.initial_mirror_mapping[seg] + + # Find all valid options + options = [] + + for p_host in range(self.n_hosts): + if primary_capacity[p_host] <= 0: + continue + + valid_mirrors = self._get_valid_mirrors( + p_host, mirror_capacity, + primary_to_mirror if self.strategy == 'grouped' else primary_to_used_mirrors + ) + + for m_host in valid_mirrors: + cost = (0 if p_host == orig_p else 1) + (0 if m_host == orig_m else 1) + options.append((cost, p_host, m_host)) + + if len(options) == 0: + continue # Skip infeasible segments + + # Sort by cost + options.sort() + + # Calculate regret + if len(options) == 1: + # Only one option = infinite regret + regret = float('inf') + else: + # Regret = difference between best and 2nd best + regret = options[1][0] - options[0][0] + + # Track segment with highest regret + if regret > best_regret: + best_regret = regret + best_seg = seg + best_assignment = options[0] # (cost, p, m) + + if best_seg is None: + break + + # Assign the segment with highest regret + _, best_p, best_m = best_assignment + + new_primary_mapping[best_seg] = best_p + new_mirror_mapping[best_seg] = best_m + primary_capacity[best_p] -= 1 + mirror_capacity[best_m] -= 1 + + # Update cache + if self.strategy == 'grouped' and best_p not in primary_to_mirror: + primary_to_mirror[best_p] = best_m + elif self.strategy == 'spread': + primary_to_used_mirrors[best_p].add(best_m) + + unassigned.remove(best_seg) + + return new_primary_mapping, new_mirror_mapping + + +class ALNS: + """ + Adaptive Large Neighborhood Search (ALNS) for refining segment + rebalancing solutions produced by GreedySolver. + + Uses: + - several destroy operators + - several repair operators + - simulated annealing for acceptance + - occasional local search (mirror swaps in grouped) + """ + def __init__(self, + solver: GreedySolver, + config: ALNSConfig = None): + self.solver = solver + self.config = config or ALNSConfig() + + self.solver = solver + self.n_segments = solver.n_segments + self.n_hosts = solver.n_hosts_target + self.strategy = solver.strategy + self.initial_primary_mapping = solver.initial_primary_mapping + self.initial_mirror_mapping = solver.initial_mirror_mapping + self.target_primary_load = solver.target_primary_load + self.printing = solver.printing + + self.start_time = None + + self.destroy_ops = ALNSDestroyOperators(self.n_segments, self.strategy, + self.initial_primary_mapping, self.initial_mirror_mapping) + + self.repair_ops = ALNSRepairOperators( + self.n_segments, self.n_hosts, self.strategy, + self.initial_primary_mapping, self.initial_mirror_mapping, + self.solver.target_primary_load + ) + + def optimize(self, + primary_mapping: List[HostId], + mirror_mapping: List[HostId]) -> Tuple[List[HostId], List[HostId]]: + """ + ALNS. + Uses all destroy/repair strategies with SA acceptance. + """ + self.start_time = time.time() + + initial_cost = self.solver._calculate_cost(primary_mapping, mirror_mapping) + + state = SearchState(current_primary_mapping=primary_mapping[:], + current_mirror_mapping=mirror_mapping[:], + current_cost=initial_cost, + best_primary_mapping=primary_mapping[:], + best_mirror_mapping=mirror_mapping[:], + best_cost=initial_cost) + + if self.solver.printing: + print(f"Initial cost: {state.best_cost}") + + for state.iteration in range(self.config.max_iterations): + if self._timeout_reached(): + if self.printing: + print(f"\n Time limit reached ({self.config.timeout}s) at {state.iteration} iteration") + return state.best_primary_mapping, state.best_mirror_mapping + + new_primary_mapping, new_mirror_mapping, destroy_method, repair_method = self._generate_neighbor(state) + + # Validate + if not self._is_valid(new_primary_mapping, new_mirror_mapping): + state.stagnation_count += 1 + continue + + new_cost = self.solver._calculate_cost(new_primary_mapping, new_mirror_mapping) + + # SA acceptance + if self._accept_move(state.current_cost, new_cost, state.get_temperature(self.config)): + # Accept move + state.current_primary_mapping = new_primary_mapping + state.current_mirror_mapping = new_mirror_mapping + state.current_cost = new_cost + state.stagnation_count = 0 + + if state.update_best(new_primary_mapping, new_mirror_mapping, new_cost): + if self.printing: + print(f"Iter {state.iteration} ({destroy_method.value}) " + f"({repair_method.value}): NEW BEST = {state.best_cost}") + else: + state.stagnation_count += 1 + + # Restart from best if stuck + if state.should_restart(self.config): + if self.printing: + print(f"Iter {state.iteration}: Restarting from best (cost={state.best_cost})") + state.restart_from_best() + + if self.printing: + print(f"Final cost: {state.best_cost}") + + return state.best_primary_mapping, state.best_mirror_mapping + + def _generate_neighbor(self, state: SearchState) -> Tuple[List[HostId], List[HostId], + ALNSDestroyMethod, ALNSRepairMethod]: + """ + Generate a neighbor solution using destroy/repair. + """ + destroy_size = state.get_destroy_size(self.config) + destroy_method = self.destroy_ops.select_method(state.stagnation_count) + + # Apply destroy + destroyed = self._apply_destroy(state.current_primary_mapping, state.current_mirror_mapping, + destroy_method, destroy_size) + + + repair_method = self.repair_ops.select_method(state.iteration, self.config.max_iterations) + + new_primary_mapping, new_mirror_mapping = self._apply_repair(state.current_primary_mapping, + state.current_mirror_mapping, + repair_method, destroyed) + + # Apply local search periodically + if (self.strategy == 'grouped' and + state.stagnation_count < 10 and + state.iteration % self.config.local_search_frequency == 0): + new_primary_mapping, new_mirror_mapping = self._local_mirror_swap(new_primary_mapping, new_mirror_mapping) + + return new_primary_mapping, new_mirror_mapping, destroy_method, repair_method + + def _apply_destroy(self, primary: List[HostId], mirror: List[HostId], + method: ALNSDestroyMethod, destroy_size: float) -> Set[ContentId]: + """ + Apply the selected destroy method. + """ + if method == ALNSDestroyMethod.GROUP_DESTROY and self.strategy == 'grouped': + return self.destroy_ops.destroy_primary_groups(primary, mirror, destroy_size) + elif method == ALNSDestroyMethod.BAD_SEGMENTS: + return self.destroy_ops.destroy_bad_segments(primary, mirror, destroy_size) + elif method == ALNSDestroyMethod.SHAW_REMOVAL: + return self.destroy_ops.shaw_removal(primary, mirror, destroy_size) + else: # RANDOM_SEGMENTS + return self.destroy_ops.destroy_random(destroy_size) + + # REPAIR + def _apply_repair(self, + primary: List[HostId], + mirror: List[HostId], + method: ALNSRepairMethod, + destroyed: Set[ContentId]) -> Tuple[List[HostId], List[HostId]]: + if method == ALNSRepairMethod.GREEDY_REPAIR: + return self.repair_ops.repair_greedy(primary, mirror, destroyed) + elif method == ALNSRepairMethod.CONSTRAINT_REPAIR: + return self.repair_ops.repair_most_constrained(primary, mirror, destroyed) + elif method == ALNSRepairMethod.REGRET_REPAIR: + return self.repair_ops.repair_regret(primary, mirror, destroyed) + + # LOCAL SEARCH + def _local_mirror_swap(self, primary_mapping: List[HostId], + mirror_mapping: List[HostId]) -> Tuple[List[HostId], List[HostId]]: + """ + Local search: swap mirror assignments between two primary groups. + Only for grouped strategy. + """ + if self.strategy != 'grouped': + return primary_mapping, mirror_mapping + + # Build primary -> mirror mapping + groups = defaultdict(list) + ph_to_mh = {} + + for seg in range(self.n_segments): + p = primary_mapping[seg] + m = mirror_mapping[seg] + groups[p].append(seg) + ph_to_mh[p] = m + + current_cost = self.solver._calculate_cost(primary_mapping, mirror_mapping) + best_mirror_mapping = mirror_mapping[:] + best_cost = current_cost + + # Try swapping mirrors between random pairs + primaries = list(groups.keys()) + + if len(primaries) < 2: + return primary_mapping, mirror_mapping + + attempts = min(20, len(primaries) * (len(primaries) - 1) // 2) + + for _ in range(attempts): + p1, p2 = random.sample(primaries, 2) + m1 = ph_to_mh[p1] + m2 = ph_to_mh[p2] + + # Check if swap is valid (doesn't violate primary != mirror) + if m1 == p2 or m2 == p1: + continue + + # Create candidate solution with swapped mirrors + candidate_mirror_mapping = mirror_mapping[:] + for seg in groups[p1]: + candidate_mirror_mapping[seg] = m2 + for seg in groups[p2]: + candidate_mirror_mapping[seg] = m1 + + candidate_cost = self.solver._calculate_cost(primary_mapping, candidate_mirror_mapping) + + if candidate_cost < best_cost: + best_mirror_mapping = candidate_mirror_mapping + best_cost = candidate_cost + + return primary_mapping, best_mirror_mapping + + # UTILITIES + def _accept_move(self, current_cost: Cost, new_cost: Cost, temperature: float) -> bool: + """ + Simulated Annealing acceptance criterion. + """ + if new_cost < current_cost: + return True + elif new_cost == current_cost: + return random.random() < 0.6 + else: + delta = (new_cost - current_cost) / self.n_segments + delta_capped = min(delta, 5.0) + return random.random() < math.exp(-delta_capped / temperature) + + def _is_valid(self, primary_mapping: List[HostId], mirror_mapping: List[HostId]) -> bool: + """ + Validation check. + """ + solution = {i: (primary_mapping[i], mirror_mapping[i]) for i in range(self.n_segments)} + return self.solver._validate_solution(solution) + + def _timeout_reached(self) -> bool: + """Check if time limit exceeded.""" + if self.start_time is None or self.config.timeout is None: + return False + return time.time() - self.start_time > self.config.timeout diff --git a/gpMgmt/bin/gprebalance_modules/test/__init__.py b/gpMgmt/bin/gprebalance_modules/test/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/gpMgmt/bin/gprebalance_modules/test/config.py b/gpMgmt/bin/gprebalance_modules/test/config.py new file mode 100644 index 000000000000..aa658cadb2a9 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/config.py @@ -0,0 +1,34 @@ +import os +import types + +from typing import List, Tuple, Dict +from collections import defaultdict +from gppylib.gparray import Segment, GpArray +from ..planner import ConfigurationEncoder, Planner + +def initGparrayFromFile(basename): + filename = os.path.dirname(__file__) + \ + "/data/" + basename + ".array" + segdbs = [] + with open(filename, 'r') as fp: + for line in fp: + if not line.lstrip().startswith('#'): + segdbs.append(Segment.initFromString(line)) + return GpArray(segdbs, segdbs) + +def getEncoding(file, strat, target_hosts, add_hosts, remove_hosts): + def inner(func): + def wrapper(self, *args, **kwargs): + gparray = initGparrayFromFile(file) + self.encoding = ConfigurationEncoder.encode_configuration(gparray, + Planner.get_target_hosts( + gparray, + types.SimpleNamespace(target_hosts=target_hosts, + add_hosts=add_hosts, + remove_hosts=remove_hosts) )[0], + strat) + res = func(self, *args, **kwargs) + self.encoding = None + return res + return wrapper + return inner diff --git a/gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_grouped.array b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_grouped.array new file mode 100644 index 000000000000..0af454dd30d1 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_grouped.array @@ -0,0 +1,2052 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +8|6|p|p|s|u|sdw1|sdw1|7006|/data/primary6 +9|7|p|p|s|u|sdw1|sdw1|7007|/data/primary7 +10|8|p|p|s|u|sdw1|sdw1|7008|/data/primary8 +11|9|p|p|s|u|sdw1|sdw1|7009|/data/primary9 +12|10|p|p|s|u|sdw1|sdw1|7010|/data/primary10 +13|11|p|p|s|u|sdw1|sdw1|7011|/data/primary11 +14|12|p|p|s|u|sdw1|sdw1|7012|/data/primary12 +15|13|p|p|s|u|sdw1|sdw1|7013|/data/primary13 +16|14|p|p|s|u|sdw1|sdw1|7014|/data/primary14 +17|15|p|p|s|u|sdw1|sdw1|7015|/data/primary15 +18|16|p|p|s|u|sdw1|sdw1|7016|/data/primary16 +19|17|p|p|s|u|sdw1|sdw1|7017|/data/primary17 +20|18|p|p|s|u|sdw1|sdw1|7018|/data/primary18 +21|19|p|p|s|u|sdw1|sdw1|7019|/data/primary19 +1982|980|m|m|s|u|sdw1|sdw1|8980|/data/mirror980 +1983|981|m|m|s|u|sdw1|sdw1|8981|/data/mirror981 +1984|982|m|m|s|u|sdw1|sdw1|8982|/data/mirror982 +1985|983|m|m|s|u|sdw1|sdw1|8983|/data/mirror983 +1986|984|m|m|s|u|sdw1|sdw1|8984|/data/mirror984 +1987|985|m|m|s|u|sdw1|sdw1|8985|/data/mirror985 +1988|986|m|m|s|u|sdw1|sdw1|8986|/data/mirror986 +1989|987|m|m|s|u|sdw1|sdw1|8987|/data/mirror987 +1990|988|m|m|s|u|sdw1|sdw1|8988|/data/mirror988 +1991|989|m|m|s|u|sdw1|sdw1|8989|/data/mirror989 +1992|990|m|m|s|u|sdw1|sdw1|8990|/data/mirror990 +1993|991|m|m|s|u|sdw1|sdw1|8991|/data/mirror991 +1994|992|m|m|s|u|sdw1|sdw1|8992|/data/mirror992 +1995|993|m|m|s|u|sdw1|sdw1|8993|/data/mirror993 +1996|994|m|m|s|u|sdw1|sdw1|8994|/data/mirror994 +1997|995|m|m|s|u|sdw1|sdw1|8995|/data/mirror995 +1998|996|m|m|s|u|sdw1|sdw1|8996|/data/mirror996 +1999|997|m|m|s|u|sdw1|sdw1|8997|/data/mirror997 +2000|998|m|m|s|u|sdw1|sdw1|8998|/data/mirror998 +2001|999|m|m|s|u|sdw1|sdw1|8999|/data/mirror999 +# SDW10 +182|180|p|p|s|u|sdw10|sdw10|7180|/data/primary180 +183|181|p|p|s|u|sdw10|sdw10|7181|/data/primary181 +184|182|p|p|s|u|sdw10|sdw10|7182|/data/primary182 +185|183|p|p|s|u|sdw10|sdw10|7183|/data/primary183 +186|184|p|p|s|u|sdw10|sdw10|7184|/data/primary184 +187|185|p|p|s|u|sdw10|sdw10|7185|/data/primary185 +188|186|p|p|s|u|sdw10|sdw10|7186|/data/primary186 +189|187|p|p|s|u|sdw10|sdw10|7187|/data/primary187 +190|188|p|p|s|u|sdw10|sdw10|7188|/data/primary188 +191|189|p|p|s|u|sdw10|sdw10|7189|/data/primary189 +192|190|p|p|s|u|sdw10|sdw10|7190|/data/primary190 +193|191|p|p|s|u|sdw10|sdw10|7191|/data/primary191 +194|192|p|p|s|u|sdw10|sdw10|7192|/data/primary192 +195|193|p|p|s|u|sdw10|sdw10|7193|/data/primary193 +196|194|p|p|s|u|sdw10|sdw10|7194|/data/primary194 +197|195|p|p|s|u|sdw10|sdw10|7195|/data/primary195 +198|196|p|p|s|u|sdw10|sdw10|7196|/data/primary196 +199|197|p|p|s|u|sdw10|sdw10|7197|/data/primary197 +200|198|p|p|s|u|sdw10|sdw10|7198|/data/primary198 +201|199|p|p|s|u|sdw10|sdw10|7199|/data/primary199 +1162|160|m|m|s|u|sdw10|sdw10|8160|/data/mirror160 +1163|161|m|m|s|u|sdw10|sdw10|8161|/data/mirror161 +1164|162|m|m|s|u|sdw10|sdw10|8162|/data/mirror162 +1165|163|m|m|s|u|sdw10|sdw10|8163|/data/mirror163 +1166|164|m|m|s|u|sdw10|sdw10|8164|/data/mirror164 +1167|165|m|m|s|u|sdw10|sdw10|8165|/data/mirror165 +1168|166|m|m|s|u|sdw10|sdw10|8166|/data/mirror166 +1169|167|m|m|s|u|sdw10|sdw10|8167|/data/mirror167 +1170|168|m|m|s|u|sdw10|sdw10|8168|/data/mirror168 +1171|169|m|m|s|u|sdw10|sdw10|8169|/data/mirror169 +1172|170|m|m|s|u|sdw10|sdw10|8170|/data/mirror170 +1173|171|m|m|s|u|sdw10|sdw10|8171|/data/mirror171 +1174|172|m|m|s|u|sdw10|sdw10|8172|/data/mirror172 +1175|173|m|m|s|u|sdw10|sdw10|8173|/data/mirror173 +1176|174|m|m|s|u|sdw10|sdw10|8174|/data/mirror174 +1177|175|m|m|s|u|sdw10|sdw10|8175|/data/mirror175 +1178|176|m|m|s|u|sdw10|sdw10|8176|/data/mirror176 +1179|177|m|m|s|u|sdw10|sdw10|8177|/data/mirror177 +1180|178|m|m|s|u|sdw10|sdw10|8178|/data/mirror178 +1181|179|m|m|s|u|sdw10|sdw10|8179|/data/mirror179 +# SDW11 +202|200|p|p|s|u|sdw11|sdw11|7200|/data/primary200 +203|201|p|p|s|u|sdw11|sdw11|7201|/data/primary201 +204|202|p|p|s|u|sdw11|sdw11|7202|/data/primary202 +205|203|p|p|s|u|sdw11|sdw11|7203|/data/primary203 +206|204|p|p|s|u|sdw11|sdw11|7204|/data/primary204 +207|205|p|p|s|u|sdw11|sdw11|7205|/data/primary205 +208|206|p|p|s|u|sdw11|sdw11|7206|/data/primary206 +209|207|p|p|s|u|sdw11|sdw11|7207|/data/primary207 +210|208|p|p|s|u|sdw11|sdw11|7208|/data/primary208 +211|209|p|p|s|u|sdw11|sdw11|7209|/data/primary209 +212|210|p|p|s|u|sdw11|sdw11|7210|/data/primary210 +213|211|p|p|s|u|sdw11|sdw11|7211|/data/primary211 +214|212|p|p|s|u|sdw11|sdw11|7212|/data/primary212 +215|213|p|p|s|u|sdw11|sdw11|7213|/data/primary213 +216|214|p|p|s|u|sdw11|sdw11|7214|/data/primary214 +217|215|p|p|s|u|sdw11|sdw11|7215|/data/primary215 +218|216|p|p|s|u|sdw11|sdw11|7216|/data/primary216 +219|217|p|p|s|u|sdw11|sdw11|7217|/data/primary217 +220|218|p|p|s|u|sdw11|sdw11|7218|/data/primary218 +221|219|p|p|s|u|sdw11|sdw11|7219|/data/primary219 +1182|180|m|m|s|u|sdw11|sdw11|8180|/data/mirror180 +1183|181|m|m|s|u|sdw11|sdw11|8181|/data/mirror181 +1184|182|m|m|s|u|sdw11|sdw11|8182|/data/mirror182 +1185|183|m|m|s|u|sdw11|sdw11|8183|/data/mirror183 +1186|184|m|m|s|u|sdw11|sdw11|8184|/data/mirror184 +1187|185|m|m|s|u|sdw11|sdw11|8185|/data/mirror185 +1188|186|m|m|s|u|sdw11|sdw11|8186|/data/mirror186 +1189|187|m|m|s|u|sdw11|sdw11|8187|/data/mirror187 +1190|188|m|m|s|u|sdw11|sdw11|8188|/data/mirror188 +1191|189|m|m|s|u|sdw11|sdw11|8189|/data/mirror189 +1192|190|m|m|s|u|sdw11|sdw11|8190|/data/mirror190 +1193|191|m|m|s|u|sdw11|sdw11|8191|/data/mirror191 +1194|192|m|m|s|u|sdw11|sdw11|8192|/data/mirror192 +1195|193|m|m|s|u|sdw11|sdw11|8193|/data/mirror193 +1196|194|m|m|s|u|sdw11|sdw11|8194|/data/mirror194 +1197|195|m|m|s|u|sdw11|sdw11|8195|/data/mirror195 +1198|196|m|m|s|u|sdw11|sdw11|8196|/data/mirror196 +1199|197|m|m|s|u|sdw11|sdw11|8197|/data/mirror197 +1200|198|m|m|s|u|sdw11|sdw11|8198|/data/mirror198 +1201|199|m|m|s|u|sdw11|sdw11|8199|/data/mirror199 +# SDW12 +222|220|p|p|s|u|sdw12|sdw12|7220|/data/primary220 +223|221|p|p|s|u|sdw12|sdw12|7221|/data/primary221 +224|222|p|p|s|u|sdw12|sdw12|7222|/data/primary222 +225|223|p|p|s|u|sdw12|sdw12|7223|/data/primary223 +226|224|p|p|s|u|sdw12|sdw12|7224|/data/primary224 +227|225|p|p|s|u|sdw12|sdw12|7225|/data/primary225 +228|226|p|p|s|u|sdw12|sdw12|7226|/data/primary226 +229|227|p|p|s|u|sdw12|sdw12|7227|/data/primary227 +230|228|p|p|s|u|sdw12|sdw12|7228|/data/primary228 +231|229|p|p|s|u|sdw12|sdw12|7229|/data/primary229 +232|230|p|p|s|u|sdw12|sdw12|7230|/data/primary230 +233|231|p|p|s|u|sdw12|sdw12|7231|/data/primary231 +234|232|p|p|s|u|sdw12|sdw12|7232|/data/primary232 +235|233|p|p|s|u|sdw12|sdw12|7233|/data/primary233 +236|234|p|p|s|u|sdw12|sdw12|7234|/data/primary234 +237|235|p|p|s|u|sdw12|sdw12|7235|/data/primary235 +238|236|p|p|s|u|sdw12|sdw12|7236|/data/primary236 +239|237|p|p|s|u|sdw12|sdw12|7237|/data/primary237 +240|238|p|p|s|u|sdw12|sdw12|7238|/data/primary238 +241|239|p|p|s|u|sdw12|sdw12|7239|/data/primary239 +1202|200|m|m|s|u|sdw12|sdw12|8200|/data/mirror200 +1203|201|m|m|s|u|sdw12|sdw12|8201|/data/mirror201 +1204|202|m|m|s|u|sdw12|sdw12|8202|/data/mirror202 +1205|203|m|m|s|u|sdw12|sdw12|8203|/data/mirror203 +1206|204|m|m|s|u|sdw12|sdw12|8204|/data/mirror204 +1207|205|m|m|s|u|sdw12|sdw12|8205|/data/mirror205 +1208|206|m|m|s|u|sdw12|sdw12|8206|/data/mirror206 +1209|207|m|m|s|u|sdw12|sdw12|8207|/data/mirror207 +1210|208|m|m|s|u|sdw12|sdw12|8208|/data/mirror208 +1211|209|m|m|s|u|sdw12|sdw12|8209|/data/mirror209 +1212|210|m|m|s|u|sdw12|sdw12|8210|/data/mirror210 +1213|211|m|m|s|u|sdw12|sdw12|8211|/data/mirror211 +1214|212|m|m|s|u|sdw12|sdw12|8212|/data/mirror212 +1215|213|m|m|s|u|sdw12|sdw12|8213|/data/mirror213 +1216|214|m|m|s|u|sdw12|sdw12|8214|/data/mirror214 +1217|215|m|m|s|u|sdw12|sdw12|8215|/data/mirror215 +1218|216|m|m|s|u|sdw12|sdw12|8216|/data/mirror216 +1219|217|m|m|s|u|sdw12|sdw12|8217|/data/mirror217 +1220|218|m|m|s|u|sdw12|sdw12|8218|/data/mirror218 +1221|219|m|m|s|u|sdw12|sdw12|8219|/data/mirror219 +# SDW13 +242|240|p|p|s|u|sdw13|sdw13|7240|/data/primary240 +243|241|p|p|s|u|sdw13|sdw13|7241|/data/primary241 +244|242|p|p|s|u|sdw13|sdw13|7242|/data/primary242 +245|243|p|p|s|u|sdw13|sdw13|7243|/data/primary243 +246|244|p|p|s|u|sdw13|sdw13|7244|/data/primary244 +247|245|p|p|s|u|sdw13|sdw13|7245|/data/primary245 +248|246|p|p|s|u|sdw13|sdw13|7246|/data/primary246 +249|247|p|p|s|u|sdw13|sdw13|7247|/data/primary247 +250|248|p|p|s|u|sdw13|sdw13|7248|/data/primary248 +251|249|p|p|s|u|sdw13|sdw13|7249|/data/primary249 +252|250|p|p|s|u|sdw13|sdw13|7250|/data/primary250 +253|251|p|p|s|u|sdw13|sdw13|7251|/data/primary251 +254|252|p|p|s|u|sdw13|sdw13|7252|/data/primary252 +255|253|p|p|s|u|sdw13|sdw13|7253|/data/primary253 +256|254|p|p|s|u|sdw13|sdw13|7254|/data/primary254 +257|255|p|p|s|u|sdw13|sdw13|7255|/data/primary255 +258|256|p|p|s|u|sdw13|sdw13|7256|/data/primary256 +259|257|p|p|s|u|sdw13|sdw13|7257|/data/primary257 +260|258|p|p|s|u|sdw13|sdw13|7258|/data/primary258 +261|259|p|p|s|u|sdw13|sdw13|7259|/data/primary259 +1222|220|m|m|s|u|sdw13|sdw13|8220|/data/mirror220 +1223|221|m|m|s|u|sdw13|sdw13|8221|/data/mirror221 +1224|222|m|m|s|u|sdw13|sdw13|8222|/data/mirror222 +1225|223|m|m|s|u|sdw13|sdw13|8223|/data/mirror223 +1226|224|m|m|s|u|sdw13|sdw13|8224|/data/mirror224 +1227|225|m|m|s|u|sdw13|sdw13|8225|/data/mirror225 +1228|226|m|m|s|u|sdw13|sdw13|8226|/data/mirror226 +1229|227|m|m|s|u|sdw13|sdw13|8227|/data/mirror227 +1230|228|m|m|s|u|sdw13|sdw13|8228|/data/mirror228 +1231|229|m|m|s|u|sdw13|sdw13|8229|/data/mirror229 +1232|230|m|m|s|u|sdw13|sdw13|8230|/data/mirror230 +1233|231|m|m|s|u|sdw13|sdw13|8231|/data/mirror231 +1234|232|m|m|s|u|sdw13|sdw13|8232|/data/mirror232 +1235|233|m|m|s|u|sdw13|sdw13|8233|/data/mirror233 +1236|234|m|m|s|u|sdw13|sdw13|8234|/data/mirror234 +1237|235|m|m|s|u|sdw13|sdw13|8235|/data/mirror235 +1238|236|m|m|s|u|sdw13|sdw13|8236|/data/mirror236 +1239|237|m|m|s|u|sdw13|sdw13|8237|/data/mirror237 +1240|238|m|m|s|u|sdw13|sdw13|8238|/data/mirror238 +1241|239|m|m|s|u|sdw13|sdw13|8239|/data/mirror239 +# SDW14 +262|260|p|p|s|u|sdw14|sdw14|7260|/data/primary260 +263|261|p|p|s|u|sdw14|sdw14|7261|/data/primary261 +264|262|p|p|s|u|sdw14|sdw14|7262|/data/primary262 +265|263|p|p|s|u|sdw14|sdw14|7263|/data/primary263 +266|264|p|p|s|u|sdw14|sdw14|7264|/data/primary264 +267|265|p|p|s|u|sdw14|sdw14|7265|/data/primary265 +268|266|p|p|s|u|sdw14|sdw14|7266|/data/primary266 +269|267|p|p|s|u|sdw14|sdw14|7267|/data/primary267 +270|268|p|p|s|u|sdw14|sdw14|7268|/data/primary268 +271|269|p|p|s|u|sdw14|sdw14|7269|/data/primary269 +272|270|p|p|s|u|sdw14|sdw14|7270|/data/primary270 +273|271|p|p|s|u|sdw14|sdw14|7271|/data/primary271 +274|272|p|p|s|u|sdw14|sdw14|7272|/data/primary272 +275|273|p|p|s|u|sdw14|sdw14|7273|/data/primary273 +276|274|p|p|s|u|sdw14|sdw14|7274|/data/primary274 +277|275|p|p|s|u|sdw14|sdw14|7275|/data/primary275 +278|276|p|p|s|u|sdw14|sdw14|7276|/data/primary276 +279|277|p|p|s|u|sdw14|sdw14|7277|/data/primary277 +280|278|p|p|s|u|sdw14|sdw14|7278|/data/primary278 +281|279|p|p|s|u|sdw14|sdw14|7279|/data/primary279 +1242|240|m|m|s|u|sdw14|sdw14|8240|/data/mirror240 +1243|241|m|m|s|u|sdw14|sdw14|8241|/data/mirror241 +1244|242|m|m|s|u|sdw14|sdw14|8242|/data/mirror242 +1245|243|m|m|s|u|sdw14|sdw14|8243|/data/mirror243 +1246|244|m|m|s|u|sdw14|sdw14|8244|/data/mirror244 +1247|245|m|m|s|u|sdw14|sdw14|8245|/data/mirror245 +1248|246|m|m|s|u|sdw14|sdw14|8246|/data/mirror246 +1249|247|m|m|s|u|sdw14|sdw14|8247|/data/mirror247 +1250|248|m|m|s|u|sdw14|sdw14|8248|/data/mirror248 +1251|249|m|m|s|u|sdw14|sdw14|8249|/data/mirror249 +1252|250|m|m|s|u|sdw14|sdw14|8250|/data/mirror250 +1253|251|m|m|s|u|sdw14|sdw14|8251|/data/mirror251 +1254|252|m|m|s|u|sdw14|sdw14|8252|/data/mirror252 +1255|253|m|m|s|u|sdw14|sdw14|8253|/data/mirror253 +1256|254|m|m|s|u|sdw14|sdw14|8254|/data/mirror254 +1257|255|m|m|s|u|sdw14|sdw14|8255|/data/mirror255 +1258|256|m|m|s|u|sdw14|sdw14|8256|/data/mirror256 +1259|257|m|m|s|u|sdw14|sdw14|8257|/data/mirror257 +1260|258|m|m|s|u|sdw14|sdw14|8258|/data/mirror258 +1261|259|m|m|s|u|sdw14|sdw14|8259|/data/mirror259 +# SDW15 +282|280|p|p|s|u|sdw15|sdw15|7280|/data/primary280 +283|281|p|p|s|u|sdw15|sdw15|7281|/data/primary281 +284|282|p|p|s|u|sdw15|sdw15|7282|/data/primary282 +285|283|p|p|s|u|sdw15|sdw15|7283|/data/primary283 +286|284|p|p|s|u|sdw15|sdw15|7284|/data/primary284 +287|285|p|p|s|u|sdw15|sdw15|7285|/data/primary285 +288|286|p|p|s|u|sdw15|sdw15|7286|/data/primary286 +289|287|p|p|s|u|sdw15|sdw15|7287|/data/primary287 +290|288|p|p|s|u|sdw15|sdw15|7288|/data/primary288 +291|289|p|p|s|u|sdw15|sdw15|7289|/data/primary289 +292|290|p|p|s|u|sdw15|sdw15|7290|/data/primary290 +293|291|p|p|s|u|sdw15|sdw15|7291|/data/primary291 +294|292|p|p|s|u|sdw15|sdw15|7292|/data/primary292 +295|293|p|p|s|u|sdw15|sdw15|7293|/data/primary293 +296|294|p|p|s|u|sdw15|sdw15|7294|/data/primary294 +297|295|p|p|s|u|sdw15|sdw15|7295|/data/primary295 +298|296|p|p|s|u|sdw15|sdw15|7296|/data/primary296 +299|297|p|p|s|u|sdw15|sdw15|7297|/data/primary297 +300|298|p|p|s|u|sdw15|sdw15|7298|/data/primary298 +301|299|p|p|s|u|sdw15|sdw15|7299|/data/primary299 +1262|260|m|m|s|u|sdw15|sdw15|8260|/data/mirror260 +1263|261|m|m|s|u|sdw15|sdw15|8261|/data/mirror261 +1264|262|m|m|s|u|sdw15|sdw15|8262|/data/mirror262 +1265|263|m|m|s|u|sdw15|sdw15|8263|/data/mirror263 +1266|264|m|m|s|u|sdw15|sdw15|8264|/data/mirror264 +1267|265|m|m|s|u|sdw15|sdw15|8265|/data/mirror265 +1268|266|m|m|s|u|sdw15|sdw15|8266|/data/mirror266 +1269|267|m|m|s|u|sdw15|sdw15|8267|/data/mirror267 +1270|268|m|m|s|u|sdw15|sdw15|8268|/data/mirror268 +1271|269|m|m|s|u|sdw15|sdw15|8269|/data/mirror269 +1272|270|m|m|s|u|sdw15|sdw15|8270|/data/mirror270 +1273|271|m|m|s|u|sdw15|sdw15|8271|/data/mirror271 +1274|272|m|m|s|u|sdw15|sdw15|8272|/data/mirror272 +1275|273|m|m|s|u|sdw15|sdw15|8273|/data/mirror273 +1276|274|m|m|s|u|sdw15|sdw15|8274|/data/mirror274 +1277|275|m|m|s|u|sdw15|sdw15|8275|/data/mirror275 +1278|276|m|m|s|u|sdw15|sdw15|8276|/data/mirror276 +1279|277|m|m|s|u|sdw15|sdw15|8277|/data/mirror277 +1280|278|m|m|s|u|sdw15|sdw15|8278|/data/mirror278 +1281|279|m|m|s|u|sdw15|sdw15|8279|/data/mirror279 +# SDW16 +302|300|p|p|s|u|sdw16|sdw16|7300|/data/primary300 +303|301|p|p|s|u|sdw16|sdw16|7301|/data/primary301 +304|302|p|p|s|u|sdw16|sdw16|7302|/data/primary302 +305|303|p|p|s|u|sdw16|sdw16|7303|/data/primary303 +306|304|p|p|s|u|sdw16|sdw16|7304|/data/primary304 +307|305|p|p|s|u|sdw16|sdw16|7305|/data/primary305 +308|306|p|p|s|u|sdw16|sdw16|7306|/data/primary306 +309|307|p|p|s|u|sdw16|sdw16|7307|/data/primary307 +310|308|p|p|s|u|sdw16|sdw16|7308|/data/primary308 +311|309|p|p|s|u|sdw16|sdw16|7309|/data/primary309 +312|310|p|p|s|u|sdw16|sdw16|7310|/data/primary310 +313|311|p|p|s|u|sdw16|sdw16|7311|/data/primary311 +314|312|p|p|s|u|sdw16|sdw16|7312|/data/primary312 +315|313|p|p|s|u|sdw16|sdw16|7313|/data/primary313 +316|314|p|p|s|u|sdw16|sdw16|7314|/data/primary314 +317|315|p|p|s|u|sdw16|sdw16|7315|/data/primary315 +318|316|p|p|s|u|sdw16|sdw16|7316|/data/primary316 +319|317|p|p|s|u|sdw16|sdw16|7317|/data/primary317 +320|318|p|p|s|u|sdw16|sdw16|7318|/data/primary318 +321|319|p|p|s|u|sdw16|sdw16|7319|/data/primary319 +1282|280|m|m|s|u|sdw16|sdw16|8280|/data/mirror280 +1283|281|m|m|s|u|sdw16|sdw16|8281|/data/mirror281 +1284|282|m|m|s|u|sdw16|sdw16|8282|/data/mirror282 +1285|283|m|m|s|u|sdw16|sdw16|8283|/data/mirror283 +1286|284|m|m|s|u|sdw16|sdw16|8284|/data/mirror284 +1287|285|m|m|s|u|sdw16|sdw16|8285|/data/mirror285 +1288|286|m|m|s|u|sdw16|sdw16|8286|/data/mirror286 +1289|287|m|m|s|u|sdw16|sdw16|8287|/data/mirror287 +1290|288|m|m|s|u|sdw16|sdw16|8288|/data/mirror288 +1291|289|m|m|s|u|sdw16|sdw16|8289|/data/mirror289 +1292|290|m|m|s|u|sdw16|sdw16|8290|/data/mirror290 +1293|291|m|m|s|u|sdw16|sdw16|8291|/data/mirror291 +1294|292|m|m|s|u|sdw16|sdw16|8292|/data/mirror292 +1295|293|m|m|s|u|sdw16|sdw16|8293|/data/mirror293 +1296|294|m|m|s|u|sdw16|sdw16|8294|/data/mirror294 +1297|295|m|m|s|u|sdw16|sdw16|8295|/data/mirror295 +1298|296|m|m|s|u|sdw16|sdw16|8296|/data/mirror296 +1299|297|m|m|s|u|sdw16|sdw16|8297|/data/mirror297 +1300|298|m|m|s|u|sdw16|sdw16|8298|/data/mirror298 +1301|299|m|m|s|u|sdw16|sdw16|8299|/data/mirror299 +# SDW17 +322|320|p|p|s|u|sdw17|sdw17|7320|/data/primary320 +323|321|p|p|s|u|sdw17|sdw17|7321|/data/primary321 +324|322|p|p|s|u|sdw17|sdw17|7322|/data/primary322 +325|323|p|p|s|u|sdw17|sdw17|7323|/data/primary323 +326|324|p|p|s|u|sdw17|sdw17|7324|/data/primary324 +327|325|p|p|s|u|sdw17|sdw17|7325|/data/primary325 +328|326|p|p|s|u|sdw17|sdw17|7326|/data/primary326 +329|327|p|p|s|u|sdw17|sdw17|7327|/data/primary327 +330|328|p|p|s|u|sdw17|sdw17|7328|/data/primary328 +331|329|p|p|s|u|sdw17|sdw17|7329|/data/primary329 +332|330|p|p|s|u|sdw17|sdw17|7330|/data/primary330 +333|331|p|p|s|u|sdw17|sdw17|7331|/data/primary331 +334|332|p|p|s|u|sdw17|sdw17|7332|/data/primary332 +335|333|p|p|s|u|sdw17|sdw17|7333|/data/primary333 +336|334|p|p|s|u|sdw17|sdw17|7334|/data/primary334 +337|335|p|p|s|u|sdw17|sdw17|7335|/data/primary335 +338|336|p|p|s|u|sdw17|sdw17|7336|/data/primary336 +339|337|p|p|s|u|sdw17|sdw17|7337|/data/primary337 +340|338|p|p|s|u|sdw17|sdw17|7338|/data/primary338 +341|339|p|p|s|u|sdw17|sdw17|7339|/data/primary339 +1302|300|m|m|s|u|sdw17|sdw17|8300|/data/mirror300 +1303|301|m|m|s|u|sdw17|sdw17|8301|/data/mirror301 +1304|302|m|m|s|u|sdw17|sdw17|8302|/data/mirror302 +1305|303|m|m|s|u|sdw17|sdw17|8303|/data/mirror303 +1306|304|m|m|s|u|sdw17|sdw17|8304|/data/mirror304 +1307|305|m|m|s|u|sdw17|sdw17|8305|/data/mirror305 +1308|306|m|m|s|u|sdw17|sdw17|8306|/data/mirror306 +1309|307|m|m|s|u|sdw17|sdw17|8307|/data/mirror307 +1310|308|m|m|s|u|sdw17|sdw17|8308|/data/mirror308 +1311|309|m|m|s|u|sdw17|sdw17|8309|/data/mirror309 +1312|310|m|m|s|u|sdw17|sdw17|8310|/data/mirror310 +1313|311|m|m|s|u|sdw17|sdw17|8311|/data/mirror311 +1314|312|m|m|s|u|sdw17|sdw17|8312|/data/mirror312 +1315|313|m|m|s|u|sdw17|sdw17|8313|/data/mirror313 +1316|314|m|m|s|u|sdw17|sdw17|8314|/data/mirror314 +1317|315|m|m|s|u|sdw17|sdw17|8315|/data/mirror315 +1318|316|m|m|s|u|sdw17|sdw17|8316|/data/mirror316 +1319|317|m|m|s|u|sdw17|sdw17|8317|/data/mirror317 +1320|318|m|m|s|u|sdw17|sdw17|8318|/data/mirror318 +1321|319|m|m|s|u|sdw17|sdw17|8319|/data/mirror319 +# SDW18 +342|340|p|p|s|u|sdw18|sdw18|7340|/data/primary340 +343|341|p|p|s|u|sdw18|sdw18|7341|/data/primary341 +344|342|p|p|s|u|sdw18|sdw18|7342|/data/primary342 +345|343|p|p|s|u|sdw18|sdw18|7343|/data/primary343 +346|344|p|p|s|u|sdw18|sdw18|7344|/data/primary344 +347|345|p|p|s|u|sdw18|sdw18|7345|/data/primary345 +348|346|p|p|s|u|sdw18|sdw18|7346|/data/primary346 +349|347|p|p|s|u|sdw18|sdw18|7347|/data/primary347 +350|348|p|p|s|u|sdw18|sdw18|7348|/data/primary348 +351|349|p|p|s|u|sdw18|sdw18|7349|/data/primary349 +352|350|p|p|s|u|sdw18|sdw18|7350|/data/primary350 +353|351|p|p|s|u|sdw18|sdw18|7351|/data/primary351 +354|352|p|p|s|u|sdw18|sdw18|7352|/data/primary352 +355|353|p|p|s|u|sdw18|sdw18|7353|/data/primary353 +356|354|p|p|s|u|sdw18|sdw18|7354|/data/primary354 +357|355|p|p|s|u|sdw18|sdw18|7355|/data/primary355 +358|356|p|p|s|u|sdw18|sdw18|7356|/data/primary356 +359|357|p|p|s|u|sdw18|sdw18|7357|/data/primary357 +360|358|p|p|s|u|sdw18|sdw18|7358|/data/primary358 +361|359|p|p|s|u|sdw18|sdw18|7359|/data/primary359 +1322|320|m|m|s|u|sdw18|sdw18|8320|/data/mirror320 +1323|321|m|m|s|u|sdw18|sdw18|8321|/data/mirror321 +1324|322|m|m|s|u|sdw18|sdw18|8322|/data/mirror322 +1325|323|m|m|s|u|sdw18|sdw18|8323|/data/mirror323 +1326|324|m|m|s|u|sdw18|sdw18|8324|/data/mirror324 +1327|325|m|m|s|u|sdw18|sdw18|8325|/data/mirror325 +1328|326|m|m|s|u|sdw18|sdw18|8326|/data/mirror326 +1329|327|m|m|s|u|sdw18|sdw18|8327|/data/mirror327 +1330|328|m|m|s|u|sdw18|sdw18|8328|/data/mirror328 +1331|329|m|m|s|u|sdw18|sdw18|8329|/data/mirror329 +1332|330|m|m|s|u|sdw18|sdw18|8330|/data/mirror330 +1333|331|m|m|s|u|sdw18|sdw18|8331|/data/mirror331 +1334|332|m|m|s|u|sdw18|sdw18|8332|/data/mirror332 +1335|333|m|m|s|u|sdw18|sdw18|8333|/data/mirror333 +1336|334|m|m|s|u|sdw18|sdw18|8334|/data/mirror334 +1337|335|m|m|s|u|sdw18|sdw18|8335|/data/mirror335 +1338|336|m|m|s|u|sdw18|sdw18|8336|/data/mirror336 +1339|337|m|m|s|u|sdw18|sdw18|8337|/data/mirror337 +1340|338|m|m|s|u|sdw18|sdw18|8338|/data/mirror338 +1341|339|m|m|s|u|sdw18|sdw18|8339|/data/mirror339 +# SDW19 +362|360|p|p|s|u|sdw19|sdw19|7360|/data/primary360 +363|361|p|p|s|u|sdw19|sdw19|7361|/data/primary361 +364|362|p|p|s|u|sdw19|sdw19|7362|/data/primary362 +365|363|p|p|s|u|sdw19|sdw19|7363|/data/primary363 +366|364|p|p|s|u|sdw19|sdw19|7364|/data/primary364 +367|365|p|p|s|u|sdw19|sdw19|7365|/data/primary365 +368|366|p|p|s|u|sdw19|sdw19|7366|/data/primary366 +369|367|p|p|s|u|sdw19|sdw19|7367|/data/primary367 +370|368|p|p|s|u|sdw19|sdw19|7368|/data/primary368 +371|369|p|p|s|u|sdw19|sdw19|7369|/data/primary369 +372|370|p|p|s|u|sdw19|sdw19|7370|/data/primary370 +373|371|p|p|s|u|sdw19|sdw19|7371|/data/primary371 +374|372|p|p|s|u|sdw19|sdw19|7372|/data/primary372 +375|373|p|p|s|u|sdw19|sdw19|7373|/data/primary373 +376|374|p|p|s|u|sdw19|sdw19|7374|/data/primary374 +377|375|p|p|s|u|sdw19|sdw19|7375|/data/primary375 +378|376|p|p|s|u|sdw19|sdw19|7376|/data/primary376 +379|377|p|p|s|u|sdw19|sdw19|7377|/data/primary377 +380|378|p|p|s|u|sdw19|sdw19|7378|/data/primary378 +381|379|p|p|s|u|sdw19|sdw19|7379|/data/primary379 +1342|340|m|m|s|u|sdw19|sdw19|8340|/data/mirror340 +1343|341|m|m|s|u|sdw19|sdw19|8341|/data/mirror341 +1344|342|m|m|s|u|sdw19|sdw19|8342|/data/mirror342 +1345|343|m|m|s|u|sdw19|sdw19|8343|/data/mirror343 +1346|344|m|m|s|u|sdw19|sdw19|8344|/data/mirror344 +1347|345|m|m|s|u|sdw19|sdw19|8345|/data/mirror345 +1348|346|m|m|s|u|sdw19|sdw19|8346|/data/mirror346 +1349|347|m|m|s|u|sdw19|sdw19|8347|/data/mirror347 +1350|348|m|m|s|u|sdw19|sdw19|8348|/data/mirror348 +1351|349|m|m|s|u|sdw19|sdw19|8349|/data/mirror349 +1352|350|m|m|s|u|sdw19|sdw19|8350|/data/mirror350 +1353|351|m|m|s|u|sdw19|sdw19|8351|/data/mirror351 +1354|352|m|m|s|u|sdw19|sdw19|8352|/data/mirror352 +1355|353|m|m|s|u|sdw19|sdw19|8353|/data/mirror353 +1356|354|m|m|s|u|sdw19|sdw19|8354|/data/mirror354 +1357|355|m|m|s|u|sdw19|sdw19|8355|/data/mirror355 +1358|356|m|m|s|u|sdw19|sdw19|8356|/data/mirror356 +1359|357|m|m|s|u|sdw19|sdw19|8357|/data/mirror357 +1360|358|m|m|s|u|sdw19|sdw19|8358|/data/mirror358 +1361|359|m|m|s|u|sdw19|sdw19|8359|/data/mirror359 +# SDW2 +22|20|p|p|s|u|sdw2|sdw2|7020|/data/primary20 +23|21|p|p|s|u|sdw2|sdw2|7021|/data/primary21 +24|22|p|p|s|u|sdw2|sdw2|7022|/data/primary22 +25|23|p|p|s|u|sdw2|sdw2|7023|/data/primary23 +26|24|p|p|s|u|sdw2|sdw2|7024|/data/primary24 +27|25|p|p|s|u|sdw2|sdw2|7025|/data/primary25 +28|26|p|p|s|u|sdw2|sdw2|7026|/data/primary26 +29|27|p|p|s|u|sdw2|sdw2|7027|/data/primary27 +30|28|p|p|s|u|sdw2|sdw2|7028|/data/primary28 +31|29|p|p|s|u|sdw2|sdw2|7029|/data/primary29 +32|30|p|p|s|u|sdw2|sdw2|7030|/data/primary30 +33|31|p|p|s|u|sdw2|sdw2|7031|/data/primary31 +34|32|p|p|s|u|sdw2|sdw2|7032|/data/primary32 +35|33|p|p|s|u|sdw2|sdw2|7033|/data/primary33 +36|34|p|p|s|u|sdw2|sdw2|7034|/data/primary34 +37|35|p|p|s|u|sdw2|sdw2|7035|/data/primary35 +38|36|p|p|s|u|sdw2|sdw2|7036|/data/primary36 +39|37|p|p|s|u|sdw2|sdw2|7037|/data/primary37 +40|38|p|p|s|u|sdw2|sdw2|7038|/data/primary38 +41|39|p|p|s|u|sdw2|sdw2|7039|/data/primary39 +1002|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +1003|1|m|m|s|u|sdw2|sdw2|8001|/data/mirror1 +1004|2|m|m|s|u|sdw2|sdw2|8002|/data/mirror2 +1005|3|m|m|s|u|sdw2|sdw2|8003|/data/mirror3 +1006|4|m|m|s|u|sdw2|sdw2|8004|/data/mirror4 +1007|5|m|m|s|u|sdw2|sdw2|8005|/data/mirror5 +1008|6|m|m|s|u|sdw2|sdw2|8006|/data/mirror6 +1009|7|m|m|s|u|sdw2|sdw2|8007|/data/mirror7 +1010|8|m|m|s|u|sdw2|sdw2|8008|/data/mirror8 +1011|9|m|m|s|u|sdw2|sdw2|8009|/data/mirror9 +1012|10|m|m|s|u|sdw2|sdw2|8010|/data/mirror10 +1013|11|m|m|s|u|sdw2|sdw2|8011|/data/mirror11 +1014|12|m|m|s|u|sdw2|sdw2|8012|/data/mirror12 +1015|13|m|m|s|u|sdw2|sdw2|8013|/data/mirror13 +1016|14|m|m|s|u|sdw2|sdw2|8014|/data/mirror14 +1017|15|m|m|s|u|sdw2|sdw2|8015|/data/mirror15 +1018|16|m|m|s|u|sdw2|sdw2|8016|/data/mirror16 +1019|17|m|m|s|u|sdw2|sdw2|8017|/data/mirror17 +1020|18|m|m|s|u|sdw2|sdw2|8018|/data/mirror18 +1021|19|m|m|s|u|sdw2|sdw2|8019|/data/mirror19 +# SDW20 +382|380|p|p|s|u|sdw20|sdw20|7380|/data/primary380 +383|381|p|p|s|u|sdw20|sdw20|7381|/data/primary381 +384|382|p|p|s|u|sdw20|sdw20|7382|/data/primary382 +385|383|p|p|s|u|sdw20|sdw20|7383|/data/primary383 +386|384|p|p|s|u|sdw20|sdw20|7384|/data/primary384 +387|385|p|p|s|u|sdw20|sdw20|7385|/data/primary385 +388|386|p|p|s|u|sdw20|sdw20|7386|/data/primary386 +389|387|p|p|s|u|sdw20|sdw20|7387|/data/primary387 +390|388|p|p|s|u|sdw20|sdw20|7388|/data/primary388 +391|389|p|p|s|u|sdw20|sdw20|7389|/data/primary389 +392|390|p|p|s|u|sdw20|sdw20|7390|/data/primary390 +393|391|p|p|s|u|sdw20|sdw20|7391|/data/primary391 +394|392|p|p|s|u|sdw20|sdw20|7392|/data/primary392 +395|393|p|p|s|u|sdw20|sdw20|7393|/data/primary393 +396|394|p|p|s|u|sdw20|sdw20|7394|/data/primary394 +397|395|p|p|s|u|sdw20|sdw20|7395|/data/primary395 +398|396|p|p|s|u|sdw20|sdw20|7396|/data/primary396 +399|397|p|p|s|u|sdw20|sdw20|7397|/data/primary397 +400|398|p|p|s|u|sdw20|sdw20|7398|/data/primary398 +401|399|p|p|s|u|sdw20|sdw20|7399|/data/primary399 +1362|360|m|m|s|u|sdw20|sdw20|8360|/data/mirror360 +1363|361|m|m|s|u|sdw20|sdw20|8361|/data/mirror361 +1364|362|m|m|s|u|sdw20|sdw20|8362|/data/mirror362 +1365|363|m|m|s|u|sdw20|sdw20|8363|/data/mirror363 +1366|364|m|m|s|u|sdw20|sdw20|8364|/data/mirror364 +1367|365|m|m|s|u|sdw20|sdw20|8365|/data/mirror365 +1368|366|m|m|s|u|sdw20|sdw20|8366|/data/mirror366 +1369|367|m|m|s|u|sdw20|sdw20|8367|/data/mirror367 +1370|368|m|m|s|u|sdw20|sdw20|8368|/data/mirror368 +1371|369|m|m|s|u|sdw20|sdw20|8369|/data/mirror369 +1372|370|m|m|s|u|sdw20|sdw20|8370|/data/mirror370 +1373|371|m|m|s|u|sdw20|sdw20|8371|/data/mirror371 +1374|372|m|m|s|u|sdw20|sdw20|8372|/data/mirror372 +1375|373|m|m|s|u|sdw20|sdw20|8373|/data/mirror373 +1376|374|m|m|s|u|sdw20|sdw20|8374|/data/mirror374 +1377|375|m|m|s|u|sdw20|sdw20|8375|/data/mirror375 +1378|376|m|m|s|u|sdw20|sdw20|8376|/data/mirror376 +1379|377|m|m|s|u|sdw20|sdw20|8377|/data/mirror377 +1380|378|m|m|s|u|sdw20|sdw20|8378|/data/mirror378 +1381|379|m|m|s|u|sdw20|sdw20|8379|/data/mirror379 +# SDW21 +402|400|p|p|s|u|sdw21|sdw21|7400|/data/primary400 +403|401|p|p|s|u|sdw21|sdw21|7401|/data/primary401 +404|402|p|p|s|u|sdw21|sdw21|7402|/data/primary402 +405|403|p|p|s|u|sdw21|sdw21|7403|/data/primary403 +406|404|p|p|s|u|sdw21|sdw21|7404|/data/primary404 +407|405|p|p|s|u|sdw21|sdw21|7405|/data/primary405 +408|406|p|p|s|u|sdw21|sdw21|7406|/data/primary406 +409|407|p|p|s|u|sdw21|sdw21|7407|/data/primary407 +410|408|p|p|s|u|sdw21|sdw21|7408|/data/primary408 +411|409|p|p|s|u|sdw21|sdw21|7409|/data/primary409 +412|410|p|p|s|u|sdw21|sdw21|7410|/data/primary410 +413|411|p|p|s|u|sdw21|sdw21|7411|/data/primary411 +414|412|p|p|s|u|sdw21|sdw21|7412|/data/primary412 +415|413|p|p|s|u|sdw21|sdw21|7413|/data/primary413 +416|414|p|p|s|u|sdw21|sdw21|7414|/data/primary414 +417|415|p|p|s|u|sdw21|sdw21|7415|/data/primary415 +418|416|p|p|s|u|sdw21|sdw21|7416|/data/primary416 +419|417|p|p|s|u|sdw21|sdw21|7417|/data/primary417 +420|418|p|p|s|u|sdw21|sdw21|7418|/data/primary418 +421|419|p|p|s|u|sdw21|sdw21|7419|/data/primary419 +1382|380|m|m|s|u|sdw21|sdw21|8380|/data/mirror380 +1383|381|m|m|s|u|sdw21|sdw21|8381|/data/mirror381 +1384|382|m|m|s|u|sdw21|sdw21|8382|/data/mirror382 +1385|383|m|m|s|u|sdw21|sdw21|8383|/data/mirror383 +1386|384|m|m|s|u|sdw21|sdw21|8384|/data/mirror384 +1387|385|m|m|s|u|sdw21|sdw21|8385|/data/mirror385 +1388|386|m|m|s|u|sdw21|sdw21|8386|/data/mirror386 +1389|387|m|m|s|u|sdw21|sdw21|8387|/data/mirror387 +1390|388|m|m|s|u|sdw21|sdw21|8388|/data/mirror388 +1391|389|m|m|s|u|sdw21|sdw21|8389|/data/mirror389 +1392|390|m|m|s|u|sdw21|sdw21|8390|/data/mirror390 +1393|391|m|m|s|u|sdw21|sdw21|8391|/data/mirror391 +1394|392|m|m|s|u|sdw21|sdw21|8392|/data/mirror392 +1395|393|m|m|s|u|sdw21|sdw21|8393|/data/mirror393 +1396|394|m|m|s|u|sdw21|sdw21|8394|/data/mirror394 +1397|395|m|m|s|u|sdw21|sdw21|8395|/data/mirror395 +1398|396|m|m|s|u|sdw21|sdw21|8396|/data/mirror396 +1399|397|m|m|s|u|sdw21|sdw21|8397|/data/mirror397 +1400|398|m|m|s|u|sdw21|sdw21|8398|/data/mirror398 +1401|399|m|m|s|u|sdw21|sdw21|8399|/data/mirror399 +# SDW22 +422|420|p|p|s|u|sdw22|sdw22|7420|/data/primary420 +423|421|p|p|s|u|sdw22|sdw22|7421|/data/primary421 +424|422|p|p|s|u|sdw22|sdw22|7422|/data/primary422 +425|423|p|p|s|u|sdw22|sdw22|7423|/data/primary423 +426|424|p|p|s|u|sdw22|sdw22|7424|/data/primary424 +427|425|p|p|s|u|sdw22|sdw22|7425|/data/primary425 +428|426|p|p|s|u|sdw22|sdw22|7426|/data/primary426 +429|427|p|p|s|u|sdw22|sdw22|7427|/data/primary427 +430|428|p|p|s|u|sdw22|sdw22|7428|/data/primary428 +431|429|p|p|s|u|sdw22|sdw22|7429|/data/primary429 +432|430|p|p|s|u|sdw22|sdw22|7430|/data/primary430 +433|431|p|p|s|u|sdw22|sdw22|7431|/data/primary431 +434|432|p|p|s|u|sdw22|sdw22|7432|/data/primary432 +435|433|p|p|s|u|sdw22|sdw22|7433|/data/primary433 +436|434|p|p|s|u|sdw22|sdw22|7434|/data/primary434 +437|435|p|p|s|u|sdw22|sdw22|7435|/data/primary435 +438|436|p|p|s|u|sdw22|sdw22|7436|/data/primary436 +439|437|p|p|s|u|sdw22|sdw22|7437|/data/primary437 +440|438|p|p|s|u|sdw22|sdw22|7438|/data/primary438 +441|439|p|p|s|u|sdw22|sdw22|7439|/data/primary439 +1402|400|m|m|s|u|sdw22|sdw22|8400|/data/mirror400 +1403|401|m|m|s|u|sdw22|sdw22|8401|/data/mirror401 +1404|402|m|m|s|u|sdw22|sdw22|8402|/data/mirror402 +1405|403|m|m|s|u|sdw22|sdw22|8403|/data/mirror403 +1406|404|m|m|s|u|sdw22|sdw22|8404|/data/mirror404 +1407|405|m|m|s|u|sdw22|sdw22|8405|/data/mirror405 +1408|406|m|m|s|u|sdw22|sdw22|8406|/data/mirror406 +1409|407|m|m|s|u|sdw22|sdw22|8407|/data/mirror407 +1410|408|m|m|s|u|sdw22|sdw22|8408|/data/mirror408 +1411|409|m|m|s|u|sdw22|sdw22|8409|/data/mirror409 +1412|410|m|m|s|u|sdw22|sdw22|8410|/data/mirror410 +1413|411|m|m|s|u|sdw22|sdw22|8411|/data/mirror411 +1414|412|m|m|s|u|sdw22|sdw22|8412|/data/mirror412 +1415|413|m|m|s|u|sdw22|sdw22|8413|/data/mirror413 +1416|414|m|m|s|u|sdw22|sdw22|8414|/data/mirror414 +1417|415|m|m|s|u|sdw22|sdw22|8415|/data/mirror415 +1418|416|m|m|s|u|sdw22|sdw22|8416|/data/mirror416 +1419|417|m|m|s|u|sdw22|sdw22|8417|/data/mirror417 +1420|418|m|m|s|u|sdw22|sdw22|8418|/data/mirror418 +1421|419|m|m|s|u|sdw22|sdw22|8419|/data/mirror419 +# SDW23 +442|440|p|p|s|u|sdw23|sdw23|7440|/data/primary440 +443|441|p|p|s|u|sdw23|sdw23|7441|/data/primary441 +444|442|p|p|s|u|sdw23|sdw23|7442|/data/primary442 +445|443|p|p|s|u|sdw23|sdw23|7443|/data/primary443 +446|444|p|p|s|u|sdw23|sdw23|7444|/data/primary444 +447|445|p|p|s|u|sdw23|sdw23|7445|/data/primary445 +448|446|p|p|s|u|sdw23|sdw23|7446|/data/primary446 +449|447|p|p|s|u|sdw23|sdw23|7447|/data/primary447 +450|448|p|p|s|u|sdw23|sdw23|7448|/data/primary448 +451|449|p|p|s|u|sdw23|sdw23|7449|/data/primary449 +452|450|p|p|s|u|sdw23|sdw23|7450|/data/primary450 +453|451|p|p|s|u|sdw23|sdw23|7451|/data/primary451 +454|452|p|p|s|u|sdw23|sdw23|7452|/data/primary452 +455|453|p|p|s|u|sdw23|sdw23|7453|/data/primary453 +456|454|p|p|s|u|sdw23|sdw23|7454|/data/primary454 +457|455|p|p|s|u|sdw23|sdw23|7455|/data/primary455 +458|456|p|p|s|u|sdw23|sdw23|7456|/data/primary456 +459|457|p|p|s|u|sdw23|sdw23|7457|/data/primary457 +460|458|p|p|s|u|sdw23|sdw23|7458|/data/primary458 +461|459|p|p|s|u|sdw23|sdw23|7459|/data/primary459 +1422|420|m|m|s|u|sdw23|sdw23|8420|/data/mirror420 +1423|421|m|m|s|u|sdw23|sdw23|8421|/data/mirror421 +1424|422|m|m|s|u|sdw23|sdw23|8422|/data/mirror422 +1425|423|m|m|s|u|sdw23|sdw23|8423|/data/mirror423 +1426|424|m|m|s|u|sdw23|sdw23|8424|/data/mirror424 +1427|425|m|m|s|u|sdw23|sdw23|8425|/data/mirror425 +1428|426|m|m|s|u|sdw23|sdw23|8426|/data/mirror426 +1429|427|m|m|s|u|sdw23|sdw23|8427|/data/mirror427 +1430|428|m|m|s|u|sdw23|sdw23|8428|/data/mirror428 +1431|429|m|m|s|u|sdw23|sdw23|8429|/data/mirror429 +1432|430|m|m|s|u|sdw23|sdw23|8430|/data/mirror430 +1433|431|m|m|s|u|sdw23|sdw23|8431|/data/mirror431 +1434|432|m|m|s|u|sdw23|sdw23|8432|/data/mirror432 +1435|433|m|m|s|u|sdw23|sdw23|8433|/data/mirror433 +1436|434|m|m|s|u|sdw23|sdw23|8434|/data/mirror434 +1437|435|m|m|s|u|sdw23|sdw23|8435|/data/mirror435 +1438|436|m|m|s|u|sdw23|sdw23|8436|/data/mirror436 +1439|437|m|m|s|u|sdw23|sdw23|8437|/data/mirror437 +1440|438|m|m|s|u|sdw23|sdw23|8438|/data/mirror438 +1441|439|m|m|s|u|sdw23|sdw23|8439|/data/mirror439 +# SDW24 +462|460|p|p|s|u|sdw24|sdw24|7460|/data/primary460 +463|461|p|p|s|u|sdw24|sdw24|7461|/data/primary461 +464|462|p|p|s|u|sdw24|sdw24|7462|/data/primary462 +465|463|p|p|s|u|sdw24|sdw24|7463|/data/primary463 +466|464|p|p|s|u|sdw24|sdw24|7464|/data/primary464 +467|465|p|p|s|u|sdw24|sdw24|7465|/data/primary465 +468|466|p|p|s|u|sdw24|sdw24|7466|/data/primary466 +469|467|p|p|s|u|sdw24|sdw24|7467|/data/primary467 +470|468|p|p|s|u|sdw24|sdw24|7468|/data/primary468 +471|469|p|p|s|u|sdw24|sdw24|7469|/data/primary469 +472|470|p|p|s|u|sdw24|sdw24|7470|/data/primary470 +473|471|p|p|s|u|sdw24|sdw24|7471|/data/primary471 +474|472|p|p|s|u|sdw24|sdw24|7472|/data/primary472 +475|473|p|p|s|u|sdw24|sdw24|7473|/data/primary473 +476|474|p|p|s|u|sdw24|sdw24|7474|/data/primary474 +477|475|p|p|s|u|sdw24|sdw24|7475|/data/primary475 +478|476|p|p|s|u|sdw24|sdw24|7476|/data/primary476 +479|477|p|p|s|u|sdw24|sdw24|7477|/data/primary477 +480|478|p|p|s|u|sdw24|sdw24|7478|/data/primary478 +481|479|p|p|s|u|sdw24|sdw24|7479|/data/primary479 +1442|440|m|m|s|u|sdw24|sdw24|8440|/data/mirror440 +1443|441|m|m|s|u|sdw24|sdw24|8441|/data/mirror441 +1444|442|m|m|s|u|sdw24|sdw24|8442|/data/mirror442 +1445|443|m|m|s|u|sdw24|sdw24|8443|/data/mirror443 +1446|444|m|m|s|u|sdw24|sdw24|8444|/data/mirror444 +1447|445|m|m|s|u|sdw24|sdw24|8445|/data/mirror445 +1448|446|m|m|s|u|sdw24|sdw24|8446|/data/mirror446 +1449|447|m|m|s|u|sdw24|sdw24|8447|/data/mirror447 +1450|448|m|m|s|u|sdw24|sdw24|8448|/data/mirror448 +1451|449|m|m|s|u|sdw24|sdw24|8449|/data/mirror449 +1452|450|m|m|s|u|sdw24|sdw24|8450|/data/mirror450 +1453|451|m|m|s|u|sdw24|sdw24|8451|/data/mirror451 +1454|452|m|m|s|u|sdw24|sdw24|8452|/data/mirror452 +1455|453|m|m|s|u|sdw24|sdw24|8453|/data/mirror453 +1456|454|m|m|s|u|sdw24|sdw24|8454|/data/mirror454 +1457|455|m|m|s|u|sdw24|sdw24|8455|/data/mirror455 +1458|456|m|m|s|u|sdw24|sdw24|8456|/data/mirror456 +1459|457|m|m|s|u|sdw24|sdw24|8457|/data/mirror457 +1460|458|m|m|s|u|sdw24|sdw24|8458|/data/mirror458 +1461|459|m|m|s|u|sdw24|sdw24|8459|/data/mirror459 +# SDW25 +482|480|p|p|s|u|sdw25|sdw25|7480|/data/primary480 +483|481|p|p|s|u|sdw25|sdw25|7481|/data/primary481 +484|482|p|p|s|u|sdw25|sdw25|7482|/data/primary482 +485|483|p|p|s|u|sdw25|sdw25|7483|/data/primary483 +486|484|p|p|s|u|sdw25|sdw25|7484|/data/primary484 +487|485|p|p|s|u|sdw25|sdw25|7485|/data/primary485 +488|486|p|p|s|u|sdw25|sdw25|7486|/data/primary486 +489|487|p|p|s|u|sdw25|sdw25|7487|/data/primary487 +490|488|p|p|s|u|sdw25|sdw25|7488|/data/primary488 +491|489|p|p|s|u|sdw25|sdw25|7489|/data/primary489 +492|490|p|p|s|u|sdw25|sdw25|7490|/data/primary490 +493|491|p|p|s|u|sdw25|sdw25|7491|/data/primary491 +494|492|p|p|s|u|sdw25|sdw25|7492|/data/primary492 +495|493|p|p|s|u|sdw25|sdw25|7493|/data/primary493 +496|494|p|p|s|u|sdw25|sdw25|7494|/data/primary494 +497|495|p|p|s|u|sdw25|sdw25|7495|/data/primary495 +498|496|p|p|s|u|sdw25|sdw25|7496|/data/primary496 +499|497|p|p|s|u|sdw25|sdw25|7497|/data/primary497 +500|498|p|p|s|u|sdw25|sdw25|7498|/data/primary498 +501|499|p|p|s|u|sdw25|sdw25|7499|/data/primary499 +1462|460|m|m|s|u|sdw25|sdw25|8460|/data/mirror460 +1463|461|m|m|s|u|sdw25|sdw25|8461|/data/mirror461 +1464|462|m|m|s|u|sdw25|sdw25|8462|/data/mirror462 +1465|463|m|m|s|u|sdw25|sdw25|8463|/data/mirror463 +1466|464|m|m|s|u|sdw25|sdw25|8464|/data/mirror464 +1467|465|m|m|s|u|sdw25|sdw25|8465|/data/mirror465 +1468|466|m|m|s|u|sdw25|sdw25|8466|/data/mirror466 +1469|467|m|m|s|u|sdw25|sdw25|8467|/data/mirror467 +1470|468|m|m|s|u|sdw25|sdw25|8468|/data/mirror468 +1471|469|m|m|s|u|sdw25|sdw25|8469|/data/mirror469 +1472|470|m|m|s|u|sdw25|sdw25|8470|/data/mirror470 +1473|471|m|m|s|u|sdw25|sdw25|8471|/data/mirror471 +1474|472|m|m|s|u|sdw25|sdw25|8472|/data/mirror472 +1475|473|m|m|s|u|sdw25|sdw25|8473|/data/mirror473 +1476|474|m|m|s|u|sdw25|sdw25|8474|/data/mirror474 +1477|475|m|m|s|u|sdw25|sdw25|8475|/data/mirror475 +1478|476|m|m|s|u|sdw25|sdw25|8476|/data/mirror476 +1479|477|m|m|s|u|sdw25|sdw25|8477|/data/mirror477 +1480|478|m|m|s|u|sdw25|sdw25|8478|/data/mirror478 +1481|479|m|m|s|u|sdw25|sdw25|8479|/data/mirror479 +# SDW26 +502|500|p|p|s|u|sdw26|sdw26|7500|/data/primary500 +503|501|p|p|s|u|sdw26|sdw26|7501|/data/primary501 +504|502|p|p|s|u|sdw26|sdw26|7502|/data/primary502 +505|503|p|p|s|u|sdw26|sdw26|7503|/data/primary503 +506|504|p|p|s|u|sdw26|sdw26|7504|/data/primary504 +507|505|p|p|s|u|sdw26|sdw26|7505|/data/primary505 +508|506|p|p|s|u|sdw26|sdw26|7506|/data/primary506 +509|507|p|p|s|u|sdw26|sdw26|7507|/data/primary507 +510|508|p|p|s|u|sdw26|sdw26|7508|/data/primary508 +511|509|p|p|s|u|sdw26|sdw26|7509|/data/primary509 +512|510|p|p|s|u|sdw26|sdw26|7510|/data/primary510 +513|511|p|p|s|u|sdw26|sdw26|7511|/data/primary511 +514|512|p|p|s|u|sdw26|sdw26|7512|/data/primary512 +515|513|p|p|s|u|sdw26|sdw26|7513|/data/primary513 +516|514|p|p|s|u|sdw26|sdw26|7514|/data/primary514 +517|515|p|p|s|u|sdw26|sdw26|7515|/data/primary515 +518|516|p|p|s|u|sdw26|sdw26|7516|/data/primary516 +519|517|p|p|s|u|sdw26|sdw26|7517|/data/primary517 +520|518|p|p|s|u|sdw26|sdw26|7518|/data/primary518 +521|519|p|p|s|u|sdw26|sdw26|7519|/data/primary519 +1482|480|m|m|s|u|sdw26|sdw26|8480|/data/mirror480 +1483|481|m|m|s|u|sdw26|sdw26|8481|/data/mirror481 +1484|482|m|m|s|u|sdw26|sdw26|8482|/data/mirror482 +1485|483|m|m|s|u|sdw26|sdw26|8483|/data/mirror483 +1486|484|m|m|s|u|sdw26|sdw26|8484|/data/mirror484 +1487|485|m|m|s|u|sdw26|sdw26|8485|/data/mirror485 +1488|486|m|m|s|u|sdw26|sdw26|8486|/data/mirror486 +1489|487|m|m|s|u|sdw26|sdw26|8487|/data/mirror487 +1490|488|m|m|s|u|sdw26|sdw26|8488|/data/mirror488 +1491|489|m|m|s|u|sdw26|sdw26|8489|/data/mirror489 +1492|490|m|m|s|u|sdw26|sdw26|8490|/data/mirror490 +1493|491|m|m|s|u|sdw26|sdw26|8491|/data/mirror491 +1494|492|m|m|s|u|sdw26|sdw26|8492|/data/mirror492 +1495|493|m|m|s|u|sdw26|sdw26|8493|/data/mirror493 +1496|494|m|m|s|u|sdw26|sdw26|8494|/data/mirror494 +1497|495|m|m|s|u|sdw26|sdw26|8495|/data/mirror495 +1498|496|m|m|s|u|sdw26|sdw26|8496|/data/mirror496 +1499|497|m|m|s|u|sdw26|sdw26|8497|/data/mirror497 +1500|498|m|m|s|u|sdw26|sdw26|8498|/data/mirror498 +1501|499|m|m|s|u|sdw26|sdw26|8499|/data/mirror499 +# SDW27 +522|520|p|p|s|u|sdw27|sdw27|7520|/data/primary520 +523|521|p|p|s|u|sdw27|sdw27|7521|/data/primary521 +524|522|p|p|s|u|sdw27|sdw27|7522|/data/primary522 +525|523|p|p|s|u|sdw27|sdw27|7523|/data/primary523 +526|524|p|p|s|u|sdw27|sdw27|7524|/data/primary524 +527|525|p|p|s|u|sdw27|sdw27|7525|/data/primary525 +528|526|p|p|s|u|sdw27|sdw27|7526|/data/primary526 +529|527|p|p|s|u|sdw27|sdw27|7527|/data/primary527 +530|528|p|p|s|u|sdw27|sdw27|7528|/data/primary528 +531|529|p|p|s|u|sdw27|sdw27|7529|/data/primary529 +532|530|p|p|s|u|sdw27|sdw27|7530|/data/primary530 +533|531|p|p|s|u|sdw27|sdw27|7531|/data/primary531 +534|532|p|p|s|u|sdw27|sdw27|7532|/data/primary532 +535|533|p|p|s|u|sdw27|sdw27|7533|/data/primary533 +536|534|p|p|s|u|sdw27|sdw27|7534|/data/primary534 +537|535|p|p|s|u|sdw27|sdw27|7535|/data/primary535 +538|536|p|p|s|u|sdw27|sdw27|7536|/data/primary536 +539|537|p|p|s|u|sdw27|sdw27|7537|/data/primary537 +540|538|p|p|s|u|sdw27|sdw27|7538|/data/primary538 +541|539|p|p|s|u|sdw27|sdw27|7539|/data/primary539 +1502|500|m|m|s|u|sdw27|sdw27|8500|/data/mirror500 +1503|501|m|m|s|u|sdw27|sdw27|8501|/data/mirror501 +1504|502|m|m|s|u|sdw27|sdw27|8502|/data/mirror502 +1505|503|m|m|s|u|sdw27|sdw27|8503|/data/mirror503 +1506|504|m|m|s|u|sdw27|sdw27|8504|/data/mirror504 +1507|505|m|m|s|u|sdw27|sdw27|8505|/data/mirror505 +1508|506|m|m|s|u|sdw27|sdw27|8506|/data/mirror506 +1509|507|m|m|s|u|sdw27|sdw27|8507|/data/mirror507 +1510|508|m|m|s|u|sdw27|sdw27|8508|/data/mirror508 +1511|509|m|m|s|u|sdw27|sdw27|8509|/data/mirror509 +1512|510|m|m|s|u|sdw27|sdw27|8510|/data/mirror510 +1513|511|m|m|s|u|sdw27|sdw27|8511|/data/mirror511 +1514|512|m|m|s|u|sdw27|sdw27|8512|/data/mirror512 +1515|513|m|m|s|u|sdw27|sdw27|8513|/data/mirror513 +1516|514|m|m|s|u|sdw27|sdw27|8514|/data/mirror514 +1517|515|m|m|s|u|sdw27|sdw27|8515|/data/mirror515 +1518|516|m|m|s|u|sdw27|sdw27|8516|/data/mirror516 +1519|517|m|m|s|u|sdw27|sdw27|8517|/data/mirror517 +1520|518|m|m|s|u|sdw27|sdw27|8518|/data/mirror518 +1521|519|m|m|s|u|sdw27|sdw27|8519|/data/mirror519 +# SDW28 +542|540|p|p|s|u|sdw28|sdw28|7540|/data/primary540 +543|541|p|p|s|u|sdw28|sdw28|7541|/data/primary541 +544|542|p|p|s|u|sdw28|sdw28|7542|/data/primary542 +545|543|p|p|s|u|sdw28|sdw28|7543|/data/primary543 +546|544|p|p|s|u|sdw28|sdw28|7544|/data/primary544 +547|545|p|p|s|u|sdw28|sdw28|7545|/data/primary545 +548|546|p|p|s|u|sdw28|sdw28|7546|/data/primary546 +549|547|p|p|s|u|sdw28|sdw28|7547|/data/primary547 +550|548|p|p|s|u|sdw28|sdw28|7548|/data/primary548 +551|549|p|p|s|u|sdw28|sdw28|7549|/data/primary549 +552|550|p|p|s|u|sdw28|sdw28|7550|/data/primary550 +553|551|p|p|s|u|sdw28|sdw28|7551|/data/primary551 +554|552|p|p|s|u|sdw28|sdw28|7552|/data/primary552 +555|553|p|p|s|u|sdw28|sdw28|7553|/data/primary553 +556|554|p|p|s|u|sdw28|sdw28|7554|/data/primary554 +557|555|p|p|s|u|sdw28|sdw28|7555|/data/primary555 +558|556|p|p|s|u|sdw28|sdw28|7556|/data/primary556 +559|557|p|p|s|u|sdw28|sdw28|7557|/data/primary557 +560|558|p|p|s|u|sdw28|sdw28|7558|/data/primary558 +561|559|p|p|s|u|sdw28|sdw28|7559|/data/primary559 +1522|520|m|m|s|u|sdw28|sdw28|8520|/data/mirror520 +1523|521|m|m|s|u|sdw28|sdw28|8521|/data/mirror521 +1524|522|m|m|s|u|sdw28|sdw28|8522|/data/mirror522 +1525|523|m|m|s|u|sdw28|sdw28|8523|/data/mirror523 +1526|524|m|m|s|u|sdw28|sdw28|8524|/data/mirror524 +1527|525|m|m|s|u|sdw28|sdw28|8525|/data/mirror525 +1528|526|m|m|s|u|sdw28|sdw28|8526|/data/mirror526 +1529|527|m|m|s|u|sdw28|sdw28|8527|/data/mirror527 +1530|528|m|m|s|u|sdw28|sdw28|8528|/data/mirror528 +1531|529|m|m|s|u|sdw28|sdw28|8529|/data/mirror529 +1532|530|m|m|s|u|sdw28|sdw28|8530|/data/mirror530 +1533|531|m|m|s|u|sdw28|sdw28|8531|/data/mirror531 +1534|532|m|m|s|u|sdw28|sdw28|8532|/data/mirror532 +1535|533|m|m|s|u|sdw28|sdw28|8533|/data/mirror533 +1536|534|m|m|s|u|sdw28|sdw28|8534|/data/mirror534 +1537|535|m|m|s|u|sdw28|sdw28|8535|/data/mirror535 +1538|536|m|m|s|u|sdw28|sdw28|8536|/data/mirror536 +1539|537|m|m|s|u|sdw28|sdw28|8537|/data/mirror537 +1540|538|m|m|s|u|sdw28|sdw28|8538|/data/mirror538 +1541|539|m|m|s|u|sdw28|sdw28|8539|/data/mirror539 +# SDW29 +562|560|p|p|s|u|sdw29|sdw29|7560|/data/primary560 +563|561|p|p|s|u|sdw29|sdw29|7561|/data/primary561 +564|562|p|p|s|u|sdw29|sdw29|7562|/data/primary562 +565|563|p|p|s|u|sdw29|sdw29|7563|/data/primary563 +566|564|p|p|s|u|sdw29|sdw29|7564|/data/primary564 +567|565|p|p|s|u|sdw29|sdw29|7565|/data/primary565 +568|566|p|p|s|u|sdw29|sdw29|7566|/data/primary566 +569|567|p|p|s|u|sdw29|sdw29|7567|/data/primary567 +570|568|p|p|s|u|sdw29|sdw29|7568|/data/primary568 +571|569|p|p|s|u|sdw29|sdw29|7569|/data/primary569 +572|570|p|p|s|u|sdw29|sdw29|7570|/data/primary570 +573|571|p|p|s|u|sdw29|sdw29|7571|/data/primary571 +574|572|p|p|s|u|sdw29|sdw29|7572|/data/primary572 +575|573|p|p|s|u|sdw29|sdw29|7573|/data/primary573 +576|574|p|p|s|u|sdw29|sdw29|7574|/data/primary574 +577|575|p|p|s|u|sdw29|sdw29|7575|/data/primary575 +578|576|p|p|s|u|sdw29|sdw29|7576|/data/primary576 +579|577|p|p|s|u|sdw29|sdw29|7577|/data/primary577 +580|578|p|p|s|u|sdw29|sdw29|7578|/data/primary578 +581|579|p|p|s|u|sdw29|sdw29|7579|/data/primary579 +1542|540|m|m|s|u|sdw29|sdw29|8540|/data/mirror540 +1543|541|m|m|s|u|sdw29|sdw29|8541|/data/mirror541 +1544|542|m|m|s|u|sdw29|sdw29|8542|/data/mirror542 +1545|543|m|m|s|u|sdw29|sdw29|8543|/data/mirror543 +1546|544|m|m|s|u|sdw29|sdw29|8544|/data/mirror544 +1547|545|m|m|s|u|sdw29|sdw29|8545|/data/mirror545 +1548|546|m|m|s|u|sdw29|sdw29|8546|/data/mirror546 +1549|547|m|m|s|u|sdw29|sdw29|8547|/data/mirror547 +1550|548|m|m|s|u|sdw29|sdw29|8548|/data/mirror548 +1551|549|m|m|s|u|sdw29|sdw29|8549|/data/mirror549 +1552|550|m|m|s|u|sdw29|sdw29|8550|/data/mirror550 +1553|551|m|m|s|u|sdw29|sdw29|8551|/data/mirror551 +1554|552|m|m|s|u|sdw29|sdw29|8552|/data/mirror552 +1555|553|m|m|s|u|sdw29|sdw29|8553|/data/mirror553 +1556|554|m|m|s|u|sdw29|sdw29|8554|/data/mirror554 +1557|555|m|m|s|u|sdw29|sdw29|8555|/data/mirror555 +1558|556|m|m|s|u|sdw29|sdw29|8556|/data/mirror556 +1559|557|m|m|s|u|sdw29|sdw29|8557|/data/mirror557 +1560|558|m|m|s|u|sdw29|sdw29|8558|/data/mirror558 +1561|559|m|m|s|u|sdw29|sdw29|8559|/data/mirror559 +# SDW3 +42|40|p|p|s|u|sdw3|sdw3|7040|/data/primary40 +43|41|p|p|s|u|sdw3|sdw3|7041|/data/primary41 +44|42|p|p|s|u|sdw3|sdw3|7042|/data/primary42 +45|43|p|p|s|u|sdw3|sdw3|7043|/data/primary43 +46|44|p|p|s|u|sdw3|sdw3|7044|/data/primary44 +47|45|p|p|s|u|sdw3|sdw3|7045|/data/primary45 +48|46|p|p|s|u|sdw3|sdw3|7046|/data/primary46 +49|47|p|p|s|u|sdw3|sdw3|7047|/data/primary47 +50|48|p|p|s|u|sdw3|sdw3|7048|/data/primary48 +51|49|p|p|s|u|sdw3|sdw3|7049|/data/primary49 +52|50|p|p|s|u|sdw3|sdw3|7050|/data/primary50 +53|51|p|p|s|u|sdw3|sdw3|7051|/data/primary51 +54|52|p|p|s|u|sdw3|sdw3|7052|/data/primary52 +55|53|p|p|s|u|sdw3|sdw3|7053|/data/primary53 +56|54|p|p|s|u|sdw3|sdw3|7054|/data/primary54 +57|55|p|p|s|u|sdw3|sdw3|7055|/data/primary55 +58|56|p|p|s|u|sdw3|sdw3|7056|/data/primary56 +59|57|p|p|s|u|sdw3|sdw3|7057|/data/primary57 +60|58|p|p|s|u|sdw3|sdw3|7058|/data/primary58 +61|59|p|p|s|u|sdw3|sdw3|7059|/data/primary59 +1022|20|m|m|s|u|sdw3|sdw3|8020|/data/mirror20 +1023|21|m|m|s|u|sdw3|sdw3|8021|/data/mirror21 +1024|22|m|m|s|u|sdw3|sdw3|8022|/data/mirror22 +1025|23|m|m|s|u|sdw3|sdw3|8023|/data/mirror23 +1026|24|m|m|s|u|sdw3|sdw3|8024|/data/mirror24 +1027|25|m|m|s|u|sdw3|sdw3|8025|/data/mirror25 +1028|26|m|m|s|u|sdw3|sdw3|8026|/data/mirror26 +1029|27|m|m|s|u|sdw3|sdw3|8027|/data/mirror27 +1030|28|m|m|s|u|sdw3|sdw3|8028|/data/mirror28 +1031|29|m|m|s|u|sdw3|sdw3|8029|/data/mirror29 +1032|30|m|m|s|u|sdw3|sdw3|8030|/data/mirror30 +1033|31|m|m|s|u|sdw3|sdw3|8031|/data/mirror31 +1034|32|m|m|s|u|sdw3|sdw3|8032|/data/mirror32 +1035|33|m|m|s|u|sdw3|sdw3|8033|/data/mirror33 +1036|34|m|m|s|u|sdw3|sdw3|8034|/data/mirror34 +1037|35|m|m|s|u|sdw3|sdw3|8035|/data/mirror35 +1038|36|m|m|s|u|sdw3|sdw3|8036|/data/mirror36 +1039|37|m|m|s|u|sdw3|sdw3|8037|/data/mirror37 +1040|38|m|m|s|u|sdw3|sdw3|8038|/data/mirror38 +1041|39|m|m|s|u|sdw3|sdw3|8039|/data/mirror39 +# SDW30 +582|580|p|p|s|u|sdw30|sdw30|7580|/data/primary580 +583|581|p|p|s|u|sdw30|sdw30|7581|/data/primary581 +584|582|p|p|s|u|sdw30|sdw30|7582|/data/primary582 +585|583|p|p|s|u|sdw30|sdw30|7583|/data/primary583 +586|584|p|p|s|u|sdw30|sdw30|7584|/data/primary584 +587|585|p|p|s|u|sdw30|sdw30|7585|/data/primary585 +588|586|p|p|s|u|sdw30|sdw30|7586|/data/primary586 +589|587|p|p|s|u|sdw30|sdw30|7587|/data/primary587 +590|588|p|p|s|u|sdw30|sdw30|7588|/data/primary588 +591|589|p|p|s|u|sdw30|sdw30|7589|/data/primary589 +592|590|p|p|s|u|sdw30|sdw30|7590|/data/primary590 +593|591|p|p|s|u|sdw30|sdw30|7591|/data/primary591 +594|592|p|p|s|u|sdw30|sdw30|7592|/data/primary592 +595|593|p|p|s|u|sdw30|sdw30|7593|/data/primary593 +596|594|p|p|s|u|sdw30|sdw30|7594|/data/primary594 +597|595|p|p|s|u|sdw30|sdw30|7595|/data/primary595 +598|596|p|p|s|u|sdw30|sdw30|7596|/data/primary596 +599|597|p|p|s|u|sdw30|sdw30|7597|/data/primary597 +600|598|p|p|s|u|sdw30|sdw30|7598|/data/primary598 +601|599|p|p|s|u|sdw30|sdw30|7599|/data/primary599 +1562|560|m|m|s|u|sdw30|sdw30|8560|/data/mirror560 +1563|561|m|m|s|u|sdw30|sdw30|8561|/data/mirror561 +1564|562|m|m|s|u|sdw30|sdw30|8562|/data/mirror562 +1565|563|m|m|s|u|sdw30|sdw30|8563|/data/mirror563 +1566|564|m|m|s|u|sdw30|sdw30|8564|/data/mirror564 +1567|565|m|m|s|u|sdw30|sdw30|8565|/data/mirror565 +1568|566|m|m|s|u|sdw30|sdw30|8566|/data/mirror566 +1569|567|m|m|s|u|sdw30|sdw30|8567|/data/mirror567 +1570|568|m|m|s|u|sdw30|sdw30|8568|/data/mirror568 +1571|569|m|m|s|u|sdw30|sdw30|8569|/data/mirror569 +1572|570|m|m|s|u|sdw30|sdw30|8570|/data/mirror570 +1573|571|m|m|s|u|sdw30|sdw30|8571|/data/mirror571 +1574|572|m|m|s|u|sdw30|sdw30|8572|/data/mirror572 +1575|573|m|m|s|u|sdw30|sdw30|8573|/data/mirror573 +1576|574|m|m|s|u|sdw30|sdw30|8574|/data/mirror574 +1577|575|m|m|s|u|sdw30|sdw30|8575|/data/mirror575 +1578|576|m|m|s|u|sdw30|sdw30|8576|/data/mirror576 +1579|577|m|m|s|u|sdw30|sdw30|8577|/data/mirror577 +1580|578|m|m|s|u|sdw30|sdw30|8578|/data/mirror578 +1581|579|m|m|s|u|sdw30|sdw30|8579|/data/mirror579 +# SDW31 +602|600|p|p|s|u|sdw31|sdw31|7600|/data/primary600 +603|601|p|p|s|u|sdw31|sdw31|7601|/data/primary601 +604|602|p|p|s|u|sdw31|sdw31|7602|/data/primary602 +605|603|p|p|s|u|sdw31|sdw31|7603|/data/primary603 +606|604|p|p|s|u|sdw31|sdw31|7604|/data/primary604 +607|605|p|p|s|u|sdw31|sdw31|7605|/data/primary605 +608|606|p|p|s|u|sdw31|sdw31|7606|/data/primary606 +609|607|p|p|s|u|sdw31|sdw31|7607|/data/primary607 +610|608|p|p|s|u|sdw31|sdw31|7608|/data/primary608 +611|609|p|p|s|u|sdw31|sdw31|7609|/data/primary609 +612|610|p|p|s|u|sdw31|sdw31|7610|/data/primary610 +613|611|p|p|s|u|sdw31|sdw31|7611|/data/primary611 +614|612|p|p|s|u|sdw31|sdw31|7612|/data/primary612 +615|613|p|p|s|u|sdw31|sdw31|7613|/data/primary613 +616|614|p|p|s|u|sdw31|sdw31|7614|/data/primary614 +617|615|p|p|s|u|sdw31|sdw31|7615|/data/primary615 +618|616|p|p|s|u|sdw31|sdw31|7616|/data/primary616 +619|617|p|p|s|u|sdw31|sdw31|7617|/data/primary617 +620|618|p|p|s|u|sdw31|sdw31|7618|/data/primary618 +621|619|p|p|s|u|sdw31|sdw31|7619|/data/primary619 +1582|580|m|m|s|u|sdw31|sdw31|8580|/data/mirror580 +1583|581|m|m|s|u|sdw31|sdw31|8581|/data/mirror581 +1584|582|m|m|s|u|sdw31|sdw31|8582|/data/mirror582 +1585|583|m|m|s|u|sdw31|sdw31|8583|/data/mirror583 +1586|584|m|m|s|u|sdw31|sdw31|8584|/data/mirror584 +1587|585|m|m|s|u|sdw31|sdw31|8585|/data/mirror585 +1588|586|m|m|s|u|sdw31|sdw31|8586|/data/mirror586 +1589|587|m|m|s|u|sdw31|sdw31|8587|/data/mirror587 +1590|588|m|m|s|u|sdw31|sdw31|8588|/data/mirror588 +1591|589|m|m|s|u|sdw31|sdw31|8589|/data/mirror589 +1592|590|m|m|s|u|sdw31|sdw31|8590|/data/mirror590 +1593|591|m|m|s|u|sdw31|sdw31|8591|/data/mirror591 +1594|592|m|m|s|u|sdw31|sdw31|8592|/data/mirror592 +1595|593|m|m|s|u|sdw31|sdw31|8593|/data/mirror593 +1596|594|m|m|s|u|sdw31|sdw31|8594|/data/mirror594 +1597|595|m|m|s|u|sdw31|sdw31|8595|/data/mirror595 +1598|596|m|m|s|u|sdw31|sdw31|8596|/data/mirror596 +1599|597|m|m|s|u|sdw31|sdw31|8597|/data/mirror597 +1600|598|m|m|s|u|sdw31|sdw31|8598|/data/mirror598 +1601|599|m|m|s|u|sdw31|sdw31|8599|/data/mirror599 +# SDW32 +622|620|p|p|s|u|sdw32|sdw32|7620|/data/primary620 +623|621|p|p|s|u|sdw32|sdw32|7621|/data/primary621 +624|622|p|p|s|u|sdw32|sdw32|7622|/data/primary622 +625|623|p|p|s|u|sdw32|sdw32|7623|/data/primary623 +626|624|p|p|s|u|sdw32|sdw32|7624|/data/primary624 +627|625|p|p|s|u|sdw32|sdw32|7625|/data/primary625 +628|626|p|p|s|u|sdw32|sdw32|7626|/data/primary626 +629|627|p|p|s|u|sdw32|sdw32|7627|/data/primary627 +630|628|p|p|s|u|sdw32|sdw32|7628|/data/primary628 +631|629|p|p|s|u|sdw32|sdw32|7629|/data/primary629 +632|630|p|p|s|u|sdw32|sdw32|7630|/data/primary630 +633|631|p|p|s|u|sdw32|sdw32|7631|/data/primary631 +634|632|p|p|s|u|sdw32|sdw32|7632|/data/primary632 +635|633|p|p|s|u|sdw32|sdw32|7633|/data/primary633 +636|634|p|p|s|u|sdw32|sdw32|7634|/data/primary634 +637|635|p|p|s|u|sdw32|sdw32|7635|/data/primary635 +638|636|p|p|s|u|sdw32|sdw32|7636|/data/primary636 +639|637|p|p|s|u|sdw32|sdw32|7637|/data/primary637 +640|638|p|p|s|u|sdw32|sdw32|7638|/data/primary638 +641|639|p|p|s|u|sdw32|sdw32|7639|/data/primary639 +1602|600|m|m|s|u|sdw32|sdw32|8600|/data/mirror600 +1603|601|m|m|s|u|sdw32|sdw32|8601|/data/mirror601 +1604|602|m|m|s|u|sdw32|sdw32|8602|/data/mirror602 +1605|603|m|m|s|u|sdw32|sdw32|8603|/data/mirror603 +1606|604|m|m|s|u|sdw32|sdw32|8604|/data/mirror604 +1607|605|m|m|s|u|sdw32|sdw32|8605|/data/mirror605 +1608|606|m|m|s|u|sdw32|sdw32|8606|/data/mirror606 +1609|607|m|m|s|u|sdw32|sdw32|8607|/data/mirror607 +1610|608|m|m|s|u|sdw32|sdw32|8608|/data/mirror608 +1611|609|m|m|s|u|sdw32|sdw32|8609|/data/mirror609 +1612|610|m|m|s|u|sdw32|sdw32|8610|/data/mirror610 +1613|611|m|m|s|u|sdw32|sdw32|8611|/data/mirror611 +1614|612|m|m|s|u|sdw32|sdw32|8612|/data/mirror612 +1615|613|m|m|s|u|sdw32|sdw32|8613|/data/mirror613 +1616|614|m|m|s|u|sdw32|sdw32|8614|/data/mirror614 +1617|615|m|m|s|u|sdw32|sdw32|8615|/data/mirror615 +1618|616|m|m|s|u|sdw32|sdw32|8616|/data/mirror616 +1619|617|m|m|s|u|sdw32|sdw32|8617|/data/mirror617 +1620|618|m|m|s|u|sdw32|sdw32|8618|/data/mirror618 +1621|619|m|m|s|u|sdw32|sdw32|8619|/data/mirror619 +# SDW33 +642|640|p|p|s|u|sdw33|sdw33|7640|/data/primary640 +643|641|p|p|s|u|sdw33|sdw33|7641|/data/primary641 +644|642|p|p|s|u|sdw33|sdw33|7642|/data/primary642 +645|643|p|p|s|u|sdw33|sdw33|7643|/data/primary643 +646|644|p|p|s|u|sdw33|sdw33|7644|/data/primary644 +647|645|p|p|s|u|sdw33|sdw33|7645|/data/primary645 +648|646|p|p|s|u|sdw33|sdw33|7646|/data/primary646 +649|647|p|p|s|u|sdw33|sdw33|7647|/data/primary647 +650|648|p|p|s|u|sdw33|sdw33|7648|/data/primary648 +651|649|p|p|s|u|sdw33|sdw33|7649|/data/primary649 +652|650|p|p|s|u|sdw33|sdw33|7650|/data/primary650 +653|651|p|p|s|u|sdw33|sdw33|7651|/data/primary651 +654|652|p|p|s|u|sdw33|sdw33|7652|/data/primary652 +655|653|p|p|s|u|sdw33|sdw33|7653|/data/primary653 +656|654|p|p|s|u|sdw33|sdw33|7654|/data/primary654 +657|655|p|p|s|u|sdw33|sdw33|7655|/data/primary655 +658|656|p|p|s|u|sdw33|sdw33|7656|/data/primary656 +659|657|p|p|s|u|sdw33|sdw33|7657|/data/primary657 +660|658|p|p|s|u|sdw33|sdw33|7658|/data/primary658 +661|659|p|p|s|u|sdw33|sdw33|7659|/data/primary659 +1622|620|m|m|s|u|sdw33|sdw33|8620|/data/mirror620 +1623|621|m|m|s|u|sdw33|sdw33|8621|/data/mirror621 +1624|622|m|m|s|u|sdw33|sdw33|8622|/data/mirror622 +1625|623|m|m|s|u|sdw33|sdw33|8623|/data/mirror623 +1626|624|m|m|s|u|sdw33|sdw33|8624|/data/mirror624 +1627|625|m|m|s|u|sdw33|sdw33|8625|/data/mirror625 +1628|626|m|m|s|u|sdw33|sdw33|8626|/data/mirror626 +1629|627|m|m|s|u|sdw33|sdw33|8627|/data/mirror627 +1630|628|m|m|s|u|sdw33|sdw33|8628|/data/mirror628 +1631|629|m|m|s|u|sdw33|sdw33|8629|/data/mirror629 +1632|630|m|m|s|u|sdw33|sdw33|8630|/data/mirror630 +1633|631|m|m|s|u|sdw33|sdw33|8631|/data/mirror631 +1634|632|m|m|s|u|sdw33|sdw33|8632|/data/mirror632 +1635|633|m|m|s|u|sdw33|sdw33|8633|/data/mirror633 +1636|634|m|m|s|u|sdw33|sdw33|8634|/data/mirror634 +1637|635|m|m|s|u|sdw33|sdw33|8635|/data/mirror635 +1638|636|m|m|s|u|sdw33|sdw33|8636|/data/mirror636 +1639|637|m|m|s|u|sdw33|sdw33|8637|/data/mirror637 +1640|638|m|m|s|u|sdw33|sdw33|8638|/data/mirror638 +1641|639|m|m|s|u|sdw33|sdw33|8639|/data/mirror639 +# SDW34 +662|660|p|p|s|u|sdw34|sdw34|7660|/data/primary660 +663|661|p|p|s|u|sdw34|sdw34|7661|/data/primary661 +664|662|p|p|s|u|sdw34|sdw34|7662|/data/primary662 +665|663|p|p|s|u|sdw34|sdw34|7663|/data/primary663 +666|664|p|p|s|u|sdw34|sdw34|7664|/data/primary664 +667|665|p|p|s|u|sdw34|sdw34|7665|/data/primary665 +668|666|p|p|s|u|sdw34|sdw34|7666|/data/primary666 +669|667|p|p|s|u|sdw34|sdw34|7667|/data/primary667 +670|668|p|p|s|u|sdw34|sdw34|7668|/data/primary668 +671|669|p|p|s|u|sdw34|sdw34|7669|/data/primary669 +672|670|p|p|s|u|sdw34|sdw34|7670|/data/primary670 +673|671|p|p|s|u|sdw34|sdw34|7671|/data/primary671 +674|672|p|p|s|u|sdw34|sdw34|7672|/data/primary672 +675|673|p|p|s|u|sdw34|sdw34|7673|/data/primary673 +676|674|p|p|s|u|sdw34|sdw34|7674|/data/primary674 +677|675|p|p|s|u|sdw34|sdw34|7675|/data/primary675 +678|676|p|p|s|u|sdw34|sdw34|7676|/data/primary676 +679|677|p|p|s|u|sdw34|sdw34|7677|/data/primary677 +680|678|p|p|s|u|sdw34|sdw34|7678|/data/primary678 +681|679|p|p|s|u|sdw34|sdw34|7679|/data/primary679 +1642|640|m|m|s|u|sdw34|sdw34|8640|/data/mirror640 +1643|641|m|m|s|u|sdw34|sdw34|8641|/data/mirror641 +1644|642|m|m|s|u|sdw34|sdw34|8642|/data/mirror642 +1645|643|m|m|s|u|sdw34|sdw34|8643|/data/mirror643 +1646|644|m|m|s|u|sdw34|sdw34|8644|/data/mirror644 +1647|645|m|m|s|u|sdw34|sdw34|8645|/data/mirror645 +1648|646|m|m|s|u|sdw34|sdw34|8646|/data/mirror646 +1649|647|m|m|s|u|sdw34|sdw34|8647|/data/mirror647 +1650|648|m|m|s|u|sdw34|sdw34|8648|/data/mirror648 +1651|649|m|m|s|u|sdw34|sdw34|8649|/data/mirror649 +1652|650|m|m|s|u|sdw34|sdw34|8650|/data/mirror650 +1653|651|m|m|s|u|sdw34|sdw34|8651|/data/mirror651 +1654|652|m|m|s|u|sdw34|sdw34|8652|/data/mirror652 +1655|653|m|m|s|u|sdw34|sdw34|8653|/data/mirror653 +1656|654|m|m|s|u|sdw34|sdw34|8654|/data/mirror654 +1657|655|m|m|s|u|sdw34|sdw34|8655|/data/mirror655 +1658|656|m|m|s|u|sdw34|sdw34|8656|/data/mirror656 +1659|657|m|m|s|u|sdw34|sdw34|8657|/data/mirror657 +1660|658|m|m|s|u|sdw34|sdw34|8658|/data/mirror658 +1661|659|m|m|s|u|sdw34|sdw34|8659|/data/mirror659 +# SDW35 +682|680|p|p|s|u|sdw35|sdw35|7680|/data/primary680 +683|681|p|p|s|u|sdw35|sdw35|7681|/data/primary681 +684|682|p|p|s|u|sdw35|sdw35|7682|/data/primary682 +685|683|p|p|s|u|sdw35|sdw35|7683|/data/primary683 +686|684|p|p|s|u|sdw35|sdw35|7684|/data/primary684 +687|685|p|p|s|u|sdw35|sdw35|7685|/data/primary685 +688|686|p|p|s|u|sdw35|sdw35|7686|/data/primary686 +689|687|p|p|s|u|sdw35|sdw35|7687|/data/primary687 +690|688|p|p|s|u|sdw35|sdw35|7688|/data/primary688 +691|689|p|p|s|u|sdw35|sdw35|7689|/data/primary689 +692|690|p|p|s|u|sdw35|sdw35|7690|/data/primary690 +693|691|p|p|s|u|sdw35|sdw35|7691|/data/primary691 +694|692|p|p|s|u|sdw35|sdw35|7692|/data/primary692 +695|693|p|p|s|u|sdw35|sdw35|7693|/data/primary693 +696|694|p|p|s|u|sdw35|sdw35|7694|/data/primary694 +697|695|p|p|s|u|sdw35|sdw35|7695|/data/primary695 +698|696|p|p|s|u|sdw35|sdw35|7696|/data/primary696 +699|697|p|p|s|u|sdw35|sdw35|7697|/data/primary697 +700|698|p|p|s|u|sdw35|sdw35|7698|/data/primary698 +701|699|p|p|s|u|sdw35|sdw35|7699|/data/primary699 +1662|660|m|m|s|u|sdw35|sdw35|8660|/data/mirror660 +1663|661|m|m|s|u|sdw35|sdw35|8661|/data/mirror661 +1664|662|m|m|s|u|sdw35|sdw35|8662|/data/mirror662 +1665|663|m|m|s|u|sdw35|sdw35|8663|/data/mirror663 +1666|664|m|m|s|u|sdw35|sdw35|8664|/data/mirror664 +1667|665|m|m|s|u|sdw35|sdw35|8665|/data/mirror665 +1668|666|m|m|s|u|sdw35|sdw35|8666|/data/mirror666 +1669|667|m|m|s|u|sdw35|sdw35|8667|/data/mirror667 +1670|668|m|m|s|u|sdw35|sdw35|8668|/data/mirror668 +1671|669|m|m|s|u|sdw35|sdw35|8669|/data/mirror669 +1672|670|m|m|s|u|sdw35|sdw35|8670|/data/mirror670 +1673|671|m|m|s|u|sdw35|sdw35|8671|/data/mirror671 +1674|672|m|m|s|u|sdw35|sdw35|8672|/data/mirror672 +1675|673|m|m|s|u|sdw35|sdw35|8673|/data/mirror673 +1676|674|m|m|s|u|sdw35|sdw35|8674|/data/mirror674 +1677|675|m|m|s|u|sdw35|sdw35|8675|/data/mirror675 +1678|676|m|m|s|u|sdw35|sdw35|8676|/data/mirror676 +1679|677|m|m|s|u|sdw35|sdw35|8677|/data/mirror677 +1680|678|m|m|s|u|sdw35|sdw35|8678|/data/mirror678 +1681|679|m|m|s|u|sdw35|sdw35|8679|/data/mirror679 +# SDW36 +702|700|p|p|s|u|sdw36|sdw36|7700|/data/primary700 +703|701|p|p|s|u|sdw36|sdw36|7701|/data/primary701 +704|702|p|p|s|u|sdw36|sdw36|7702|/data/primary702 +705|703|p|p|s|u|sdw36|sdw36|7703|/data/primary703 +706|704|p|p|s|u|sdw36|sdw36|7704|/data/primary704 +707|705|p|p|s|u|sdw36|sdw36|7705|/data/primary705 +708|706|p|p|s|u|sdw36|sdw36|7706|/data/primary706 +709|707|p|p|s|u|sdw36|sdw36|7707|/data/primary707 +710|708|p|p|s|u|sdw36|sdw36|7708|/data/primary708 +711|709|p|p|s|u|sdw36|sdw36|7709|/data/primary709 +712|710|p|p|s|u|sdw36|sdw36|7710|/data/primary710 +713|711|p|p|s|u|sdw36|sdw36|7711|/data/primary711 +714|712|p|p|s|u|sdw36|sdw36|7712|/data/primary712 +715|713|p|p|s|u|sdw36|sdw36|7713|/data/primary713 +716|714|p|p|s|u|sdw36|sdw36|7714|/data/primary714 +717|715|p|p|s|u|sdw36|sdw36|7715|/data/primary715 +718|716|p|p|s|u|sdw36|sdw36|7716|/data/primary716 +719|717|p|p|s|u|sdw36|sdw36|7717|/data/primary717 +720|718|p|p|s|u|sdw36|sdw36|7718|/data/primary718 +721|719|p|p|s|u|sdw36|sdw36|7719|/data/primary719 +1682|680|m|m|s|u|sdw36|sdw36|8680|/data/mirror680 +1683|681|m|m|s|u|sdw36|sdw36|8681|/data/mirror681 +1684|682|m|m|s|u|sdw36|sdw36|8682|/data/mirror682 +1685|683|m|m|s|u|sdw36|sdw36|8683|/data/mirror683 +1686|684|m|m|s|u|sdw36|sdw36|8684|/data/mirror684 +1687|685|m|m|s|u|sdw36|sdw36|8685|/data/mirror685 +1688|686|m|m|s|u|sdw36|sdw36|8686|/data/mirror686 +1689|687|m|m|s|u|sdw36|sdw36|8687|/data/mirror687 +1690|688|m|m|s|u|sdw36|sdw36|8688|/data/mirror688 +1691|689|m|m|s|u|sdw36|sdw36|8689|/data/mirror689 +1692|690|m|m|s|u|sdw36|sdw36|8690|/data/mirror690 +1693|691|m|m|s|u|sdw36|sdw36|8691|/data/mirror691 +1694|692|m|m|s|u|sdw36|sdw36|8692|/data/mirror692 +1695|693|m|m|s|u|sdw36|sdw36|8693|/data/mirror693 +1696|694|m|m|s|u|sdw36|sdw36|8694|/data/mirror694 +1697|695|m|m|s|u|sdw36|sdw36|8695|/data/mirror695 +1698|696|m|m|s|u|sdw36|sdw36|8696|/data/mirror696 +1699|697|m|m|s|u|sdw36|sdw36|8697|/data/mirror697 +1700|698|m|m|s|u|sdw36|sdw36|8698|/data/mirror698 +1701|699|m|m|s|u|sdw36|sdw36|8699|/data/mirror699 +# SDW37 +722|720|p|p|s|u|sdw37|sdw37|7720|/data/primary720 +723|721|p|p|s|u|sdw37|sdw37|7721|/data/primary721 +724|722|p|p|s|u|sdw37|sdw37|7722|/data/primary722 +725|723|p|p|s|u|sdw37|sdw37|7723|/data/primary723 +726|724|p|p|s|u|sdw37|sdw37|7724|/data/primary724 +727|725|p|p|s|u|sdw37|sdw37|7725|/data/primary725 +728|726|p|p|s|u|sdw37|sdw37|7726|/data/primary726 +729|727|p|p|s|u|sdw37|sdw37|7727|/data/primary727 +730|728|p|p|s|u|sdw37|sdw37|7728|/data/primary728 +731|729|p|p|s|u|sdw37|sdw37|7729|/data/primary729 +732|730|p|p|s|u|sdw37|sdw37|7730|/data/primary730 +733|731|p|p|s|u|sdw37|sdw37|7731|/data/primary731 +734|732|p|p|s|u|sdw37|sdw37|7732|/data/primary732 +735|733|p|p|s|u|sdw37|sdw37|7733|/data/primary733 +736|734|p|p|s|u|sdw37|sdw37|7734|/data/primary734 +737|735|p|p|s|u|sdw37|sdw37|7735|/data/primary735 +738|736|p|p|s|u|sdw37|sdw37|7736|/data/primary736 +739|737|p|p|s|u|sdw37|sdw37|7737|/data/primary737 +740|738|p|p|s|u|sdw37|sdw37|7738|/data/primary738 +741|739|p|p|s|u|sdw37|sdw37|7739|/data/primary739 +1702|700|m|m|s|u|sdw37|sdw37|8700|/data/mirror700 +1703|701|m|m|s|u|sdw37|sdw37|8701|/data/mirror701 +1704|702|m|m|s|u|sdw37|sdw37|8702|/data/mirror702 +1705|703|m|m|s|u|sdw37|sdw37|8703|/data/mirror703 +1706|704|m|m|s|u|sdw37|sdw37|8704|/data/mirror704 +1707|705|m|m|s|u|sdw37|sdw37|8705|/data/mirror705 +1708|706|m|m|s|u|sdw37|sdw37|8706|/data/mirror706 +1709|707|m|m|s|u|sdw37|sdw37|8707|/data/mirror707 +1710|708|m|m|s|u|sdw37|sdw37|8708|/data/mirror708 +1711|709|m|m|s|u|sdw37|sdw37|8709|/data/mirror709 +1712|710|m|m|s|u|sdw37|sdw37|8710|/data/mirror710 +1713|711|m|m|s|u|sdw37|sdw37|8711|/data/mirror711 +1714|712|m|m|s|u|sdw37|sdw37|8712|/data/mirror712 +1715|713|m|m|s|u|sdw37|sdw37|8713|/data/mirror713 +1716|714|m|m|s|u|sdw37|sdw37|8714|/data/mirror714 +1717|715|m|m|s|u|sdw37|sdw37|8715|/data/mirror715 +1718|716|m|m|s|u|sdw37|sdw37|8716|/data/mirror716 +1719|717|m|m|s|u|sdw37|sdw37|8717|/data/mirror717 +1720|718|m|m|s|u|sdw37|sdw37|8718|/data/mirror718 +1721|719|m|m|s|u|sdw37|sdw37|8719|/data/mirror719 +# SDW38 +742|740|p|p|s|u|sdw38|sdw38|7740|/data/primary740 +743|741|p|p|s|u|sdw38|sdw38|7741|/data/primary741 +744|742|p|p|s|u|sdw38|sdw38|7742|/data/primary742 +745|743|p|p|s|u|sdw38|sdw38|7743|/data/primary743 +746|744|p|p|s|u|sdw38|sdw38|7744|/data/primary744 +747|745|p|p|s|u|sdw38|sdw38|7745|/data/primary745 +748|746|p|p|s|u|sdw38|sdw38|7746|/data/primary746 +749|747|p|p|s|u|sdw38|sdw38|7747|/data/primary747 +750|748|p|p|s|u|sdw38|sdw38|7748|/data/primary748 +751|749|p|p|s|u|sdw38|sdw38|7749|/data/primary749 +752|750|p|p|s|u|sdw38|sdw38|7750|/data/primary750 +753|751|p|p|s|u|sdw38|sdw38|7751|/data/primary751 +754|752|p|p|s|u|sdw38|sdw38|7752|/data/primary752 +755|753|p|p|s|u|sdw38|sdw38|7753|/data/primary753 +756|754|p|p|s|u|sdw38|sdw38|7754|/data/primary754 +757|755|p|p|s|u|sdw38|sdw38|7755|/data/primary755 +758|756|p|p|s|u|sdw38|sdw38|7756|/data/primary756 +759|757|p|p|s|u|sdw38|sdw38|7757|/data/primary757 +760|758|p|p|s|u|sdw38|sdw38|7758|/data/primary758 +761|759|p|p|s|u|sdw38|sdw38|7759|/data/primary759 +1722|720|m|m|s|u|sdw38|sdw38|8720|/data/mirror720 +1723|721|m|m|s|u|sdw38|sdw38|8721|/data/mirror721 +1724|722|m|m|s|u|sdw38|sdw38|8722|/data/mirror722 +1725|723|m|m|s|u|sdw38|sdw38|8723|/data/mirror723 +1726|724|m|m|s|u|sdw38|sdw38|8724|/data/mirror724 +1727|725|m|m|s|u|sdw38|sdw38|8725|/data/mirror725 +1728|726|m|m|s|u|sdw38|sdw38|8726|/data/mirror726 +1729|727|m|m|s|u|sdw38|sdw38|8727|/data/mirror727 +1730|728|m|m|s|u|sdw38|sdw38|8728|/data/mirror728 +1731|729|m|m|s|u|sdw38|sdw38|8729|/data/mirror729 +1732|730|m|m|s|u|sdw38|sdw38|8730|/data/mirror730 +1733|731|m|m|s|u|sdw38|sdw38|8731|/data/mirror731 +1734|732|m|m|s|u|sdw38|sdw38|8732|/data/mirror732 +1735|733|m|m|s|u|sdw38|sdw38|8733|/data/mirror733 +1736|734|m|m|s|u|sdw38|sdw38|8734|/data/mirror734 +1737|735|m|m|s|u|sdw38|sdw38|8735|/data/mirror735 +1738|736|m|m|s|u|sdw38|sdw38|8736|/data/mirror736 +1739|737|m|m|s|u|sdw38|sdw38|8737|/data/mirror737 +1740|738|m|m|s|u|sdw38|sdw38|8738|/data/mirror738 +1741|739|m|m|s|u|sdw38|sdw38|8739|/data/mirror739 +# SDW39 +762|760|p|p|s|u|sdw39|sdw39|7760|/data/primary760 +763|761|p|p|s|u|sdw39|sdw39|7761|/data/primary761 +764|762|p|p|s|u|sdw39|sdw39|7762|/data/primary762 +765|763|p|p|s|u|sdw39|sdw39|7763|/data/primary763 +766|764|p|p|s|u|sdw39|sdw39|7764|/data/primary764 +767|765|p|p|s|u|sdw39|sdw39|7765|/data/primary765 +768|766|p|p|s|u|sdw39|sdw39|7766|/data/primary766 +769|767|p|p|s|u|sdw39|sdw39|7767|/data/primary767 +770|768|p|p|s|u|sdw39|sdw39|7768|/data/primary768 +771|769|p|p|s|u|sdw39|sdw39|7769|/data/primary769 +772|770|p|p|s|u|sdw39|sdw39|7770|/data/primary770 +773|771|p|p|s|u|sdw39|sdw39|7771|/data/primary771 +774|772|p|p|s|u|sdw39|sdw39|7772|/data/primary772 +775|773|p|p|s|u|sdw39|sdw39|7773|/data/primary773 +776|774|p|p|s|u|sdw39|sdw39|7774|/data/primary774 +777|775|p|p|s|u|sdw39|sdw39|7775|/data/primary775 +778|776|p|p|s|u|sdw39|sdw39|7776|/data/primary776 +779|777|p|p|s|u|sdw39|sdw39|7777|/data/primary777 +780|778|p|p|s|u|sdw39|sdw39|7778|/data/primary778 +781|779|p|p|s|u|sdw39|sdw39|7779|/data/primary779 +1742|740|m|m|s|u|sdw39|sdw39|8740|/data/mirror740 +1743|741|m|m|s|u|sdw39|sdw39|8741|/data/mirror741 +1744|742|m|m|s|u|sdw39|sdw39|8742|/data/mirror742 +1745|743|m|m|s|u|sdw39|sdw39|8743|/data/mirror743 +1746|744|m|m|s|u|sdw39|sdw39|8744|/data/mirror744 +1747|745|m|m|s|u|sdw39|sdw39|8745|/data/mirror745 +1748|746|m|m|s|u|sdw39|sdw39|8746|/data/mirror746 +1749|747|m|m|s|u|sdw39|sdw39|8747|/data/mirror747 +1750|748|m|m|s|u|sdw39|sdw39|8748|/data/mirror748 +1751|749|m|m|s|u|sdw39|sdw39|8749|/data/mirror749 +1752|750|m|m|s|u|sdw39|sdw39|8750|/data/mirror750 +1753|751|m|m|s|u|sdw39|sdw39|8751|/data/mirror751 +1754|752|m|m|s|u|sdw39|sdw39|8752|/data/mirror752 +1755|753|m|m|s|u|sdw39|sdw39|8753|/data/mirror753 +1756|754|m|m|s|u|sdw39|sdw39|8754|/data/mirror754 +1757|755|m|m|s|u|sdw39|sdw39|8755|/data/mirror755 +1758|756|m|m|s|u|sdw39|sdw39|8756|/data/mirror756 +1759|757|m|m|s|u|sdw39|sdw39|8757|/data/mirror757 +1760|758|m|m|s|u|sdw39|sdw39|8758|/data/mirror758 +1761|759|m|m|s|u|sdw39|sdw39|8759|/data/mirror759 +# SDW4 +62|60|p|p|s|u|sdw4|sdw4|7060|/data/primary60 +63|61|p|p|s|u|sdw4|sdw4|7061|/data/primary61 +64|62|p|p|s|u|sdw4|sdw4|7062|/data/primary62 +65|63|p|p|s|u|sdw4|sdw4|7063|/data/primary63 +66|64|p|p|s|u|sdw4|sdw4|7064|/data/primary64 +67|65|p|p|s|u|sdw4|sdw4|7065|/data/primary65 +68|66|p|p|s|u|sdw4|sdw4|7066|/data/primary66 +69|67|p|p|s|u|sdw4|sdw4|7067|/data/primary67 +70|68|p|p|s|u|sdw4|sdw4|7068|/data/primary68 +71|69|p|p|s|u|sdw4|sdw4|7069|/data/primary69 +72|70|p|p|s|u|sdw4|sdw4|7070|/data/primary70 +73|71|p|p|s|u|sdw4|sdw4|7071|/data/primary71 +74|72|p|p|s|u|sdw4|sdw4|7072|/data/primary72 +75|73|p|p|s|u|sdw4|sdw4|7073|/data/primary73 +76|74|p|p|s|u|sdw4|sdw4|7074|/data/primary74 +77|75|p|p|s|u|sdw4|sdw4|7075|/data/primary75 +78|76|p|p|s|u|sdw4|sdw4|7076|/data/primary76 +79|77|p|p|s|u|sdw4|sdw4|7077|/data/primary77 +80|78|p|p|s|u|sdw4|sdw4|7078|/data/primary78 +81|79|p|p|s|u|sdw4|sdw4|7079|/data/primary79 +1042|40|m|m|s|u|sdw4|sdw4|8040|/data/mirror40 +1043|41|m|m|s|u|sdw4|sdw4|8041|/data/mirror41 +1044|42|m|m|s|u|sdw4|sdw4|8042|/data/mirror42 +1045|43|m|m|s|u|sdw4|sdw4|8043|/data/mirror43 +1046|44|m|m|s|u|sdw4|sdw4|8044|/data/mirror44 +1047|45|m|m|s|u|sdw4|sdw4|8045|/data/mirror45 +1048|46|m|m|s|u|sdw4|sdw4|8046|/data/mirror46 +1049|47|m|m|s|u|sdw4|sdw4|8047|/data/mirror47 +1050|48|m|m|s|u|sdw4|sdw4|8048|/data/mirror48 +1051|49|m|m|s|u|sdw4|sdw4|8049|/data/mirror49 +1052|50|m|m|s|u|sdw4|sdw4|8050|/data/mirror50 +1053|51|m|m|s|u|sdw4|sdw4|8051|/data/mirror51 +1054|52|m|m|s|u|sdw4|sdw4|8052|/data/mirror52 +1055|53|m|m|s|u|sdw4|sdw4|8053|/data/mirror53 +1056|54|m|m|s|u|sdw4|sdw4|8054|/data/mirror54 +1057|55|m|m|s|u|sdw4|sdw4|8055|/data/mirror55 +1058|56|m|m|s|u|sdw4|sdw4|8056|/data/mirror56 +1059|57|m|m|s|u|sdw4|sdw4|8057|/data/mirror57 +1060|58|m|m|s|u|sdw4|sdw4|8058|/data/mirror58 +1061|59|m|m|s|u|sdw4|sdw4|8059|/data/mirror59 +# SDW40 +782|780|p|p|s|u|sdw40|sdw40|7780|/data/primary780 +783|781|p|p|s|u|sdw40|sdw40|7781|/data/primary781 +784|782|p|p|s|u|sdw40|sdw40|7782|/data/primary782 +785|783|p|p|s|u|sdw40|sdw40|7783|/data/primary783 +786|784|p|p|s|u|sdw40|sdw40|7784|/data/primary784 +787|785|p|p|s|u|sdw40|sdw40|7785|/data/primary785 +788|786|p|p|s|u|sdw40|sdw40|7786|/data/primary786 +789|787|p|p|s|u|sdw40|sdw40|7787|/data/primary787 +790|788|p|p|s|u|sdw40|sdw40|7788|/data/primary788 +791|789|p|p|s|u|sdw40|sdw40|7789|/data/primary789 +792|790|p|p|s|u|sdw40|sdw40|7790|/data/primary790 +793|791|p|p|s|u|sdw40|sdw40|7791|/data/primary791 +794|792|p|p|s|u|sdw40|sdw40|7792|/data/primary792 +795|793|p|p|s|u|sdw40|sdw40|7793|/data/primary793 +796|794|p|p|s|u|sdw40|sdw40|7794|/data/primary794 +797|795|p|p|s|u|sdw40|sdw40|7795|/data/primary795 +798|796|p|p|s|u|sdw40|sdw40|7796|/data/primary796 +799|797|p|p|s|u|sdw40|sdw40|7797|/data/primary797 +800|798|p|p|s|u|sdw40|sdw40|7798|/data/primary798 +801|799|p|p|s|u|sdw40|sdw40|7799|/data/primary799 +1762|760|m|m|s|u|sdw40|sdw40|8760|/data/mirror760 +1763|761|m|m|s|u|sdw40|sdw40|8761|/data/mirror761 +1764|762|m|m|s|u|sdw40|sdw40|8762|/data/mirror762 +1765|763|m|m|s|u|sdw40|sdw40|8763|/data/mirror763 +1766|764|m|m|s|u|sdw40|sdw40|8764|/data/mirror764 +1767|765|m|m|s|u|sdw40|sdw40|8765|/data/mirror765 +1768|766|m|m|s|u|sdw40|sdw40|8766|/data/mirror766 +1769|767|m|m|s|u|sdw40|sdw40|8767|/data/mirror767 +1770|768|m|m|s|u|sdw40|sdw40|8768|/data/mirror768 +1771|769|m|m|s|u|sdw40|sdw40|8769|/data/mirror769 +1772|770|m|m|s|u|sdw40|sdw40|8770|/data/mirror770 +1773|771|m|m|s|u|sdw40|sdw40|8771|/data/mirror771 +1774|772|m|m|s|u|sdw40|sdw40|8772|/data/mirror772 +1775|773|m|m|s|u|sdw40|sdw40|8773|/data/mirror773 +1776|774|m|m|s|u|sdw40|sdw40|8774|/data/mirror774 +1777|775|m|m|s|u|sdw40|sdw40|8775|/data/mirror775 +1778|776|m|m|s|u|sdw40|sdw40|8776|/data/mirror776 +1779|777|m|m|s|u|sdw40|sdw40|8777|/data/mirror777 +1780|778|m|m|s|u|sdw40|sdw40|8778|/data/mirror778 +1781|779|m|m|s|u|sdw40|sdw40|8779|/data/mirror779 +# SDW41 +802|800|p|p|s|u|sdw41|sdw41|7800|/data/primary800 +803|801|p|p|s|u|sdw41|sdw41|7801|/data/primary801 +804|802|p|p|s|u|sdw41|sdw41|7802|/data/primary802 +805|803|p|p|s|u|sdw41|sdw41|7803|/data/primary803 +806|804|p|p|s|u|sdw41|sdw41|7804|/data/primary804 +807|805|p|p|s|u|sdw41|sdw41|7805|/data/primary805 +808|806|p|p|s|u|sdw41|sdw41|7806|/data/primary806 +809|807|p|p|s|u|sdw41|sdw41|7807|/data/primary807 +810|808|p|p|s|u|sdw41|sdw41|7808|/data/primary808 +811|809|p|p|s|u|sdw41|sdw41|7809|/data/primary809 +812|810|p|p|s|u|sdw41|sdw41|7810|/data/primary810 +813|811|p|p|s|u|sdw41|sdw41|7811|/data/primary811 +814|812|p|p|s|u|sdw41|sdw41|7812|/data/primary812 +815|813|p|p|s|u|sdw41|sdw41|7813|/data/primary813 +816|814|p|p|s|u|sdw41|sdw41|7814|/data/primary814 +817|815|p|p|s|u|sdw41|sdw41|7815|/data/primary815 +818|816|p|p|s|u|sdw41|sdw41|7816|/data/primary816 +819|817|p|p|s|u|sdw41|sdw41|7817|/data/primary817 +820|818|p|p|s|u|sdw41|sdw41|7818|/data/primary818 +821|819|p|p|s|u|sdw41|sdw41|7819|/data/primary819 +1782|780|m|m|s|u|sdw41|sdw41|8780|/data/mirror780 +1783|781|m|m|s|u|sdw41|sdw41|8781|/data/mirror781 +1784|782|m|m|s|u|sdw41|sdw41|8782|/data/mirror782 +1785|783|m|m|s|u|sdw41|sdw41|8783|/data/mirror783 +1786|784|m|m|s|u|sdw41|sdw41|8784|/data/mirror784 +1787|785|m|m|s|u|sdw41|sdw41|8785|/data/mirror785 +1788|786|m|m|s|u|sdw41|sdw41|8786|/data/mirror786 +1789|787|m|m|s|u|sdw41|sdw41|8787|/data/mirror787 +1790|788|m|m|s|u|sdw41|sdw41|8788|/data/mirror788 +1791|789|m|m|s|u|sdw41|sdw41|8789|/data/mirror789 +1792|790|m|m|s|u|sdw41|sdw41|8790|/data/mirror790 +1793|791|m|m|s|u|sdw41|sdw41|8791|/data/mirror791 +1794|792|m|m|s|u|sdw41|sdw41|8792|/data/mirror792 +1795|793|m|m|s|u|sdw41|sdw41|8793|/data/mirror793 +1796|794|m|m|s|u|sdw41|sdw41|8794|/data/mirror794 +1797|795|m|m|s|u|sdw41|sdw41|8795|/data/mirror795 +1798|796|m|m|s|u|sdw41|sdw41|8796|/data/mirror796 +1799|797|m|m|s|u|sdw41|sdw41|8797|/data/mirror797 +1800|798|m|m|s|u|sdw41|sdw41|8798|/data/mirror798 +1801|799|m|m|s|u|sdw41|sdw41|8799|/data/mirror799 +# SDW42 +822|820|p|p|s|u|sdw42|sdw42|7820|/data/primary820 +823|821|p|p|s|u|sdw42|sdw42|7821|/data/primary821 +824|822|p|p|s|u|sdw42|sdw42|7822|/data/primary822 +825|823|p|p|s|u|sdw42|sdw42|7823|/data/primary823 +826|824|p|p|s|u|sdw42|sdw42|7824|/data/primary824 +827|825|p|p|s|u|sdw42|sdw42|7825|/data/primary825 +828|826|p|p|s|u|sdw42|sdw42|7826|/data/primary826 +829|827|p|p|s|u|sdw42|sdw42|7827|/data/primary827 +830|828|p|p|s|u|sdw42|sdw42|7828|/data/primary828 +831|829|p|p|s|u|sdw42|sdw42|7829|/data/primary829 +832|830|p|p|s|u|sdw42|sdw42|7830|/data/primary830 +833|831|p|p|s|u|sdw42|sdw42|7831|/data/primary831 +834|832|p|p|s|u|sdw42|sdw42|7832|/data/primary832 +835|833|p|p|s|u|sdw42|sdw42|7833|/data/primary833 +836|834|p|p|s|u|sdw42|sdw42|7834|/data/primary834 +837|835|p|p|s|u|sdw42|sdw42|7835|/data/primary835 +838|836|p|p|s|u|sdw42|sdw42|7836|/data/primary836 +839|837|p|p|s|u|sdw42|sdw42|7837|/data/primary837 +840|838|p|p|s|u|sdw42|sdw42|7838|/data/primary838 +841|839|p|p|s|u|sdw42|sdw42|7839|/data/primary839 +1802|800|m|m|s|u|sdw42|sdw42|8800|/data/mirror800 +1803|801|m|m|s|u|sdw42|sdw42|8801|/data/mirror801 +1804|802|m|m|s|u|sdw42|sdw42|8802|/data/mirror802 +1805|803|m|m|s|u|sdw42|sdw42|8803|/data/mirror803 +1806|804|m|m|s|u|sdw42|sdw42|8804|/data/mirror804 +1807|805|m|m|s|u|sdw42|sdw42|8805|/data/mirror805 +1808|806|m|m|s|u|sdw42|sdw42|8806|/data/mirror806 +1809|807|m|m|s|u|sdw42|sdw42|8807|/data/mirror807 +1810|808|m|m|s|u|sdw42|sdw42|8808|/data/mirror808 +1811|809|m|m|s|u|sdw42|sdw42|8809|/data/mirror809 +1812|810|m|m|s|u|sdw42|sdw42|8810|/data/mirror810 +1813|811|m|m|s|u|sdw42|sdw42|8811|/data/mirror811 +1814|812|m|m|s|u|sdw42|sdw42|8812|/data/mirror812 +1815|813|m|m|s|u|sdw42|sdw42|8813|/data/mirror813 +1816|814|m|m|s|u|sdw42|sdw42|8814|/data/mirror814 +1817|815|m|m|s|u|sdw42|sdw42|8815|/data/mirror815 +1818|816|m|m|s|u|sdw42|sdw42|8816|/data/mirror816 +1819|817|m|m|s|u|sdw42|sdw42|8817|/data/mirror817 +1820|818|m|m|s|u|sdw42|sdw42|8818|/data/mirror818 +1821|819|m|m|s|u|sdw42|sdw42|8819|/data/mirror819 +# SDW43 +842|840|p|p|s|u|sdw43|sdw43|7840|/data/primary840 +843|841|p|p|s|u|sdw43|sdw43|7841|/data/primary841 +844|842|p|p|s|u|sdw43|sdw43|7842|/data/primary842 +845|843|p|p|s|u|sdw43|sdw43|7843|/data/primary843 +846|844|p|p|s|u|sdw43|sdw43|7844|/data/primary844 +847|845|p|p|s|u|sdw43|sdw43|7845|/data/primary845 +848|846|p|p|s|u|sdw43|sdw43|7846|/data/primary846 +849|847|p|p|s|u|sdw43|sdw43|7847|/data/primary847 +850|848|p|p|s|u|sdw43|sdw43|7848|/data/primary848 +851|849|p|p|s|u|sdw43|sdw43|7849|/data/primary849 +852|850|p|p|s|u|sdw43|sdw43|7850|/data/primary850 +853|851|p|p|s|u|sdw43|sdw43|7851|/data/primary851 +854|852|p|p|s|u|sdw43|sdw43|7852|/data/primary852 +855|853|p|p|s|u|sdw43|sdw43|7853|/data/primary853 +856|854|p|p|s|u|sdw43|sdw43|7854|/data/primary854 +857|855|p|p|s|u|sdw43|sdw43|7855|/data/primary855 +858|856|p|p|s|u|sdw43|sdw43|7856|/data/primary856 +859|857|p|p|s|u|sdw43|sdw43|7857|/data/primary857 +860|858|p|p|s|u|sdw43|sdw43|7858|/data/primary858 +861|859|p|p|s|u|sdw43|sdw43|7859|/data/primary859 +1822|820|m|m|s|u|sdw43|sdw43|8820|/data/mirror820 +1823|821|m|m|s|u|sdw43|sdw43|8821|/data/mirror821 +1824|822|m|m|s|u|sdw43|sdw43|8822|/data/mirror822 +1825|823|m|m|s|u|sdw43|sdw43|8823|/data/mirror823 +1826|824|m|m|s|u|sdw43|sdw43|8824|/data/mirror824 +1827|825|m|m|s|u|sdw43|sdw43|8825|/data/mirror825 +1828|826|m|m|s|u|sdw43|sdw43|8826|/data/mirror826 +1829|827|m|m|s|u|sdw43|sdw43|8827|/data/mirror827 +1830|828|m|m|s|u|sdw43|sdw43|8828|/data/mirror828 +1831|829|m|m|s|u|sdw43|sdw43|8829|/data/mirror829 +1832|830|m|m|s|u|sdw43|sdw43|8830|/data/mirror830 +1833|831|m|m|s|u|sdw43|sdw43|8831|/data/mirror831 +1834|832|m|m|s|u|sdw43|sdw43|8832|/data/mirror832 +1835|833|m|m|s|u|sdw43|sdw43|8833|/data/mirror833 +1836|834|m|m|s|u|sdw43|sdw43|8834|/data/mirror834 +1837|835|m|m|s|u|sdw43|sdw43|8835|/data/mirror835 +1838|836|m|m|s|u|sdw43|sdw43|8836|/data/mirror836 +1839|837|m|m|s|u|sdw43|sdw43|8837|/data/mirror837 +1840|838|m|m|s|u|sdw43|sdw43|8838|/data/mirror838 +1841|839|m|m|s|u|sdw43|sdw43|8839|/data/mirror839 +# SDW44 +862|860|p|p|s|u|sdw44|sdw44|7860|/data/primary860 +863|861|p|p|s|u|sdw44|sdw44|7861|/data/primary861 +864|862|p|p|s|u|sdw44|sdw44|7862|/data/primary862 +865|863|p|p|s|u|sdw44|sdw44|7863|/data/primary863 +866|864|p|p|s|u|sdw44|sdw44|7864|/data/primary864 +867|865|p|p|s|u|sdw44|sdw44|7865|/data/primary865 +868|866|p|p|s|u|sdw44|sdw44|7866|/data/primary866 +869|867|p|p|s|u|sdw44|sdw44|7867|/data/primary867 +870|868|p|p|s|u|sdw44|sdw44|7868|/data/primary868 +871|869|p|p|s|u|sdw44|sdw44|7869|/data/primary869 +872|870|p|p|s|u|sdw44|sdw44|7870|/data/primary870 +873|871|p|p|s|u|sdw44|sdw44|7871|/data/primary871 +874|872|p|p|s|u|sdw44|sdw44|7872|/data/primary872 +875|873|p|p|s|u|sdw44|sdw44|7873|/data/primary873 +876|874|p|p|s|u|sdw44|sdw44|7874|/data/primary874 +877|875|p|p|s|u|sdw44|sdw44|7875|/data/primary875 +878|876|p|p|s|u|sdw44|sdw44|7876|/data/primary876 +879|877|p|p|s|u|sdw44|sdw44|7877|/data/primary877 +880|878|p|p|s|u|sdw44|sdw44|7878|/data/primary878 +881|879|p|p|s|u|sdw44|sdw44|7879|/data/primary879 +1842|840|m|m|s|u|sdw44|sdw44|8840|/data/mirror840 +1843|841|m|m|s|u|sdw44|sdw44|8841|/data/mirror841 +1844|842|m|m|s|u|sdw44|sdw44|8842|/data/mirror842 +1845|843|m|m|s|u|sdw44|sdw44|8843|/data/mirror843 +1846|844|m|m|s|u|sdw44|sdw44|8844|/data/mirror844 +1847|845|m|m|s|u|sdw44|sdw44|8845|/data/mirror845 +1848|846|m|m|s|u|sdw44|sdw44|8846|/data/mirror846 +1849|847|m|m|s|u|sdw44|sdw44|8847|/data/mirror847 +1850|848|m|m|s|u|sdw44|sdw44|8848|/data/mirror848 +1851|849|m|m|s|u|sdw44|sdw44|8849|/data/mirror849 +1852|850|m|m|s|u|sdw44|sdw44|8850|/data/mirror850 +1853|851|m|m|s|u|sdw44|sdw44|8851|/data/mirror851 +1854|852|m|m|s|u|sdw44|sdw44|8852|/data/mirror852 +1855|853|m|m|s|u|sdw44|sdw44|8853|/data/mirror853 +1856|854|m|m|s|u|sdw44|sdw44|8854|/data/mirror854 +1857|855|m|m|s|u|sdw44|sdw44|8855|/data/mirror855 +1858|856|m|m|s|u|sdw44|sdw44|8856|/data/mirror856 +1859|857|m|m|s|u|sdw44|sdw44|8857|/data/mirror857 +1860|858|m|m|s|u|sdw44|sdw44|8858|/data/mirror858 +1861|859|m|m|s|u|sdw44|sdw44|8859|/data/mirror859 +# SDW45 +882|880|p|p|s|u|sdw45|sdw45|7880|/data/primary880 +883|881|p|p|s|u|sdw45|sdw45|7881|/data/primary881 +884|882|p|p|s|u|sdw45|sdw45|7882|/data/primary882 +885|883|p|p|s|u|sdw45|sdw45|7883|/data/primary883 +886|884|p|p|s|u|sdw45|sdw45|7884|/data/primary884 +887|885|p|p|s|u|sdw45|sdw45|7885|/data/primary885 +888|886|p|p|s|u|sdw45|sdw45|7886|/data/primary886 +889|887|p|p|s|u|sdw45|sdw45|7887|/data/primary887 +890|888|p|p|s|u|sdw45|sdw45|7888|/data/primary888 +891|889|p|p|s|u|sdw45|sdw45|7889|/data/primary889 +892|890|p|p|s|u|sdw45|sdw45|7890|/data/primary890 +893|891|p|p|s|u|sdw45|sdw45|7891|/data/primary891 +894|892|p|p|s|u|sdw45|sdw45|7892|/data/primary892 +895|893|p|p|s|u|sdw45|sdw45|7893|/data/primary893 +896|894|p|p|s|u|sdw45|sdw45|7894|/data/primary894 +897|895|p|p|s|u|sdw45|sdw45|7895|/data/primary895 +898|896|p|p|s|u|sdw45|sdw45|7896|/data/primary896 +899|897|p|p|s|u|sdw45|sdw45|7897|/data/primary897 +900|898|p|p|s|u|sdw45|sdw45|7898|/data/primary898 +901|899|p|p|s|u|sdw45|sdw45|7899|/data/primary899 +1862|860|m|m|s|u|sdw45|sdw45|8860|/data/mirror860 +1863|861|m|m|s|u|sdw45|sdw45|8861|/data/mirror861 +1864|862|m|m|s|u|sdw45|sdw45|8862|/data/mirror862 +1865|863|m|m|s|u|sdw45|sdw45|8863|/data/mirror863 +1866|864|m|m|s|u|sdw45|sdw45|8864|/data/mirror864 +1867|865|m|m|s|u|sdw45|sdw45|8865|/data/mirror865 +1868|866|m|m|s|u|sdw45|sdw45|8866|/data/mirror866 +1869|867|m|m|s|u|sdw45|sdw45|8867|/data/mirror867 +1870|868|m|m|s|u|sdw45|sdw45|8868|/data/mirror868 +1871|869|m|m|s|u|sdw45|sdw45|8869|/data/mirror869 +1872|870|m|m|s|u|sdw45|sdw45|8870|/data/mirror870 +1873|871|m|m|s|u|sdw45|sdw45|8871|/data/mirror871 +1874|872|m|m|s|u|sdw45|sdw45|8872|/data/mirror872 +1875|873|m|m|s|u|sdw45|sdw45|8873|/data/mirror873 +1876|874|m|m|s|u|sdw45|sdw45|8874|/data/mirror874 +1877|875|m|m|s|u|sdw45|sdw45|8875|/data/mirror875 +1878|876|m|m|s|u|sdw45|sdw45|8876|/data/mirror876 +1879|877|m|m|s|u|sdw45|sdw45|8877|/data/mirror877 +1880|878|m|m|s|u|sdw45|sdw45|8878|/data/mirror878 +1881|879|m|m|s|u|sdw45|sdw45|8879|/data/mirror879 +# SDW46 +902|900|p|p|s|u|sdw46|sdw46|7900|/data/primary900 +903|901|p|p|s|u|sdw46|sdw46|7901|/data/primary901 +904|902|p|p|s|u|sdw46|sdw46|7902|/data/primary902 +905|903|p|p|s|u|sdw46|sdw46|7903|/data/primary903 +906|904|p|p|s|u|sdw46|sdw46|7904|/data/primary904 +907|905|p|p|s|u|sdw46|sdw46|7905|/data/primary905 +908|906|p|p|s|u|sdw46|sdw46|7906|/data/primary906 +909|907|p|p|s|u|sdw46|sdw46|7907|/data/primary907 +910|908|p|p|s|u|sdw46|sdw46|7908|/data/primary908 +911|909|p|p|s|u|sdw46|sdw46|7909|/data/primary909 +912|910|p|p|s|u|sdw46|sdw46|7910|/data/primary910 +913|911|p|p|s|u|sdw46|sdw46|7911|/data/primary911 +914|912|p|p|s|u|sdw46|sdw46|7912|/data/primary912 +915|913|p|p|s|u|sdw46|sdw46|7913|/data/primary913 +916|914|p|p|s|u|sdw46|sdw46|7914|/data/primary914 +917|915|p|p|s|u|sdw46|sdw46|7915|/data/primary915 +918|916|p|p|s|u|sdw46|sdw46|7916|/data/primary916 +919|917|p|p|s|u|sdw46|sdw46|7917|/data/primary917 +920|918|p|p|s|u|sdw46|sdw46|7918|/data/primary918 +921|919|p|p|s|u|sdw46|sdw46|7919|/data/primary919 +1882|880|m|m|s|u|sdw46|sdw46|8880|/data/mirror880 +1883|881|m|m|s|u|sdw46|sdw46|8881|/data/mirror881 +1884|882|m|m|s|u|sdw46|sdw46|8882|/data/mirror882 +1885|883|m|m|s|u|sdw46|sdw46|8883|/data/mirror883 +1886|884|m|m|s|u|sdw46|sdw46|8884|/data/mirror884 +1887|885|m|m|s|u|sdw46|sdw46|8885|/data/mirror885 +1888|886|m|m|s|u|sdw46|sdw46|8886|/data/mirror886 +1889|887|m|m|s|u|sdw46|sdw46|8887|/data/mirror887 +1890|888|m|m|s|u|sdw46|sdw46|8888|/data/mirror888 +1891|889|m|m|s|u|sdw46|sdw46|8889|/data/mirror889 +1892|890|m|m|s|u|sdw46|sdw46|8890|/data/mirror890 +1893|891|m|m|s|u|sdw46|sdw46|8891|/data/mirror891 +1894|892|m|m|s|u|sdw46|sdw46|8892|/data/mirror892 +1895|893|m|m|s|u|sdw46|sdw46|8893|/data/mirror893 +1896|894|m|m|s|u|sdw46|sdw46|8894|/data/mirror894 +1897|895|m|m|s|u|sdw46|sdw46|8895|/data/mirror895 +1898|896|m|m|s|u|sdw46|sdw46|8896|/data/mirror896 +1899|897|m|m|s|u|sdw46|sdw46|8897|/data/mirror897 +1900|898|m|m|s|u|sdw46|sdw46|8898|/data/mirror898 +1901|899|m|m|s|u|sdw46|sdw46|8899|/data/mirror899 +# SDW47 +922|920|p|p|s|u|sdw47|sdw47|7920|/data/primary920 +923|921|p|p|s|u|sdw47|sdw47|7921|/data/primary921 +924|922|p|p|s|u|sdw47|sdw47|7922|/data/primary922 +925|923|p|p|s|u|sdw47|sdw47|7923|/data/primary923 +926|924|p|p|s|u|sdw47|sdw47|7924|/data/primary924 +927|925|p|p|s|u|sdw47|sdw47|7925|/data/primary925 +928|926|p|p|s|u|sdw47|sdw47|7926|/data/primary926 +929|927|p|p|s|u|sdw47|sdw47|7927|/data/primary927 +930|928|p|p|s|u|sdw47|sdw47|7928|/data/primary928 +931|929|p|p|s|u|sdw47|sdw47|7929|/data/primary929 +932|930|p|p|s|u|sdw47|sdw47|7930|/data/primary930 +933|931|p|p|s|u|sdw47|sdw47|7931|/data/primary931 +934|932|p|p|s|u|sdw47|sdw47|7932|/data/primary932 +935|933|p|p|s|u|sdw47|sdw47|7933|/data/primary933 +936|934|p|p|s|u|sdw47|sdw47|7934|/data/primary934 +937|935|p|p|s|u|sdw47|sdw47|7935|/data/primary935 +938|936|p|p|s|u|sdw47|sdw47|7936|/data/primary936 +939|937|p|p|s|u|sdw47|sdw47|7937|/data/primary937 +940|938|p|p|s|u|sdw47|sdw47|7938|/data/primary938 +941|939|p|p|s|u|sdw47|sdw47|7939|/data/primary939 +1902|900|m|m|s|u|sdw47|sdw47|8900|/data/mirror900 +1903|901|m|m|s|u|sdw47|sdw47|8901|/data/mirror901 +1904|902|m|m|s|u|sdw47|sdw47|8902|/data/mirror902 +1905|903|m|m|s|u|sdw47|sdw47|8903|/data/mirror903 +1906|904|m|m|s|u|sdw47|sdw47|8904|/data/mirror904 +1907|905|m|m|s|u|sdw47|sdw47|8905|/data/mirror905 +1908|906|m|m|s|u|sdw47|sdw47|8906|/data/mirror906 +1909|907|m|m|s|u|sdw47|sdw47|8907|/data/mirror907 +1910|908|m|m|s|u|sdw47|sdw47|8908|/data/mirror908 +1911|909|m|m|s|u|sdw47|sdw47|8909|/data/mirror909 +1912|910|m|m|s|u|sdw47|sdw47|8910|/data/mirror910 +1913|911|m|m|s|u|sdw47|sdw47|8911|/data/mirror911 +1914|912|m|m|s|u|sdw47|sdw47|8912|/data/mirror912 +1915|913|m|m|s|u|sdw47|sdw47|8913|/data/mirror913 +1916|914|m|m|s|u|sdw47|sdw47|8914|/data/mirror914 +1917|915|m|m|s|u|sdw47|sdw47|8915|/data/mirror915 +1918|916|m|m|s|u|sdw47|sdw47|8916|/data/mirror916 +1919|917|m|m|s|u|sdw47|sdw47|8917|/data/mirror917 +1920|918|m|m|s|u|sdw47|sdw47|8918|/data/mirror918 +1921|919|m|m|s|u|sdw47|sdw47|8919|/data/mirror919 +# SDW48 +942|940|p|p|s|u|sdw48|sdw48|7940|/data/primary940 +943|941|p|p|s|u|sdw48|sdw48|7941|/data/primary941 +944|942|p|p|s|u|sdw48|sdw48|7942|/data/primary942 +945|943|p|p|s|u|sdw48|sdw48|7943|/data/primary943 +946|944|p|p|s|u|sdw48|sdw48|7944|/data/primary944 +947|945|p|p|s|u|sdw48|sdw48|7945|/data/primary945 +948|946|p|p|s|u|sdw48|sdw48|7946|/data/primary946 +949|947|p|p|s|u|sdw48|sdw48|7947|/data/primary947 +950|948|p|p|s|u|sdw48|sdw48|7948|/data/primary948 +951|949|p|p|s|u|sdw48|sdw48|7949|/data/primary949 +952|950|p|p|s|u|sdw48|sdw48|7950|/data/primary950 +953|951|p|p|s|u|sdw48|sdw48|7951|/data/primary951 +954|952|p|p|s|u|sdw48|sdw48|7952|/data/primary952 +955|953|p|p|s|u|sdw48|sdw48|7953|/data/primary953 +956|954|p|p|s|u|sdw48|sdw48|7954|/data/primary954 +957|955|p|p|s|u|sdw48|sdw48|7955|/data/primary955 +958|956|p|p|s|u|sdw48|sdw48|7956|/data/primary956 +959|957|p|p|s|u|sdw48|sdw48|7957|/data/primary957 +960|958|p|p|s|u|sdw48|sdw48|7958|/data/primary958 +961|959|p|p|s|u|sdw48|sdw48|7959|/data/primary959 +1922|920|m|m|s|u|sdw48|sdw48|8920|/data/mirror920 +1923|921|m|m|s|u|sdw48|sdw48|8921|/data/mirror921 +1924|922|m|m|s|u|sdw48|sdw48|8922|/data/mirror922 +1925|923|m|m|s|u|sdw48|sdw48|8923|/data/mirror923 +1926|924|m|m|s|u|sdw48|sdw48|8924|/data/mirror924 +1927|925|m|m|s|u|sdw48|sdw48|8925|/data/mirror925 +1928|926|m|m|s|u|sdw48|sdw48|8926|/data/mirror926 +1929|927|m|m|s|u|sdw48|sdw48|8927|/data/mirror927 +1930|928|m|m|s|u|sdw48|sdw48|8928|/data/mirror928 +1931|929|m|m|s|u|sdw48|sdw48|8929|/data/mirror929 +1932|930|m|m|s|u|sdw48|sdw48|8930|/data/mirror930 +1933|931|m|m|s|u|sdw48|sdw48|8931|/data/mirror931 +1934|932|m|m|s|u|sdw48|sdw48|8932|/data/mirror932 +1935|933|m|m|s|u|sdw48|sdw48|8933|/data/mirror933 +1936|934|m|m|s|u|sdw48|sdw48|8934|/data/mirror934 +1937|935|m|m|s|u|sdw48|sdw48|8935|/data/mirror935 +1938|936|m|m|s|u|sdw48|sdw48|8936|/data/mirror936 +1939|937|m|m|s|u|sdw48|sdw48|8937|/data/mirror937 +1940|938|m|m|s|u|sdw48|sdw48|8938|/data/mirror938 +1941|939|m|m|s|u|sdw48|sdw48|8939|/data/mirror939 +# SDW49 +962|960|p|p|s|u|sdw49|sdw49|7960|/data/primary960 +963|961|p|p|s|u|sdw49|sdw49|7961|/data/primary961 +964|962|p|p|s|u|sdw49|sdw49|7962|/data/primary962 +965|963|p|p|s|u|sdw49|sdw49|7963|/data/primary963 +966|964|p|p|s|u|sdw49|sdw49|7964|/data/primary964 +967|965|p|p|s|u|sdw49|sdw49|7965|/data/primary965 +968|966|p|p|s|u|sdw49|sdw49|7966|/data/primary966 +969|967|p|p|s|u|sdw49|sdw49|7967|/data/primary967 +970|968|p|p|s|u|sdw49|sdw49|7968|/data/primary968 +971|969|p|p|s|u|sdw49|sdw49|7969|/data/primary969 +972|970|p|p|s|u|sdw49|sdw49|7970|/data/primary970 +973|971|p|p|s|u|sdw49|sdw49|7971|/data/primary971 +974|972|p|p|s|u|sdw49|sdw49|7972|/data/primary972 +975|973|p|p|s|u|sdw49|sdw49|7973|/data/primary973 +976|974|p|p|s|u|sdw49|sdw49|7974|/data/primary974 +977|975|p|p|s|u|sdw49|sdw49|7975|/data/primary975 +978|976|p|p|s|u|sdw49|sdw49|7976|/data/primary976 +979|977|p|p|s|u|sdw49|sdw49|7977|/data/primary977 +980|978|p|p|s|u|sdw49|sdw49|7978|/data/primary978 +981|979|p|p|s|u|sdw49|sdw49|7979|/data/primary979 +1942|940|m|m|s|u|sdw49|sdw49|8940|/data/mirror940 +1943|941|m|m|s|u|sdw49|sdw49|8941|/data/mirror941 +1944|942|m|m|s|u|sdw49|sdw49|8942|/data/mirror942 +1945|943|m|m|s|u|sdw49|sdw49|8943|/data/mirror943 +1946|944|m|m|s|u|sdw49|sdw49|8944|/data/mirror944 +1947|945|m|m|s|u|sdw49|sdw49|8945|/data/mirror945 +1948|946|m|m|s|u|sdw49|sdw49|8946|/data/mirror946 +1949|947|m|m|s|u|sdw49|sdw49|8947|/data/mirror947 +1950|948|m|m|s|u|sdw49|sdw49|8948|/data/mirror948 +1951|949|m|m|s|u|sdw49|sdw49|8949|/data/mirror949 +1952|950|m|m|s|u|sdw49|sdw49|8950|/data/mirror950 +1953|951|m|m|s|u|sdw49|sdw49|8951|/data/mirror951 +1954|952|m|m|s|u|sdw49|sdw49|8952|/data/mirror952 +1955|953|m|m|s|u|sdw49|sdw49|8953|/data/mirror953 +1956|954|m|m|s|u|sdw49|sdw49|8954|/data/mirror954 +1957|955|m|m|s|u|sdw49|sdw49|8955|/data/mirror955 +1958|956|m|m|s|u|sdw49|sdw49|8956|/data/mirror956 +1959|957|m|m|s|u|sdw49|sdw49|8957|/data/mirror957 +1960|958|m|m|s|u|sdw49|sdw49|8958|/data/mirror958 +1961|959|m|m|s|u|sdw49|sdw49|8959|/data/mirror959 +# SDW5 +82|80|p|p|s|u|sdw5|sdw5|7080|/data/primary80 +83|81|p|p|s|u|sdw5|sdw5|7081|/data/primary81 +84|82|p|p|s|u|sdw5|sdw5|7082|/data/primary82 +85|83|p|p|s|u|sdw5|sdw5|7083|/data/primary83 +86|84|p|p|s|u|sdw5|sdw5|7084|/data/primary84 +87|85|p|p|s|u|sdw5|sdw5|7085|/data/primary85 +88|86|p|p|s|u|sdw5|sdw5|7086|/data/primary86 +89|87|p|p|s|u|sdw5|sdw5|7087|/data/primary87 +90|88|p|p|s|u|sdw5|sdw5|7088|/data/primary88 +91|89|p|p|s|u|sdw5|sdw5|7089|/data/primary89 +92|90|p|p|s|u|sdw5|sdw5|7090|/data/primary90 +93|91|p|p|s|u|sdw5|sdw5|7091|/data/primary91 +94|92|p|p|s|u|sdw5|sdw5|7092|/data/primary92 +95|93|p|p|s|u|sdw5|sdw5|7093|/data/primary93 +96|94|p|p|s|u|sdw5|sdw5|7094|/data/primary94 +97|95|p|p|s|u|sdw5|sdw5|7095|/data/primary95 +98|96|p|p|s|u|sdw5|sdw5|7096|/data/primary96 +99|97|p|p|s|u|sdw5|sdw5|7097|/data/primary97 +100|98|p|p|s|u|sdw5|sdw5|7098|/data/primary98 +101|99|p|p|s|u|sdw5|sdw5|7099|/data/primary99 +1062|60|m|m|s|u|sdw5|sdw5|8060|/data/mirror60 +1063|61|m|m|s|u|sdw5|sdw5|8061|/data/mirror61 +1064|62|m|m|s|u|sdw5|sdw5|8062|/data/mirror62 +1065|63|m|m|s|u|sdw5|sdw5|8063|/data/mirror63 +1066|64|m|m|s|u|sdw5|sdw5|8064|/data/mirror64 +1067|65|m|m|s|u|sdw5|sdw5|8065|/data/mirror65 +1068|66|m|m|s|u|sdw5|sdw5|8066|/data/mirror66 +1069|67|m|m|s|u|sdw5|sdw5|8067|/data/mirror67 +1070|68|m|m|s|u|sdw5|sdw5|8068|/data/mirror68 +1071|69|m|m|s|u|sdw5|sdw5|8069|/data/mirror69 +1072|70|m|m|s|u|sdw5|sdw5|8070|/data/mirror70 +1073|71|m|m|s|u|sdw5|sdw5|8071|/data/mirror71 +1074|72|m|m|s|u|sdw5|sdw5|8072|/data/mirror72 +1075|73|m|m|s|u|sdw5|sdw5|8073|/data/mirror73 +1076|74|m|m|s|u|sdw5|sdw5|8074|/data/mirror74 +1077|75|m|m|s|u|sdw5|sdw5|8075|/data/mirror75 +1078|76|m|m|s|u|sdw5|sdw5|8076|/data/mirror76 +1079|77|m|m|s|u|sdw5|sdw5|8077|/data/mirror77 +1080|78|m|m|s|u|sdw5|sdw5|8078|/data/mirror78 +1081|79|m|m|s|u|sdw5|sdw5|8079|/data/mirror79 +# SDW50 +982|980|p|p|s|u|sdw50|sdw50|7980|/data/primary980 +983|981|p|p|s|u|sdw50|sdw50|7981|/data/primary981 +984|982|p|p|s|u|sdw50|sdw50|7982|/data/primary982 +985|983|p|p|s|u|sdw50|sdw50|7983|/data/primary983 +986|984|p|p|s|u|sdw50|sdw50|7984|/data/primary984 +987|985|p|p|s|u|sdw50|sdw50|7985|/data/primary985 +988|986|p|p|s|u|sdw50|sdw50|7986|/data/primary986 +989|987|p|p|s|u|sdw50|sdw50|7987|/data/primary987 +990|988|p|p|s|u|sdw50|sdw50|7988|/data/primary988 +991|989|p|p|s|u|sdw50|sdw50|7989|/data/primary989 +992|990|p|p|s|u|sdw50|sdw50|7990|/data/primary990 +993|991|p|p|s|u|sdw50|sdw50|7991|/data/primary991 +994|992|p|p|s|u|sdw50|sdw50|7992|/data/primary992 +995|993|p|p|s|u|sdw50|sdw50|7993|/data/primary993 +996|994|p|p|s|u|sdw50|sdw50|7994|/data/primary994 +997|995|p|p|s|u|sdw50|sdw50|7995|/data/primary995 +998|996|p|p|s|u|sdw50|sdw50|7996|/data/primary996 +999|997|p|p|s|u|sdw50|sdw50|7997|/data/primary997 +1000|998|p|p|s|u|sdw50|sdw50|7998|/data/primary998 +1001|999|p|p|s|u|sdw50|sdw50|7999|/data/primary999 +1962|960|m|m|s|u|sdw50|sdw50|8960|/data/mirror960 +1963|961|m|m|s|u|sdw50|sdw50|8961|/data/mirror961 +1964|962|m|m|s|u|sdw50|sdw50|8962|/data/mirror962 +1965|963|m|m|s|u|sdw50|sdw50|8963|/data/mirror963 +1966|964|m|m|s|u|sdw50|sdw50|8964|/data/mirror964 +1967|965|m|m|s|u|sdw50|sdw50|8965|/data/mirror965 +1968|966|m|m|s|u|sdw50|sdw50|8966|/data/mirror966 +1969|967|m|m|s|u|sdw50|sdw50|8967|/data/mirror967 +1970|968|m|m|s|u|sdw50|sdw50|8968|/data/mirror968 +1971|969|m|m|s|u|sdw50|sdw50|8969|/data/mirror969 +1972|970|m|m|s|u|sdw50|sdw50|8970|/data/mirror970 +1973|971|m|m|s|u|sdw50|sdw50|8971|/data/mirror971 +1974|972|m|m|s|u|sdw50|sdw50|8972|/data/mirror972 +1975|973|m|m|s|u|sdw50|sdw50|8973|/data/mirror973 +1976|974|m|m|s|u|sdw50|sdw50|8974|/data/mirror974 +1977|975|m|m|s|u|sdw50|sdw50|8975|/data/mirror975 +1978|976|m|m|s|u|sdw50|sdw50|8976|/data/mirror976 +1979|977|m|m|s|u|sdw50|sdw50|8977|/data/mirror977 +1980|978|m|m|s|u|sdw50|sdw50|8978|/data/mirror978 +1981|979|m|m|s|u|sdw50|sdw50|8979|/data/mirror979 +# SDW6 +102|100|p|p|s|u|sdw6|sdw6|7100|/data/primary100 +103|101|p|p|s|u|sdw6|sdw6|7101|/data/primary101 +104|102|p|p|s|u|sdw6|sdw6|7102|/data/primary102 +105|103|p|p|s|u|sdw6|sdw6|7103|/data/primary103 +106|104|p|p|s|u|sdw6|sdw6|7104|/data/primary104 +107|105|p|p|s|u|sdw6|sdw6|7105|/data/primary105 +108|106|p|p|s|u|sdw6|sdw6|7106|/data/primary106 +109|107|p|p|s|u|sdw6|sdw6|7107|/data/primary107 +110|108|p|p|s|u|sdw6|sdw6|7108|/data/primary108 +111|109|p|p|s|u|sdw6|sdw6|7109|/data/primary109 +112|110|p|p|s|u|sdw6|sdw6|7110|/data/primary110 +113|111|p|p|s|u|sdw6|sdw6|7111|/data/primary111 +114|112|p|p|s|u|sdw6|sdw6|7112|/data/primary112 +115|113|p|p|s|u|sdw6|sdw6|7113|/data/primary113 +116|114|p|p|s|u|sdw6|sdw6|7114|/data/primary114 +117|115|p|p|s|u|sdw6|sdw6|7115|/data/primary115 +118|116|p|p|s|u|sdw6|sdw6|7116|/data/primary116 +119|117|p|p|s|u|sdw6|sdw6|7117|/data/primary117 +120|118|p|p|s|u|sdw6|sdw6|7118|/data/primary118 +121|119|p|p|s|u|sdw6|sdw6|7119|/data/primary119 +1082|80|m|m|s|u|sdw6|sdw6|8080|/data/mirror80 +1083|81|m|m|s|u|sdw6|sdw6|8081|/data/mirror81 +1084|82|m|m|s|u|sdw6|sdw6|8082|/data/mirror82 +1085|83|m|m|s|u|sdw6|sdw6|8083|/data/mirror83 +1086|84|m|m|s|u|sdw6|sdw6|8084|/data/mirror84 +1087|85|m|m|s|u|sdw6|sdw6|8085|/data/mirror85 +1088|86|m|m|s|u|sdw6|sdw6|8086|/data/mirror86 +1089|87|m|m|s|u|sdw6|sdw6|8087|/data/mirror87 +1090|88|m|m|s|u|sdw6|sdw6|8088|/data/mirror88 +1091|89|m|m|s|u|sdw6|sdw6|8089|/data/mirror89 +1092|90|m|m|s|u|sdw6|sdw6|8090|/data/mirror90 +1093|91|m|m|s|u|sdw6|sdw6|8091|/data/mirror91 +1094|92|m|m|s|u|sdw6|sdw6|8092|/data/mirror92 +1095|93|m|m|s|u|sdw6|sdw6|8093|/data/mirror93 +1096|94|m|m|s|u|sdw6|sdw6|8094|/data/mirror94 +1097|95|m|m|s|u|sdw6|sdw6|8095|/data/mirror95 +1098|96|m|m|s|u|sdw6|sdw6|8096|/data/mirror96 +1099|97|m|m|s|u|sdw6|sdw6|8097|/data/mirror97 +1100|98|m|m|s|u|sdw6|sdw6|8098|/data/mirror98 +1101|99|m|m|s|u|sdw6|sdw6|8099|/data/mirror99 +# SDW7 +122|120|p|p|s|u|sdw7|sdw7|7120|/data/primary120 +123|121|p|p|s|u|sdw7|sdw7|7121|/data/primary121 +124|122|p|p|s|u|sdw7|sdw7|7122|/data/primary122 +125|123|p|p|s|u|sdw7|sdw7|7123|/data/primary123 +126|124|p|p|s|u|sdw7|sdw7|7124|/data/primary124 +127|125|p|p|s|u|sdw7|sdw7|7125|/data/primary125 +128|126|p|p|s|u|sdw7|sdw7|7126|/data/primary126 +129|127|p|p|s|u|sdw7|sdw7|7127|/data/primary127 +130|128|p|p|s|u|sdw7|sdw7|7128|/data/primary128 +131|129|p|p|s|u|sdw7|sdw7|7129|/data/primary129 +132|130|p|p|s|u|sdw7|sdw7|7130|/data/primary130 +133|131|p|p|s|u|sdw7|sdw7|7131|/data/primary131 +134|132|p|p|s|u|sdw7|sdw7|7132|/data/primary132 +135|133|p|p|s|u|sdw7|sdw7|7133|/data/primary133 +136|134|p|p|s|u|sdw7|sdw7|7134|/data/primary134 +137|135|p|p|s|u|sdw7|sdw7|7135|/data/primary135 +138|136|p|p|s|u|sdw7|sdw7|7136|/data/primary136 +139|137|p|p|s|u|sdw7|sdw7|7137|/data/primary137 +140|138|p|p|s|u|sdw7|sdw7|7138|/data/primary138 +141|139|p|p|s|u|sdw7|sdw7|7139|/data/primary139 +1102|100|m|m|s|u|sdw7|sdw7|8100|/data/mirror100 +1103|101|m|m|s|u|sdw7|sdw7|8101|/data/mirror101 +1104|102|m|m|s|u|sdw7|sdw7|8102|/data/mirror102 +1105|103|m|m|s|u|sdw7|sdw7|8103|/data/mirror103 +1106|104|m|m|s|u|sdw7|sdw7|8104|/data/mirror104 +1107|105|m|m|s|u|sdw7|sdw7|8105|/data/mirror105 +1108|106|m|m|s|u|sdw7|sdw7|8106|/data/mirror106 +1109|107|m|m|s|u|sdw7|sdw7|8107|/data/mirror107 +1110|108|m|m|s|u|sdw7|sdw7|8108|/data/mirror108 +1111|109|m|m|s|u|sdw7|sdw7|8109|/data/mirror109 +1112|110|m|m|s|u|sdw7|sdw7|8110|/data/mirror110 +1113|111|m|m|s|u|sdw7|sdw7|8111|/data/mirror111 +1114|112|m|m|s|u|sdw7|sdw7|8112|/data/mirror112 +1115|113|m|m|s|u|sdw7|sdw7|8113|/data/mirror113 +1116|114|m|m|s|u|sdw7|sdw7|8114|/data/mirror114 +1117|115|m|m|s|u|sdw7|sdw7|8115|/data/mirror115 +1118|116|m|m|s|u|sdw7|sdw7|8116|/data/mirror116 +1119|117|m|m|s|u|sdw7|sdw7|8117|/data/mirror117 +1120|118|m|m|s|u|sdw7|sdw7|8118|/data/mirror118 +1121|119|m|m|s|u|sdw7|sdw7|8119|/data/mirror119 +# SDW8 +142|140|p|p|s|u|sdw8|sdw8|7140|/data/primary140 +143|141|p|p|s|u|sdw8|sdw8|7141|/data/primary141 +144|142|p|p|s|u|sdw8|sdw8|7142|/data/primary142 +145|143|p|p|s|u|sdw8|sdw8|7143|/data/primary143 +146|144|p|p|s|u|sdw8|sdw8|7144|/data/primary144 +147|145|p|p|s|u|sdw8|sdw8|7145|/data/primary145 +148|146|p|p|s|u|sdw8|sdw8|7146|/data/primary146 +149|147|p|p|s|u|sdw8|sdw8|7147|/data/primary147 +150|148|p|p|s|u|sdw8|sdw8|7148|/data/primary148 +151|149|p|p|s|u|sdw8|sdw8|7149|/data/primary149 +152|150|p|p|s|u|sdw8|sdw8|7150|/data/primary150 +153|151|p|p|s|u|sdw8|sdw8|7151|/data/primary151 +154|152|p|p|s|u|sdw8|sdw8|7152|/data/primary152 +155|153|p|p|s|u|sdw8|sdw8|7153|/data/primary153 +156|154|p|p|s|u|sdw8|sdw8|7154|/data/primary154 +157|155|p|p|s|u|sdw8|sdw8|7155|/data/primary155 +158|156|p|p|s|u|sdw8|sdw8|7156|/data/primary156 +159|157|p|p|s|u|sdw8|sdw8|7157|/data/primary157 +160|158|p|p|s|u|sdw8|sdw8|7158|/data/primary158 +161|159|p|p|s|u|sdw8|sdw8|7159|/data/primary159 +1122|120|m|m|s|u|sdw8|sdw8|8120|/data/mirror120 +1123|121|m|m|s|u|sdw8|sdw8|8121|/data/mirror121 +1124|122|m|m|s|u|sdw8|sdw8|8122|/data/mirror122 +1125|123|m|m|s|u|sdw8|sdw8|8123|/data/mirror123 +1126|124|m|m|s|u|sdw8|sdw8|8124|/data/mirror124 +1127|125|m|m|s|u|sdw8|sdw8|8125|/data/mirror125 +1128|126|m|m|s|u|sdw8|sdw8|8126|/data/mirror126 +1129|127|m|m|s|u|sdw8|sdw8|8127|/data/mirror127 +1130|128|m|m|s|u|sdw8|sdw8|8128|/data/mirror128 +1131|129|m|m|s|u|sdw8|sdw8|8129|/data/mirror129 +1132|130|m|m|s|u|sdw8|sdw8|8130|/data/mirror130 +1133|131|m|m|s|u|sdw8|sdw8|8131|/data/mirror131 +1134|132|m|m|s|u|sdw8|sdw8|8132|/data/mirror132 +1135|133|m|m|s|u|sdw8|sdw8|8133|/data/mirror133 +1136|134|m|m|s|u|sdw8|sdw8|8134|/data/mirror134 +1137|135|m|m|s|u|sdw8|sdw8|8135|/data/mirror135 +1138|136|m|m|s|u|sdw8|sdw8|8136|/data/mirror136 +1139|137|m|m|s|u|sdw8|sdw8|8137|/data/mirror137 +1140|138|m|m|s|u|sdw8|sdw8|8138|/data/mirror138 +1141|139|m|m|s|u|sdw8|sdw8|8139|/data/mirror139 +# SDW9 +162|160|p|p|s|u|sdw9|sdw9|7160|/data/primary160 +163|161|p|p|s|u|sdw9|sdw9|7161|/data/primary161 +164|162|p|p|s|u|sdw9|sdw9|7162|/data/primary162 +165|163|p|p|s|u|sdw9|sdw9|7163|/data/primary163 +166|164|p|p|s|u|sdw9|sdw9|7164|/data/primary164 +167|165|p|p|s|u|sdw9|sdw9|7165|/data/primary165 +168|166|p|p|s|u|sdw9|sdw9|7166|/data/primary166 +169|167|p|p|s|u|sdw9|sdw9|7167|/data/primary167 +170|168|p|p|s|u|sdw9|sdw9|7168|/data/primary168 +171|169|p|p|s|u|sdw9|sdw9|7169|/data/primary169 +172|170|p|p|s|u|sdw9|sdw9|7170|/data/primary170 +173|171|p|p|s|u|sdw9|sdw9|7171|/data/primary171 +174|172|p|p|s|u|sdw9|sdw9|7172|/data/primary172 +175|173|p|p|s|u|sdw9|sdw9|7173|/data/primary173 +176|174|p|p|s|u|sdw9|sdw9|7174|/data/primary174 +177|175|p|p|s|u|sdw9|sdw9|7175|/data/primary175 +178|176|p|p|s|u|sdw9|sdw9|7176|/data/primary176 +179|177|p|p|s|u|sdw9|sdw9|7177|/data/primary177 +180|178|p|p|s|u|sdw9|sdw9|7178|/data/primary178 +181|179|p|p|s|u|sdw9|sdw9|7179|/data/primary179 +1142|140|m|m|s|u|sdw9|sdw9|8140|/data/mirror140 +1143|141|m|m|s|u|sdw9|sdw9|8141|/data/mirror141 +1144|142|m|m|s|u|sdw9|sdw9|8142|/data/mirror142 +1145|143|m|m|s|u|sdw9|sdw9|8143|/data/mirror143 +1146|144|m|m|s|u|sdw9|sdw9|8144|/data/mirror144 +1147|145|m|m|s|u|sdw9|sdw9|8145|/data/mirror145 +1148|146|m|m|s|u|sdw9|sdw9|8146|/data/mirror146 +1149|147|m|m|s|u|sdw9|sdw9|8147|/data/mirror147 +1150|148|m|m|s|u|sdw9|sdw9|8148|/data/mirror148 +1151|149|m|m|s|u|sdw9|sdw9|8149|/data/mirror149 +1152|150|m|m|s|u|sdw9|sdw9|8150|/data/mirror150 +1153|151|m|m|s|u|sdw9|sdw9|8151|/data/mirror151 +1154|152|m|m|s|u|sdw9|sdw9|8152|/data/mirror152 +1155|153|m|m|s|u|sdw9|sdw9|8153|/data/mirror153 +1156|154|m|m|s|u|sdw9|sdw9|8154|/data/mirror154 +1157|155|m|m|s|u|sdw9|sdw9|8155|/data/mirror155 +1158|156|m|m|s|u|sdw9|sdw9|8156|/data/mirror156 +1159|157|m|m|s|u|sdw9|sdw9|8157|/data/mirror157 +1160|158|m|m|s|u|sdw9|sdw9|8158|/data/mirror158 +1161|159|m|m|s|u|sdw9|sdw9|8159|/data/mirror159 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_spread.array new file mode 100644 index 000000000000..4d62750e35f5 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_spread.array @@ -0,0 +1,2052 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +8|6|p|p|s|u|sdw1|sdw1|7006|/data/primary6 +9|7|p|p|s|u|sdw1|sdw1|7007|/data/primary7 +10|8|p|p|s|u|sdw1|sdw1|7008|/data/primary8 +11|9|p|p|s|u|sdw1|sdw1|7009|/data/primary9 +12|10|p|p|s|u|sdw1|sdw1|7010|/data/primary10 +13|11|p|p|s|u|sdw1|sdw1|7011|/data/primary11 +14|12|p|p|s|u|sdw1|sdw1|7012|/data/primary12 +15|13|p|p|s|u|sdw1|sdw1|7013|/data/primary13 +16|14|p|p|s|u|sdw1|sdw1|7014|/data/primary14 +17|15|p|p|s|u|sdw1|sdw1|7015|/data/primary15 +18|16|p|p|s|u|sdw1|sdw1|7016|/data/primary16 +19|17|p|p|s|u|sdw1|sdw1|7017|/data/primary17 +20|18|p|p|s|u|sdw1|sdw1|7018|/data/primary18 +21|19|p|p|s|u|sdw1|sdw1|7019|/data/primary19 +1621|619|m|m|s|u|sdw1|sdw1|8619|/data/mirror619 +1640|638|m|m|s|u|sdw1|sdw1|8638|/data/mirror638 +1659|657|m|m|s|u|sdw1|sdw1|8657|/data/mirror657 +1678|676|m|m|s|u|sdw1|sdw1|8676|/data/mirror676 +1697|695|m|m|s|u|sdw1|sdw1|8695|/data/mirror695 +1716|714|m|m|s|u|sdw1|sdw1|8714|/data/mirror714 +1735|733|m|m|s|u|sdw1|sdw1|8733|/data/mirror733 +1754|752|m|m|s|u|sdw1|sdw1|8752|/data/mirror752 +1773|771|m|m|s|u|sdw1|sdw1|8771|/data/mirror771 +1792|790|m|m|s|u|sdw1|sdw1|8790|/data/mirror790 +1811|809|m|m|s|u|sdw1|sdw1|8809|/data/mirror809 +1830|828|m|m|s|u|sdw1|sdw1|8828|/data/mirror828 +1849|847|m|m|s|u|sdw1|sdw1|8847|/data/mirror847 +1868|866|m|m|s|u|sdw1|sdw1|8866|/data/mirror866 +1887|885|m|m|s|u|sdw1|sdw1|8885|/data/mirror885 +1906|904|m|m|s|u|sdw1|sdw1|8904|/data/mirror904 +1925|923|m|m|s|u|sdw1|sdw1|8923|/data/mirror923 +1944|942|m|m|s|u|sdw1|sdw1|8942|/data/mirror942 +1963|961|m|m|s|u|sdw1|sdw1|8961|/data/mirror961 +1982|980|m|m|s|u|sdw1|sdw1|8980|/data/mirror980 +# SDW10 +182|180|p|p|s|u|sdw10|sdw10|7180|/data/primary180 +183|181|p|p|s|u|sdw10|sdw10|7181|/data/primary181 +184|182|p|p|s|u|sdw10|sdw10|7182|/data/primary182 +185|183|p|p|s|u|sdw10|sdw10|7183|/data/primary183 +186|184|p|p|s|u|sdw10|sdw10|7184|/data/primary184 +187|185|p|p|s|u|sdw10|sdw10|7185|/data/primary185 +188|186|p|p|s|u|sdw10|sdw10|7186|/data/primary186 +189|187|p|p|s|u|sdw10|sdw10|7187|/data/primary187 +190|188|p|p|s|u|sdw10|sdw10|7188|/data/primary188 +191|189|p|p|s|u|sdw10|sdw10|7189|/data/primary189 +192|190|p|p|s|u|sdw10|sdw10|7190|/data/primary190 +193|191|p|p|s|u|sdw10|sdw10|7191|/data/primary191 +194|192|p|p|s|u|sdw10|sdw10|7192|/data/primary192 +195|193|p|p|s|u|sdw10|sdw10|7193|/data/primary193 +196|194|p|p|s|u|sdw10|sdw10|7194|/data/primary194 +197|195|p|p|s|u|sdw10|sdw10|7195|/data/primary195 +198|196|p|p|s|u|sdw10|sdw10|7196|/data/primary196 +199|197|p|p|s|u|sdw10|sdw10|7197|/data/primary197 +200|198|p|p|s|u|sdw10|sdw10|7198|/data/primary198 +201|199|p|p|s|u|sdw10|sdw10|7199|/data/primary199 +1010|8|m|m|s|u|sdw10|sdw10|8008|/data/mirror8 +1029|27|m|m|s|u|sdw10|sdw10|8027|/data/mirror27 +1048|46|m|m|s|u|sdw10|sdw10|8046|/data/mirror46 +1067|65|m|m|s|u|sdw10|sdw10|8065|/data/mirror65 +1086|84|m|m|s|u|sdw10|sdw10|8084|/data/mirror84 +1105|103|m|m|s|u|sdw10|sdw10|8103|/data/mirror103 +1124|122|m|m|s|u|sdw10|sdw10|8122|/data/mirror122 +1143|141|m|m|s|u|sdw10|sdw10|8141|/data/mirror141 +1162|160|m|m|s|u|sdw10|sdw10|8160|/data/mirror160 +1801|799|m|m|s|u|sdw10|sdw10|8799|/data/mirror799 +1820|818|m|m|s|u|sdw10|sdw10|8818|/data/mirror818 +1839|837|m|m|s|u|sdw10|sdw10|8837|/data/mirror837 +1858|856|m|m|s|u|sdw10|sdw10|8856|/data/mirror856 +1877|875|m|m|s|u|sdw10|sdw10|8875|/data/mirror875 +1896|894|m|m|s|u|sdw10|sdw10|8894|/data/mirror894 +1915|913|m|m|s|u|sdw10|sdw10|8913|/data/mirror913 +1934|932|m|m|s|u|sdw10|sdw10|8932|/data/mirror932 +1953|951|m|m|s|u|sdw10|sdw10|8951|/data/mirror951 +1972|970|m|m|s|u|sdw10|sdw10|8970|/data/mirror970 +1991|989|m|m|s|u|sdw10|sdw10|8989|/data/mirror989 +# SDW11 +202|200|p|p|s|u|sdw11|sdw11|7200|/data/primary200 +203|201|p|p|s|u|sdw11|sdw11|7201|/data/primary201 +204|202|p|p|s|u|sdw11|sdw11|7202|/data/primary202 +205|203|p|p|s|u|sdw11|sdw11|7203|/data/primary203 +206|204|p|p|s|u|sdw11|sdw11|7204|/data/primary204 +207|205|p|p|s|u|sdw11|sdw11|7205|/data/primary205 +208|206|p|p|s|u|sdw11|sdw11|7206|/data/primary206 +209|207|p|p|s|u|sdw11|sdw11|7207|/data/primary207 +210|208|p|p|s|u|sdw11|sdw11|7208|/data/primary208 +211|209|p|p|s|u|sdw11|sdw11|7209|/data/primary209 +212|210|p|p|s|u|sdw11|sdw11|7210|/data/primary210 +213|211|p|p|s|u|sdw11|sdw11|7211|/data/primary211 +214|212|p|p|s|u|sdw11|sdw11|7212|/data/primary212 +215|213|p|p|s|u|sdw11|sdw11|7213|/data/primary213 +216|214|p|p|s|u|sdw11|sdw11|7214|/data/primary214 +217|215|p|p|s|u|sdw11|sdw11|7215|/data/primary215 +218|216|p|p|s|u|sdw11|sdw11|7216|/data/primary216 +219|217|p|p|s|u|sdw11|sdw11|7217|/data/primary217 +220|218|p|p|s|u|sdw11|sdw11|7218|/data/primary218 +221|219|p|p|s|u|sdw11|sdw11|7219|/data/primary219 +1011|9|m|m|s|u|sdw11|sdw11|8009|/data/mirror9 +1030|28|m|m|s|u|sdw11|sdw11|8028|/data/mirror28 +1049|47|m|m|s|u|sdw11|sdw11|8047|/data/mirror47 +1068|66|m|m|s|u|sdw11|sdw11|8066|/data/mirror66 +1087|85|m|m|s|u|sdw11|sdw11|8085|/data/mirror85 +1106|104|m|m|s|u|sdw11|sdw11|8104|/data/mirror104 +1125|123|m|m|s|u|sdw11|sdw11|8123|/data/mirror123 +1144|142|m|m|s|u|sdw11|sdw11|8142|/data/mirror142 +1163|161|m|m|s|u|sdw11|sdw11|8161|/data/mirror161 +1182|180|m|m|s|u|sdw11|sdw11|8180|/data/mirror180 +1821|819|m|m|s|u|sdw11|sdw11|8819|/data/mirror819 +1840|838|m|m|s|u|sdw11|sdw11|8838|/data/mirror838 +1859|857|m|m|s|u|sdw11|sdw11|8857|/data/mirror857 +1878|876|m|m|s|u|sdw11|sdw11|8876|/data/mirror876 +1897|895|m|m|s|u|sdw11|sdw11|8895|/data/mirror895 +1916|914|m|m|s|u|sdw11|sdw11|8914|/data/mirror914 +1935|933|m|m|s|u|sdw11|sdw11|8933|/data/mirror933 +1954|952|m|m|s|u|sdw11|sdw11|8952|/data/mirror952 +1973|971|m|m|s|u|sdw11|sdw11|8971|/data/mirror971 +1992|990|m|m|s|u|sdw11|sdw11|8990|/data/mirror990 +# SDW12 +222|220|p|p|s|u|sdw12|sdw12|7220|/data/primary220 +223|221|p|p|s|u|sdw12|sdw12|7221|/data/primary221 +224|222|p|p|s|u|sdw12|sdw12|7222|/data/primary222 +225|223|p|p|s|u|sdw12|sdw12|7223|/data/primary223 +226|224|p|p|s|u|sdw12|sdw12|7224|/data/primary224 +227|225|p|p|s|u|sdw12|sdw12|7225|/data/primary225 +228|226|p|p|s|u|sdw12|sdw12|7226|/data/primary226 +229|227|p|p|s|u|sdw12|sdw12|7227|/data/primary227 +230|228|p|p|s|u|sdw12|sdw12|7228|/data/primary228 +231|229|p|p|s|u|sdw12|sdw12|7229|/data/primary229 +232|230|p|p|s|u|sdw12|sdw12|7230|/data/primary230 +233|231|p|p|s|u|sdw12|sdw12|7231|/data/primary231 +234|232|p|p|s|u|sdw12|sdw12|7232|/data/primary232 +235|233|p|p|s|u|sdw12|sdw12|7233|/data/primary233 +236|234|p|p|s|u|sdw12|sdw12|7234|/data/primary234 +237|235|p|p|s|u|sdw12|sdw12|7235|/data/primary235 +238|236|p|p|s|u|sdw12|sdw12|7236|/data/primary236 +239|237|p|p|s|u|sdw12|sdw12|7237|/data/primary237 +240|238|p|p|s|u|sdw12|sdw12|7238|/data/primary238 +241|239|p|p|s|u|sdw12|sdw12|7239|/data/primary239 +1012|10|m|m|s|u|sdw12|sdw12|8010|/data/mirror10 +1031|29|m|m|s|u|sdw12|sdw12|8029|/data/mirror29 +1050|48|m|m|s|u|sdw12|sdw12|8048|/data/mirror48 +1069|67|m|m|s|u|sdw12|sdw12|8067|/data/mirror67 +1088|86|m|m|s|u|sdw12|sdw12|8086|/data/mirror86 +1107|105|m|m|s|u|sdw12|sdw12|8105|/data/mirror105 +1126|124|m|m|s|u|sdw12|sdw12|8124|/data/mirror124 +1145|143|m|m|s|u|sdw12|sdw12|8143|/data/mirror143 +1164|162|m|m|s|u|sdw12|sdw12|8162|/data/mirror162 +1183|181|m|m|s|u|sdw12|sdw12|8181|/data/mirror181 +1202|200|m|m|s|u|sdw12|sdw12|8200|/data/mirror200 +1841|839|m|m|s|u|sdw12|sdw12|8839|/data/mirror839 +1860|858|m|m|s|u|sdw12|sdw12|8858|/data/mirror858 +1879|877|m|m|s|u|sdw12|sdw12|8877|/data/mirror877 +1898|896|m|m|s|u|sdw12|sdw12|8896|/data/mirror896 +1917|915|m|m|s|u|sdw12|sdw12|8915|/data/mirror915 +1936|934|m|m|s|u|sdw12|sdw12|8934|/data/mirror934 +1955|953|m|m|s|u|sdw12|sdw12|8953|/data/mirror953 +1974|972|m|m|s|u|sdw12|sdw12|8972|/data/mirror972 +1993|991|m|m|s|u|sdw12|sdw12|8991|/data/mirror991 +# SDW13 +242|240|p|p|s|u|sdw13|sdw13|7240|/data/primary240 +243|241|p|p|s|u|sdw13|sdw13|7241|/data/primary241 +244|242|p|p|s|u|sdw13|sdw13|7242|/data/primary242 +245|243|p|p|s|u|sdw13|sdw13|7243|/data/primary243 +246|244|p|p|s|u|sdw13|sdw13|7244|/data/primary244 +247|245|p|p|s|u|sdw13|sdw13|7245|/data/primary245 +248|246|p|p|s|u|sdw13|sdw13|7246|/data/primary246 +249|247|p|p|s|u|sdw13|sdw13|7247|/data/primary247 +250|248|p|p|s|u|sdw13|sdw13|7248|/data/primary248 +251|249|p|p|s|u|sdw13|sdw13|7249|/data/primary249 +252|250|p|p|s|u|sdw13|sdw13|7250|/data/primary250 +253|251|p|p|s|u|sdw13|sdw13|7251|/data/primary251 +254|252|p|p|s|u|sdw13|sdw13|7252|/data/primary252 +255|253|p|p|s|u|sdw13|sdw13|7253|/data/primary253 +256|254|p|p|s|u|sdw13|sdw13|7254|/data/primary254 +257|255|p|p|s|u|sdw13|sdw13|7255|/data/primary255 +258|256|p|p|s|u|sdw13|sdw13|7256|/data/primary256 +259|257|p|p|s|u|sdw13|sdw13|7257|/data/primary257 +260|258|p|p|s|u|sdw13|sdw13|7258|/data/primary258 +261|259|p|p|s|u|sdw13|sdw13|7259|/data/primary259 +1013|11|m|m|s|u|sdw13|sdw13|8011|/data/mirror11 +1032|30|m|m|s|u|sdw13|sdw13|8030|/data/mirror30 +1051|49|m|m|s|u|sdw13|sdw13|8049|/data/mirror49 +1070|68|m|m|s|u|sdw13|sdw13|8068|/data/mirror68 +1089|87|m|m|s|u|sdw13|sdw13|8087|/data/mirror87 +1108|106|m|m|s|u|sdw13|sdw13|8106|/data/mirror106 +1127|125|m|m|s|u|sdw13|sdw13|8125|/data/mirror125 +1146|144|m|m|s|u|sdw13|sdw13|8144|/data/mirror144 +1165|163|m|m|s|u|sdw13|sdw13|8163|/data/mirror163 +1184|182|m|m|s|u|sdw13|sdw13|8182|/data/mirror182 +1203|201|m|m|s|u|sdw13|sdw13|8201|/data/mirror201 +1222|220|m|m|s|u|sdw13|sdw13|8220|/data/mirror220 +1861|859|m|m|s|u|sdw13|sdw13|8859|/data/mirror859 +1880|878|m|m|s|u|sdw13|sdw13|8878|/data/mirror878 +1899|897|m|m|s|u|sdw13|sdw13|8897|/data/mirror897 +1918|916|m|m|s|u|sdw13|sdw13|8916|/data/mirror916 +1937|935|m|m|s|u|sdw13|sdw13|8935|/data/mirror935 +1956|954|m|m|s|u|sdw13|sdw13|8954|/data/mirror954 +1975|973|m|m|s|u|sdw13|sdw13|8973|/data/mirror973 +1994|992|m|m|s|u|sdw13|sdw13|8992|/data/mirror992 +# SDW14 +262|260|p|p|s|u|sdw14|sdw14|7260|/data/primary260 +263|261|p|p|s|u|sdw14|sdw14|7261|/data/primary261 +264|262|p|p|s|u|sdw14|sdw14|7262|/data/primary262 +265|263|p|p|s|u|sdw14|sdw14|7263|/data/primary263 +266|264|p|p|s|u|sdw14|sdw14|7264|/data/primary264 +267|265|p|p|s|u|sdw14|sdw14|7265|/data/primary265 +268|266|p|p|s|u|sdw14|sdw14|7266|/data/primary266 +269|267|p|p|s|u|sdw14|sdw14|7267|/data/primary267 +270|268|p|p|s|u|sdw14|sdw14|7268|/data/primary268 +271|269|p|p|s|u|sdw14|sdw14|7269|/data/primary269 +272|270|p|p|s|u|sdw14|sdw14|7270|/data/primary270 +273|271|p|p|s|u|sdw14|sdw14|7271|/data/primary271 +274|272|p|p|s|u|sdw14|sdw14|7272|/data/primary272 +275|273|p|p|s|u|sdw14|sdw14|7273|/data/primary273 +276|274|p|p|s|u|sdw14|sdw14|7274|/data/primary274 +277|275|p|p|s|u|sdw14|sdw14|7275|/data/primary275 +278|276|p|p|s|u|sdw14|sdw14|7276|/data/primary276 +279|277|p|p|s|u|sdw14|sdw14|7277|/data/primary277 +280|278|p|p|s|u|sdw14|sdw14|7278|/data/primary278 +281|279|p|p|s|u|sdw14|sdw14|7279|/data/primary279 +1014|12|m|m|s|u|sdw14|sdw14|8012|/data/mirror12 +1033|31|m|m|s|u|sdw14|sdw14|8031|/data/mirror31 +1052|50|m|m|s|u|sdw14|sdw14|8050|/data/mirror50 +1071|69|m|m|s|u|sdw14|sdw14|8069|/data/mirror69 +1090|88|m|m|s|u|sdw14|sdw14|8088|/data/mirror88 +1109|107|m|m|s|u|sdw14|sdw14|8107|/data/mirror107 +1128|126|m|m|s|u|sdw14|sdw14|8126|/data/mirror126 +1147|145|m|m|s|u|sdw14|sdw14|8145|/data/mirror145 +1166|164|m|m|s|u|sdw14|sdw14|8164|/data/mirror164 +1185|183|m|m|s|u|sdw14|sdw14|8183|/data/mirror183 +1204|202|m|m|s|u|sdw14|sdw14|8202|/data/mirror202 +1223|221|m|m|s|u|sdw14|sdw14|8221|/data/mirror221 +1242|240|m|m|s|u|sdw14|sdw14|8240|/data/mirror240 +1881|879|m|m|s|u|sdw14|sdw14|8879|/data/mirror879 +1900|898|m|m|s|u|sdw14|sdw14|8898|/data/mirror898 +1919|917|m|m|s|u|sdw14|sdw14|8917|/data/mirror917 +1938|936|m|m|s|u|sdw14|sdw14|8936|/data/mirror936 +1957|955|m|m|s|u|sdw14|sdw14|8955|/data/mirror955 +1976|974|m|m|s|u|sdw14|sdw14|8974|/data/mirror974 +1995|993|m|m|s|u|sdw14|sdw14|8993|/data/mirror993 +# SDW15 +282|280|p|p|s|u|sdw15|sdw15|7280|/data/primary280 +283|281|p|p|s|u|sdw15|sdw15|7281|/data/primary281 +284|282|p|p|s|u|sdw15|sdw15|7282|/data/primary282 +285|283|p|p|s|u|sdw15|sdw15|7283|/data/primary283 +286|284|p|p|s|u|sdw15|sdw15|7284|/data/primary284 +287|285|p|p|s|u|sdw15|sdw15|7285|/data/primary285 +288|286|p|p|s|u|sdw15|sdw15|7286|/data/primary286 +289|287|p|p|s|u|sdw15|sdw15|7287|/data/primary287 +290|288|p|p|s|u|sdw15|sdw15|7288|/data/primary288 +291|289|p|p|s|u|sdw15|sdw15|7289|/data/primary289 +292|290|p|p|s|u|sdw15|sdw15|7290|/data/primary290 +293|291|p|p|s|u|sdw15|sdw15|7291|/data/primary291 +294|292|p|p|s|u|sdw15|sdw15|7292|/data/primary292 +295|293|p|p|s|u|sdw15|sdw15|7293|/data/primary293 +296|294|p|p|s|u|sdw15|sdw15|7294|/data/primary294 +297|295|p|p|s|u|sdw15|sdw15|7295|/data/primary295 +298|296|p|p|s|u|sdw15|sdw15|7296|/data/primary296 +299|297|p|p|s|u|sdw15|sdw15|7297|/data/primary297 +300|298|p|p|s|u|sdw15|sdw15|7298|/data/primary298 +301|299|p|p|s|u|sdw15|sdw15|7299|/data/primary299 +1015|13|m|m|s|u|sdw15|sdw15|8013|/data/mirror13 +1034|32|m|m|s|u|sdw15|sdw15|8032|/data/mirror32 +1053|51|m|m|s|u|sdw15|sdw15|8051|/data/mirror51 +1072|70|m|m|s|u|sdw15|sdw15|8070|/data/mirror70 +1091|89|m|m|s|u|sdw15|sdw15|8089|/data/mirror89 +1110|108|m|m|s|u|sdw15|sdw15|8108|/data/mirror108 +1129|127|m|m|s|u|sdw15|sdw15|8127|/data/mirror127 +1148|146|m|m|s|u|sdw15|sdw15|8146|/data/mirror146 +1167|165|m|m|s|u|sdw15|sdw15|8165|/data/mirror165 +1186|184|m|m|s|u|sdw15|sdw15|8184|/data/mirror184 +1205|203|m|m|s|u|sdw15|sdw15|8203|/data/mirror203 +1224|222|m|m|s|u|sdw15|sdw15|8222|/data/mirror222 +1243|241|m|m|s|u|sdw15|sdw15|8241|/data/mirror241 +1262|260|m|m|s|u|sdw15|sdw15|8260|/data/mirror260 +1901|899|m|m|s|u|sdw15|sdw15|8899|/data/mirror899 +1920|918|m|m|s|u|sdw15|sdw15|8918|/data/mirror918 +1939|937|m|m|s|u|sdw15|sdw15|8937|/data/mirror937 +1958|956|m|m|s|u|sdw15|sdw15|8956|/data/mirror956 +1977|975|m|m|s|u|sdw15|sdw15|8975|/data/mirror975 +1996|994|m|m|s|u|sdw15|sdw15|8994|/data/mirror994 +# SDW16 +302|300|p|p|s|u|sdw16|sdw16|7300|/data/primary300 +303|301|p|p|s|u|sdw16|sdw16|7301|/data/primary301 +304|302|p|p|s|u|sdw16|sdw16|7302|/data/primary302 +305|303|p|p|s|u|sdw16|sdw16|7303|/data/primary303 +306|304|p|p|s|u|sdw16|sdw16|7304|/data/primary304 +307|305|p|p|s|u|sdw16|sdw16|7305|/data/primary305 +308|306|p|p|s|u|sdw16|sdw16|7306|/data/primary306 +309|307|p|p|s|u|sdw16|sdw16|7307|/data/primary307 +310|308|p|p|s|u|sdw16|sdw16|7308|/data/primary308 +311|309|p|p|s|u|sdw16|sdw16|7309|/data/primary309 +312|310|p|p|s|u|sdw16|sdw16|7310|/data/primary310 +313|311|p|p|s|u|sdw16|sdw16|7311|/data/primary311 +314|312|p|p|s|u|sdw16|sdw16|7312|/data/primary312 +315|313|p|p|s|u|sdw16|sdw16|7313|/data/primary313 +316|314|p|p|s|u|sdw16|sdw16|7314|/data/primary314 +317|315|p|p|s|u|sdw16|sdw16|7315|/data/primary315 +318|316|p|p|s|u|sdw16|sdw16|7316|/data/primary316 +319|317|p|p|s|u|sdw16|sdw16|7317|/data/primary317 +320|318|p|p|s|u|sdw16|sdw16|7318|/data/primary318 +321|319|p|p|s|u|sdw16|sdw16|7319|/data/primary319 +1016|14|m|m|s|u|sdw16|sdw16|8014|/data/mirror14 +1035|33|m|m|s|u|sdw16|sdw16|8033|/data/mirror33 +1054|52|m|m|s|u|sdw16|sdw16|8052|/data/mirror52 +1073|71|m|m|s|u|sdw16|sdw16|8071|/data/mirror71 +1092|90|m|m|s|u|sdw16|sdw16|8090|/data/mirror90 +1111|109|m|m|s|u|sdw16|sdw16|8109|/data/mirror109 +1130|128|m|m|s|u|sdw16|sdw16|8128|/data/mirror128 +1149|147|m|m|s|u|sdw16|sdw16|8147|/data/mirror147 +1168|166|m|m|s|u|sdw16|sdw16|8166|/data/mirror166 +1187|185|m|m|s|u|sdw16|sdw16|8185|/data/mirror185 +1206|204|m|m|s|u|sdw16|sdw16|8204|/data/mirror204 +1225|223|m|m|s|u|sdw16|sdw16|8223|/data/mirror223 +1244|242|m|m|s|u|sdw16|sdw16|8242|/data/mirror242 +1263|261|m|m|s|u|sdw16|sdw16|8261|/data/mirror261 +1282|280|m|m|s|u|sdw16|sdw16|8280|/data/mirror280 +1921|919|m|m|s|u|sdw16|sdw16|8919|/data/mirror919 +1940|938|m|m|s|u|sdw16|sdw16|8938|/data/mirror938 +1959|957|m|m|s|u|sdw16|sdw16|8957|/data/mirror957 +1978|976|m|m|s|u|sdw16|sdw16|8976|/data/mirror976 +1997|995|m|m|s|u|sdw16|sdw16|8995|/data/mirror995 +# SDW17 +322|320|p|p|s|u|sdw17|sdw17|7320|/data/primary320 +323|321|p|p|s|u|sdw17|sdw17|7321|/data/primary321 +324|322|p|p|s|u|sdw17|sdw17|7322|/data/primary322 +325|323|p|p|s|u|sdw17|sdw17|7323|/data/primary323 +326|324|p|p|s|u|sdw17|sdw17|7324|/data/primary324 +327|325|p|p|s|u|sdw17|sdw17|7325|/data/primary325 +328|326|p|p|s|u|sdw17|sdw17|7326|/data/primary326 +329|327|p|p|s|u|sdw17|sdw17|7327|/data/primary327 +330|328|p|p|s|u|sdw17|sdw17|7328|/data/primary328 +331|329|p|p|s|u|sdw17|sdw17|7329|/data/primary329 +332|330|p|p|s|u|sdw17|sdw17|7330|/data/primary330 +333|331|p|p|s|u|sdw17|sdw17|7331|/data/primary331 +334|332|p|p|s|u|sdw17|sdw17|7332|/data/primary332 +335|333|p|p|s|u|sdw17|sdw17|7333|/data/primary333 +336|334|p|p|s|u|sdw17|sdw17|7334|/data/primary334 +337|335|p|p|s|u|sdw17|sdw17|7335|/data/primary335 +338|336|p|p|s|u|sdw17|sdw17|7336|/data/primary336 +339|337|p|p|s|u|sdw17|sdw17|7337|/data/primary337 +340|338|p|p|s|u|sdw17|sdw17|7338|/data/primary338 +341|339|p|p|s|u|sdw17|sdw17|7339|/data/primary339 +1017|15|m|m|s|u|sdw17|sdw17|8015|/data/mirror15 +1036|34|m|m|s|u|sdw17|sdw17|8034|/data/mirror34 +1055|53|m|m|s|u|sdw17|sdw17|8053|/data/mirror53 +1074|72|m|m|s|u|sdw17|sdw17|8072|/data/mirror72 +1093|91|m|m|s|u|sdw17|sdw17|8091|/data/mirror91 +1112|110|m|m|s|u|sdw17|sdw17|8110|/data/mirror110 +1131|129|m|m|s|u|sdw17|sdw17|8129|/data/mirror129 +1150|148|m|m|s|u|sdw17|sdw17|8148|/data/mirror148 +1169|167|m|m|s|u|sdw17|sdw17|8167|/data/mirror167 +1188|186|m|m|s|u|sdw17|sdw17|8186|/data/mirror186 +1207|205|m|m|s|u|sdw17|sdw17|8205|/data/mirror205 +1226|224|m|m|s|u|sdw17|sdw17|8224|/data/mirror224 +1245|243|m|m|s|u|sdw17|sdw17|8243|/data/mirror243 +1264|262|m|m|s|u|sdw17|sdw17|8262|/data/mirror262 +1283|281|m|m|s|u|sdw17|sdw17|8281|/data/mirror281 +1302|300|m|m|s|u|sdw17|sdw17|8300|/data/mirror300 +1941|939|m|m|s|u|sdw17|sdw17|8939|/data/mirror939 +1960|958|m|m|s|u|sdw17|sdw17|8958|/data/mirror958 +1979|977|m|m|s|u|sdw17|sdw17|8977|/data/mirror977 +1998|996|m|m|s|u|sdw17|sdw17|8996|/data/mirror996 +# SDW18 +342|340|p|p|s|u|sdw18|sdw18|7340|/data/primary340 +343|341|p|p|s|u|sdw18|sdw18|7341|/data/primary341 +344|342|p|p|s|u|sdw18|sdw18|7342|/data/primary342 +345|343|p|p|s|u|sdw18|sdw18|7343|/data/primary343 +346|344|p|p|s|u|sdw18|sdw18|7344|/data/primary344 +347|345|p|p|s|u|sdw18|sdw18|7345|/data/primary345 +348|346|p|p|s|u|sdw18|sdw18|7346|/data/primary346 +349|347|p|p|s|u|sdw18|sdw18|7347|/data/primary347 +350|348|p|p|s|u|sdw18|sdw18|7348|/data/primary348 +351|349|p|p|s|u|sdw18|sdw18|7349|/data/primary349 +352|350|p|p|s|u|sdw18|sdw18|7350|/data/primary350 +353|351|p|p|s|u|sdw18|sdw18|7351|/data/primary351 +354|352|p|p|s|u|sdw18|sdw18|7352|/data/primary352 +355|353|p|p|s|u|sdw18|sdw18|7353|/data/primary353 +356|354|p|p|s|u|sdw18|sdw18|7354|/data/primary354 +357|355|p|p|s|u|sdw18|sdw18|7355|/data/primary355 +358|356|p|p|s|u|sdw18|sdw18|7356|/data/primary356 +359|357|p|p|s|u|sdw18|sdw18|7357|/data/primary357 +360|358|p|p|s|u|sdw18|sdw18|7358|/data/primary358 +361|359|p|p|s|u|sdw18|sdw18|7359|/data/primary359 +1018|16|m|m|s|u|sdw18|sdw18|8016|/data/mirror16 +1037|35|m|m|s|u|sdw18|sdw18|8035|/data/mirror35 +1056|54|m|m|s|u|sdw18|sdw18|8054|/data/mirror54 +1075|73|m|m|s|u|sdw18|sdw18|8073|/data/mirror73 +1094|92|m|m|s|u|sdw18|sdw18|8092|/data/mirror92 +1113|111|m|m|s|u|sdw18|sdw18|8111|/data/mirror111 +1132|130|m|m|s|u|sdw18|sdw18|8130|/data/mirror130 +1151|149|m|m|s|u|sdw18|sdw18|8149|/data/mirror149 +1170|168|m|m|s|u|sdw18|sdw18|8168|/data/mirror168 +1189|187|m|m|s|u|sdw18|sdw18|8187|/data/mirror187 +1208|206|m|m|s|u|sdw18|sdw18|8206|/data/mirror206 +1227|225|m|m|s|u|sdw18|sdw18|8225|/data/mirror225 +1246|244|m|m|s|u|sdw18|sdw18|8244|/data/mirror244 +1265|263|m|m|s|u|sdw18|sdw18|8263|/data/mirror263 +1284|282|m|m|s|u|sdw18|sdw18|8282|/data/mirror282 +1303|301|m|m|s|u|sdw18|sdw18|8301|/data/mirror301 +1322|320|m|m|s|u|sdw18|sdw18|8320|/data/mirror320 +1961|959|m|m|s|u|sdw18|sdw18|8959|/data/mirror959 +1980|978|m|m|s|u|sdw18|sdw18|8978|/data/mirror978 +1999|997|m|m|s|u|sdw18|sdw18|8997|/data/mirror997 +# SDW19 +362|360|p|p|s|u|sdw19|sdw19|7360|/data/primary360 +363|361|p|p|s|u|sdw19|sdw19|7361|/data/primary361 +364|362|p|p|s|u|sdw19|sdw19|7362|/data/primary362 +365|363|p|p|s|u|sdw19|sdw19|7363|/data/primary363 +366|364|p|p|s|u|sdw19|sdw19|7364|/data/primary364 +367|365|p|p|s|u|sdw19|sdw19|7365|/data/primary365 +368|366|p|p|s|u|sdw19|sdw19|7366|/data/primary366 +369|367|p|p|s|u|sdw19|sdw19|7367|/data/primary367 +370|368|p|p|s|u|sdw19|sdw19|7368|/data/primary368 +371|369|p|p|s|u|sdw19|sdw19|7369|/data/primary369 +372|370|p|p|s|u|sdw19|sdw19|7370|/data/primary370 +373|371|p|p|s|u|sdw19|sdw19|7371|/data/primary371 +374|372|p|p|s|u|sdw19|sdw19|7372|/data/primary372 +375|373|p|p|s|u|sdw19|sdw19|7373|/data/primary373 +376|374|p|p|s|u|sdw19|sdw19|7374|/data/primary374 +377|375|p|p|s|u|sdw19|sdw19|7375|/data/primary375 +378|376|p|p|s|u|sdw19|sdw19|7376|/data/primary376 +379|377|p|p|s|u|sdw19|sdw19|7377|/data/primary377 +380|378|p|p|s|u|sdw19|sdw19|7378|/data/primary378 +381|379|p|p|s|u|sdw19|sdw19|7379|/data/primary379 +1019|17|m|m|s|u|sdw19|sdw19|8017|/data/mirror17 +1038|36|m|m|s|u|sdw19|sdw19|8036|/data/mirror36 +1057|55|m|m|s|u|sdw19|sdw19|8055|/data/mirror55 +1076|74|m|m|s|u|sdw19|sdw19|8074|/data/mirror74 +1095|93|m|m|s|u|sdw19|sdw19|8093|/data/mirror93 +1114|112|m|m|s|u|sdw19|sdw19|8112|/data/mirror112 +1133|131|m|m|s|u|sdw19|sdw19|8131|/data/mirror131 +1152|150|m|m|s|u|sdw19|sdw19|8150|/data/mirror150 +1171|169|m|m|s|u|sdw19|sdw19|8169|/data/mirror169 +1190|188|m|m|s|u|sdw19|sdw19|8188|/data/mirror188 +1209|207|m|m|s|u|sdw19|sdw19|8207|/data/mirror207 +1228|226|m|m|s|u|sdw19|sdw19|8226|/data/mirror226 +1247|245|m|m|s|u|sdw19|sdw19|8245|/data/mirror245 +1266|264|m|m|s|u|sdw19|sdw19|8264|/data/mirror264 +1285|283|m|m|s|u|sdw19|sdw19|8283|/data/mirror283 +1304|302|m|m|s|u|sdw19|sdw19|8302|/data/mirror302 +1323|321|m|m|s|u|sdw19|sdw19|8321|/data/mirror321 +1342|340|m|m|s|u|sdw19|sdw19|8340|/data/mirror340 +1981|979|m|m|s|u|sdw19|sdw19|8979|/data/mirror979 +2000|998|m|m|s|u|sdw19|sdw19|8998|/data/mirror998 +# SDW2 +22|20|p|p|s|u|sdw2|sdw2|7020|/data/primary20 +23|21|p|p|s|u|sdw2|sdw2|7021|/data/primary21 +24|22|p|p|s|u|sdw2|sdw2|7022|/data/primary22 +25|23|p|p|s|u|sdw2|sdw2|7023|/data/primary23 +26|24|p|p|s|u|sdw2|sdw2|7024|/data/primary24 +27|25|p|p|s|u|sdw2|sdw2|7025|/data/primary25 +28|26|p|p|s|u|sdw2|sdw2|7026|/data/primary26 +29|27|p|p|s|u|sdw2|sdw2|7027|/data/primary27 +30|28|p|p|s|u|sdw2|sdw2|7028|/data/primary28 +31|29|p|p|s|u|sdw2|sdw2|7029|/data/primary29 +32|30|p|p|s|u|sdw2|sdw2|7030|/data/primary30 +33|31|p|p|s|u|sdw2|sdw2|7031|/data/primary31 +34|32|p|p|s|u|sdw2|sdw2|7032|/data/primary32 +35|33|p|p|s|u|sdw2|sdw2|7033|/data/primary33 +36|34|p|p|s|u|sdw2|sdw2|7034|/data/primary34 +37|35|p|p|s|u|sdw2|sdw2|7035|/data/primary35 +38|36|p|p|s|u|sdw2|sdw2|7036|/data/primary36 +39|37|p|p|s|u|sdw2|sdw2|7037|/data/primary37 +40|38|p|p|s|u|sdw2|sdw2|7038|/data/primary38 +41|39|p|p|s|u|sdw2|sdw2|7039|/data/primary39 +1002|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +1641|639|m|m|s|u|sdw2|sdw2|8639|/data/mirror639 +1660|658|m|m|s|u|sdw2|sdw2|8658|/data/mirror658 +1679|677|m|m|s|u|sdw2|sdw2|8677|/data/mirror677 +1698|696|m|m|s|u|sdw2|sdw2|8696|/data/mirror696 +1717|715|m|m|s|u|sdw2|sdw2|8715|/data/mirror715 +1736|734|m|m|s|u|sdw2|sdw2|8734|/data/mirror734 +1755|753|m|m|s|u|sdw2|sdw2|8753|/data/mirror753 +1774|772|m|m|s|u|sdw2|sdw2|8772|/data/mirror772 +1793|791|m|m|s|u|sdw2|sdw2|8791|/data/mirror791 +1812|810|m|m|s|u|sdw2|sdw2|8810|/data/mirror810 +1831|829|m|m|s|u|sdw2|sdw2|8829|/data/mirror829 +1850|848|m|m|s|u|sdw2|sdw2|8848|/data/mirror848 +1869|867|m|m|s|u|sdw2|sdw2|8867|/data/mirror867 +1888|886|m|m|s|u|sdw2|sdw2|8886|/data/mirror886 +1907|905|m|m|s|u|sdw2|sdw2|8905|/data/mirror905 +1926|924|m|m|s|u|sdw2|sdw2|8924|/data/mirror924 +1945|943|m|m|s|u|sdw2|sdw2|8943|/data/mirror943 +1964|962|m|m|s|u|sdw2|sdw2|8962|/data/mirror962 +1983|981|m|m|s|u|sdw2|sdw2|8981|/data/mirror981 +# SDW20 +382|380|p|p|s|u|sdw20|sdw20|7380|/data/primary380 +383|381|p|p|s|u|sdw20|sdw20|7381|/data/primary381 +384|382|p|p|s|u|sdw20|sdw20|7382|/data/primary382 +385|383|p|p|s|u|sdw20|sdw20|7383|/data/primary383 +386|384|p|p|s|u|sdw20|sdw20|7384|/data/primary384 +387|385|p|p|s|u|sdw20|sdw20|7385|/data/primary385 +388|386|p|p|s|u|sdw20|sdw20|7386|/data/primary386 +389|387|p|p|s|u|sdw20|sdw20|7387|/data/primary387 +390|388|p|p|s|u|sdw20|sdw20|7388|/data/primary388 +391|389|p|p|s|u|sdw20|sdw20|7389|/data/primary389 +392|390|p|p|s|u|sdw20|sdw20|7390|/data/primary390 +393|391|p|p|s|u|sdw20|sdw20|7391|/data/primary391 +394|392|p|p|s|u|sdw20|sdw20|7392|/data/primary392 +395|393|p|p|s|u|sdw20|sdw20|7393|/data/primary393 +396|394|p|p|s|u|sdw20|sdw20|7394|/data/primary394 +397|395|p|p|s|u|sdw20|sdw20|7395|/data/primary395 +398|396|p|p|s|u|sdw20|sdw20|7396|/data/primary396 +399|397|p|p|s|u|sdw20|sdw20|7397|/data/primary397 +400|398|p|p|s|u|sdw20|sdw20|7398|/data/primary398 +401|399|p|p|s|u|sdw20|sdw20|7399|/data/primary399 +1020|18|m|m|s|u|sdw20|sdw20|8018|/data/mirror18 +1039|37|m|m|s|u|sdw20|sdw20|8037|/data/mirror37 +1058|56|m|m|s|u|sdw20|sdw20|8056|/data/mirror56 +1077|75|m|m|s|u|sdw20|sdw20|8075|/data/mirror75 +1096|94|m|m|s|u|sdw20|sdw20|8094|/data/mirror94 +1115|113|m|m|s|u|sdw20|sdw20|8113|/data/mirror113 +1134|132|m|m|s|u|sdw20|sdw20|8132|/data/mirror132 +1153|151|m|m|s|u|sdw20|sdw20|8151|/data/mirror151 +1172|170|m|m|s|u|sdw20|sdw20|8170|/data/mirror170 +1191|189|m|m|s|u|sdw20|sdw20|8189|/data/mirror189 +1210|208|m|m|s|u|sdw20|sdw20|8208|/data/mirror208 +1229|227|m|m|s|u|sdw20|sdw20|8227|/data/mirror227 +1248|246|m|m|s|u|sdw20|sdw20|8246|/data/mirror246 +1267|265|m|m|s|u|sdw20|sdw20|8265|/data/mirror265 +1286|284|m|m|s|u|sdw20|sdw20|8284|/data/mirror284 +1305|303|m|m|s|u|sdw20|sdw20|8303|/data/mirror303 +1324|322|m|m|s|u|sdw20|sdw20|8322|/data/mirror322 +1343|341|m|m|s|u|sdw20|sdw20|8341|/data/mirror341 +1362|360|m|m|s|u|sdw20|sdw20|8360|/data/mirror360 +2001|999|m|m|s|u|sdw20|sdw20|8999|/data/mirror999 +# SDW21 +402|400|p|p|s|u|sdw21|sdw21|7400|/data/primary400 +403|401|p|p|s|u|sdw21|sdw21|7401|/data/primary401 +404|402|p|p|s|u|sdw21|sdw21|7402|/data/primary402 +405|403|p|p|s|u|sdw21|sdw21|7403|/data/primary403 +406|404|p|p|s|u|sdw21|sdw21|7404|/data/primary404 +407|405|p|p|s|u|sdw21|sdw21|7405|/data/primary405 +408|406|p|p|s|u|sdw21|sdw21|7406|/data/primary406 +409|407|p|p|s|u|sdw21|sdw21|7407|/data/primary407 +410|408|p|p|s|u|sdw21|sdw21|7408|/data/primary408 +411|409|p|p|s|u|sdw21|sdw21|7409|/data/primary409 +412|410|p|p|s|u|sdw21|sdw21|7410|/data/primary410 +413|411|p|p|s|u|sdw21|sdw21|7411|/data/primary411 +414|412|p|p|s|u|sdw21|sdw21|7412|/data/primary412 +415|413|p|p|s|u|sdw21|sdw21|7413|/data/primary413 +416|414|p|p|s|u|sdw21|sdw21|7414|/data/primary414 +417|415|p|p|s|u|sdw21|sdw21|7415|/data/primary415 +418|416|p|p|s|u|sdw21|sdw21|7416|/data/primary416 +419|417|p|p|s|u|sdw21|sdw21|7417|/data/primary417 +420|418|p|p|s|u|sdw21|sdw21|7418|/data/primary418 +421|419|p|p|s|u|sdw21|sdw21|7419|/data/primary419 +1021|19|m|m|s|u|sdw21|sdw21|8019|/data/mirror19 +1040|38|m|m|s|u|sdw21|sdw21|8038|/data/mirror38 +1059|57|m|m|s|u|sdw21|sdw21|8057|/data/mirror57 +1078|76|m|m|s|u|sdw21|sdw21|8076|/data/mirror76 +1097|95|m|m|s|u|sdw21|sdw21|8095|/data/mirror95 +1116|114|m|m|s|u|sdw21|sdw21|8114|/data/mirror114 +1135|133|m|m|s|u|sdw21|sdw21|8133|/data/mirror133 +1154|152|m|m|s|u|sdw21|sdw21|8152|/data/mirror152 +1173|171|m|m|s|u|sdw21|sdw21|8171|/data/mirror171 +1192|190|m|m|s|u|sdw21|sdw21|8190|/data/mirror190 +1211|209|m|m|s|u|sdw21|sdw21|8209|/data/mirror209 +1230|228|m|m|s|u|sdw21|sdw21|8228|/data/mirror228 +1249|247|m|m|s|u|sdw21|sdw21|8247|/data/mirror247 +1268|266|m|m|s|u|sdw21|sdw21|8266|/data/mirror266 +1287|285|m|m|s|u|sdw21|sdw21|8285|/data/mirror285 +1306|304|m|m|s|u|sdw21|sdw21|8304|/data/mirror304 +1325|323|m|m|s|u|sdw21|sdw21|8323|/data/mirror323 +1344|342|m|m|s|u|sdw21|sdw21|8342|/data/mirror342 +1363|361|m|m|s|u|sdw21|sdw21|8361|/data/mirror361 +1382|380|m|m|s|u|sdw21|sdw21|8380|/data/mirror380 +# SDW22 +422|420|p|p|s|u|sdw22|sdw22|7420|/data/primary420 +423|421|p|p|s|u|sdw22|sdw22|7421|/data/primary421 +424|422|p|p|s|u|sdw22|sdw22|7422|/data/primary422 +425|423|p|p|s|u|sdw22|sdw22|7423|/data/primary423 +426|424|p|p|s|u|sdw22|sdw22|7424|/data/primary424 +427|425|p|p|s|u|sdw22|sdw22|7425|/data/primary425 +428|426|p|p|s|u|sdw22|sdw22|7426|/data/primary426 +429|427|p|p|s|u|sdw22|sdw22|7427|/data/primary427 +430|428|p|p|s|u|sdw22|sdw22|7428|/data/primary428 +431|429|p|p|s|u|sdw22|sdw22|7429|/data/primary429 +432|430|p|p|s|u|sdw22|sdw22|7430|/data/primary430 +433|431|p|p|s|u|sdw22|sdw22|7431|/data/primary431 +434|432|p|p|s|u|sdw22|sdw22|7432|/data/primary432 +435|433|p|p|s|u|sdw22|sdw22|7433|/data/primary433 +436|434|p|p|s|u|sdw22|sdw22|7434|/data/primary434 +437|435|p|p|s|u|sdw22|sdw22|7435|/data/primary435 +438|436|p|p|s|u|sdw22|sdw22|7436|/data/primary436 +439|437|p|p|s|u|sdw22|sdw22|7437|/data/primary437 +440|438|p|p|s|u|sdw22|sdw22|7438|/data/primary438 +441|439|p|p|s|u|sdw22|sdw22|7439|/data/primary439 +1041|39|m|m|s|u|sdw22|sdw22|8039|/data/mirror39 +1060|58|m|m|s|u|sdw22|sdw22|8058|/data/mirror58 +1079|77|m|m|s|u|sdw22|sdw22|8077|/data/mirror77 +1098|96|m|m|s|u|sdw22|sdw22|8096|/data/mirror96 +1117|115|m|m|s|u|sdw22|sdw22|8115|/data/mirror115 +1136|134|m|m|s|u|sdw22|sdw22|8134|/data/mirror134 +1155|153|m|m|s|u|sdw22|sdw22|8153|/data/mirror153 +1174|172|m|m|s|u|sdw22|sdw22|8172|/data/mirror172 +1193|191|m|m|s|u|sdw22|sdw22|8191|/data/mirror191 +1212|210|m|m|s|u|sdw22|sdw22|8210|/data/mirror210 +1231|229|m|m|s|u|sdw22|sdw22|8229|/data/mirror229 +1250|248|m|m|s|u|sdw22|sdw22|8248|/data/mirror248 +1269|267|m|m|s|u|sdw22|sdw22|8267|/data/mirror267 +1288|286|m|m|s|u|sdw22|sdw22|8286|/data/mirror286 +1307|305|m|m|s|u|sdw22|sdw22|8305|/data/mirror305 +1326|324|m|m|s|u|sdw22|sdw22|8324|/data/mirror324 +1345|343|m|m|s|u|sdw22|sdw22|8343|/data/mirror343 +1364|362|m|m|s|u|sdw22|sdw22|8362|/data/mirror362 +1383|381|m|m|s|u|sdw22|sdw22|8381|/data/mirror381 +1402|400|m|m|s|u|sdw22|sdw22|8400|/data/mirror400 +# SDW23 +442|440|p|p|s|u|sdw23|sdw23|7440|/data/primary440 +443|441|p|p|s|u|sdw23|sdw23|7441|/data/primary441 +444|442|p|p|s|u|sdw23|sdw23|7442|/data/primary442 +445|443|p|p|s|u|sdw23|sdw23|7443|/data/primary443 +446|444|p|p|s|u|sdw23|sdw23|7444|/data/primary444 +447|445|p|p|s|u|sdw23|sdw23|7445|/data/primary445 +448|446|p|p|s|u|sdw23|sdw23|7446|/data/primary446 +449|447|p|p|s|u|sdw23|sdw23|7447|/data/primary447 +450|448|p|p|s|u|sdw23|sdw23|7448|/data/primary448 +451|449|p|p|s|u|sdw23|sdw23|7449|/data/primary449 +452|450|p|p|s|u|sdw23|sdw23|7450|/data/primary450 +453|451|p|p|s|u|sdw23|sdw23|7451|/data/primary451 +454|452|p|p|s|u|sdw23|sdw23|7452|/data/primary452 +455|453|p|p|s|u|sdw23|sdw23|7453|/data/primary453 +456|454|p|p|s|u|sdw23|sdw23|7454|/data/primary454 +457|455|p|p|s|u|sdw23|sdw23|7455|/data/primary455 +458|456|p|p|s|u|sdw23|sdw23|7456|/data/primary456 +459|457|p|p|s|u|sdw23|sdw23|7457|/data/primary457 +460|458|p|p|s|u|sdw23|sdw23|7458|/data/primary458 +461|459|p|p|s|u|sdw23|sdw23|7459|/data/primary459 +1061|59|m|m|s|u|sdw23|sdw23|8059|/data/mirror59 +1080|78|m|m|s|u|sdw23|sdw23|8078|/data/mirror78 +1099|97|m|m|s|u|sdw23|sdw23|8097|/data/mirror97 +1118|116|m|m|s|u|sdw23|sdw23|8116|/data/mirror116 +1137|135|m|m|s|u|sdw23|sdw23|8135|/data/mirror135 +1156|154|m|m|s|u|sdw23|sdw23|8154|/data/mirror154 +1175|173|m|m|s|u|sdw23|sdw23|8173|/data/mirror173 +1194|192|m|m|s|u|sdw23|sdw23|8192|/data/mirror192 +1213|211|m|m|s|u|sdw23|sdw23|8211|/data/mirror211 +1232|230|m|m|s|u|sdw23|sdw23|8230|/data/mirror230 +1251|249|m|m|s|u|sdw23|sdw23|8249|/data/mirror249 +1270|268|m|m|s|u|sdw23|sdw23|8268|/data/mirror268 +1289|287|m|m|s|u|sdw23|sdw23|8287|/data/mirror287 +1308|306|m|m|s|u|sdw23|sdw23|8306|/data/mirror306 +1327|325|m|m|s|u|sdw23|sdw23|8325|/data/mirror325 +1346|344|m|m|s|u|sdw23|sdw23|8344|/data/mirror344 +1365|363|m|m|s|u|sdw23|sdw23|8363|/data/mirror363 +1384|382|m|m|s|u|sdw23|sdw23|8382|/data/mirror382 +1403|401|m|m|s|u|sdw23|sdw23|8401|/data/mirror401 +1422|420|m|m|s|u|sdw23|sdw23|8420|/data/mirror420 +# SDW24 +462|460|p|p|s|u|sdw24|sdw24|7460|/data/primary460 +463|461|p|p|s|u|sdw24|sdw24|7461|/data/primary461 +464|462|p|p|s|u|sdw24|sdw24|7462|/data/primary462 +465|463|p|p|s|u|sdw24|sdw24|7463|/data/primary463 +466|464|p|p|s|u|sdw24|sdw24|7464|/data/primary464 +467|465|p|p|s|u|sdw24|sdw24|7465|/data/primary465 +468|466|p|p|s|u|sdw24|sdw24|7466|/data/primary466 +469|467|p|p|s|u|sdw24|sdw24|7467|/data/primary467 +470|468|p|p|s|u|sdw24|sdw24|7468|/data/primary468 +471|469|p|p|s|u|sdw24|sdw24|7469|/data/primary469 +472|470|p|p|s|u|sdw24|sdw24|7470|/data/primary470 +473|471|p|p|s|u|sdw24|sdw24|7471|/data/primary471 +474|472|p|p|s|u|sdw24|sdw24|7472|/data/primary472 +475|473|p|p|s|u|sdw24|sdw24|7473|/data/primary473 +476|474|p|p|s|u|sdw24|sdw24|7474|/data/primary474 +477|475|p|p|s|u|sdw24|sdw24|7475|/data/primary475 +478|476|p|p|s|u|sdw24|sdw24|7476|/data/primary476 +479|477|p|p|s|u|sdw24|sdw24|7477|/data/primary477 +480|478|p|p|s|u|sdw24|sdw24|7478|/data/primary478 +481|479|p|p|s|u|sdw24|sdw24|7479|/data/primary479 +1081|79|m|m|s|u|sdw24|sdw24|8079|/data/mirror79 +1100|98|m|m|s|u|sdw24|sdw24|8098|/data/mirror98 +1119|117|m|m|s|u|sdw24|sdw24|8117|/data/mirror117 +1138|136|m|m|s|u|sdw24|sdw24|8136|/data/mirror136 +1157|155|m|m|s|u|sdw24|sdw24|8155|/data/mirror155 +1176|174|m|m|s|u|sdw24|sdw24|8174|/data/mirror174 +1195|193|m|m|s|u|sdw24|sdw24|8193|/data/mirror193 +1214|212|m|m|s|u|sdw24|sdw24|8212|/data/mirror212 +1233|231|m|m|s|u|sdw24|sdw24|8231|/data/mirror231 +1252|250|m|m|s|u|sdw24|sdw24|8250|/data/mirror250 +1271|269|m|m|s|u|sdw24|sdw24|8269|/data/mirror269 +1290|288|m|m|s|u|sdw24|sdw24|8288|/data/mirror288 +1309|307|m|m|s|u|sdw24|sdw24|8307|/data/mirror307 +1328|326|m|m|s|u|sdw24|sdw24|8326|/data/mirror326 +1347|345|m|m|s|u|sdw24|sdw24|8345|/data/mirror345 +1366|364|m|m|s|u|sdw24|sdw24|8364|/data/mirror364 +1385|383|m|m|s|u|sdw24|sdw24|8383|/data/mirror383 +1404|402|m|m|s|u|sdw24|sdw24|8402|/data/mirror402 +1423|421|m|m|s|u|sdw24|sdw24|8421|/data/mirror421 +1442|440|m|m|s|u|sdw24|sdw24|8440|/data/mirror440 +# SDW25 +482|480|p|p|s|u|sdw25|sdw25|7480|/data/primary480 +483|481|p|p|s|u|sdw25|sdw25|7481|/data/primary481 +484|482|p|p|s|u|sdw25|sdw25|7482|/data/primary482 +485|483|p|p|s|u|sdw25|sdw25|7483|/data/primary483 +486|484|p|p|s|u|sdw25|sdw25|7484|/data/primary484 +487|485|p|p|s|u|sdw25|sdw25|7485|/data/primary485 +488|486|p|p|s|u|sdw25|sdw25|7486|/data/primary486 +489|487|p|p|s|u|sdw25|sdw25|7487|/data/primary487 +490|488|p|p|s|u|sdw25|sdw25|7488|/data/primary488 +491|489|p|p|s|u|sdw25|sdw25|7489|/data/primary489 +492|490|p|p|s|u|sdw25|sdw25|7490|/data/primary490 +493|491|p|p|s|u|sdw25|sdw25|7491|/data/primary491 +494|492|p|p|s|u|sdw25|sdw25|7492|/data/primary492 +495|493|p|p|s|u|sdw25|sdw25|7493|/data/primary493 +496|494|p|p|s|u|sdw25|sdw25|7494|/data/primary494 +497|495|p|p|s|u|sdw25|sdw25|7495|/data/primary495 +498|496|p|p|s|u|sdw25|sdw25|7496|/data/primary496 +499|497|p|p|s|u|sdw25|sdw25|7497|/data/primary497 +500|498|p|p|s|u|sdw25|sdw25|7498|/data/primary498 +501|499|p|p|s|u|sdw25|sdw25|7499|/data/primary499 +1101|99|m|m|s|u|sdw25|sdw25|8099|/data/mirror99 +1120|118|m|m|s|u|sdw25|sdw25|8118|/data/mirror118 +1139|137|m|m|s|u|sdw25|sdw25|8137|/data/mirror137 +1158|156|m|m|s|u|sdw25|sdw25|8156|/data/mirror156 +1177|175|m|m|s|u|sdw25|sdw25|8175|/data/mirror175 +1196|194|m|m|s|u|sdw25|sdw25|8194|/data/mirror194 +1215|213|m|m|s|u|sdw25|sdw25|8213|/data/mirror213 +1234|232|m|m|s|u|sdw25|sdw25|8232|/data/mirror232 +1253|251|m|m|s|u|sdw25|sdw25|8251|/data/mirror251 +1272|270|m|m|s|u|sdw25|sdw25|8270|/data/mirror270 +1291|289|m|m|s|u|sdw25|sdw25|8289|/data/mirror289 +1310|308|m|m|s|u|sdw25|sdw25|8308|/data/mirror308 +1329|327|m|m|s|u|sdw25|sdw25|8327|/data/mirror327 +1348|346|m|m|s|u|sdw25|sdw25|8346|/data/mirror346 +1367|365|m|m|s|u|sdw25|sdw25|8365|/data/mirror365 +1386|384|m|m|s|u|sdw25|sdw25|8384|/data/mirror384 +1405|403|m|m|s|u|sdw25|sdw25|8403|/data/mirror403 +1424|422|m|m|s|u|sdw25|sdw25|8422|/data/mirror422 +1443|441|m|m|s|u|sdw25|sdw25|8441|/data/mirror441 +1462|460|m|m|s|u|sdw25|sdw25|8460|/data/mirror460 +# SDW26 +502|500|p|p|s|u|sdw26|sdw26|7500|/data/primary500 +503|501|p|p|s|u|sdw26|sdw26|7501|/data/primary501 +504|502|p|p|s|u|sdw26|sdw26|7502|/data/primary502 +505|503|p|p|s|u|sdw26|sdw26|7503|/data/primary503 +506|504|p|p|s|u|sdw26|sdw26|7504|/data/primary504 +507|505|p|p|s|u|sdw26|sdw26|7505|/data/primary505 +508|506|p|p|s|u|sdw26|sdw26|7506|/data/primary506 +509|507|p|p|s|u|sdw26|sdw26|7507|/data/primary507 +510|508|p|p|s|u|sdw26|sdw26|7508|/data/primary508 +511|509|p|p|s|u|sdw26|sdw26|7509|/data/primary509 +512|510|p|p|s|u|sdw26|sdw26|7510|/data/primary510 +513|511|p|p|s|u|sdw26|sdw26|7511|/data/primary511 +514|512|p|p|s|u|sdw26|sdw26|7512|/data/primary512 +515|513|p|p|s|u|sdw26|sdw26|7513|/data/primary513 +516|514|p|p|s|u|sdw26|sdw26|7514|/data/primary514 +517|515|p|p|s|u|sdw26|sdw26|7515|/data/primary515 +518|516|p|p|s|u|sdw26|sdw26|7516|/data/primary516 +519|517|p|p|s|u|sdw26|sdw26|7517|/data/primary517 +520|518|p|p|s|u|sdw26|sdw26|7518|/data/primary518 +521|519|p|p|s|u|sdw26|sdw26|7519|/data/primary519 +1121|119|m|m|s|u|sdw26|sdw26|8119|/data/mirror119 +1140|138|m|m|s|u|sdw26|sdw26|8138|/data/mirror138 +1159|157|m|m|s|u|sdw26|sdw26|8157|/data/mirror157 +1178|176|m|m|s|u|sdw26|sdw26|8176|/data/mirror176 +1197|195|m|m|s|u|sdw26|sdw26|8195|/data/mirror195 +1216|214|m|m|s|u|sdw26|sdw26|8214|/data/mirror214 +1235|233|m|m|s|u|sdw26|sdw26|8233|/data/mirror233 +1254|252|m|m|s|u|sdw26|sdw26|8252|/data/mirror252 +1273|271|m|m|s|u|sdw26|sdw26|8271|/data/mirror271 +1292|290|m|m|s|u|sdw26|sdw26|8290|/data/mirror290 +1311|309|m|m|s|u|sdw26|sdw26|8309|/data/mirror309 +1330|328|m|m|s|u|sdw26|sdw26|8328|/data/mirror328 +1349|347|m|m|s|u|sdw26|sdw26|8347|/data/mirror347 +1368|366|m|m|s|u|sdw26|sdw26|8366|/data/mirror366 +1387|385|m|m|s|u|sdw26|sdw26|8385|/data/mirror385 +1406|404|m|m|s|u|sdw26|sdw26|8404|/data/mirror404 +1425|423|m|m|s|u|sdw26|sdw26|8423|/data/mirror423 +1444|442|m|m|s|u|sdw26|sdw26|8442|/data/mirror442 +1463|461|m|m|s|u|sdw26|sdw26|8461|/data/mirror461 +1482|480|m|m|s|u|sdw26|sdw26|8480|/data/mirror480 +# SDW27 +522|520|p|p|s|u|sdw27|sdw27|7520|/data/primary520 +523|521|p|p|s|u|sdw27|sdw27|7521|/data/primary521 +524|522|p|p|s|u|sdw27|sdw27|7522|/data/primary522 +525|523|p|p|s|u|sdw27|sdw27|7523|/data/primary523 +526|524|p|p|s|u|sdw27|sdw27|7524|/data/primary524 +527|525|p|p|s|u|sdw27|sdw27|7525|/data/primary525 +528|526|p|p|s|u|sdw27|sdw27|7526|/data/primary526 +529|527|p|p|s|u|sdw27|sdw27|7527|/data/primary527 +530|528|p|p|s|u|sdw27|sdw27|7528|/data/primary528 +531|529|p|p|s|u|sdw27|sdw27|7529|/data/primary529 +532|530|p|p|s|u|sdw27|sdw27|7530|/data/primary530 +533|531|p|p|s|u|sdw27|sdw27|7531|/data/primary531 +534|532|p|p|s|u|sdw27|sdw27|7532|/data/primary532 +535|533|p|p|s|u|sdw27|sdw27|7533|/data/primary533 +536|534|p|p|s|u|sdw27|sdw27|7534|/data/primary534 +537|535|p|p|s|u|sdw27|sdw27|7535|/data/primary535 +538|536|p|p|s|u|sdw27|sdw27|7536|/data/primary536 +539|537|p|p|s|u|sdw27|sdw27|7537|/data/primary537 +540|538|p|p|s|u|sdw27|sdw27|7538|/data/primary538 +541|539|p|p|s|u|sdw27|sdw27|7539|/data/primary539 +1141|139|m|m|s|u|sdw27|sdw27|8139|/data/mirror139 +1160|158|m|m|s|u|sdw27|sdw27|8158|/data/mirror158 +1179|177|m|m|s|u|sdw27|sdw27|8177|/data/mirror177 +1198|196|m|m|s|u|sdw27|sdw27|8196|/data/mirror196 +1217|215|m|m|s|u|sdw27|sdw27|8215|/data/mirror215 +1236|234|m|m|s|u|sdw27|sdw27|8234|/data/mirror234 +1255|253|m|m|s|u|sdw27|sdw27|8253|/data/mirror253 +1274|272|m|m|s|u|sdw27|sdw27|8272|/data/mirror272 +1293|291|m|m|s|u|sdw27|sdw27|8291|/data/mirror291 +1312|310|m|m|s|u|sdw27|sdw27|8310|/data/mirror310 +1331|329|m|m|s|u|sdw27|sdw27|8329|/data/mirror329 +1350|348|m|m|s|u|sdw27|sdw27|8348|/data/mirror348 +1369|367|m|m|s|u|sdw27|sdw27|8367|/data/mirror367 +1388|386|m|m|s|u|sdw27|sdw27|8386|/data/mirror386 +1407|405|m|m|s|u|sdw27|sdw27|8405|/data/mirror405 +1426|424|m|m|s|u|sdw27|sdw27|8424|/data/mirror424 +1445|443|m|m|s|u|sdw27|sdw27|8443|/data/mirror443 +1464|462|m|m|s|u|sdw27|sdw27|8462|/data/mirror462 +1483|481|m|m|s|u|sdw27|sdw27|8481|/data/mirror481 +1502|500|m|m|s|u|sdw27|sdw27|8500|/data/mirror500 +# SDW28 +542|540|p|p|s|u|sdw28|sdw28|7540|/data/primary540 +543|541|p|p|s|u|sdw28|sdw28|7541|/data/primary541 +544|542|p|p|s|u|sdw28|sdw28|7542|/data/primary542 +545|543|p|p|s|u|sdw28|sdw28|7543|/data/primary543 +546|544|p|p|s|u|sdw28|sdw28|7544|/data/primary544 +547|545|p|p|s|u|sdw28|sdw28|7545|/data/primary545 +548|546|p|p|s|u|sdw28|sdw28|7546|/data/primary546 +549|547|p|p|s|u|sdw28|sdw28|7547|/data/primary547 +550|548|p|p|s|u|sdw28|sdw28|7548|/data/primary548 +551|549|p|p|s|u|sdw28|sdw28|7549|/data/primary549 +552|550|p|p|s|u|sdw28|sdw28|7550|/data/primary550 +553|551|p|p|s|u|sdw28|sdw28|7551|/data/primary551 +554|552|p|p|s|u|sdw28|sdw28|7552|/data/primary552 +555|553|p|p|s|u|sdw28|sdw28|7553|/data/primary553 +556|554|p|p|s|u|sdw28|sdw28|7554|/data/primary554 +557|555|p|p|s|u|sdw28|sdw28|7555|/data/primary555 +558|556|p|p|s|u|sdw28|sdw28|7556|/data/primary556 +559|557|p|p|s|u|sdw28|sdw28|7557|/data/primary557 +560|558|p|p|s|u|sdw28|sdw28|7558|/data/primary558 +561|559|p|p|s|u|sdw28|sdw28|7559|/data/primary559 +1161|159|m|m|s|u|sdw28|sdw28|8159|/data/mirror159 +1180|178|m|m|s|u|sdw28|sdw28|8178|/data/mirror178 +1199|197|m|m|s|u|sdw28|sdw28|8197|/data/mirror197 +1218|216|m|m|s|u|sdw28|sdw28|8216|/data/mirror216 +1237|235|m|m|s|u|sdw28|sdw28|8235|/data/mirror235 +1256|254|m|m|s|u|sdw28|sdw28|8254|/data/mirror254 +1275|273|m|m|s|u|sdw28|sdw28|8273|/data/mirror273 +1294|292|m|m|s|u|sdw28|sdw28|8292|/data/mirror292 +1313|311|m|m|s|u|sdw28|sdw28|8311|/data/mirror311 +1332|330|m|m|s|u|sdw28|sdw28|8330|/data/mirror330 +1351|349|m|m|s|u|sdw28|sdw28|8349|/data/mirror349 +1370|368|m|m|s|u|sdw28|sdw28|8368|/data/mirror368 +1389|387|m|m|s|u|sdw28|sdw28|8387|/data/mirror387 +1408|406|m|m|s|u|sdw28|sdw28|8406|/data/mirror406 +1427|425|m|m|s|u|sdw28|sdw28|8425|/data/mirror425 +1446|444|m|m|s|u|sdw28|sdw28|8444|/data/mirror444 +1465|463|m|m|s|u|sdw28|sdw28|8463|/data/mirror463 +1484|482|m|m|s|u|sdw28|sdw28|8482|/data/mirror482 +1503|501|m|m|s|u|sdw28|sdw28|8501|/data/mirror501 +1522|520|m|m|s|u|sdw28|sdw28|8520|/data/mirror520 +# SDW29 +562|560|p|p|s|u|sdw29|sdw29|7560|/data/primary560 +563|561|p|p|s|u|sdw29|sdw29|7561|/data/primary561 +564|562|p|p|s|u|sdw29|sdw29|7562|/data/primary562 +565|563|p|p|s|u|sdw29|sdw29|7563|/data/primary563 +566|564|p|p|s|u|sdw29|sdw29|7564|/data/primary564 +567|565|p|p|s|u|sdw29|sdw29|7565|/data/primary565 +568|566|p|p|s|u|sdw29|sdw29|7566|/data/primary566 +569|567|p|p|s|u|sdw29|sdw29|7567|/data/primary567 +570|568|p|p|s|u|sdw29|sdw29|7568|/data/primary568 +571|569|p|p|s|u|sdw29|sdw29|7569|/data/primary569 +572|570|p|p|s|u|sdw29|sdw29|7570|/data/primary570 +573|571|p|p|s|u|sdw29|sdw29|7571|/data/primary571 +574|572|p|p|s|u|sdw29|sdw29|7572|/data/primary572 +575|573|p|p|s|u|sdw29|sdw29|7573|/data/primary573 +576|574|p|p|s|u|sdw29|sdw29|7574|/data/primary574 +577|575|p|p|s|u|sdw29|sdw29|7575|/data/primary575 +578|576|p|p|s|u|sdw29|sdw29|7576|/data/primary576 +579|577|p|p|s|u|sdw29|sdw29|7577|/data/primary577 +580|578|p|p|s|u|sdw29|sdw29|7578|/data/primary578 +581|579|p|p|s|u|sdw29|sdw29|7579|/data/primary579 +1181|179|m|m|s|u|sdw29|sdw29|8179|/data/mirror179 +1200|198|m|m|s|u|sdw29|sdw29|8198|/data/mirror198 +1219|217|m|m|s|u|sdw29|sdw29|8217|/data/mirror217 +1238|236|m|m|s|u|sdw29|sdw29|8236|/data/mirror236 +1257|255|m|m|s|u|sdw29|sdw29|8255|/data/mirror255 +1276|274|m|m|s|u|sdw29|sdw29|8274|/data/mirror274 +1295|293|m|m|s|u|sdw29|sdw29|8293|/data/mirror293 +1314|312|m|m|s|u|sdw29|sdw29|8312|/data/mirror312 +1333|331|m|m|s|u|sdw29|sdw29|8331|/data/mirror331 +1352|350|m|m|s|u|sdw29|sdw29|8350|/data/mirror350 +1371|369|m|m|s|u|sdw29|sdw29|8369|/data/mirror369 +1390|388|m|m|s|u|sdw29|sdw29|8388|/data/mirror388 +1409|407|m|m|s|u|sdw29|sdw29|8407|/data/mirror407 +1428|426|m|m|s|u|sdw29|sdw29|8426|/data/mirror426 +1447|445|m|m|s|u|sdw29|sdw29|8445|/data/mirror445 +1466|464|m|m|s|u|sdw29|sdw29|8464|/data/mirror464 +1485|483|m|m|s|u|sdw29|sdw29|8483|/data/mirror483 +1504|502|m|m|s|u|sdw29|sdw29|8502|/data/mirror502 +1523|521|m|m|s|u|sdw29|sdw29|8521|/data/mirror521 +1542|540|m|m|s|u|sdw29|sdw29|8540|/data/mirror540 +# SDW3 +42|40|p|p|s|u|sdw3|sdw3|7040|/data/primary40 +43|41|p|p|s|u|sdw3|sdw3|7041|/data/primary41 +44|42|p|p|s|u|sdw3|sdw3|7042|/data/primary42 +45|43|p|p|s|u|sdw3|sdw3|7043|/data/primary43 +46|44|p|p|s|u|sdw3|sdw3|7044|/data/primary44 +47|45|p|p|s|u|sdw3|sdw3|7045|/data/primary45 +48|46|p|p|s|u|sdw3|sdw3|7046|/data/primary46 +49|47|p|p|s|u|sdw3|sdw3|7047|/data/primary47 +50|48|p|p|s|u|sdw3|sdw3|7048|/data/primary48 +51|49|p|p|s|u|sdw3|sdw3|7049|/data/primary49 +52|50|p|p|s|u|sdw3|sdw3|7050|/data/primary50 +53|51|p|p|s|u|sdw3|sdw3|7051|/data/primary51 +54|52|p|p|s|u|sdw3|sdw3|7052|/data/primary52 +55|53|p|p|s|u|sdw3|sdw3|7053|/data/primary53 +56|54|p|p|s|u|sdw3|sdw3|7054|/data/primary54 +57|55|p|p|s|u|sdw3|sdw3|7055|/data/primary55 +58|56|p|p|s|u|sdw3|sdw3|7056|/data/primary56 +59|57|p|p|s|u|sdw3|sdw3|7057|/data/primary57 +60|58|p|p|s|u|sdw3|sdw3|7058|/data/primary58 +61|59|p|p|s|u|sdw3|sdw3|7059|/data/primary59 +1003|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +1022|20|m|m|s|u|sdw3|sdw3|8020|/data/mirror20 +1661|659|m|m|s|u|sdw3|sdw3|8659|/data/mirror659 +1680|678|m|m|s|u|sdw3|sdw3|8678|/data/mirror678 +1699|697|m|m|s|u|sdw3|sdw3|8697|/data/mirror697 +1718|716|m|m|s|u|sdw3|sdw3|8716|/data/mirror716 +1737|735|m|m|s|u|sdw3|sdw3|8735|/data/mirror735 +1756|754|m|m|s|u|sdw3|sdw3|8754|/data/mirror754 +1775|773|m|m|s|u|sdw3|sdw3|8773|/data/mirror773 +1794|792|m|m|s|u|sdw3|sdw3|8792|/data/mirror792 +1813|811|m|m|s|u|sdw3|sdw3|8811|/data/mirror811 +1832|830|m|m|s|u|sdw3|sdw3|8830|/data/mirror830 +1851|849|m|m|s|u|sdw3|sdw3|8849|/data/mirror849 +1870|868|m|m|s|u|sdw3|sdw3|8868|/data/mirror868 +1889|887|m|m|s|u|sdw3|sdw3|8887|/data/mirror887 +1908|906|m|m|s|u|sdw3|sdw3|8906|/data/mirror906 +1927|925|m|m|s|u|sdw3|sdw3|8925|/data/mirror925 +1946|944|m|m|s|u|sdw3|sdw3|8944|/data/mirror944 +1965|963|m|m|s|u|sdw3|sdw3|8963|/data/mirror963 +1984|982|m|m|s|u|sdw3|sdw3|8982|/data/mirror982 +# SDW30 +582|580|p|p|s|u|sdw30|sdw30|7580|/data/primary580 +583|581|p|p|s|u|sdw30|sdw30|7581|/data/primary581 +584|582|p|p|s|u|sdw30|sdw30|7582|/data/primary582 +585|583|p|p|s|u|sdw30|sdw30|7583|/data/primary583 +586|584|p|p|s|u|sdw30|sdw30|7584|/data/primary584 +587|585|p|p|s|u|sdw30|sdw30|7585|/data/primary585 +588|586|p|p|s|u|sdw30|sdw30|7586|/data/primary586 +589|587|p|p|s|u|sdw30|sdw30|7587|/data/primary587 +590|588|p|p|s|u|sdw30|sdw30|7588|/data/primary588 +591|589|p|p|s|u|sdw30|sdw30|7589|/data/primary589 +592|590|p|p|s|u|sdw30|sdw30|7590|/data/primary590 +593|591|p|p|s|u|sdw30|sdw30|7591|/data/primary591 +594|592|p|p|s|u|sdw30|sdw30|7592|/data/primary592 +595|593|p|p|s|u|sdw30|sdw30|7593|/data/primary593 +596|594|p|p|s|u|sdw30|sdw30|7594|/data/primary594 +597|595|p|p|s|u|sdw30|sdw30|7595|/data/primary595 +598|596|p|p|s|u|sdw30|sdw30|7596|/data/primary596 +599|597|p|p|s|u|sdw30|sdw30|7597|/data/primary597 +600|598|p|p|s|u|sdw30|sdw30|7598|/data/primary598 +601|599|p|p|s|u|sdw30|sdw30|7599|/data/primary599 +1201|199|m|m|s|u|sdw30|sdw30|8199|/data/mirror199 +1220|218|m|m|s|u|sdw30|sdw30|8218|/data/mirror218 +1239|237|m|m|s|u|sdw30|sdw30|8237|/data/mirror237 +1258|256|m|m|s|u|sdw30|sdw30|8256|/data/mirror256 +1277|275|m|m|s|u|sdw30|sdw30|8275|/data/mirror275 +1296|294|m|m|s|u|sdw30|sdw30|8294|/data/mirror294 +1315|313|m|m|s|u|sdw30|sdw30|8313|/data/mirror313 +1334|332|m|m|s|u|sdw30|sdw30|8332|/data/mirror332 +1353|351|m|m|s|u|sdw30|sdw30|8351|/data/mirror351 +1372|370|m|m|s|u|sdw30|sdw30|8370|/data/mirror370 +1391|389|m|m|s|u|sdw30|sdw30|8389|/data/mirror389 +1410|408|m|m|s|u|sdw30|sdw30|8408|/data/mirror408 +1429|427|m|m|s|u|sdw30|sdw30|8427|/data/mirror427 +1448|446|m|m|s|u|sdw30|sdw30|8446|/data/mirror446 +1467|465|m|m|s|u|sdw30|sdw30|8465|/data/mirror465 +1486|484|m|m|s|u|sdw30|sdw30|8484|/data/mirror484 +1505|503|m|m|s|u|sdw30|sdw30|8503|/data/mirror503 +1524|522|m|m|s|u|sdw30|sdw30|8522|/data/mirror522 +1543|541|m|m|s|u|sdw30|sdw30|8541|/data/mirror541 +1562|560|m|m|s|u|sdw30|sdw30|8560|/data/mirror560 +# SDW31 +602|600|p|p|s|u|sdw31|sdw31|7600|/data/primary600 +603|601|p|p|s|u|sdw31|sdw31|7601|/data/primary601 +604|602|p|p|s|u|sdw31|sdw31|7602|/data/primary602 +605|603|p|p|s|u|sdw31|sdw31|7603|/data/primary603 +606|604|p|p|s|u|sdw31|sdw31|7604|/data/primary604 +607|605|p|p|s|u|sdw31|sdw31|7605|/data/primary605 +608|606|p|p|s|u|sdw31|sdw31|7606|/data/primary606 +609|607|p|p|s|u|sdw31|sdw31|7607|/data/primary607 +610|608|p|p|s|u|sdw31|sdw31|7608|/data/primary608 +611|609|p|p|s|u|sdw31|sdw31|7609|/data/primary609 +612|610|p|p|s|u|sdw31|sdw31|7610|/data/primary610 +613|611|p|p|s|u|sdw31|sdw31|7611|/data/primary611 +614|612|p|p|s|u|sdw31|sdw31|7612|/data/primary612 +615|613|p|p|s|u|sdw31|sdw31|7613|/data/primary613 +616|614|p|p|s|u|sdw31|sdw31|7614|/data/primary614 +617|615|p|p|s|u|sdw31|sdw31|7615|/data/primary615 +618|616|p|p|s|u|sdw31|sdw31|7616|/data/primary616 +619|617|p|p|s|u|sdw31|sdw31|7617|/data/primary617 +620|618|p|p|s|u|sdw31|sdw31|7618|/data/primary618 +621|619|p|p|s|u|sdw31|sdw31|7619|/data/primary619 +1221|219|m|m|s|u|sdw31|sdw31|8219|/data/mirror219 +1240|238|m|m|s|u|sdw31|sdw31|8238|/data/mirror238 +1259|257|m|m|s|u|sdw31|sdw31|8257|/data/mirror257 +1278|276|m|m|s|u|sdw31|sdw31|8276|/data/mirror276 +1297|295|m|m|s|u|sdw31|sdw31|8295|/data/mirror295 +1316|314|m|m|s|u|sdw31|sdw31|8314|/data/mirror314 +1335|333|m|m|s|u|sdw31|sdw31|8333|/data/mirror333 +1354|352|m|m|s|u|sdw31|sdw31|8352|/data/mirror352 +1373|371|m|m|s|u|sdw31|sdw31|8371|/data/mirror371 +1392|390|m|m|s|u|sdw31|sdw31|8390|/data/mirror390 +1411|409|m|m|s|u|sdw31|sdw31|8409|/data/mirror409 +1430|428|m|m|s|u|sdw31|sdw31|8428|/data/mirror428 +1449|447|m|m|s|u|sdw31|sdw31|8447|/data/mirror447 +1468|466|m|m|s|u|sdw31|sdw31|8466|/data/mirror466 +1487|485|m|m|s|u|sdw31|sdw31|8485|/data/mirror485 +1506|504|m|m|s|u|sdw31|sdw31|8504|/data/mirror504 +1525|523|m|m|s|u|sdw31|sdw31|8523|/data/mirror523 +1544|542|m|m|s|u|sdw31|sdw31|8542|/data/mirror542 +1563|561|m|m|s|u|sdw31|sdw31|8561|/data/mirror561 +1582|580|m|m|s|u|sdw31|sdw31|8580|/data/mirror580 +# SDW32 +622|620|p|p|s|u|sdw32|sdw32|7620|/data/primary620 +623|621|p|p|s|u|sdw32|sdw32|7621|/data/primary621 +624|622|p|p|s|u|sdw32|sdw32|7622|/data/primary622 +625|623|p|p|s|u|sdw32|sdw32|7623|/data/primary623 +626|624|p|p|s|u|sdw32|sdw32|7624|/data/primary624 +627|625|p|p|s|u|sdw32|sdw32|7625|/data/primary625 +628|626|p|p|s|u|sdw32|sdw32|7626|/data/primary626 +629|627|p|p|s|u|sdw32|sdw32|7627|/data/primary627 +630|628|p|p|s|u|sdw32|sdw32|7628|/data/primary628 +631|629|p|p|s|u|sdw32|sdw32|7629|/data/primary629 +632|630|p|p|s|u|sdw32|sdw32|7630|/data/primary630 +633|631|p|p|s|u|sdw32|sdw32|7631|/data/primary631 +634|632|p|p|s|u|sdw32|sdw32|7632|/data/primary632 +635|633|p|p|s|u|sdw32|sdw32|7633|/data/primary633 +636|634|p|p|s|u|sdw32|sdw32|7634|/data/primary634 +637|635|p|p|s|u|sdw32|sdw32|7635|/data/primary635 +638|636|p|p|s|u|sdw32|sdw32|7636|/data/primary636 +639|637|p|p|s|u|sdw32|sdw32|7637|/data/primary637 +640|638|p|p|s|u|sdw32|sdw32|7638|/data/primary638 +641|639|p|p|s|u|sdw32|sdw32|7639|/data/primary639 +1241|239|m|m|s|u|sdw32|sdw32|8239|/data/mirror239 +1260|258|m|m|s|u|sdw32|sdw32|8258|/data/mirror258 +1279|277|m|m|s|u|sdw32|sdw32|8277|/data/mirror277 +1298|296|m|m|s|u|sdw32|sdw32|8296|/data/mirror296 +1317|315|m|m|s|u|sdw32|sdw32|8315|/data/mirror315 +1336|334|m|m|s|u|sdw32|sdw32|8334|/data/mirror334 +1355|353|m|m|s|u|sdw32|sdw32|8353|/data/mirror353 +1374|372|m|m|s|u|sdw32|sdw32|8372|/data/mirror372 +1393|391|m|m|s|u|sdw32|sdw32|8391|/data/mirror391 +1412|410|m|m|s|u|sdw32|sdw32|8410|/data/mirror410 +1431|429|m|m|s|u|sdw32|sdw32|8429|/data/mirror429 +1450|448|m|m|s|u|sdw32|sdw32|8448|/data/mirror448 +1469|467|m|m|s|u|sdw32|sdw32|8467|/data/mirror467 +1488|486|m|m|s|u|sdw32|sdw32|8486|/data/mirror486 +1507|505|m|m|s|u|sdw32|sdw32|8505|/data/mirror505 +1526|524|m|m|s|u|sdw32|sdw32|8524|/data/mirror524 +1545|543|m|m|s|u|sdw32|sdw32|8543|/data/mirror543 +1564|562|m|m|s|u|sdw32|sdw32|8562|/data/mirror562 +1583|581|m|m|s|u|sdw32|sdw32|8581|/data/mirror581 +1602|600|m|m|s|u|sdw32|sdw32|8600|/data/mirror600 +# SDW33 +642|640|p|p|s|u|sdw33|sdw33|7640|/data/primary640 +643|641|p|p|s|u|sdw33|sdw33|7641|/data/primary641 +644|642|p|p|s|u|sdw33|sdw33|7642|/data/primary642 +645|643|p|p|s|u|sdw33|sdw33|7643|/data/primary643 +646|644|p|p|s|u|sdw33|sdw33|7644|/data/primary644 +647|645|p|p|s|u|sdw33|sdw33|7645|/data/primary645 +648|646|p|p|s|u|sdw33|sdw33|7646|/data/primary646 +649|647|p|p|s|u|sdw33|sdw33|7647|/data/primary647 +650|648|p|p|s|u|sdw33|sdw33|7648|/data/primary648 +651|649|p|p|s|u|sdw33|sdw33|7649|/data/primary649 +652|650|p|p|s|u|sdw33|sdw33|7650|/data/primary650 +653|651|p|p|s|u|sdw33|sdw33|7651|/data/primary651 +654|652|p|p|s|u|sdw33|sdw33|7652|/data/primary652 +655|653|p|p|s|u|sdw33|sdw33|7653|/data/primary653 +656|654|p|p|s|u|sdw33|sdw33|7654|/data/primary654 +657|655|p|p|s|u|sdw33|sdw33|7655|/data/primary655 +658|656|p|p|s|u|sdw33|sdw33|7656|/data/primary656 +659|657|p|p|s|u|sdw33|sdw33|7657|/data/primary657 +660|658|p|p|s|u|sdw33|sdw33|7658|/data/primary658 +661|659|p|p|s|u|sdw33|sdw33|7659|/data/primary659 +1261|259|m|m|s|u|sdw33|sdw33|8259|/data/mirror259 +1280|278|m|m|s|u|sdw33|sdw33|8278|/data/mirror278 +1299|297|m|m|s|u|sdw33|sdw33|8297|/data/mirror297 +1318|316|m|m|s|u|sdw33|sdw33|8316|/data/mirror316 +1337|335|m|m|s|u|sdw33|sdw33|8335|/data/mirror335 +1356|354|m|m|s|u|sdw33|sdw33|8354|/data/mirror354 +1375|373|m|m|s|u|sdw33|sdw33|8373|/data/mirror373 +1394|392|m|m|s|u|sdw33|sdw33|8392|/data/mirror392 +1413|411|m|m|s|u|sdw33|sdw33|8411|/data/mirror411 +1432|430|m|m|s|u|sdw33|sdw33|8430|/data/mirror430 +1451|449|m|m|s|u|sdw33|sdw33|8449|/data/mirror449 +1470|468|m|m|s|u|sdw33|sdw33|8468|/data/mirror468 +1489|487|m|m|s|u|sdw33|sdw33|8487|/data/mirror487 +1508|506|m|m|s|u|sdw33|sdw33|8506|/data/mirror506 +1527|525|m|m|s|u|sdw33|sdw33|8525|/data/mirror525 +1546|544|m|m|s|u|sdw33|sdw33|8544|/data/mirror544 +1565|563|m|m|s|u|sdw33|sdw33|8563|/data/mirror563 +1584|582|m|m|s|u|sdw33|sdw33|8582|/data/mirror582 +1603|601|m|m|s|u|sdw33|sdw33|8601|/data/mirror601 +1622|620|m|m|s|u|sdw33|sdw33|8620|/data/mirror620 +# SDW34 +662|660|p|p|s|u|sdw34|sdw34|7660|/data/primary660 +663|661|p|p|s|u|sdw34|sdw34|7661|/data/primary661 +664|662|p|p|s|u|sdw34|sdw34|7662|/data/primary662 +665|663|p|p|s|u|sdw34|sdw34|7663|/data/primary663 +666|664|p|p|s|u|sdw34|sdw34|7664|/data/primary664 +667|665|p|p|s|u|sdw34|sdw34|7665|/data/primary665 +668|666|p|p|s|u|sdw34|sdw34|7666|/data/primary666 +669|667|p|p|s|u|sdw34|sdw34|7667|/data/primary667 +670|668|p|p|s|u|sdw34|sdw34|7668|/data/primary668 +671|669|p|p|s|u|sdw34|sdw34|7669|/data/primary669 +672|670|p|p|s|u|sdw34|sdw34|7670|/data/primary670 +673|671|p|p|s|u|sdw34|sdw34|7671|/data/primary671 +674|672|p|p|s|u|sdw34|sdw34|7672|/data/primary672 +675|673|p|p|s|u|sdw34|sdw34|7673|/data/primary673 +676|674|p|p|s|u|sdw34|sdw34|7674|/data/primary674 +677|675|p|p|s|u|sdw34|sdw34|7675|/data/primary675 +678|676|p|p|s|u|sdw34|sdw34|7676|/data/primary676 +679|677|p|p|s|u|sdw34|sdw34|7677|/data/primary677 +680|678|p|p|s|u|sdw34|sdw34|7678|/data/primary678 +681|679|p|p|s|u|sdw34|sdw34|7679|/data/primary679 +1281|279|m|m|s|u|sdw34|sdw34|8279|/data/mirror279 +1300|298|m|m|s|u|sdw34|sdw34|8298|/data/mirror298 +1319|317|m|m|s|u|sdw34|sdw34|8317|/data/mirror317 +1338|336|m|m|s|u|sdw34|sdw34|8336|/data/mirror336 +1357|355|m|m|s|u|sdw34|sdw34|8355|/data/mirror355 +1376|374|m|m|s|u|sdw34|sdw34|8374|/data/mirror374 +1395|393|m|m|s|u|sdw34|sdw34|8393|/data/mirror393 +1414|412|m|m|s|u|sdw34|sdw34|8412|/data/mirror412 +1433|431|m|m|s|u|sdw34|sdw34|8431|/data/mirror431 +1452|450|m|m|s|u|sdw34|sdw34|8450|/data/mirror450 +1471|469|m|m|s|u|sdw34|sdw34|8469|/data/mirror469 +1490|488|m|m|s|u|sdw34|sdw34|8488|/data/mirror488 +1509|507|m|m|s|u|sdw34|sdw34|8507|/data/mirror507 +1528|526|m|m|s|u|sdw34|sdw34|8526|/data/mirror526 +1547|545|m|m|s|u|sdw34|sdw34|8545|/data/mirror545 +1566|564|m|m|s|u|sdw34|sdw34|8564|/data/mirror564 +1585|583|m|m|s|u|sdw34|sdw34|8583|/data/mirror583 +1604|602|m|m|s|u|sdw34|sdw34|8602|/data/mirror602 +1623|621|m|m|s|u|sdw34|sdw34|8621|/data/mirror621 +1642|640|m|m|s|u|sdw34|sdw34|8640|/data/mirror640 +# SDW35 +682|680|p|p|s|u|sdw35|sdw35|7680|/data/primary680 +683|681|p|p|s|u|sdw35|sdw35|7681|/data/primary681 +684|682|p|p|s|u|sdw35|sdw35|7682|/data/primary682 +685|683|p|p|s|u|sdw35|sdw35|7683|/data/primary683 +686|684|p|p|s|u|sdw35|sdw35|7684|/data/primary684 +687|685|p|p|s|u|sdw35|sdw35|7685|/data/primary685 +688|686|p|p|s|u|sdw35|sdw35|7686|/data/primary686 +689|687|p|p|s|u|sdw35|sdw35|7687|/data/primary687 +690|688|p|p|s|u|sdw35|sdw35|7688|/data/primary688 +691|689|p|p|s|u|sdw35|sdw35|7689|/data/primary689 +692|690|p|p|s|u|sdw35|sdw35|7690|/data/primary690 +693|691|p|p|s|u|sdw35|sdw35|7691|/data/primary691 +694|692|p|p|s|u|sdw35|sdw35|7692|/data/primary692 +695|693|p|p|s|u|sdw35|sdw35|7693|/data/primary693 +696|694|p|p|s|u|sdw35|sdw35|7694|/data/primary694 +697|695|p|p|s|u|sdw35|sdw35|7695|/data/primary695 +698|696|p|p|s|u|sdw35|sdw35|7696|/data/primary696 +699|697|p|p|s|u|sdw35|sdw35|7697|/data/primary697 +700|698|p|p|s|u|sdw35|sdw35|7698|/data/primary698 +701|699|p|p|s|u|sdw35|sdw35|7699|/data/primary699 +1301|299|m|m|s|u|sdw35|sdw35|8299|/data/mirror299 +1320|318|m|m|s|u|sdw35|sdw35|8318|/data/mirror318 +1339|337|m|m|s|u|sdw35|sdw35|8337|/data/mirror337 +1358|356|m|m|s|u|sdw35|sdw35|8356|/data/mirror356 +1377|375|m|m|s|u|sdw35|sdw35|8375|/data/mirror375 +1396|394|m|m|s|u|sdw35|sdw35|8394|/data/mirror394 +1415|413|m|m|s|u|sdw35|sdw35|8413|/data/mirror413 +1434|432|m|m|s|u|sdw35|sdw35|8432|/data/mirror432 +1453|451|m|m|s|u|sdw35|sdw35|8451|/data/mirror451 +1472|470|m|m|s|u|sdw35|sdw35|8470|/data/mirror470 +1491|489|m|m|s|u|sdw35|sdw35|8489|/data/mirror489 +1510|508|m|m|s|u|sdw35|sdw35|8508|/data/mirror508 +1529|527|m|m|s|u|sdw35|sdw35|8527|/data/mirror527 +1548|546|m|m|s|u|sdw35|sdw35|8546|/data/mirror546 +1567|565|m|m|s|u|sdw35|sdw35|8565|/data/mirror565 +1586|584|m|m|s|u|sdw35|sdw35|8584|/data/mirror584 +1605|603|m|m|s|u|sdw35|sdw35|8603|/data/mirror603 +1624|622|m|m|s|u|sdw35|sdw35|8622|/data/mirror622 +1643|641|m|m|s|u|sdw35|sdw35|8641|/data/mirror641 +1662|660|m|m|s|u|sdw35|sdw35|8660|/data/mirror660 +# SDW36 +702|700|p|p|s|u|sdw36|sdw36|7700|/data/primary700 +703|701|p|p|s|u|sdw36|sdw36|7701|/data/primary701 +704|702|p|p|s|u|sdw36|sdw36|7702|/data/primary702 +705|703|p|p|s|u|sdw36|sdw36|7703|/data/primary703 +706|704|p|p|s|u|sdw36|sdw36|7704|/data/primary704 +707|705|p|p|s|u|sdw36|sdw36|7705|/data/primary705 +708|706|p|p|s|u|sdw36|sdw36|7706|/data/primary706 +709|707|p|p|s|u|sdw36|sdw36|7707|/data/primary707 +710|708|p|p|s|u|sdw36|sdw36|7708|/data/primary708 +711|709|p|p|s|u|sdw36|sdw36|7709|/data/primary709 +712|710|p|p|s|u|sdw36|sdw36|7710|/data/primary710 +713|711|p|p|s|u|sdw36|sdw36|7711|/data/primary711 +714|712|p|p|s|u|sdw36|sdw36|7712|/data/primary712 +715|713|p|p|s|u|sdw36|sdw36|7713|/data/primary713 +716|714|p|p|s|u|sdw36|sdw36|7714|/data/primary714 +717|715|p|p|s|u|sdw36|sdw36|7715|/data/primary715 +718|716|p|p|s|u|sdw36|sdw36|7716|/data/primary716 +719|717|p|p|s|u|sdw36|sdw36|7717|/data/primary717 +720|718|p|p|s|u|sdw36|sdw36|7718|/data/primary718 +721|719|p|p|s|u|sdw36|sdw36|7719|/data/primary719 +1321|319|m|m|s|u|sdw36|sdw36|8319|/data/mirror319 +1340|338|m|m|s|u|sdw36|sdw36|8338|/data/mirror338 +1359|357|m|m|s|u|sdw36|sdw36|8357|/data/mirror357 +1378|376|m|m|s|u|sdw36|sdw36|8376|/data/mirror376 +1397|395|m|m|s|u|sdw36|sdw36|8395|/data/mirror395 +1416|414|m|m|s|u|sdw36|sdw36|8414|/data/mirror414 +1435|433|m|m|s|u|sdw36|sdw36|8433|/data/mirror433 +1454|452|m|m|s|u|sdw36|sdw36|8452|/data/mirror452 +1473|471|m|m|s|u|sdw36|sdw36|8471|/data/mirror471 +1492|490|m|m|s|u|sdw36|sdw36|8490|/data/mirror490 +1511|509|m|m|s|u|sdw36|sdw36|8509|/data/mirror509 +1530|528|m|m|s|u|sdw36|sdw36|8528|/data/mirror528 +1549|547|m|m|s|u|sdw36|sdw36|8547|/data/mirror547 +1568|566|m|m|s|u|sdw36|sdw36|8566|/data/mirror566 +1587|585|m|m|s|u|sdw36|sdw36|8585|/data/mirror585 +1606|604|m|m|s|u|sdw36|sdw36|8604|/data/mirror604 +1625|623|m|m|s|u|sdw36|sdw36|8623|/data/mirror623 +1644|642|m|m|s|u|sdw36|sdw36|8642|/data/mirror642 +1663|661|m|m|s|u|sdw36|sdw36|8661|/data/mirror661 +1682|680|m|m|s|u|sdw36|sdw36|8680|/data/mirror680 +# SDW37 +722|720|p|p|s|u|sdw37|sdw37|7720|/data/primary720 +723|721|p|p|s|u|sdw37|sdw37|7721|/data/primary721 +724|722|p|p|s|u|sdw37|sdw37|7722|/data/primary722 +725|723|p|p|s|u|sdw37|sdw37|7723|/data/primary723 +726|724|p|p|s|u|sdw37|sdw37|7724|/data/primary724 +727|725|p|p|s|u|sdw37|sdw37|7725|/data/primary725 +728|726|p|p|s|u|sdw37|sdw37|7726|/data/primary726 +729|727|p|p|s|u|sdw37|sdw37|7727|/data/primary727 +730|728|p|p|s|u|sdw37|sdw37|7728|/data/primary728 +731|729|p|p|s|u|sdw37|sdw37|7729|/data/primary729 +732|730|p|p|s|u|sdw37|sdw37|7730|/data/primary730 +733|731|p|p|s|u|sdw37|sdw37|7731|/data/primary731 +734|732|p|p|s|u|sdw37|sdw37|7732|/data/primary732 +735|733|p|p|s|u|sdw37|sdw37|7733|/data/primary733 +736|734|p|p|s|u|sdw37|sdw37|7734|/data/primary734 +737|735|p|p|s|u|sdw37|sdw37|7735|/data/primary735 +738|736|p|p|s|u|sdw37|sdw37|7736|/data/primary736 +739|737|p|p|s|u|sdw37|sdw37|7737|/data/primary737 +740|738|p|p|s|u|sdw37|sdw37|7738|/data/primary738 +741|739|p|p|s|u|sdw37|sdw37|7739|/data/primary739 +1341|339|m|m|s|u|sdw37|sdw37|8339|/data/mirror339 +1360|358|m|m|s|u|sdw37|sdw37|8358|/data/mirror358 +1379|377|m|m|s|u|sdw37|sdw37|8377|/data/mirror377 +1398|396|m|m|s|u|sdw37|sdw37|8396|/data/mirror396 +1417|415|m|m|s|u|sdw37|sdw37|8415|/data/mirror415 +1436|434|m|m|s|u|sdw37|sdw37|8434|/data/mirror434 +1455|453|m|m|s|u|sdw37|sdw37|8453|/data/mirror453 +1474|472|m|m|s|u|sdw37|sdw37|8472|/data/mirror472 +1493|491|m|m|s|u|sdw37|sdw37|8491|/data/mirror491 +1512|510|m|m|s|u|sdw37|sdw37|8510|/data/mirror510 +1531|529|m|m|s|u|sdw37|sdw37|8529|/data/mirror529 +1550|548|m|m|s|u|sdw37|sdw37|8548|/data/mirror548 +1569|567|m|m|s|u|sdw37|sdw37|8567|/data/mirror567 +1588|586|m|m|s|u|sdw37|sdw37|8586|/data/mirror586 +1607|605|m|m|s|u|sdw37|sdw37|8605|/data/mirror605 +1626|624|m|m|s|u|sdw37|sdw37|8624|/data/mirror624 +1645|643|m|m|s|u|sdw37|sdw37|8643|/data/mirror643 +1664|662|m|m|s|u|sdw37|sdw37|8662|/data/mirror662 +1683|681|m|m|s|u|sdw37|sdw37|8681|/data/mirror681 +1702|700|m|m|s|u|sdw37|sdw37|8700|/data/mirror700 +# SDW38 +742|740|p|p|s|u|sdw38|sdw38|7740|/data/primary740 +743|741|p|p|s|u|sdw38|sdw38|7741|/data/primary741 +744|742|p|p|s|u|sdw38|sdw38|7742|/data/primary742 +745|743|p|p|s|u|sdw38|sdw38|7743|/data/primary743 +746|744|p|p|s|u|sdw38|sdw38|7744|/data/primary744 +747|745|p|p|s|u|sdw38|sdw38|7745|/data/primary745 +748|746|p|p|s|u|sdw38|sdw38|7746|/data/primary746 +749|747|p|p|s|u|sdw38|sdw38|7747|/data/primary747 +750|748|p|p|s|u|sdw38|sdw38|7748|/data/primary748 +751|749|p|p|s|u|sdw38|sdw38|7749|/data/primary749 +752|750|p|p|s|u|sdw38|sdw38|7750|/data/primary750 +753|751|p|p|s|u|sdw38|sdw38|7751|/data/primary751 +754|752|p|p|s|u|sdw38|sdw38|7752|/data/primary752 +755|753|p|p|s|u|sdw38|sdw38|7753|/data/primary753 +756|754|p|p|s|u|sdw38|sdw38|7754|/data/primary754 +757|755|p|p|s|u|sdw38|sdw38|7755|/data/primary755 +758|756|p|p|s|u|sdw38|sdw38|7756|/data/primary756 +759|757|p|p|s|u|sdw38|sdw38|7757|/data/primary757 +760|758|p|p|s|u|sdw38|sdw38|7758|/data/primary758 +761|759|p|p|s|u|sdw38|sdw38|7759|/data/primary759 +1361|359|m|m|s|u|sdw38|sdw38|8359|/data/mirror359 +1380|378|m|m|s|u|sdw38|sdw38|8378|/data/mirror378 +1399|397|m|m|s|u|sdw38|sdw38|8397|/data/mirror397 +1418|416|m|m|s|u|sdw38|sdw38|8416|/data/mirror416 +1437|435|m|m|s|u|sdw38|sdw38|8435|/data/mirror435 +1456|454|m|m|s|u|sdw38|sdw38|8454|/data/mirror454 +1475|473|m|m|s|u|sdw38|sdw38|8473|/data/mirror473 +1494|492|m|m|s|u|sdw38|sdw38|8492|/data/mirror492 +1513|511|m|m|s|u|sdw38|sdw38|8511|/data/mirror511 +1532|530|m|m|s|u|sdw38|sdw38|8530|/data/mirror530 +1551|549|m|m|s|u|sdw38|sdw38|8549|/data/mirror549 +1570|568|m|m|s|u|sdw38|sdw38|8568|/data/mirror568 +1589|587|m|m|s|u|sdw38|sdw38|8587|/data/mirror587 +1608|606|m|m|s|u|sdw38|sdw38|8606|/data/mirror606 +1627|625|m|m|s|u|sdw38|sdw38|8625|/data/mirror625 +1646|644|m|m|s|u|sdw38|sdw38|8644|/data/mirror644 +1665|663|m|m|s|u|sdw38|sdw38|8663|/data/mirror663 +1684|682|m|m|s|u|sdw38|sdw38|8682|/data/mirror682 +1703|701|m|m|s|u|sdw38|sdw38|8701|/data/mirror701 +1722|720|m|m|s|u|sdw38|sdw38|8720|/data/mirror720 +# SDW39 +762|760|p|p|s|u|sdw39|sdw39|7760|/data/primary760 +763|761|p|p|s|u|sdw39|sdw39|7761|/data/primary761 +764|762|p|p|s|u|sdw39|sdw39|7762|/data/primary762 +765|763|p|p|s|u|sdw39|sdw39|7763|/data/primary763 +766|764|p|p|s|u|sdw39|sdw39|7764|/data/primary764 +767|765|p|p|s|u|sdw39|sdw39|7765|/data/primary765 +768|766|p|p|s|u|sdw39|sdw39|7766|/data/primary766 +769|767|p|p|s|u|sdw39|sdw39|7767|/data/primary767 +770|768|p|p|s|u|sdw39|sdw39|7768|/data/primary768 +771|769|p|p|s|u|sdw39|sdw39|7769|/data/primary769 +772|770|p|p|s|u|sdw39|sdw39|7770|/data/primary770 +773|771|p|p|s|u|sdw39|sdw39|7771|/data/primary771 +774|772|p|p|s|u|sdw39|sdw39|7772|/data/primary772 +775|773|p|p|s|u|sdw39|sdw39|7773|/data/primary773 +776|774|p|p|s|u|sdw39|sdw39|7774|/data/primary774 +777|775|p|p|s|u|sdw39|sdw39|7775|/data/primary775 +778|776|p|p|s|u|sdw39|sdw39|7776|/data/primary776 +779|777|p|p|s|u|sdw39|sdw39|7777|/data/primary777 +780|778|p|p|s|u|sdw39|sdw39|7778|/data/primary778 +781|779|p|p|s|u|sdw39|sdw39|7779|/data/primary779 +1381|379|m|m|s|u|sdw39|sdw39|8379|/data/mirror379 +1400|398|m|m|s|u|sdw39|sdw39|8398|/data/mirror398 +1419|417|m|m|s|u|sdw39|sdw39|8417|/data/mirror417 +1438|436|m|m|s|u|sdw39|sdw39|8436|/data/mirror436 +1457|455|m|m|s|u|sdw39|sdw39|8455|/data/mirror455 +1476|474|m|m|s|u|sdw39|sdw39|8474|/data/mirror474 +1495|493|m|m|s|u|sdw39|sdw39|8493|/data/mirror493 +1514|512|m|m|s|u|sdw39|sdw39|8512|/data/mirror512 +1533|531|m|m|s|u|sdw39|sdw39|8531|/data/mirror531 +1552|550|m|m|s|u|sdw39|sdw39|8550|/data/mirror550 +1571|569|m|m|s|u|sdw39|sdw39|8569|/data/mirror569 +1590|588|m|m|s|u|sdw39|sdw39|8588|/data/mirror588 +1609|607|m|m|s|u|sdw39|sdw39|8607|/data/mirror607 +1628|626|m|m|s|u|sdw39|sdw39|8626|/data/mirror626 +1647|645|m|m|s|u|sdw39|sdw39|8645|/data/mirror645 +1666|664|m|m|s|u|sdw39|sdw39|8664|/data/mirror664 +1685|683|m|m|s|u|sdw39|sdw39|8683|/data/mirror683 +1704|702|m|m|s|u|sdw39|sdw39|8702|/data/mirror702 +1723|721|m|m|s|u|sdw39|sdw39|8721|/data/mirror721 +1742|740|m|m|s|u|sdw39|sdw39|8740|/data/mirror740 +# SDW4 +62|60|p|p|s|u|sdw4|sdw4|7060|/data/primary60 +63|61|p|p|s|u|sdw4|sdw4|7061|/data/primary61 +64|62|p|p|s|u|sdw4|sdw4|7062|/data/primary62 +65|63|p|p|s|u|sdw4|sdw4|7063|/data/primary63 +66|64|p|p|s|u|sdw4|sdw4|7064|/data/primary64 +67|65|p|p|s|u|sdw4|sdw4|7065|/data/primary65 +68|66|p|p|s|u|sdw4|sdw4|7066|/data/primary66 +69|67|p|p|s|u|sdw4|sdw4|7067|/data/primary67 +70|68|p|p|s|u|sdw4|sdw4|7068|/data/primary68 +71|69|p|p|s|u|sdw4|sdw4|7069|/data/primary69 +72|70|p|p|s|u|sdw4|sdw4|7070|/data/primary70 +73|71|p|p|s|u|sdw4|sdw4|7071|/data/primary71 +74|72|p|p|s|u|sdw4|sdw4|7072|/data/primary72 +75|73|p|p|s|u|sdw4|sdw4|7073|/data/primary73 +76|74|p|p|s|u|sdw4|sdw4|7074|/data/primary74 +77|75|p|p|s|u|sdw4|sdw4|7075|/data/primary75 +78|76|p|p|s|u|sdw4|sdw4|7076|/data/primary76 +79|77|p|p|s|u|sdw4|sdw4|7077|/data/primary77 +80|78|p|p|s|u|sdw4|sdw4|7078|/data/primary78 +81|79|p|p|s|u|sdw4|sdw4|7079|/data/primary79 +1004|2|m|m|s|u|sdw4|sdw4|8002|/data/mirror2 +1023|21|m|m|s|u|sdw4|sdw4|8021|/data/mirror21 +1042|40|m|m|s|u|sdw4|sdw4|8040|/data/mirror40 +1681|679|m|m|s|u|sdw4|sdw4|8679|/data/mirror679 +1700|698|m|m|s|u|sdw4|sdw4|8698|/data/mirror698 +1719|717|m|m|s|u|sdw4|sdw4|8717|/data/mirror717 +1738|736|m|m|s|u|sdw4|sdw4|8736|/data/mirror736 +1757|755|m|m|s|u|sdw4|sdw4|8755|/data/mirror755 +1776|774|m|m|s|u|sdw4|sdw4|8774|/data/mirror774 +1795|793|m|m|s|u|sdw4|sdw4|8793|/data/mirror793 +1814|812|m|m|s|u|sdw4|sdw4|8812|/data/mirror812 +1833|831|m|m|s|u|sdw4|sdw4|8831|/data/mirror831 +1852|850|m|m|s|u|sdw4|sdw4|8850|/data/mirror850 +1871|869|m|m|s|u|sdw4|sdw4|8869|/data/mirror869 +1890|888|m|m|s|u|sdw4|sdw4|8888|/data/mirror888 +1909|907|m|m|s|u|sdw4|sdw4|8907|/data/mirror907 +1928|926|m|m|s|u|sdw4|sdw4|8926|/data/mirror926 +1947|945|m|m|s|u|sdw4|sdw4|8945|/data/mirror945 +1966|964|m|m|s|u|sdw4|sdw4|8964|/data/mirror964 +1985|983|m|m|s|u|sdw4|sdw4|8983|/data/mirror983 +# SDW40 +782|780|p|p|s|u|sdw40|sdw40|7780|/data/primary780 +783|781|p|p|s|u|sdw40|sdw40|7781|/data/primary781 +784|782|p|p|s|u|sdw40|sdw40|7782|/data/primary782 +785|783|p|p|s|u|sdw40|sdw40|7783|/data/primary783 +786|784|p|p|s|u|sdw40|sdw40|7784|/data/primary784 +787|785|p|p|s|u|sdw40|sdw40|7785|/data/primary785 +788|786|p|p|s|u|sdw40|sdw40|7786|/data/primary786 +789|787|p|p|s|u|sdw40|sdw40|7787|/data/primary787 +790|788|p|p|s|u|sdw40|sdw40|7788|/data/primary788 +791|789|p|p|s|u|sdw40|sdw40|7789|/data/primary789 +792|790|p|p|s|u|sdw40|sdw40|7790|/data/primary790 +793|791|p|p|s|u|sdw40|sdw40|7791|/data/primary791 +794|792|p|p|s|u|sdw40|sdw40|7792|/data/primary792 +795|793|p|p|s|u|sdw40|sdw40|7793|/data/primary793 +796|794|p|p|s|u|sdw40|sdw40|7794|/data/primary794 +797|795|p|p|s|u|sdw40|sdw40|7795|/data/primary795 +798|796|p|p|s|u|sdw40|sdw40|7796|/data/primary796 +799|797|p|p|s|u|sdw40|sdw40|7797|/data/primary797 +800|798|p|p|s|u|sdw40|sdw40|7798|/data/primary798 +801|799|p|p|s|u|sdw40|sdw40|7799|/data/primary799 +1401|399|m|m|s|u|sdw40|sdw40|8399|/data/mirror399 +1420|418|m|m|s|u|sdw40|sdw40|8418|/data/mirror418 +1439|437|m|m|s|u|sdw40|sdw40|8437|/data/mirror437 +1458|456|m|m|s|u|sdw40|sdw40|8456|/data/mirror456 +1477|475|m|m|s|u|sdw40|sdw40|8475|/data/mirror475 +1496|494|m|m|s|u|sdw40|sdw40|8494|/data/mirror494 +1515|513|m|m|s|u|sdw40|sdw40|8513|/data/mirror513 +1534|532|m|m|s|u|sdw40|sdw40|8532|/data/mirror532 +1553|551|m|m|s|u|sdw40|sdw40|8551|/data/mirror551 +1572|570|m|m|s|u|sdw40|sdw40|8570|/data/mirror570 +1591|589|m|m|s|u|sdw40|sdw40|8589|/data/mirror589 +1610|608|m|m|s|u|sdw40|sdw40|8608|/data/mirror608 +1629|627|m|m|s|u|sdw40|sdw40|8627|/data/mirror627 +1648|646|m|m|s|u|sdw40|sdw40|8646|/data/mirror646 +1667|665|m|m|s|u|sdw40|sdw40|8665|/data/mirror665 +1686|684|m|m|s|u|sdw40|sdw40|8684|/data/mirror684 +1705|703|m|m|s|u|sdw40|sdw40|8703|/data/mirror703 +1724|722|m|m|s|u|sdw40|sdw40|8722|/data/mirror722 +1743|741|m|m|s|u|sdw40|sdw40|8741|/data/mirror741 +1762|760|m|m|s|u|sdw40|sdw40|8760|/data/mirror760 +# SDW41 +802|800|p|p|s|u|sdw41|sdw41|7800|/data/primary800 +803|801|p|p|s|u|sdw41|sdw41|7801|/data/primary801 +804|802|p|p|s|u|sdw41|sdw41|7802|/data/primary802 +805|803|p|p|s|u|sdw41|sdw41|7803|/data/primary803 +806|804|p|p|s|u|sdw41|sdw41|7804|/data/primary804 +807|805|p|p|s|u|sdw41|sdw41|7805|/data/primary805 +808|806|p|p|s|u|sdw41|sdw41|7806|/data/primary806 +809|807|p|p|s|u|sdw41|sdw41|7807|/data/primary807 +810|808|p|p|s|u|sdw41|sdw41|7808|/data/primary808 +811|809|p|p|s|u|sdw41|sdw41|7809|/data/primary809 +812|810|p|p|s|u|sdw41|sdw41|7810|/data/primary810 +813|811|p|p|s|u|sdw41|sdw41|7811|/data/primary811 +814|812|p|p|s|u|sdw41|sdw41|7812|/data/primary812 +815|813|p|p|s|u|sdw41|sdw41|7813|/data/primary813 +816|814|p|p|s|u|sdw41|sdw41|7814|/data/primary814 +817|815|p|p|s|u|sdw41|sdw41|7815|/data/primary815 +818|816|p|p|s|u|sdw41|sdw41|7816|/data/primary816 +819|817|p|p|s|u|sdw41|sdw41|7817|/data/primary817 +820|818|p|p|s|u|sdw41|sdw41|7818|/data/primary818 +821|819|p|p|s|u|sdw41|sdw41|7819|/data/primary819 +1421|419|m|m|s|u|sdw41|sdw41|8419|/data/mirror419 +1440|438|m|m|s|u|sdw41|sdw41|8438|/data/mirror438 +1459|457|m|m|s|u|sdw41|sdw41|8457|/data/mirror457 +1478|476|m|m|s|u|sdw41|sdw41|8476|/data/mirror476 +1497|495|m|m|s|u|sdw41|sdw41|8495|/data/mirror495 +1516|514|m|m|s|u|sdw41|sdw41|8514|/data/mirror514 +1535|533|m|m|s|u|sdw41|sdw41|8533|/data/mirror533 +1554|552|m|m|s|u|sdw41|sdw41|8552|/data/mirror552 +1573|571|m|m|s|u|sdw41|sdw41|8571|/data/mirror571 +1592|590|m|m|s|u|sdw41|sdw41|8590|/data/mirror590 +1611|609|m|m|s|u|sdw41|sdw41|8609|/data/mirror609 +1630|628|m|m|s|u|sdw41|sdw41|8628|/data/mirror628 +1649|647|m|m|s|u|sdw41|sdw41|8647|/data/mirror647 +1668|666|m|m|s|u|sdw41|sdw41|8666|/data/mirror666 +1687|685|m|m|s|u|sdw41|sdw41|8685|/data/mirror685 +1706|704|m|m|s|u|sdw41|sdw41|8704|/data/mirror704 +1725|723|m|m|s|u|sdw41|sdw41|8723|/data/mirror723 +1744|742|m|m|s|u|sdw41|sdw41|8742|/data/mirror742 +1763|761|m|m|s|u|sdw41|sdw41|8761|/data/mirror761 +1782|780|m|m|s|u|sdw41|sdw41|8780|/data/mirror780 +# SDW42 +822|820|p|p|s|u|sdw42|sdw42|7820|/data/primary820 +823|821|p|p|s|u|sdw42|sdw42|7821|/data/primary821 +824|822|p|p|s|u|sdw42|sdw42|7822|/data/primary822 +825|823|p|p|s|u|sdw42|sdw42|7823|/data/primary823 +826|824|p|p|s|u|sdw42|sdw42|7824|/data/primary824 +827|825|p|p|s|u|sdw42|sdw42|7825|/data/primary825 +828|826|p|p|s|u|sdw42|sdw42|7826|/data/primary826 +829|827|p|p|s|u|sdw42|sdw42|7827|/data/primary827 +830|828|p|p|s|u|sdw42|sdw42|7828|/data/primary828 +831|829|p|p|s|u|sdw42|sdw42|7829|/data/primary829 +832|830|p|p|s|u|sdw42|sdw42|7830|/data/primary830 +833|831|p|p|s|u|sdw42|sdw42|7831|/data/primary831 +834|832|p|p|s|u|sdw42|sdw42|7832|/data/primary832 +835|833|p|p|s|u|sdw42|sdw42|7833|/data/primary833 +836|834|p|p|s|u|sdw42|sdw42|7834|/data/primary834 +837|835|p|p|s|u|sdw42|sdw42|7835|/data/primary835 +838|836|p|p|s|u|sdw42|sdw42|7836|/data/primary836 +839|837|p|p|s|u|sdw42|sdw42|7837|/data/primary837 +840|838|p|p|s|u|sdw42|sdw42|7838|/data/primary838 +841|839|p|p|s|u|sdw42|sdw42|7839|/data/primary839 +1441|439|m|m|s|u|sdw42|sdw42|8439|/data/mirror439 +1460|458|m|m|s|u|sdw42|sdw42|8458|/data/mirror458 +1479|477|m|m|s|u|sdw42|sdw42|8477|/data/mirror477 +1498|496|m|m|s|u|sdw42|sdw42|8496|/data/mirror496 +1517|515|m|m|s|u|sdw42|sdw42|8515|/data/mirror515 +1536|534|m|m|s|u|sdw42|sdw42|8534|/data/mirror534 +1555|553|m|m|s|u|sdw42|sdw42|8553|/data/mirror553 +1574|572|m|m|s|u|sdw42|sdw42|8572|/data/mirror572 +1593|591|m|m|s|u|sdw42|sdw42|8591|/data/mirror591 +1612|610|m|m|s|u|sdw42|sdw42|8610|/data/mirror610 +1631|629|m|m|s|u|sdw42|sdw42|8629|/data/mirror629 +1650|648|m|m|s|u|sdw42|sdw42|8648|/data/mirror648 +1669|667|m|m|s|u|sdw42|sdw42|8667|/data/mirror667 +1688|686|m|m|s|u|sdw42|sdw42|8686|/data/mirror686 +1707|705|m|m|s|u|sdw42|sdw42|8705|/data/mirror705 +1726|724|m|m|s|u|sdw42|sdw42|8724|/data/mirror724 +1745|743|m|m|s|u|sdw42|sdw42|8743|/data/mirror743 +1764|762|m|m|s|u|sdw42|sdw42|8762|/data/mirror762 +1783|781|m|m|s|u|sdw42|sdw42|8781|/data/mirror781 +1802|800|m|m|s|u|sdw42|sdw42|8800|/data/mirror800 +# SDW43 +842|840|p|p|s|u|sdw43|sdw43|7840|/data/primary840 +843|841|p|p|s|u|sdw43|sdw43|7841|/data/primary841 +844|842|p|p|s|u|sdw43|sdw43|7842|/data/primary842 +845|843|p|p|s|u|sdw43|sdw43|7843|/data/primary843 +846|844|p|p|s|u|sdw43|sdw43|7844|/data/primary844 +847|845|p|p|s|u|sdw43|sdw43|7845|/data/primary845 +848|846|p|p|s|u|sdw43|sdw43|7846|/data/primary846 +849|847|p|p|s|u|sdw43|sdw43|7847|/data/primary847 +850|848|p|p|s|u|sdw43|sdw43|7848|/data/primary848 +851|849|p|p|s|u|sdw43|sdw43|7849|/data/primary849 +852|850|p|p|s|u|sdw43|sdw43|7850|/data/primary850 +853|851|p|p|s|u|sdw43|sdw43|7851|/data/primary851 +854|852|p|p|s|u|sdw43|sdw43|7852|/data/primary852 +855|853|p|p|s|u|sdw43|sdw43|7853|/data/primary853 +856|854|p|p|s|u|sdw43|sdw43|7854|/data/primary854 +857|855|p|p|s|u|sdw43|sdw43|7855|/data/primary855 +858|856|p|p|s|u|sdw43|sdw43|7856|/data/primary856 +859|857|p|p|s|u|sdw43|sdw43|7857|/data/primary857 +860|858|p|p|s|u|sdw43|sdw43|7858|/data/primary858 +861|859|p|p|s|u|sdw43|sdw43|7859|/data/primary859 +1461|459|m|m|s|u|sdw43|sdw43|8459|/data/mirror459 +1480|478|m|m|s|u|sdw43|sdw43|8478|/data/mirror478 +1499|497|m|m|s|u|sdw43|sdw43|8497|/data/mirror497 +1518|516|m|m|s|u|sdw43|sdw43|8516|/data/mirror516 +1537|535|m|m|s|u|sdw43|sdw43|8535|/data/mirror535 +1556|554|m|m|s|u|sdw43|sdw43|8554|/data/mirror554 +1575|573|m|m|s|u|sdw43|sdw43|8573|/data/mirror573 +1594|592|m|m|s|u|sdw43|sdw43|8592|/data/mirror592 +1613|611|m|m|s|u|sdw43|sdw43|8611|/data/mirror611 +1632|630|m|m|s|u|sdw43|sdw43|8630|/data/mirror630 +1651|649|m|m|s|u|sdw43|sdw43|8649|/data/mirror649 +1670|668|m|m|s|u|sdw43|sdw43|8668|/data/mirror668 +1689|687|m|m|s|u|sdw43|sdw43|8687|/data/mirror687 +1708|706|m|m|s|u|sdw43|sdw43|8706|/data/mirror706 +1727|725|m|m|s|u|sdw43|sdw43|8725|/data/mirror725 +1746|744|m|m|s|u|sdw43|sdw43|8744|/data/mirror744 +1765|763|m|m|s|u|sdw43|sdw43|8763|/data/mirror763 +1784|782|m|m|s|u|sdw43|sdw43|8782|/data/mirror782 +1803|801|m|m|s|u|sdw43|sdw43|8801|/data/mirror801 +1822|820|m|m|s|u|sdw43|sdw43|8820|/data/mirror820 +# SDW44 +862|860|p|p|s|u|sdw44|sdw44|7860|/data/primary860 +863|861|p|p|s|u|sdw44|sdw44|7861|/data/primary861 +864|862|p|p|s|u|sdw44|sdw44|7862|/data/primary862 +865|863|p|p|s|u|sdw44|sdw44|7863|/data/primary863 +866|864|p|p|s|u|sdw44|sdw44|7864|/data/primary864 +867|865|p|p|s|u|sdw44|sdw44|7865|/data/primary865 +868|866|p|p|s|u|sdw44|sdw44|7866|/data/primary866 +869|867|p|p|s|u|sdw44|sdw44|7867|/data/primary867 +870|868|p|p|s|u|sdw44|sdw44|7868|/data/primary868 +871|869|p|p|s|u|sdw44|sdw44|7869|/data/primary869 +872|870|p|p|s|u|sdw44|sdw44|7870|/data/primary870 +873|871|p|p|s|u|sdw44|sdw44|7871|/data/primary871 +874|872|p|p|s|u|sdw44|sdw44|7872|/data/primary872 +875|873|p|p|s|u|sdw44|sdw44|7873|/data/primary873 +876|874|p|p|s|u|sdw44|sdw44|7874|/data/primary874 +877|875|p|p|s|u|sdw44|sdw44|7875|/data/primary875 +878|876|p|p|s|u|sdw44|sdw44|7876|/data/primary876 +879|877|p|p|s|u|sdw44|sdw44|7877|/data/primary877 +880|878|p|p|s|u|sdw44|sdw44|7878|/data/primary878 +881|879|p|p|s|u|sdw44|sdw44|7879|/data/primary879 +1481|479|m|m|s|u|sdw44|sdw44|8479|/data/mirror479 +1500|498|m|m|s|u|sdw44|sdw44|8498|/data/mirror498 +1519|517|m|m|s|u|sdw44|sdw44|8517|/data/mirror517 +1538|536|m|m|s|u|sdw44|sdw44|8536|/data/mirror536 +1557|555|m|m|s|u|sdw44|sdw44|8555|/data/mirror555 +1576|574|m|m|s|u|sdw44|sdw44|8574|/data/mirror574 +1595|593|m|m|s|u|sdw44|sdw44|8593|/data/mirror593 +1614|612|m|m|s|u|sdw44|sdw44|8612|/data/mirror612 +1633|631|m|m|s|u|sdw44|sdw44|8631|/data/mirror631 +1652|650|m|m|s|u|sdw44|sdw44|8650|/data/mirror650 +1671|669|m|m|s|u|sdw44|sdw44|8669|/data/mirror669 +1690|688|m|m|s|u|sdw44|sdw44|8688|/data/mirror688 +1709|707|m|m|s|u|sdw44|sdw44|8707|/data/mirror707 +1728|726|m|m|s|u|sdw44|sdw44|8726|/data/mirror726 +1747|745|m|m|s|u|sdw44|sdw44|8745|/data/mirror745 +1766|764|m|m|s|u|sdw44|sdw44|8764|/data/mirror764 +1785|783|m|m|s|u|sdw44|sdw44|8783|/data/mirror783 +1804|802|m|m|s|u|sdw44|sdw44|8802|/data/mirror802 +1823|821|m|m|s|u|sdw44|sdw44|8821|/data/mirror821 +1842|840|m|m|s|u|sdw44|sdw44|8840|/data/mirror840 +# SDW45 +882|880|p|p|s|u|sdw45|sdw45|7880|/data/primary880 +883|881|p|p|s|u|sdw45|sdw45|7881|/data/primary881 +884|882|p|p|s|u|sdw45|sdw45|7882|/data/primary882 +885|883|p|p|s|u|sdw45|sdw45|7883|/data/primary883 +886|884|p|p|s|u|sdw45|sdw45|7884|/data/primary884 +887|885|p|p|s|u|sdw45|sdw45|7885|/data/primary885 +888|886|p|p|s|u|sdw45|sdw45|7886|/data/primary886 +889|887|p|p|s|u|sdw45|sdw45|7887|/data/primary887 +890|888|p|p|s|u|sdw45|sdw45|7888|/data/primary888 +891|889|p|p|s|u|sdw45|sdw45|7889|/data/primary889 +892|890|p|p|s|u|sdw45|sdw45|7890|/data/primary890 +893|891|p|p|s|u|sdw45|sdw45|7891|/data/primary891 +894|892|p|p|s|u|sdw45|sdw45|7892|/data/primary892 +895|893|p|p|s|u|sdw45|sdw45|7893|/data/primary893 +896|894|p|p|s|u|sdw45|sdw45|7894|/data/primary894 +897|895|p|p|s|u|sdw45|sdw45|7895|/data/primary895 +898|896|p|p|s|u|sdw45|sdw45|7896|/data/primary896 +899|897|p|p|s|u|sdw45|sdw45|7897|/data/primary897 +900|898|p|p|s|u|sdw45|sdw45|7898|/data/primary898 +901|899|p|p|s|u|sdw45|sdw45|7899|/data/primary899 +1501|499|m|m|s|u|sdw45|sdw45|8499|/data/mirror499 +1520|518|m|m|s|u|sdw45|sdw45|8518|/data/mirror518 +1539|537|m|m|s|u|sdw45|sdw45|8537|/data/mirror537 +1558|556|m|m|s|u|sdw45|sdw45|8556|/data/mirror556 +1577|575|m|m|s|u|sdw45|sdw45|8575|/data/mirror575 +1596|594|m|m|s|u|sdw45|sdw45|8594|/data/mirror594 +1615|613|m|m|s|u|sdw45|sdw45|8613|/data/mirror613 +1634|632|m|m|s|u|sdw45|sdw45|8632|/data/mirror632 +1653|651|m|m|s|u|sdw45|sdw45|8651|/data/mirror651 +1672|670|m|m|s|u|sdw45|sdw45|8670|/data/mirror670 +1691|689|m|m|s|u|sdw45|sdw45|8689|/data/mirror689 +1710|708|m|m|s|u|sdw45|sdw45|8708|/data/mirror708 +1729|727|m|m|s|u|sdw45|sdw45|8727|/data/mirror727 +1748|746|m|m|s|u|sdw45|sdw45|8746|/data/mirror746 +1767|765|m|m|s|u|sdw45|sdw45|8765|/data/mirror765 +1786|784|m|m|s|u|sdw45|sdw45|8784|/data/mirror784 +1805|803|m|m|s|u|sdw45|sdw45|8803|/data/mirror803 +1824|822|m|m|s|u|sdw45|sdw45|8822|/data/mirror822 +1843|841|m|m|s|u|sdw45|sdw45|8841|/data/mirror841 +1862|860|m|m|s|u|sdw45|sdw45|8860|/data/mirror860 +# SDW46 +902|900|p|p|s|u|sdw46|sdw46|7900|/data/primary900 +903|901|p|p|s|u|sdw46|sdw46|7901|/data/primary901 +904|902|p|p|s|u|sdw46|sdw46|7902|/data/primary902 +905|903|p|p|s|u|sdw46|sdw46|7903|/data/primary903 +906|904|p|p|s|u|sdw46|sdw46|7904|/data/primary904 +907|905|p|p|s|u|sdw46|sdw46|7905|/data/primary905 +908|906|p|p|s|u|sdw46|sdw46|7906|/data/primary906 +909|907|p|p|s|u|sdw46|sdw46|7907|/data/primary907 +910|908|p|p|s|u|sdw46|sdw46|7908|/data/primary908 +911|909|p|p|s|u|sdw46|sdw46|7909|/data/primary909 +912|910|p|p|s|u|sdw46|sdw46|7910|/data/primary910 +913|911|p|p|s|u|sdw46|sdw46|7911|/data/primary911 +914|912|p|p|s|u|sdw46|sdw46|7912|/data/primary912 +915|913|p|p|s|u|sdw46|sdw46|7913|/data/primary913 +916|914|p|p|s|u|sdw46|sdw46|7914|/data/primary914 +917|915|p|p|s|u|sdw46|sdw46|7915|/data/primary915 +918|916|p|p|s|u|sdw46|sdw46|7916|/data/primary916 +919|917|p|p|s|u|sdw46|sdw46|7917|/data/primary917 +920|918|p|p|s|u|sdw46|sdw46|7918|/data/primary918 +921|919|p|p|s|u|sdw46|sdw46|7919|/data/primary919 +1521|519|m|m|s|u|sdw46|sdw46|8519|/data/mirror519 +1540|538|m|m|s|u|sdw46|sdw46|8538|/data/mirror538 +1559|557|m|m|s|u|sdw46|sdw46|8557|/data/mirror557 +1578|576|m|m|s|u|sdw46|sdw46|8576|/data/mirror576 +1597|595|m|m|s|u|sdw46|sdw46|8595|/data/mirror595 +1616|614|m|m|s|u|sdw46|sdw46|8614|/data/mirror614 +1635|633|m|m|s|u|sdw46|sdw46|8633|/data/mirror633 +1654|652|m|m|s|u|sdw46|sdw46|8652|/data/mirror652 +1673|671|m|m|s|u|sdw46|sdw46|8671|/data/mirror671 +1692|690|m|m|s|u|sdw46|sdw46|8690|/data/mirror690 +1711|709|m|m|s|u|sdw46|sdw46|8709|/data/mirror709 +1730|728|m|m|s|u|sdw46|sdw46|8728|/data/mirror728 +1749|747|m|m|s|u|sdw46|sdw46|8747|/data/mirror747 +1768|766|m|m|s|u|sdw46|sdw46|8766|/data/mirror766 +1787|785|m|m|s|u|sdw46|sdw46|8785|/data/mirror785 +1806|804|m|m|s|u|sdw46|sdw46|8804|/data/mirror804 +1825|823|m|m|s|u|sdw46|sdw46|8823|/data/mirror823 +1844|842|m|m|s|u|sdw46|sdw46|8842|/data/mirror842 +1863|861|m|m|s|u|sdw46|sdw46|8861|/data/mirror861 +1882|880|m|m|s|u|sdw46|sdw46|8880|/data/mirror880 +# SDW47 +922|920|p|p|s|u|sdw47|sdw47|7920|/data/primary920 +923|921|p|p|s|u|sdw47|sdw47|7921|/data/primary921 +924|922|p|p|s|u|sdw47|sdw47|7922|/data/primary922 +925|923|p|p|s|u|sdw47|sdw47|7923|/data/primary923 +926|924|p|p|s|u|sdw47|sdw47|7924|/data/primary924 +927|925|p|p|s|u|sdw47|sdw47|7925|/data/primary925 +928|926|p|p|s|u|sdw47|sdw47|7926|/data/primary926 +929|927|p|p|s|u|sdw47|sdw47|7927|/data/primary927 +930|928|p|p|s|u|sdw47|sdw47|7928|/data/primary928 +931|929|p|p|s|u|sdw47|sdw47|7929|/data/primary929 +932|930|p|p|s|u|sdw47|sdw47|7930|/data/primary930 +933|931|p|p|s|u|sdw47|sdw47|7931|/data/primary931 +934|932|p|p|s|u|sdw47|sdw47|7932|/data/primary932 +935|933|p|p|s|u|sdw47|sdw47|7933|/data/primary933 +936|934|p|p|s|u|sdw47|sdw47|7934|/data/primary934 +937|935|p|p|s|u|sdw47|sdw47|7935|/data/primary935 +938|936|p|p|s|u|sdw47|sdw47|7936|/data/primary936 +939|937|p|p|s|u|sdw47|sdw47|7937|/data/primary937 +940|938|p|p|s|u|sdw47|sdw47|7938|/data/primary938 +941|939|p|p|s|u|sdw47|sdw47|7939|/data/primary939 +1541|539|m|m|s|u|sdw47|sdw47|8539|/data/mirror539 +1560|558|m|m|s|u|sdw47|sdw47|8558|/data/mirror558 +1579|577|m|m|s|u|sdw47|sdw47|8577|/data/mirror577 +1598|596|m|m|s|u|sdw47|sdw47|8596|/data/mirror596 +1617|615|m|m|s|u|sdw47|sdw47|8615|/data/mirror615 +1636|634|m|m|s|u|sdw47|sdw47|8634|/data/mirror634 +1655|653|m|m|s|u|sdw47|sdw47|8653|/data/mirror653 +1674|672|m|m|s|u|sdw47|sdw47|8672|/data/mirror672 +1693|691|m|m|s|u|sdw47|sdw47|8691|/data/mirror691 +1712|710|m|m|s|u|sdw47|sdw47|8710|/data/mirror710 +1731|729|m|m|s|u|sdw47|sdw47|8729|/data/mirror729 +1750|748|m|m|s|u|sdw47|sdw47|8748|/data/mirror748 +1769|767|m|m|s|u|sdw47|sdw47|8767|/data/mirror767 +1788|786|m|m|s|u|sdw47|sdw47|8786|/data/mirror786 +1807|805|m|m|s|u|sdw47|sdw47|8805|/data/mirror805 +1826|824|m|m|s|u|sdw47|sdw47|8824|/data/mirror824 +1845|843|m|m|s|u|sdw47|sdw47|8843|/data/mirror843 +1864|862|m|m|s|u|sdw47|sdw47|8862|/data/mirror862 +1883|881|m|m|s|u|sdw47|sdw47|8881|/data/mirror881 +1902|900|m|m|s|u|sdw47|sdw47|8900|/data/mirror900 +# SDW48 +942|940|p|p|s|u|sdw48|sdw48|7940|/data/primary940 +943|941|p|p|s|u|sdw48|sdw48|7941|/data/primary941 +944|942|p|p|s|u|sdw48|sdw48|7942|/data/primary942 +945|943|p|p|s|u|sdw48|sdw48|7943|/data/primary943 +946|944|p|p|s|u|sdw48|sdw48|7944|/data/primary944 +947|945|p|p|s|u|sdw48|sdw48|7945|/data/primary945 +948|946|p|p|s|u|sdw48|sdw48|7946|/data/primary946 +949|947|p|p|s|u|sdw48|sdw48|7947|/data/primary947 +950|948|p|p|s|u|sdw48|sdw48|7948|/data/primary948 +951|949|p|p|s|u|sdw48|sdw48|7949|/data/primary949 +952|950|p|p|s|u|sdw48|sdw48|7950|/data/primary950 +953|951|p|p|s|u|sdw48|sdw48|7951|/data/primary951 +954|952|p|p|s|u|sdw48|sdw48|7952|/data/primary952 +955|953|p|p|s|u|sdw48|sdw48|7953|/data/primary953 +956|954|p|p|s|u|sdw48|sdw48|7954|/data/primary954 +957|955|p|p|s|u|sdw48|sdw48|7955|/data/primary955 +958|956|p|p|s|u|sdw48|sdw48|7956|/data/primary956 +959|957|p|p|s|u|sdw48|sdw48|7957|/data/primary957 +960|958|p|p|s|u|sdw48|sdw48|7958|/data/primary958 +961|959|p|p|s|u|sdw48|sdw48|7959|/data/primary959 +1561|559|m|m|s|u|sdw48|sdw48|8559|/data/mirror559 +1580|578|m|m|s|u|sdw48|sdw48|8578|/data/mirror578 +1599|597|m|m|s|u|sdw48|sdw48|8597|/data/mirror597 +1618|616|m|m|s|u|sdw48|sdw48|8616|/data/mirror616 +1637|635|m|m|s|u|sdw48|sdw48|8635|/data/mirror635 +1656|654|m|m|s|u|sdw48|sdw48|8654|/data/mirror654 +1675|673|m|m|s|u|sdw48|sdw48|8673|/data/mirror673 +1694|692|m|m|s|u|sdw48|sdw48|8692|/data/mirror692 +1713|711|m|m|s|u|sdw48|sdw48|8711|/data/mirror711 +1732|730|m|m|s|u|sdw48|sdw48|8730|/data/mirror730 +1751|749|m|m|s|u|sdw48|sdw48|8749|/data/mirror749 +1770|768|m|m|s|u|sdw48|sdw48|8768|/data/mirror768 +1789|787|m|m|s|u|sdw48|sdw48|8787|/data/mirror787 +1808|806|m|m|s|u|sdw48|sdw48|8806|/data/mirror806 +1827|825|m|m|s|u|sdw48|sdw48|8825|/data/mirror825 +1846|844|m|m|s|u|sdw48|sdw48|8844|/data/mirror844 +1865|863|m|m|s|u|sdw48|sdw48|8863|/data/mirror863 +1884|882|m|m|s|u|sdw48|sdw48|8882|/data/mirror882 +1903|901|m|m|s|u|sdw48|sdw48|8901|/data/mirror901 +1922|920|m|m|s|u|sdw48|sdw48|8920|/data/mirror920 +# SDW49 +962|960|p|p|s|u|sdw49|sdw49|7960|/data/primary960 +963|961|p|p|s|u|sdw49|sdw49|7961|/data/primary961 +964|962|p|p|s|u|sdw49|sdw49|7962|/data/primary962 +965|963|p|p|s|u|sdw49|sdw49|7963|/data/primary963 +966|964|p|p|s|u|sdw49|sdw49|7964|/data/primary964 +967|965|p|p|s|u|sdw49|sdw49|7965|/data/primary965 +968|966|p|p|s|u|sdw49|sdw49|7966|/data/primary966 +969|967|p|p|s|u|sdw49|sdw49|7967|/data/primary967 +970|968|p|p|s|u|sdw49|sdw49|7968|/data/primary968 +971|969|p|p|s|u|sdw49|sdw49|7969|/data/primary969 +972|970|p|p|s|u|sdw49|sdw49|7970|/data/primary970 +973|971|p|p|s|u|sdw49|sdw49|7971|/data/primary971 +974|972|p|p|s|u|sdw49|sdw49|7972|/data/primary972 +975|973|p|p|s|u|sdw49|sdw49|7973|/data/primary973 +976|974|p|p|s|u|sdw49|sdw49|7974|/data/primary974 +977|975|p|p|s|u|sdw49|sdw49|7975|/data/primary975 +978|976|p|p|s|u|sdw49|sdw49|7976|/data/primary976 +979|977|p|p|s|u|sdw49|sdw49|7977|/data/primary977 +980|978|p|p|s|u|sdw49|sdw49|7978|/data/primary978 +981|979|p|p|s|u|sdw49|sdw49|7979|/data/primary979 +1581|579|m|m|s|u|sdw49|sdw49|8579|/data/mirror579 +1600|598|m|m|s|u|sdw49|sdw49|8598|/data/mirror598 +1619|617|m|m|s|u|sdw49|sdw49|8617|/data/mirror617 +1638|636|m|m|s|u|sdw49|sdw49|8636|/data/mirror636 +1657|655|m|m|s|u|sdw49|sdw49|8655|/data/mirror655 +1676|674|m|m|s|u|sdw49|sdw49|8674|/data/mirror674 +1695|693|m|m|s|u|sdw49|sdw49|8693|/data/mirror693 +1714|712|m|m|s|u|sdw49|sdw49|8712|/data/mirror712 +1733|731|m|m|s|u|sdw49|sdw49|8731|/data/mirror731 +1752|750|m|m|s|u|sdw49|sdw49|8750|/data/mirror750 +1771|769|m|m|s|u|sdw49|sdw49|8769|/data/mirror769 +1790|788|m|m|s|u|sdw49|sdw49|8788|/data/mirror788 +1809|807|m|m|s|u|sdw49|sdw49|8807|/data/mirror807 +1828|826|m|m|s|u|sdw49|sdw49|8826|/data/mirror826 +1847|845|m|m|s|u|sdw49|sdw49|8845|/data/mirror845 +1866|864|m|m|s|u|sdw49|sdw49|8864|/data/mirror864 +1885|883|m|m|s|u|sdw49|sdw49|8883|/data/mirror883 +1904|902|m|m|s|u|sdw49|sdw49|8902|/data/mirror902 +1923|921|m|m|s|u|sdw49|sdw49|8921|/data/mirror921 +1942|940|m|m|s|u|sdw49|sdw49|8940|/data/mirror940 +# SDW5 +82|80|p|p|s|u|sdw5|sdw5|7080|/data/primary80 +83|81|p|p|s|u|sdw5|sdw5|7081|/data/primary81 +84|82|p|p|s|u|sdw5|sdw5|7082|/data/primary82 +85|83|p|p|s|u|sdw5|sdw5|7083|/data/primary83 +86|84|p|p|s|u|sdw5|sdw5|7084|/data/primary84 +87|85|p|p|s|u|sdw5|sdw5|7085|/data/primary85 +88|86|p|p|s|u|sdw5|sdw5|7086|/data/primary86 +89|87|p|p|s|u|sdw5|sdw5|7087|/data/primary87 +90|88|p|p|s|u|sdw5|sdw5|7088|/data/primary88 +91|89|p|p|s|u|sdw5|sdw5|7089|/data/primary89 +92|90|p|p|s|u|sdw5|sdw5|7090|/data/primary90 +93|91|p|p|s|u|sdw5|sdw5|7091|/data/primary91 +94|92|p|p|s|u|sdw5|sdw5|7092|/data/primary92 +95|93|p|p|s|u|sdw5|sdw5|7093|/data/primary93 +96|94|p|p|s|u|sdw5|sdw5|7094|/data/primary94 +97|95|p|p|s|u|sdw5|sdw5|7095|/data/primary95 +98|96|p|p|s|u|sdw5|sdw5|7096|/data/primary96 +99|97|p|p|s|u|sdw5|sdw5|7097|/data/primary97 +100|98|p|p|s|u|sdw5|sdw5|7098|/data/primary98 +101|99|p|p|s|u|sdw5|sdw5|7099|/data/primary99 +1005|3|m|m|s|u|sdw5|sdw5|8003|/data/mirror3 +1024|22|m|m|s|u|sdw5|sdw5|8022|/data/mirror22 +1043|41|m|m|s|u|sdw5|sdw5|8041|/data/mirror41 +1062|60|m|m|s|u|sdw5|sdw5|8060|/data/mirror60 +1701|699|m|m|s|u|sdw5|sdw5|8699|/data/mirror699 +1720|718|m|m|s|u|sdw5|sdw5|8718|/data/mirror718 +1739|737|m|m|s|u|sdw5|sdw5|8737|/data/mirror737 +1758|756|m|m|s|u|sdw5|sdw5|8756|/data/mirror756 +1777|775|m|m|s|u|sdw5|sdw5|8775|/data/mirror775 +1796|794|m|m|s|u|sdw5|sdw5|8794|/data/mirror794 +1815|813|m|m|s|u|sdw5|sdw5|8813|/data/mirror813 +1834|832|m|m|s|u|sdw5|sdw5|8832|/data/mirror832 +1853|851|m|m|s|u|sdw5|sdw5|8851|/data/mirror851 +1872|870|m|m|s|u|sdw5|sdw5|8870|/data/mirror870 +1891|889|m|m|s|u|sdw5|sdw5|8889|/data/mirror889 +1910|908|m|m|s|u|sdw5|sdw5|8908|/data/mirror908 +1929|927|m|m|s|u|sdw5|sdw5|8927|/data/mirror927 +1948|946|m|m|s|u|sdw5|sdw5|8946|/data/mirror946 +1967|965|m|m|s|u|sdw5|sdw5|8965|/data/mirror965 +1986|984|m|m|s|u|sdw5|sdw5|8984|/data/mirror984 +# SDW50 +982|980|p|p|s|u|sdw50|sdw50|7980|/data/primary980 +983|981|p|p|s|u|sdw50|sdw50|7981|/data/primary981 +984|982|p|p|s|u|sdw50|sdw50|7982|/data/primary982 +985|983|p|p|s|u|sdw50|sdw50|7983|/data/primary983 +986|984|p|p|s|u|sdw50|sdw50|7984|/data/primary984 +987|985|p|p|s|u|sdw50|sdw50|7985|/data/primary985 +988|986|p|p|s|u|sdw50|sdw50|7986|/data/primary986 +989|987|p|p|s|u|sdw50|sdw50|7987|/data/primary987 +990|988|p|p|s|u|sdw50|sdw50|7988|/data/primary988 +991|989|p|p|s|u|sdw50|sdw50|7989|/data/primary989 +992|990|p|p|s|u|sdw50|sdw50|7990|/data/primary990 +993|991|p|p|s|u|sdw50|sdw50|7991|/data/primary991 +994|992|p|p|s|u|sdw50|sdw50|7992|/data/primary992 +995|993|p|p|s|u|sdw50|sdw50|7993|/data/primary993 +996|994|p|p|s|u|sdw50|sdw50|7994|/data/primary994 +997|995|p|p|s|u|sdw50|sdw50|7995|/data/primary995 +998|996|p|p|s|u|sdw50|sdw50|7996|/data/primary996 +999|997|p|p|s|u|sdw50|sdw50|7997|/data/primary997 +1000|998|p|p|s|u|sdw50|sdw50|7998|/data/primary998 +1001|999|p|p|s|u|sdw50|sdw50|7999|/data/primary999 +1601|599|m|m|s|u|sdw50|sdw50|8599|/data/mirror599 +1620|618|m|m|s|u|sdw50|sdw50|8618|/data/mirror618 +1639|637|m|m|s|u|sdw50|sdw50|8637|/data/mirror637 +1658|656|m|m|s|u|sdw50|sdw50|8656|/data/mirror656 +1677|675|m|m|s|u|sdw50|sdw50|8675|/data/mirror675 +1696|694|m|m|s|u|sdw50|sdw50|8694|/data/mirror694 +1715|713|m|m|s|u|sdw50|sdw50|8713|/data/mirror713 +1734|732|m|m|s|u|sdw50|sdw50|8732|/data/mirror732 +1753|751|m|m|s|u|sdw50|sdw50|8751|/data/mirror751 +1772|770|m|m|s|u|sdw50|sdw50|8770|/data/mirror770 +1791|789|m|m|s|u|sdw50|sdw50|8789|/data/mirror789 +1810|808|m|m|s|u|sdw50|sdw50|8808|/data/mirror808 +1829|827|m|m|s|u|sdw50|sdw50|8827|/data/mirror827 +1848|846|m|m|s|u|sdw50|sdw50|8846|/data/mirror846 +1867|865|m|m|s|u|sdw50|sdw50|8865|/data/mirror865 +1886|884|m|m|s|u|sdw50|sdw50|8884|/data/mirror884 +1905|903|m|m|s|u|sdw50|sdw50|8903|/data/mirror903 +1924|922|m|m|s|u|sdw50|sdw50|8922|/data/mirror922 +1943|941|m|m|s|u|sdw50|sdw50|8941|/data/mirror941 +1962|960|m|m|s|u|sdw50|sdw50|8960|/data/mirror960 +# SDW6 +102|100|p|p|s|u|sdw6|sdw6|7100|/data/primary100 +103|101|p|p|s|u|sdw6|sdw6|7101|/data/primary101 +104|102|p|p|s|u|sdw6|sdw6|7102|/data/primary102 +105|103|p|p|s|u|sdw6|sdw6|7103|/data/primary103 +106|104|p|p|s|u|sdw6|sdw6|7104|/data/primary104 +107|105|p|p|s|u|sdw6|sdw6|7105|/data/primary105 +108|106|p|p|s|u|sdw6|sdw6|7106|/data/primary106 +109|107|p|p|s|u|sdw6|sdw6|7107|/data/primary107 +110|108|p|p|s|u|sdw6|sdw6|7108|/data/primary108 +111|109|p|p|s|u|sdw6|sdw6|7109|/data/primary109 +112|110|p|p|s|u|sdw6|sdw6|7110|/data/primary110 +113|111|p|p|s|u|sdw6|sdw6|7111|/data/primary111 +114|112|p|p|s|u|sdw6|sdw6|7112|/data/primary112 +115|113|p|p|s|u|sdw6|sdw6|7113|/data/primary113 +116|114|p|p|s|u|sdw6|sdw6|7114|/data/primary114 +117|115|p|p|s|u|sdw6|sdw6|7115|/data/primary115 +118|116|p|p|s|u|sdw6|sdw6|7116|/data/primary116 +119|117|p|p|s|u|sdw6|sdw6|7117|/data/primary117 +120|118|p|p|s|u|sdw6|sdw6|7118|/data/primary118 +121|119|p|p|s|u|sdw6|sdw6|7119|/data/primary119 +1006|4|m|m|s|u|sdw6|sdw6|8004|/data/mirror4 +1025|23|m|m|s|u|sdw6|sdw6|8023|/data/mirror23 +1044|42|m|m|s|u|sdw6|sdw6|8042|/data/mirror42 +1063|61|m|m|s|u|sdw6|sdw6|8061|/data/mirror61 +1082|80|m|m|s|u|sdw6|sdw6|8080|/data/mirror80 +1721|719|m|m|s|u|sdw6|sdw6|8719|/data/mirror719 +1740|738|m|m|s|u|sdw6|sdw6|8738|/data/mirror738 +1759|757|m|m|s|u|sdw6|sdw6|8757|/data/mirror757 +1778|776|m|m|s|u|sdw6|sdw6|8776|/data/mirror776 +1797|795|m|m|s|u|sdw6|sdw6|8795|/data/mirror795 +1816|814|m|m|s|u|sdw6|sdw6|8814|/data/mirror814 +1835|833|m|m|s|u|sdw6|sdw6|8833|/data/mirror833 +1854|852|m|m|s|u|sdw6|sdw6|8852|/data/mirror852 +1873|871|m|m|s|u|sdw6|sdw6|8871|/data/mirror871 +1892|890|m|m|s|u|sdw6|sdw6|8890|/data/mirror890 +1911|909|m|m|s|u|sdw6|sdw6|8909|/data/mirror909 +1930|928|m|m|s|u|sdw6|sdw6|8928|/data/mirror928 +1949|947|m|m|s|u|sdw6|sdw6|8947|/data/mirror947 +1968|966|m|m|s|u|sdw6|sdw6|8966|/data/mirror966 +1987|985|m|m|s|u|sdw6|sdw6|8985|/data/mirror985 +# SDW7 +122|120|p|p|s|u|sdw7|sdw7|7120|/data/primary120 +123|121|p|p|s|u|sdw7|sdw7|7121|/data/primary121 +124|122|p|p|s|u|sdw7|sdw7|7122|/data/primary122 +125|123|p|p|s|u|sdw7|sdw7|7123|/data/primary123 +126|124|p|p|s|u|sdw7|sdw7|7124|/data/primary124 +127|125|p|p|s|u|sdw7|sdw7|7125|/data/primary125 +128|126|p|p|s|u|sdw7|sdw7|7126|/data/primary126 +129|127|p|p|s|u|sdw7|sdw7|7127|/data/primary127 +130|128|p|p|s|u|sdw7|sdw7|7128|/data/primary128 +131|129|p|p|s|u|sdw7|sdw7|7129|/data/primary129 +132|130|p|p|s|u|sdw7|sdw7|7130|/data/primary130 +133|131|p|p|s|u|sdw7|sdw7|7131|/data/primary131 +134|132|p|p|s|u|sdw7|sdw7|7132|/data/primary132 +135|133|p|p|s|u|sdw7|sdw7|7133|/data/primary133 +136|134|p|p|s|u|sdw7|sdw7|7134|/data/primary134 +137|135|p|p|s|u|sdw7|sdw7|7135|/data/primary135 +138|136|p|p|s|u|sdw7|sdw7|7136|/data/primary136 +139|137|p|p|s|u|sdw7|sdw7|7137|/data/primary137 +140|138|p|p|s|u|sdw7|sdw7|7138|/data/primary138 +141|139|p|p|s|u|sdw7|sdw7|7139|/data/primary139 +1007|5|m|m|s|u|sdw7|sdw7|8005|/data/mirror5 +1026|24|m|m|s|u|sdw7|sdw7|8024|/data/mirror24 +1045|43|m|m|s|u|sdw7|sdw7|8043|/data/mirror43 +1064|62|m|m|s|u|sdw7|sdw7|8062|/data/mirror62 +1083|81|m|m|s|u|sdw7|sdw7|8081|/data/mirror81 +1102|100|m|m|s|u|sdw7|sdw7|8100|/data/mirror100 +1741|739|m|m|s|u|sdw7|sdw7|8739|/data/mirror739 +1760|758|m|m|s|u|sdw7|sdw7|8758|/data/mirror758 +1779|777|m|m|s|u|sdw7|sdw7|8777|/data/mirror777 +1798|796|m|m|s|u|sdw7|sdw7|8796|/data/mirror796 +1817|815|m|m|s|u|sdw7|sdw7|8815|/data/mirror815 +1836|834|m|m|s|u|sdw7|sdw7|8834|/data/mirror834 +1855|853|m|m|s|u|sdw7|sdw7|8853|/data/mirror853 +1874|872|m|m|s|u|sdw7|sdw7|8872|/data/mirror872 +1893|891|m|m|s|u|sdw7|sdw7|8891|/data/mirror891 +1912|910|m|m|s|u|sdw7|sdw7|8910|/data/mirror910 +1931|929|m|m|s|u|sdw7|sdw7|8929|/data/mirror929 +1950|948|m|m|s|u|sdw7|sdw7|8948|/data/mirror948 +1969|967|m|m|s|u|sdw7|sdw7|8967|/data/mirror967 +1988|986|m|m|s|u|sdw7|sdw7|8986|/data/mirror986 +# SDW8 +142|140|p|p|s|u|sdw8|sdw8|7140|/data/primary140 +143|141|p|p|s|u|sdw8|sdw8|7141|/data/primary141 +144|142|p|p|s|u|sdw8|sdw8|7142|/data/primary142 +145|143|p|p|s|u|sdw8|sdw8|7143|/data/primary143 +146|144|p|p|s|u|sdw8|sdw8|7144|/data/primary144 +147|145|p|p|s|u|sdw8|sdw8|7145|/data/primary145 +148|146|p|p|s|u|sdw8|sdw8|7146|/data/primary146 +149|147|p|p|s|u|sdw8|sdw8|7147|/data/primary147 +150|148|p|p|s|u|sdw8|sdw8|7148|/data/primary148 +151|149|p|p|s|u|sdw8|sdw8|7149|/data/primary149 +152|150|p|p|s|u|sdw8|sdw8|7150|/data/primary150 +153|151|p|p|s|u|sdw8|sdw8|7151|/data/primary151 +154|152|p|p|s|u|sdw8|sdw8|7152|/data/primary152 +155|153|p|p|s|u|sdw8|sdw8|7153|/data/primary153 +156|154|p|p|s|u|sdw8|sdw8|7154|/data/primary154 +157|155|p|p|s|u|sdw8|sdw8|7155|/data/primary155 +158|156|p|p|s|u|sdw8|sdw8|7156|/data/primary156 +159|157|p|p|s|u|sdw8|sdw8|7157|/data/primary157 +160|158|p|p|s|u|sdw8|sdw8|7158|/data/primary158 +161|159|p|p|s|u|sdw8|sdw8|7159|/data/primary159 +1008|6|m|m|s|u|sdw8|sdw8|8006|/data/mirror6 +1027|25|m|m|s|u|sdw8|sdw8|8025|/data/mirror25 +1046|44|m|m|s|u|sdw8|sdw8|8044|/data/mirror44 +1065|63|m|m|s|u|sdw8|sdw8|8063|/data/mirror63 +1084|82|m|m|s|u|sdw8|sdw8|8082|/data/mirror82 +1103|101|m|m|s|u|sdw8|sdw8|8101|/data/mirror101 +1122|120|m|m|s|u|sdw8|sdw8|8120|/data/mirror120 +1761|759|m|m|s|u|sdw8|sdw8|8759|/data/mirror759 +1780|778|m|m|s|u|sdw8|sdw8|8778|/data/mirror778 +1799|797|m|m|s|u|sdw8|sdw8|8797|/data/mirror797 +1818|816|m|m|s|u|sdw8|sdw8|8816|/data/mirror816 +1837|835|m|m|s|u|sdw8|sdw8|8835|/data/mirror835 +1856|854|m|m|s|u|sdw8|sdw8|8854|/data/mirror854 +1875|873|m|m|s|u|sdw8|sdw8|8873|/data/mirror873 +1894|892|m|m|s|u|sdw8|sdw8|8892|/data/mirror892 +1913|911|m|m|s|u|sdw8|sdw8|8911|/data/mirror911 +1932|930|m|m|s|u|sdw8|sdw8|8930|/data/mirror930 +1951|949|m|m|s|u|sdw8|sdw8|8949|/data/mirror949 +1970|968|m|m|s|u|sdw8|sdw8|8968|/data/mirror968 +1989|987|m|m|s|u|sdw8|sdw8|8987|/data/mirror987 +# SDW9 +162|160|p|p|s|u|sdw9|sdw9|7160|/data/primary160 +163|161|p|p|s|u|sdw9|sdw9|7161|/data/primary161 +164|162|p|p|s|u|sdw9|sdw9|7162|/data/primary162 +165|163|p|p|s|u|sdw9|sdw9|7163|/data/primary163 +166|164|p|p|s|u|sdw9|sdw9|7164|/data/primary164 +167|165|p|p|s|u|sdw9|sdw9|7165|/data/primary165 +168|166|p|p|s|u|sdw9|sdw9|7166|/data/primary166 +169|167|p|p|s|u|sdw9|sdw9|7167|/data/primary167 +170|168|p|p|s|u|sdw9|sdw9|7168|/data/primary168 +171|169|p|p|s|u|sdw9|sdw9|7169|/data/primary169 +172|170|p|p|s|u|sdw9|sdw9|7170|/data/primary170 +173|171|p|p|s|u|sdw9|sdw9|7171|/data/primary171 +174|172|p|p|s|u|sdw9|sdw9|7172|/data/primary172 +175|173|p|p|s|u|sdw9|sdw9|7173|/data/primary173 +176|174|p|p|s|u|sdw9|sdw9|7174|/data/primary174 +177|175|p|p|s|u|sdw9|sdw9|7175|/data/primary175 +178|176|p|p|s|u|sdw9|sdw9|7176|/data/primary176 +179|177|p|p|s|u|sdw9|sdw9|7177|/data/primary177 +180|178|p|p|s|u|sdw9|sdw9|7178|/data/primary178 +181|179|p|p|s|u|sdw9|sdw9|7179|/data/primary179 +1009|7|m|m|s|u|sdw9|sdw9|8007|/data/mirror7 +1028|26|m|m|s|u|sdw9|sdw9|8026|/data/mirror26 +1047|45|m|m|s|u|sdw9|sdw9|8045|/data/mirror45 +1066|64|m|m|s|u|sdw9|sdw9|8064|/data/mirror64 +1085|83|m|m|s|u|sdw9|sdw9|8083|/data/mirror83 +1104|102|m|m|s|u|sdw9|sdw9|8102|/data/mirror102 +1123|121|m|m|s|u|sdw9|sdw9|8121|/data/mirror121 +1142|140|m|m|s|u|sdw9|sdw9|8140|/data/mirror140 +1781|779|m|m|s|u|sdw9|sdw9|8779|/data/mirror779 +1800|798|m|m|s|u|sdw9|sdw9|8798|/data/mirror798 +1819|817|m|m|s|u|sdw9|sdw9|8817|/data/mirror817 +1838|836|m|m|s|u|sdw9|sdw9|8836|/data/mirror836 +1857|855|m|m|s|u|sdw9|sdw9|8855|/data/mirror855 +1876|874|m|m|s|u|sdw9|sdw9|8874|/data/mirror874 +1895|893|m|m|s|u|sdw9|sdw9|8893|/data/mirror893 +1914|912|m|m|s|u|sdw9|sdw9|8912|/data/mirror912 +1933|931|m|m|s|u|sdw9|sdw9|8931|/data/mirror931 +1952|950|m|m|s|u|sdw9|sdw9|8950|/data/mirror950 +1971|969|m|m|s|u|sdw9|sdw9|8969|/data/mirror969 +1990|988|m|m|s|u|sdw9|sdw9|8988|/data/mirror988 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_grouped.array b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_grouped.array new file mode 100644 index 000000000000..e986a3f9df95 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_grouped.array @@ -0,0 +1,2052 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +8|6|p|p|s|u|sdw1|sdw1|7006|/data/primary6 +9|7|p|p|s|u|sdw1|sdw1|7007|/data/primary7 +10|8|p|p|s|u|sdw1|sdw1|7008|/data/primary8 +11|9|p|p|s|u|sdw1|sdw1|7009|/data/primary9 +12|10|p|p|s|u|sdw1|sdw1|7010|/data/primary10 +13|11|p|p|s|u|sdw1|sdw1|7011|/data/primary11 +14|12|p|p|s|u|sdw1|sdw1|7012|/data/primary12 +15|13|p|p|s|u|sdw1|sdw1|7013|/data/primary13 +16|14|p|p|s|u|sdw1|sdw1|7014|/data/primary14 +17|15|p|p|s|u|sdw1|sdw1|7015|/data/primary15 +18|16|p|p|s|u|sdw1|sdw1|7016|/data/primary16 +19|17|p|p|s|u|sdw1|sdw1|7017|/data/primary17 +20|18|p|p|s|u|sdw1|sdw1|7018|/data/primary18 +1097|95|m|m|s|u|sdw1|sdw1|8095|/data/mirror95 +1098|96|m|m|s|u|sdw1|sdw1|8096|/data/mirror96 +1099|97|m|m|s|u|sdw1|sdw1|8097|/data/mirror97 +1100|98|m|m|s|u|sdw1|sdw1|8098|/data/mirror98 +1101|99|m|m|s|u|sdw1|sdw1|8099|/data/mirror99 +1102|100|m|m|s|u|sdw1|sdw1|8100|/data/mirror100 +1103|101|m|m|s|u|sdw1|sdw1|8101|/data/mirror101 +1104|102|m|m|s|u|sdw1|sdw1|8102|/data/mirror102 +1105|103|m|m|s|u|sdw1|sdw1|8103|/data/mirror103 +1106|104|m|m|s|u|sdw1|sdw1|8104|/data/mirror104 +1107|105|m|m|s|u|sdw1|sdw1|8105|/data/mirror105 +1108|106|m|m|s|u|sdw1|sdw1|8106|/data/mirror106 +1109|107|m|m|s|u|sdw1|sdw1|8107|/data/mirror107 +1110|108|m|m|s|u|sdw1|sdw1|8108|/data/mirror108 +1111|109|m|m|s|u|sdw1|sdw1|8109|/data/mirror109 +1112|110|m|m|s|u|sdw1|sdw1|8110|/data/mirror110 +1113|111|m|m|s|u|sdw1|sdw1|8111|/data/mirror111 +# SDW10 +175|173|p|p|s|u|sdw10|sdw10|7173|/data/primary173 +176|174|p|p|s|u|sdw10|sdw10|7174|/data/primary174 +177|175|p|p|s|u|sdw10|sdw10|7175|/data/primary175 +178|176|p|p|s|u|sdw10|sdw10|7176|/data/primary176 +179|177|p|p|s|u|sdw10|sdw10|7177|/data/primary177 +180|178|p|p|s|u|sdw10|sdw10|7178|/data/primary178 +181|179|p|p|s|u|sdw10|sdw10|7179|/data/primary179 +182|180|p|p|s|u|sdw10|sdw10|7180|/data/primary180 +183|181|p|p|s|u|sdw10|sdw10|7181|/data/primary181 +184|182|p|p|s|u|sdw10|sdw10|7182|/data/primary182 +185|183|p|p|s|u|sdw10|sdw10|7183|/data/primary183 +186|184|p|p|s|u|sdw10|sdw10|7184|/data/primary184 +187|185|p|p|s|u|sdw10|sdw10|7185|/data/primary185 +188|186|p|p|s|u|sdw10|sdw10|7186|/data/primary186 +189|187|p|p|s|u|sdw10|sdw10|7187|/data/primary187 +190|188|p|p|s|u|sdw10|sdw10|7188|/data/primary188 +191|189|p|p|s|u|sdw10|sdw10|7189|/data/primary189 +192|190|p|p|s|u|sdw10|sdw10|7190|/data/primary190 +193|191|p|p|s|u|sdw10|sdw10|7191|/data/primary191 +194|192|p|p|s|u|sdw10|sdw10|7192|/data/primary192 +195|193|p|p|s|u|sdw10|sdw10|7193|/data/primary193 +196|194|p|p|s|u|sdw10|sdw10|7194|/data/primary194 +197|195|p|p|s|u|sdw10|sdw10|7195|/data/primary195 +198|196|p|p|s|u|sdw10|sdw10|7196|/data/primary196 +1132|130|m|m|s|u|sdw10|sdw10|8130|/data/mirror130 +1133|131|m|m|s|u|sdw10|sdw10|8131|/data/mirror131 +1134|132|m|m|s|u|sdw10|sdw10|8132|/data/mirror132 +1135|133|m|m|s|u|sdw10|sdw10|8133|/data/mirror133 +1136|134|m|m|s|u|sdw10|sdw10|8134|/data/mirror134 +1137|135|m|m|s|u|sdw10|sdw10|8135|/data/mirror135 +1138|136|m|m|s|u|sdw10|sdw10|8136|/data/mirror136 +1139|137|m|m|s|u|sdw10|sdw10|8137|/data/mirror137 +1140|138|m|m|s|u|sdw10|sdw10|8138|/data/mirror138 +1141|139|m|m|s|u|sdw10|sdw10|8139|/data/mirror139 +1142|140|m|m|s|u|sdw10|sdw10|8140|/data/mirror140 +1143|141|m|m|s|u|sdw10|sdw10|8141|/data/mirror141 +1144|142|m|m|s|u|sdw10|sdw10|8142|/data/mirror142 +1145|143|m|m|s|u|sdw10|sdw10|8143|/data/mirror143 +1146|144|m|m|s|u|sdw10|sdw10|8144|/data/mirror144 +1147|145|m|m|s|u|sdw10|sdw10|8145|/data/mirror145 +1148|146|m|m|s|u|sdw10|sdw10|8146|/data/mirror146 +1149|147|m|m|s|u|sdw10|sdw10|8147|/data/mirror147 +1150|148|m|m|s|u|sdw10|sdw10|8148|/data/mirror148 +1151|149|m|m|s|u|sdw10|sdw10|8149|/data/mirror149 +# SDW11 +199|197|p|p|s|u|sdw11|sdw11|7197|/data/primary197 +200|198|p|p|s|u|sdw11|sdw11|7198|/data/primary198 +201|199|p|p|s|u|sdw11|sdw11|7199|/data/primary199 +202|200|p|p|s|u|sdw11|sdw11|7200|/data/primary200 +203|201|p|p|s|u|sdw11|sdw11|7201|/data/primary201 +204|202|p|p|s|u|sdw11|sdw11|7202|/data/primary202 +205|203|p|p|s|u|sdw11|sdw11|7203|/data/primary203 +206|204|p|p|s|u|sdw11|sdw11|7204|/data/primary204 +207|205|p|p|s|u|sdw11|sdw11|7205|/data/primary205 +208|206|p|p|s|u|sdw11|sdw11|7206|/data/primary206 +209|207|p|p|s|u|sdw11|sdw11|7207|/data/primary207 +210|208|p|p|s|u|sdw11|sdw11|7208|/data/primary208 +211|209|p|p|s|u|sdw11|sdw11|7209|/data/primary209 +212|210|p|p|s|u|sdw11|sdw11|7210|/data/primary210 +213|211|p|p|s|u|sdw11|sdw11|7211|/data/primary211 +214|212|p|p|s|u|sdw11|sdw11|7212|/data/primary212 +215|213|p|p|s|u|sdw11|sdw11|7213|/data/primary213 +216|214|p|p|s|u|sdw11|sdw11|7214|/data/primary214 +217|215|p|p|s|u|sdw11|sdw11|7215|/data/primary215 +218|216|p|p|s|u|sdw11|sdw11|7216|/data/primary216 +1152|150|m|m|s|u|sdw11|sdw11|8150|/data/mirror150 +1153|151|m|m|s|u|sdw11|sdw11|8151|/data/mirror151 +1154|152|m|m|s|u|sdw11|sdw11|8152|/data/mirror152 +1155|153|m|m|s|u|sdw11|sdw11|8153|/data/mirror153 +1156|154|m|m|s|u|sdw11|sdw11|8154|/data/mirror154 +1157|155|m|m|s|u|sdw11|sdw11|8155|/data/mirror155 +1158|156|m|m|s|u|sdw11|sdw11|8156|/data/mirror156 +1159|157|m|m|s|u|sdw11|sdw11|8157|/data/mirror157 +1160|158|m|m|s|u|sdw11|sdw11|8158|/data/mirror158 +1161|159|m|m|s|u|sdw11|sdw11|8159|/data/mirror159 +1162|160|m|m|s|u|sdw11|sdw11|8160|/data/mirror160 +1163|161|m|m|s|u|sdw11|sdw11|8161|/data/mirror161 +1164|162|m|m|s|u|sdw11|sdw11|8162|/data/mirror162 +1165|163|m|m|s|u|sdw11|sdw11|8163|/data/mirror163 +1166|164|m|m|s|u|sdw11|sdw11|8164|/data/mirror164 +1167|165|m|m|s|u|sdw11|sdw11|8165|/data/mirror165 +1168|166|m|m|s|u|sdw11|sdw11|8166|/data/mirror166 +1169|167|m|m|s|u|sdw11|sdw11|8167|/data/mirror167 +1170|168|m|m|s|u|sdw11|sdw11|8168|/data/mirror168 +1171|169|m|m|s|u|sdw11|sdw11|8169|/data/mirror169 +1172|170|m|m|s|u|sdw11|sdw11|8170|/data/mirror170 +1173|171|m|m|s|u|sdw11|sdw11|8171|/data/mirror171 +1174|172|m|m|s|u|sdw11|sdw11|8172|/data/mirror172 +# SDW12 +219|217|p|p|s|u|sdw12|sdw12|7217|/data/primary217 +220|218|p|p|s|u|sdw12|sdw12|7218|/data/primary218 +221|219|p|p|s|u|sdw12|sdw12|7219|/data/primary219 +222|220|p|p|s|u|sdw12|sdw12|7220|/data/primary220 +223|221|p|p|s|u|sdw12|sdw12|7221|/data/primary221 +224|222|p|p|s|u|sdw12|sdw12|7222|/data/primary222 +225|223|p|p|s|u|sdw12|sdw12|7223|/data/primary223 +226|224|p|p|s|u|sdw12|sdw12|7224|/data/primary224 +227|225|p|p|s|u|sdw12|sdw12|7225|/data/primary225 +228|226|p|p|s|u|sdw12|sdw12|7226|/data/primary226 +229|227|p|p|s|u|sdw12|sdw12|7227|/data/primary227 +230|228|p|p|s|u|sdw12|sdw12|7228|/data/primary228 +231|229|p|p|s|u|sdw12|sdw12|7229|/data/primary229 +232|230|p|p|s|u|sdw12|sdw12|7230|/data/primary230 +233|231|p|p|s|u|sdw12|sdw12|7231|/data/primary231 +234|232|p|p|s|u|sdw12|sdw12|7232|/data/primary232 +235|233|p|p|s|u|sdw12|sdw12|7233|/data/primary233 +236|234|p|p|s|u|sdw12|sdw12|7234|/data/primary234 +237|235|p|p|s|u|sdw12|sdw12|7235|/data/primary235 +238|236|p|p|s|u|sdw12|sdw12|7236|/data/primary236 +239|237|p|p|s|u|sdw12|sdw12|7237|/data/primary237 +240|238|p|p|s|u|sdw12|sdw12|7238|/data/primary238 +1199|197|m|m|s|u|sdw12|sdw12|8197|/data/mirror197 +1200|198|m|m|s|u|sdw12|sdw12|8198|/data/mirror198 +1201|199|m|m|s|u|sdw12|sdw12|8199|/data/mirror199 +1202|200|m|m|s|u|sdw12|sdw12|8200|/data/mirror200 +1203|201|m|m|s|u|sdw12|sdw12|8201|/data/mirror201 +1204|202|m|m|s|u|sdw12|sdw12|8202|/data/mirror202 +1205|203|m|m|s|u|sdw12|sdw12|8203|/data/mirror203 +1206|204|m|m|s|u|sdw12|sdw12|8204|/data/mirror204 +1207|205|m|m|s|u|sdw12|sdw12|8205|/data/mirror205 +1208|206|m|m|s|u|sdw12|sdw12|8206|/data/mirror206 +1209|207|m|m|s|u|sdw12|sdw12|8207|/data/mirror207 +1210|208|m|m|s|u|sdw12|sdw12|8208|/data/mirror208 +1211|209|m|m|s|u|sdw12|sdw12|8209|/data/mirror209 +1212|210|m|m|s|u|sdw12|sdw12|8210|/data/mirror210 +1213|211|m|m|s|u|sdw12|sdw12|8211|/data/mirror211 +1214|212|m|m|s|u|sdw12|sdw12|8212|/data/mirror212 +1215|213|m|m|s|u|sdw12|sdw12|8213|/data/mirror213 +1216|214|m|m|s|u|sdw12|sdw12|8214|/data/mirror214 +1217|215|m|m|s|u|sdw12|sdw12|8215|/data/mirror215 +1218|216|m|m|s|u|sdw12|sdw12|8216|/data/mirror216 +# SDW13 +241|239|p|p|s|u|sdw13|sdw13|7239|/data/primary239 +242|240|p|p|s|u|sdw13|sdw13|7240|/data/primary240 +243|241|p|p|s|u|sdw13|sdw13|7241|/data/primary241 +244|242|p|p|s|u|sdw13|sdw13|7242|/data/primary242 +245|243|p|p|s|u|sdw13|sdw13|7243|/data/primary243 +246|244|p|p|s|u|sdw13|sdw13|7244|/data/primary244 +247|245|p|p|s|u|sdw13|sdw13|7245|/data/primary245 +248|246|p|p|s|u|sdw13|sdw13|7246|/data/primary246 +249|247|p|p|s|u|sdw13|sdw13|7247|/data/primary247 +250|248|p|p|s|u|sdw13|sdw13|7248|/data/primary248 +251|249|p|p|s|u|sdw13|sdw13|7249|/data/primary249 +252|250|p|p|s|u|sdw13|sdw13|7250|/data/primary250 +253|251|p|p|s|u|sdw13|sdw13|7251|/data/primary251 +254|252|p|p|s|u|sdw13|sdw13|7252|/data/primary252 +255|253|p|p|s|u|sdw13|sdw13|7253|/data/primary253 +256|254|p|p|s|u|sdw13|sdw13|7254|/data/primary254 +257|255|p|p|s|u|sdw13|sdw13|7255|/data/primary255 +258|256|p|p|s|u|sdw13|sdw13|7256|/data/primary256 +259|257|p|p|s|u|sdw13|sdw13|7257|/data/primary257 +260|258|p|p|s|u|sdw13|sdw13|7258|/data/primary258 +261|259|p|p|s|u|sdw13|sdw13|7259|/data/primary259 +262|260|p|p|s|u|sdw13|sdw13|7260|/data/primary260 +263|261|p|p|s|u|sdw13|sdw13|7261|/data/primary261 +264|262|p|p|s|u|sdw13|sdw13|7262|/data/primary262 +1283|281|m|m|s|u|sdw13|sdw13|8281|/data/mirror281 +1284|282|m|m|s|u|sdw13|sdw13|8282|/data/mirror282 +1285|283|m|m|s|u|sdw13|sdw13|8283|/data/mirror283 +1286|284|m|m|s|u|sdw13|sdw13|8284|/data/mirror284 +1287|285|m|m|s|u|sdw13|sdw13|8285|/data/mirror285 +1288|286|m|m|s|u|sdw13|sdw13|8286|/data/mirror286 +1289|287|m|m|s|u|sdw13|sdw13|8287|/data/mirror287 +1290|288|m|m|s|u|sdw13|sdw13|8288|/data/mirror288 +1291|289|m|m|s|u|sdw13|sdw13|8289|/data/mirror289 +1292|290|m|m|s|u|sdw13|sdw13|8290|/data/mirror290 +1293|291|m|m|s|u|sdw13|sdw13|8291|/data/mirror291 +1294|292|m|m|s|u|sdw13|sdw13|8292|/data/mirror292 +1295|293|m|m|s|u|sdw13|sdw13|8293|/data/mirror293 +1296|294|m|m|s|u|sdw13|sdw13|8294|/data/mirror294 +1297|295|m|m|s|u|sdw13|sdw13|8295|/data/mirror295 +1298|296|m|m|s|u|sdw13|sdw13|8296|/data/mirror296 +1299|297|m|m|s|u|sdw13|sdw13|8297|/data/mirror297 +1300|298|m|m|s|u|sdw13|sdw13|8298|/data/mirror298 +1301|299|m|m|s|u|sdw13|sdw13|8299|/data/mirror299 +1302|300|m|m|s|u|sdw13|sdw13|8300|/data/mirror300 +1303|301|m|m|s|u|sdw13|sdw13|8301|/data/mirror301 +# SDW14 +265|263|p|p|s|u|sdw14|sdw14|7263|/data/primary263 +266|264|p|p|s|u|sdw14|sdw14|7264|/data/primary264 +267|265|p|p|s|u|sdw14|sdw14|7265|/data/primary265 +268|266|p|p|s|u|sdw14|sdw14|7266|/data/primary266 +269|267|p|p|s|u|sdw14|sdw14|7267|/data/primary267 +270|268|p|p|s|u|sdw14|sdw14|7268|/data/primary268 +271|269|p|p|s|u|sdw14|sdw14|7269|/data/primary269 +272|270|p|p|s|u|sdw14|sdw14|7270|/data/primary270 +273|271|p|p|s|u|sdw14|sdw14|7271|/data/primary271 +274|272|p|p|s|u|sdw14|sdw14|7272|/data/primary272 +275|273|p|p|s|u|sdw14|sdw14|7273|/data/primary273 +276|274|p|p|s|u|sdw14|sdw14|7274|/data/primary274 +277|275|p|p|s|u|sdw14|sdw14|7275|/data/primary275 +278|276|p|p|s|u|sdw14|sdw14|7276|/data/primary276 +279|277|p|p|s|u|sdw14|sdw14|7277|/data/primary277 +280|278|p|p|s|u|sdw14|sdw14|7278|/data/primary278 +281|279|p|p|s|u|sdw14|sdw14|7279|/data/primary279 +282|280|p|p|s|u|sdw14|sdw14|7280|/data/primary280 +1241|239|m|m|s|u|sdw14|sdw14|8239|/data/mirror239 +1242|240|m|m|s|u|sdw14|sdw14|8240|/data/mirror240 +1243|241|m|m|s|u|sdw14|sdw14|8241|/data/mirror241 +1244|242|m|m|s|u|sdw14|sdw14|8242|/data/mirror242 +1245|243|m|m|s|u|sdw14|sdw14|8243|/data/mirror243 +1246|244|m|m|s|u|sdw14|sdw14|8244|/data/mirror244 +1247|245|m|m|s|u|sdw14|sdw14|8245|/data/mirror245 +1248|246|m|m|s|u|sdw14|sdw14|8246|/data/mirror246 +1249|247|m|m|s|u|sdw14|sdw14|8247|/data/mirror247 +1250|248|m|m|s|u|sdw14|sdw14|8248|/data/mirror248 +1251|249|m|m|s|u|sdw14|sdw14|8249|/data/mirror249 +1252|250|m|m|s|u|sdw14|sdw14|8250|/data/mirror250 +1253|251|m|m|s|u|sdw14|sdw14|8251|/data/mirror251 +1254|252|m|m|s|u|sdw14|sdw14|8252|/data/mirror252 +1255|253|m|m|s|u|sdw14|sdw14|8253|/data/mirror253 +1256|254|m|m|s|u|sdw14|sdw14|8254|/data/mirror254 +1257|255|m|m|s|u|sdw14|sdw14|8255|/data/mirror255 +1258|256|m|m|s|u|sdw14|sdw14|8256|/data/mirror256 +1259|257|m|m|s|u|sdw14|sdw14|8257|/data/mirror257 +1260|258|m|m|s|u|sdw14|sdw14|8258|/data/mirror258 +1261|259|m|m|s|u|sdw14|sdw14|8259|/data/mirror259 +1262|260|m|m|s|u|sdw14|sdw14|8260|/data/mirror260 +1263|261|m|m|s|u|sdw14|sdw14|8261|/data/mirror261 +1264|262|m|m|s|u|sdw14|sdw14|8262|/data/mirror262 +# SDW15 +283|281|p|p|s|u|sdw15|sdw15|7281|/data/primary281 +284|282|p|p|s|u|sdw15|sdw15|7282|/data/primary282 +285|283|p|p|s|u|sdw15|sdw15|7283|/data/primary283 +286|284|p|p|s|u|sdw15|sdw15|7284|/data/primary284 +287|285|p|p|s|u|sdw15|sdw15|7285|/data/primary285 +288|286|p|p|s|u|sdw15|sdw15|7286|/data/primary286 +289|287|p|p|s|u|sdw15|sdw15|7287|/data/primary287 +290|288|p|p|s|u|sdw15|sdw15|7288|/data/primary288 +291|289|p|p|s|u|sdw15|sdw15|7289|/data/primary289 +292|290|p|p|s|u|sdw15|sdw15|7290|/data/primary290 +293|291|p|p|s|u|sdw15|sdw15|7291|/data/primary291 +294|292|p|p|s|u|sdw15|sdw15|7292|/data/primary292 +295|293|p|p|s|u|sdw15|sdw15|7293|/data/primary293 +296|294|p|p|s|u|sdw15|sdw15|7294|/data/primary294 +297|295|p|p|s|u|sdw15|sdw15|7295|/data/primary295 +298|296|p|p|s|u|sdw15|sdw15|7296|/data/primary296 +299|297|p|p|s|u|sdw15|sdw15|7297|/data/primary297 +300|298|p|p|s|u|sdw15|sdw15|7298|/data/primary298 +301|299|p|p|s|u|sdw15|sdw15|7299|/data/primary299 +302|300|p|p|s|u|sdw15|sdw15|7300|/data/primary300 +303|301|p|p|s|u|sdw15|sdw15|7301|/data/primary301 +1265|263|m|m|s|u|sdw15|sdw15|8263|/data/mirror263 +1266|264|m|m|s|u|sdw15|sdw15|8264|/data/mirror264 +1267|265|m|m|s|u|sdw15|sdw15|8265|/data/mirror265 +1268|266|m|m|s|u|sdw15|sdw15|8266|/data/mirror266 +1269|267|m|m|s|u|sdw15|sdw15|8267|/data/mirror267 +1270|268|m|m|s|u|sdw15|sdw15|8268|/data/mirror268 +1271|269|m|m|s|u|sdw15|sdw15|8269|/data/mirror269 +1272|270|m|m|s|u|sdw15|sdw15|8270|/data/mirror270 +1273|271|m|m|s|u|sdw15|sdw15|8271|/data/mirror271 +1274|272|m|m|s|u|sdw15|sdw15|8272|/data/mirror272 +1275|273|m|m|s|u|sdw15|sdw15|8273|/data/mirror273 +1276|274|m|m|s|u|sdw15|sdw15|8274|/data/mirror274 +1277|275|m|m|s|u|sdw15|sdw15|8275|/data/mirror275 +1278|276|m|m|s|u|sdw15|sdw15|8276|/data/mirror276 +1279|277|m|m|s|u|sdw15|sdw15|8277|/data/mirror277 +1280|278|m|m|s|u|sdw15|sdw15|8278|/data/mirror278 +1281|279|m|m|s|u|sdw15|sdw15|8279|/data/mirror279 +1282|280|m|m|s|u|sdw15|sdw15|8280|/data/mirror280 +# SDW16 +304|302|p|p|s|u|sdw16|sdw16|7302|/data/primary302 +305|303|p|p|s|u|sdw16|sdw16|7303|/data/primary303 +306|304|p|p|s|u|sdw16|sdw16|7304|/data/primary304 +307|305|p|p|s|u|sdw16|sdw16|7305|/data/primary305 +308|306|p|p|s|u|sdw16|sdw16|7306|/data/primary306 +309|307|p|p|s|u|sdw16|sdw16|7307|/data/primary307 +310|308|p|p|s|u|sdw16|sdw16|7308|/data/primary308 +311|309|p|p|s|u|sdw16|sdw16|7309|/data/primary309 +312|310|p|p|s|u|sdw16|sdw16|7310|/data/primary310 +313|311|p|p|s|u|sdw16|sdw16|7311|/data/primary311 +314|312|p|p|s|u|sdw16|sdw16|7312|/data/primary312 +315|313|p|p|s|u|sdw16|sdw16|7313|/data/primary313 +316|314|p|p|s|u|sdw16|sdw16|7314|/data/primary314 +317|315|p|p|s|u|sdw16|sdw16|7315|/data/primary315 +318|316|p|p|s|u|sdw16|sdw16|7316|/data/primary316 +319|317|p|p|s|u|sdw16|sdw16|7317|/data/primary317 +320|318|p|p|s|u|sdw16|sdw16|7318|/data/primary318 +321|319|p|p|s|u|sdw16|sdw16|7319|/data/primary319 +322|320|p|p|s|u|sdw16|sdw16|7320|/data/primary320 +323|321|p|p|s|u|sdw16|sdw16|7321|/data/primary321 +324|322|p|p|s|u|sdw16|sdw16|7322|/data/primary322 +325|323|p|p|s|u|sdw16|sdw16|7323|/data/primary323 +326|324|p|p|s|u|sdw16|sdw16|7324|/data/primary324 +327|325|p|p|s|u|sdw16|sdw16|7325|/data/primary325 +1343|341|m|m|s|u|sdw16|sdw16|8341|/data/mirror341 +1344|342|m|m|s|u|sdw16|sdw16|8342|/data/mirror342 +1345|343|m|m|s|u|sdw16|sdw16|8343|/data/mirror343 +1346|344|m|m|s|u|sdw16|sdw16|8344|/data/mirror344 +1347|345|m|m|s|u|sdw16|sdw16|8345|/data/mirror345 +1348|346|m|m|s|u|sdw16|sdw16|8346|/data/mirror346 +1349|347|m|m|s|u|sdw16|sdw16|8347|/data/mirror347 +1350|348|m|m|s|u|sdw16|sdw16|8348|/data/mirror348 +1351|349|m|m|s|u|sdw16|sdw16|8349|/data/mirror349 +1352|350|m|m|s|u|sdw16|sdw16|8350|/data/mirror350 +1353|351|m|m|s|u|sdw16|sdw16|8351|/data/mirror351 +1354|352|m|m|s|u|sdw16|sdw16|8352|/data/mirror352 +1355|353|m|m|s|u|sdw16|sdw16|8353|/data/mirror353 +1356|354|m|m|s|u|sdw16|sdw16|8354|/data/mirror354 +1357|355|m|m|s|u|sdw16|sdw16|8355|/data/mirror355 +1358|356|m|m|s|u|sdw16|sdw16|8356|/data/mirror356 +1359|357|m|m|s|u|sdw16|sdw16|8357|/data/mirror357 +1360|358|m|m|s|u|sdw16|sdw16|8358|/data/mirror358 +# SDW17 +328|326|p|p|s|u|sdw17|sdw17|7326|/data/primary326 +329|327|p|p|s|u|sdw17|sdw17|7327|/data/primary327 +330|328|p|p|s|u|sdw17|sdw17|7328|/data/primary328 +331|329|p|p|s|u|sdw17|sdw17|7329|/data/primary329 +332|330|p|p|s|u|sdw17|sdw17|7330|/data/primary330 +333|331|p|p|s|u|sdw17|sdw17|7331|/data/primary331 +334|332|p|p|s|u|sdw17|sdw17|7332|/data/primary332 +335|333|p|p|s|u|sdw17|sdw17|7333|/data/primary333 +336|334|p|p|s|u|sdw17|sdw17|7334|/data/primary334 +337|335|p|p|s|u|sdw17|sdw17|7335|/data/primary335 +338|336|p|p|s|u|sdw17|sdw17|7336|/data/primary336 +339|337|p|p|s|u|sdw17|sdw17|7337|/data/primary337 +340|338|p|p|s|u|sdw17|sdw17|7338|/data/primary338 +341|339|p|p|s|u|sdw17|sdw17|7339|/data/primary339 +342|340|p|p|s|u|sdw17|sdw17|7340|/data/primary340 +1361|359|m|m|s|u|sdw17|sdw17|8359|/data/mirror359 +1362|360|m|m|s|u|sdw17|sdw17|8360|/data/mirror360 +1363|361|m|m|s|u|sdw17|sdw17|8361|/data/mirror361 +1364|362|m|m|s|u|sdw17|sdw17|8362|/data/mirror362 +1365|363|m|m|s|u|sdw17|sdw17|8363|/data/mirror363 +1366|364|m|m|s|u|sdw17|sdw17|8364|/data/mirror364 +1367|365|m|m|s|u|sdw17|sdw17|8365|/data/mirror365 +1368|366|m|m|s|u|sdw17|sdw17|8366|/data/mirror366 +1369|367|m|m|s|u|sdw17|sdw17|8367|/data/mirror367 +1370|368|m|m|s|u|sdw17|sdw17|8368|/data/mirror368 +1371|369|m|m|s|u|sdw17|sdw17|8369|/data/mirror369 +1372|370|m|m|s|u|sdw17|sdw17|8370|/data/mirror370 +1373|371|m|m|s|u|sdw17|sdw17|8371|/data/mirror371 +1374|372|m|m|s|u|sdw17|sdw17|8372|/data/mirror372 +1375|373|m|m|s|u|sdw17|sdw17|8373|/data/mirror373 +1376|374|m|m|s|u|sdw17|sdw17|8374|/data/mirror374 +1377|375|m|m|s|u|sdw17|sdw17|8375|/data/mirror375 +1378|376|m|m|s|u|sdw17|sdw17|8376|/data/mirror376 +1379|377|m|m|s|u|sdw17|sdw17|8377|/data/mirror377 +# SDW18 +343|341|p|p|s|u|sdw18|sdw18|7341|/data/primary341 +344|342|p|p|s|u|sdw18|sdw18|7342|/data/primary342 +345|343|p|p|s|u|sdw18|sdw18|7343|/data/primary343 +346|344|p|p|s|u|sdw18|sdw18|7344|/data/primary344 +347|345|p|p|s|u|sdw18|sdw18|7345|/data/primary345 +348|346|p|p|s|u|sdw18|sdw18|7346|/data/primary346 +349|347|p|p|s|u|sdw18|sdw18|7347|/data/primary347 +350|348|p|p|s|u|sdw18|sdw18|7348|/data/primary348 +351|349|p|p|s|u|sdw18|sdw18|7349|/data/primary349 +352|350|p|p|s|u|sdw18|sdw18|7350|/data/primary350 +353|351|p|p|s|u|sdw18|sdw18|7351|/data/primary351 +354|352|p|p|s|u|sdw18|sdw18|7352|/data/primary352 +355|353|p|p|s|u|sdw18|sdw18|7353|/data/primary353 +356|354|p|p|s|u|sdw18|sdw18|7354|/data/primary354 +357|355|p|p|s|u|sdw18|sdw18|7355|/data/primary355 +358|356|p|p|s|u|sdw18|sdw18|7356|/data/primary356 +359|357|p|p|s|u|sdw18|sdw18|7357|/data/primary357 +360|358|p|p|s|u|sdw18|sdw18|7358|/data/primary358 +1328|326|m|m|s|u|sdw18|sdw18|8326|/data/mirror326 +1329|327|m|m|s|u|sdw18|sdw18|8327|/data/mirror327 +1330|328|m|m|s|u|sdw18|sdw18|8328|/data/mirror328 +1331|329|m|m|s|u|sdw18|sdw18|8329|/data/mirror329 +1332|330|m|m|s|u|sdw18|sdw18|8330|/data/mirror330 +1333|331|m|m|s|u|sdw18|sdw18|8331|/data/mirror331 +1334|332|m|m|s|u|sdw18|sdw18|8332|/data/mirror332 +1335|333|m|m|s|u|sdw18|sdw18|8333|/data/mirror333 +1336|334|m|m|s|u|sdw18|sdw18|8334|/data/mirror334 +1337|335|m|m|s|u|sdw18|sdw18|8335|/data/mirror335 +1338|336|m|m|s|u|sdw18|sdw18|8336|/data/mirror336 +1339|337|m|m|s|u|sdw18|sdw18|8337|/data/mirror337 +1340|338|m|m|s|u|sdw18|sdw18|8338|/data/mirror338 +1341|339|m|m|s|u|sdw18|sdw18|8339|/data/mirror339 +1342|340|m|m|s|u|sdw18|sdw18|8340|/data/mirror340 +# SDW19 +361|359|p|p|s|u|sdw19|sdw19|7359|/data/primary359 +362|360|p|p|s|u|sdw19|sdw19|7360|/data/primary360 +363|361|p|p|s|u|sdw19|sdw19|7361|/data/primary361 +364|362|p|p|s|u|sdw19|sdw19|7362|/data/primary362 +365|363|p|p|s|u|sdw19|sdw19|7363|/data/primary363 +366|364|p|p|s|u|sdw19|sdw19|7364|/data/primary364 +367|365|p|p|s|u|sdw19|sdw19|7365|/data/primary365 +368|366|p|p|s|u|sdw19|sdw19|7366|/data/primary366 +369|367|p|p|s|u|sdw19|sdw19|7367|/data/primary367 +370|368|p|p|s|u|sdw19|sdw19|7368|/data/primary368 +371|369|p|p|s|u|sdw19|sdw19|7369|/data/primary369 +372|370|p|p|s|u|sdw19|sdw19|7370|/data/primary370 +373|371|p|p|s|u|sdw19|sdw19|7371|/data/primary371 +374|372|p|p|s|u|sdw19|sdw19|7372|/data/primary372 +375|373|p|p|s|u|sdw19|sdw19|7373|/data/primary373 +376|374|p|p|s|u|sdw19|sdw19|7374|/data/primary374 +377|375|p|p|s|u|sdw19|sdw19|7375|/data/primary375 +378|376|p|p|s|u|sdw19|sdw19|7376|/data/primary376 +379|377|p|p|s|u|sdw19|sdw19|7377|/data/primary377 +1304|302|m|m|s|u|sdw19|sdw19|8302|/data/mirror302 +1305|303|m|m|s|u|sdw19|sdw19|8303|/data/mirror303 +1306|304|m|m|s|u|sdw19|sdw19|8304|/data/mirror304 +1307|305|m|m|s|u|sdw19|sdw19|8305|/data/mirror305 +1308|306|m|m|s|u|sdw19|sdw19|8306|/data/mirror306 +1309|307|m|m|s|u|sdw19|sdw19|8307|/data/mirror307 +1310|308|m|m|s|u|sdw19|sdw19|8308|/data/mirror308 +1311|309|m|m|s|u|sdw19|sdw19|8309|/data/mirror309 +1312|310|m|m|s|u|sdw19|sdw19|8310|/data/mirror310 +1313|311|m|m|s|u|sdw19|sdw19|8311|/data/mirror311 +1314|312|m|m|s|u|sdw19|sdw19|8312|/data/mirror312 +1315|313|m|m|s|u|sdw19|sdw19|8313|/data/mirror313 +1316|314|m|m|s|u|sdw19|sdw19|8314|/data/mirror314 +1317|315|m|m|s|u|sdw19|sdw19|8315|/data/mirror315 +1318|316|m|m|s|u|sdw19|sdw19|8316|/data/mirror316 +1319|317|m|m|s|u|sdw19|sdw19|8317|/data/mirror317 +1320|318|m|m|s|u|sdw19|sdw19|8318|/data/mirror318 +1321|319|m|m|s|u|sdw19|sdw19|8319|/data/mirror319 +1322|320|m|m|s|u|sdw19|sdw19|8320|/data/mirror320 +1323|321|m|m|s|u|sdw19|sdw19|8321|/data/mirror321 +1324|322|m|m|s|u|sdw19|sdw19|8322|/data/mirror322 +1325|323|m|m|s|u|sdw19|sdw19|8323|/data/mirror323 +1326|324|m|m|s|u|sdw19|sdw19|8324|/data/mirror324 +1327|325|m|m|s|u|sdw19|sdw19|8325|/data/mirror325 +# SDW2 +21|19|p|p|s|u|sdw2|sdw2|7019|/data/primary19 +22|20|p|p|s|u|sdw2|sdw2|7020|/data/primary20 +23|21|p|p|s|u|sdw2|sdw2|7021|/data/primary21 +24|22|p|p|s|u|sdw2|sdw2|7022|/data/primary22 +25|23|p|p|s|u|sdw2|sdw2|7023|/data/primary23 +26|24|p|p|s|u|sdw2|sdw2|7024|/data/primary24 +27|25|p|p|s|u|sdw2|sdw2|7025|/data/primary25 +28|26|p|p|s|u|sdw2|sdw2|7026|/data/primary26 +29|27|p|p|s|u|sdw2|sdw2|7027|/data/primary27 +30|28|p|p|s|u|sdw2|sdw2|7028|/data/primary28 +31|29|p|p|s|u|sdw2|sdw2|7029|/data/primary29 +32|30|p|p|s|u|sdw2|sdw2|7030|/data/primary30 +33|31|p|p|s|u|sdw2|sdw2|7031|/data/primary31 +34|32|p|p|s|u|sdw2|sdw2|7032|/data/primary32 +35|33|p|p|s|u|sdw2|sdw2|7033|/data/primary33 +36|34|p|p|s|u|sdw2|sdw2|7034|/data/primary34 +37|35|p|p|s|u|sdw2|sdw2|7035|/data/primary35 +38|36|p|p|s|u|sdw2|sdw2|7036|/data/primary36 +39|37|p|p|s|u|sdw2|sdw2|7037|/data/primary37 +40|38|p|p|s|u|sdw2|sdw2|7038|/data/primary38 +41|39|p|p|s|u|sdw2|sdw2|7039|/data/primary39 +1042|40|m|m|s|u|sdw2|sdw2|8040|/data/mirror40 +1043|41|m|m|s|u|sdw2|sdw2|8041|/data/mirror41 +1044|42|m|m|s|u|sdw2|sdw2|8042|/data/mirror42 +1045|43|m|m|s|u|sdw2|sdw2|8043|/data/mirror43 +1046|44|m|m|s|u|sdw2|sdw2|8044|/data/mirror44 +1047|45|m|m|s|u|sdw2|sdw2|8045|/data/mirror45 +1048|46|m|m|s|u|sdw2|sdw2|8046|/data/mirror46 +1049|47|m|m|s|u|sdw2|sdw2|8047|/data/mirror47 +1050|48|m|m|s|u|sdw2|sdw2|8048|/data/mirror48 +1051|49|m|m|s|u|sdw2|sdw2|8049|/data/mirror49 +1052|50|m|m|s|u|sdw2|sdw2|8050|/data/mirror50 +1053|51|m|m|s|u|sdw2|sdw2|8051|/data/mirror51 +1054|52|m|m|s|u|sdw2|sdw2|8052|/data/mirror52 +1055|53|m|m|s|u|sdw2|sdw2|8053|/data/mirror53 +1965|963|m|m|s|u|sdw2|sdw2|8963|/data/mirror963 +1966|964|m|m|s|u|sdw2|sdw2|8964|/data/mirror964 +1967|965|m|m|s|u|sdw2|sdw2|8965|/data/mirror965 +1968|966|m|m|s|u|sdw2|sdw2|8966|/data/mirror966 +1969|967|m|m|s|u|sdw2|sdw2|8967|/data/mirror967 +1970|968|m|m|s|u|sdw2|sdw2|8968|/data/mirror968 +1971|969|m|m|s|u|sdw2|sdw2|8969|/data/mirror969 +1972|970|m|m|s|u|sdw2|sdw2|8970|/data/mirror970 +1973|971|m|m|s|u|sdw2|sdw2|8971|/data/mirror971 +1974|972|m|m|s|u|sdw2|sdw2|8972|/data/mirror972 +1975|973|m|m|s|u|sdw2|sdw2|8973|/data/mirror973 +1976|974|m|m|s|u|sdw2|sdw2|8974|/data/mirror974 +1977|975|m|m|s|u|sdw2|sdw2|8975|/data/mirror975 +1978|976|m|m|s|u|sdw2|sdw2|8976|/data/mirror976 +1979|977|m|m|s|u|sdw2|sdw2|8977|/data/mirror977 +1980|978|m|m|s|u|sdw2|sdw2|8978|/data/mirror978 +1981|979|m|m|s|u|sdw2|sdw2|8979|/data/mirror979 +1982|980|m|m|s|u|sdw2|sdw2|8980|/data/mirror980 +1983|981|m|m|s|u|sdw2|sdw2|8981|/data/mirror981 +1984|982|m|m|s|u|sdw2|sdw2|8982|/data/mirror982 +1985|983|m|m|s|u|sdw2|sdw2|8983|/data/mirror983 +# SDW20 +380|378|p|p|s|u|sdw20|sdw20|7378|/data/primary378 +381|379|p|p|s|u|sdw20|sdw20|7379|/data/primary379 +382|380|p|p|s|u|sdw20|sdw20|7380|/data/primary380 +383|381|p|p|s|u|sdw20|sdw20|7381|/data/primary381 +384|382|p|p|s|u|sdw20|sdw20|7382|/data/primary382 +385|383|p|p|s|u|sdw20|sdw20|7383|/data/primary383 +386|384|p|p|s|u|sdw20|sdw20|7384|/data/primary384 +387|385|p|p|s|u|sdw20|sdw20|7385|/data/primary385 +388|386|p|p|s|u|sdw20|sdw20|7386|/data/primary386 +389|387|p|p|s|u|sdw20|sdw20|7387|/data/primary387 +390|388|p|p|s|u|sdw20|sdw20|7388|/data/primary388 +391|389|p|p|s|u|sdw20|sdw20|7389|/data/primary389 +392|390|p|p|s|u|sdw20|sdw20|7390|/data/primary390 +393|391|p|p|s|u|sdw20|sdw20|7391|/data/primary391 +394|392|p|p|s|u|sdw20|sdw20|7392|/data/primary392 +395|393|p|p|s|u|sdw20|sdw20|7393|/data/primary393 +396|394|p|p|s|u|sdw20|sdw20|7394|/data/primary394 +397|395|p|p|s|u|sdw20|sdw20|7395|/data/primary395 +398|396|p|p|s|u|sdw20|sdw20|7396|/data/primary396 +399|397|p|p|s|u|sdw20|sdw20|7397|/data/primary397 +400|398|p|p|s|u|sdw20|sdw20|7398|/data/primary398 +401|399|p|p|s|u|sdw20|sdw20|7399|/data/primary399 +402|400|p|p|s|u|sdw20|sdw20|7400|/data/primary400 +1403|401|m|m|s|u|sdw20|sdw20|8401|/data/mirror401 +1404|402|m|m|s|u|sdw20|sdw20|8402|/data/mirror402 +1405|403|m|m|s|u|sdw20|sdw20|8403|/data/mirror403 +1406|404|m|m|s|u|sdw20|sdw20|8404|/data/mirror404 +1407|405|m|m|s|u|sdw20|sdw20|8405|/data/mirror405 +1408|406|m|m|s|u|sdw20|sdw20|8406|/data/mirror406 +1409|407|m|m|s|u|sdw20|sdw20|8407|/data/mirror407 +1410|408|m|m|s|u|sdw20|sdw20|8408|/data/mirror408 +1411|409|m|m|s|u|sdw20|sdw20|8409|/data/mirror409 +1412|410|m|m|s|u|sdw20|sdw20|8410|/data/mirror410 +1413|411|m|m|s|u|sdw20|sdw20|8411|/data/mirror411 +1414|412|m|m|s|u|sdw20|sdw20|8412|/data/mirror412 +1415|413|m|m|s|u|sdw20|sdw20|8413|/data/mirror413 +1416|414|m|m|s|u|sdw20|sdw20|8414|/data/mirror414 +1417|415|m|m|s|u|sdw20|sdw20|8415|/data/mirror415 +1418|416|m|m|s|u|sdw20|sdw20|8416|/data/mirror416 +1419|417|m|m|s|u|sdw20|sdw20|8417|/data/mirror417 +1420|418|m|m|s|u|sdw20|sdw20|8418|/data/mirror418 +1421|419|m|m|s|u|sdw20|sdw20|8419|/data/mirror419 +1422|420|m|m|s|u|sdw20|sdw20|8420|/data/mirror420 +1423|421|m|m|s|u|sdw20|sdw20|8421|/data/mirror421 +# SDW21 +403|401|p|p|s|u|sdw21|sdw21|7401|/data/primary401 +404|402|p|p|s|u|sdw21|sdw21|7402|/data/primary402 +405|403|p|p|s|u|sdw21|sdw21|7403|/data/primary403 +406|404|p|p|s|u|sdw21|sdw21|7404|/data/primary404 +407|405|p|p|s|u|sdw21|sdw21|7405|/data/primary405 +408|406|p|p|s|u|sdw21|sdw21|7406|/data/primary406 +409|407|p|p|s|u|sdw21|sdw21|7407|/data/primary407 +410|408|p|p|s|u|sdw21|sdw21|7408|/data/primary408 +411|409|p|p|s|u|sdw21|sdw21|7409|/data/primary409 +412|410|p|p|s|u|sdw21|sdw21|7410|/data/primary410 +413|411|p|p|s|u|sdw21|sdw21|7411|/data/primary411 +414|412|p|p|s|u|sdw21|sdw21|7412|/data/primary412 +415|413|p|p|s|u|sdw21|sdw21|7413|/data/primary413 +416|414|p|p|s|u|sdw21|sdw21|7414|/data/primary414 +417|415|p|p|s|u|sdw21|sdw21|7415|/data/primary415 +418|416|p|p|s|u|sdw21|sdw21|7416|/data/primary416 +419|417|p|p|s|u|sdw21|sdw21|7417|/data/primary417 +420|418|p|p|s|u|sdw21|sdw21|7418|/data/primary418 +421|419|p|p|s|u|sdw21|sdw21|7419|/data/primary419 +422|420|p|p|s|u|sdw21|sdw21|7420|/data/primary420 +423|421|p|p|s|u|sdw21|sdw21|7421|/data/primary421 +1444|442|m|m|s|u|sdw21|sdw21|8442|/data/mirror442 +1445|443|m|m|s|u|sdw21|sdw21|8443|/data/mirror443 +1446|444|m|m|s|u|sdw21|sdw21|8444|/data/mirror444 +1447|445|m|m|s|u|sdw21|sdw21|8445|/data/mirror445 +1448|446|m|m|s|u|sdw21|sdw21|8446|/data/mirror446 +1449|447|m|m|s|u|sdw21|sdw21|8447|/data/mirror447 +1450|448|m|m|s|u|sdw21|sdw21|8448|/data/mirror448 +1451|449|m|m|s|u|sdw21|sdw21|8449|/data/mirror449 +1452|450|m|m|s|u|sdw21|sdw21|8450|/data/mirror450 +1453|451|m|m|s|u|sdw21|sdw21|8451|/data/mirror451 +1454|452|m|m|s|u|sdw21|sdw21|8452|/data/mirror452 +1455|453|m|m|s|u|sdw21|sdw21|8453|/data/mirror453 +1456|454|m|m|s|u|sdw21|sdw21|8454|/data/mirror454 +1457|455|m|m|s|u|sdw21|sdw21|8455|/data/mirror455 +1458|456|m|m|s|u|sdw21|sdw21|8456|/data/mirror456 +1459|457|m|m|s|u|sdw21|sdw21|8457|/data/mirror457 +1460|458|m|m|s|u|sdw21|sdw21|8458|/data/mirror458 +1461|459|m|m|s|u|sdw21|sdw21|8459|/data/mirror459 +1462|460|m|m|s|u|sdw21|sdw21|8460|/data/mirror460 +# SDW22 +424|422|p|p|s|u|sdw22|sdw22|7422|/data/primary422 +425|423|p|p|s|u|sdw22|sdw22|7423|/data/primary423 +426|424|p|p|s|u|sdw22|sdw22|7424|/data/primary424 +427|425|p|p|s|u|sdw22|sdw22|7425|/data/primary425 +428|426|p|p|s|u|sdw22|sdw22|7426|/data/primary426 +429|427|p|p|s|u|sdw22|sdw22|7427|/data/primary427 +430|428|p|p|s|u|sdw22|sdw22|7428|/data/primary428 +431|429|p|p|s|u|sdw22|sdw22|7429|/data/primary429 +432|430|p|p|s|u|sdw22|sdw22|7430|/data/primary430 +433|431|p|p|s|u|sdw22|sdw22|7431|/data/primary431 +434|432|p|p|s|u|sdw22|sdw22|7432|/data/primary432 +435|433|p|p|s|u|sdw22|sdw22|7433|/data/primary433 +436|434|p|p|s|u|sdw22|sdw22|7434|/data/primary434 +437|435|p|p|s|u|sdw22|sdw22|7435|/data/primary435 +438|436|p|p|s|u|sdw22|sdw22|7436|/data/primary436 +439|437|p|p|s|u|sdw22|sdw22|7437|/data/primary437 +440|438|p|p|s|u|sdw22|sdw22|7438|/data/primary438 +441|439|p|p|s|u|sdw22|sdw22|7439|/data/primary439 +442|440|p|p|s|u|sdw22|sdw22|7440|/data/primary440 +443|441|p|p|s|u|sdw22|sdw22|7441|/data/primary441 +1380|378|m|m|s|u|sdw22|sdw22|8378|/data/mirror378 +1381|379|m|m|s|u|sdw22|sdw22|8379|/data/mirror379 +1382|380|m|m|s|u|sdw22|sdw22|8380|/data/mirror380 +1383|381|m|m|s|u|sdw22|sdw22|8381|/data/mirror381 +1384|382|m|m|s|u|sdw22|sdw22|8382|/data/mirror382 +1385|383|m|m|s|u|sdw22|sdw22|8383|/data/mirror383 +1386|384|m|m|s|u|sdw22|sdw22|8384|/data/mirror384 +1387|385|m|m|s|u|sdw22|sdw22|8385|/data/mirror385 +1388|386|m|m|s|u|sdw22|sdw22|8386|/data/mirror386 +1389|387|m|m|s|u|sdw22|sdw22|8387|/data/mirror387 +1390|388|m|m|s|u|sdw22|sdw22|8388|/data/mirror388 +1391|389|m|m|s|u|sdw22|sdw22|8389|/data/mirror389 +1392|390|m|m|s|u|sdw22|sdw22|8390|/data/mirror390 +1393|391|m|m|s|u|sdw22|sdw22|8391|/data/mirror391 +1394|392|m|m|s|u|sdw22|sdw22|8392|/data/mirror392 +1395|393|m|m|s|u|sdw22|sdw22|8393|/data/mirror393 +1396|394|m|m|s|u|sdw22|sdw22|8394|/data/mirror394 +1397|395|m|m|s|u|sdw22|sdw22|8395|/data/mirror395 +1398|396|m|m|s|u|sdw22|sdw22|8396|/data/mirror396 +1399|397|m|m|s|u|sdw22|sdw22|8397|/data/mirror397 +1400|398|m|m|s|u|sdw22|sdw22|8398|/data/mirror398 +1401|399|m|m|s|u|sdw22|sdw22|8399|/data/mirror399 +1402|400|m|m|s|u|sdw22|sdw22|8400|/data/mirror400 +# SDW23 +444|442|p|p|s|u|sdw23|sdw23|7442|/data/primary442 +445|443|p|p|s|u|sdw23|sdw23|7443|/data/primary443 +446|444|p|p|s|u|sdw23|sdw23|7444|/data/primary444 +447|445|p|p|s|u|sdw23|sdw23|7445|/data/primary445 +448|446|p|p|s|u|sdw23|sdw23|7446|/data/primary446 +449|447|p|p|s|u|sdw23|sdw23|7447|/data/primary447 +450|448|p|p|s|u|sdw23|sdw23|7448|/data/primary448 +451|449|p|p|s|u|sdw23|sdw23|7449|/data/primary449 +452|450|p|p|s|u|sdw23|sdw23|7450|/data/primary450 +453|451|p|p|s|u|sdw23|sdw23|7451|/data/primary451 +454|452|p|p|s|u|sdw23|sdw23|7452|/data/primary452 +455|453|p|p|s|u|sdw23|sdw23|7453|/data/primary453 +456|454|p|p|s|u|sdw23|sdw23|7454|/data/primary454 +457|455|p|p|s|u|sdw23|sdw23|7455|/data/primary455 +458|456|p|p|s|u|sdw23|sdw23|7456|/data/primary456 +459|457|p|p|s|u|sdw23|sdw23|7457|/data/primary457 +460|458|p|p|s|u|sdw23|sdw23|7458|/data/primary458 +461|459|p|p|s|u|sdw23|sdw23|7459|/data/primary459 +462|460|p|p|s|u|sdw23|sdw23|7460|/data/primary460 +1424|422|m|m|s|u|sdw23|sdw23|8422|/data/mirror422 +1425|423|m|m|s|u|sdw23|sdw23|8423|/data/mirror423 +1426|424|m|m|s|u|sdw23|sdw23|8424|/data/mirror424 +1427|425|m|m|s|u|sdw23|sdw23|8425|/data/mirror425 +1428|426|m|m|s|u|sdw23|sdw23|8426|/data/mirror426 +1429|427|m|m|s|u|sdw23|sdw23|8427|/data/mirror427 +1430|428|m|m|s|u|sdw23|sdw23|8428|/data/mirror428 +1431|429|m|m|s|u|sdw23|sdw23|8429|/data/mirror429 +1432|430|m|m|s|u|sdw23|sdw23|8430|/data/mirror430 +1433|431|m|m|s|u|sdw23|sdw23|8431|/data/mirror431 +1434|432|m|m|s|u|sdw23|sdw23|8432|/data/mirror432 +1435|433|m|m|s|u|sdw23|sdw23|8433|/data/mirror433 +1436|434|m|m|s|u|sdw23|sdw23|8434|/data/mirror434 +1437|435|m|m|s|u|sdw23|sdw23|8435|/data/mirror435 +1438|436|m|m|s|u|sdw23|sdw23|8436|/data/mirror436 +1439|437|m|m|s|u|sdw23|sdw23|8437|/data/mirror437 +1440|438|m|m|s|u|sdw23|sdw23|8438|/data/mirror438 +1441|439|m|m|s|u|sdw23|sdw23|8439|/data/mirror439 +1442|440|m|m|s|u|sdw23|sdw23|8440|/data/mirror440 +1443|441|m|m|s|u|sdw23|sdw23|8441|/data/mirror441 +# SDW24 +463|461|p|p|s|u|sdw24|sdw24|7461|/data/primary461 +464|462|p|p|s|u|sdw24|sdw24|7462|/data/primary462 +465|463|p|p|s|u|sdw24|sdw24|7463|/data/primary463 +466|464|p|p|s|u|sdw24|sdw24|7464|/data/primary464 +467|465|p|p|s|u|sdw24|sdw24|7465|/data/primary465 +468|466|p|p|s|u|sdw24|sdw24|7466|/data/primary466 +469|467|p|p|s|u|sdw24|sdw24|7467|/data/primary467 +470|468|p|p|s|u|sdw24|sdw24|7468|/data/primary468 +471|469|p|p|s|u|sdw24|sdw24|7469|/data/primary469 +472|470|p|p|s|u|sdw24|sdw24|7470|/data/primary470 +473|471|p|p|s|u|sdw24|sdw24|7471|/data/primary471 +474|472|p|p|s|u|sdw24|sdw24|7472|/data/primary472 +475|473|p|p|s|u|sdw24|sdw24|7473|/data/primary473 +476|474|p|p|s|u|sdw24|sdw24|7474|/data/primary474 +477|475|p|p|s|u|sdw24|sdw24|7475|/data/primary475 +478|476|p|p|s|u|sdw24|sdw24|7476|/data/primary476 +479|477|p|p|s|u|sdw24|sdw24|7477|/data/primary477 +480|478|p|p|s|u|sdw24|sdw24|7478|/data/primary478 +481|479|p|p|s|u|sdw24|sdw24|7479|/data/primary479 +482|480|p|p|s|u|sdw24|sdw24|7480|/data/primary480 +483|481|p|p|s|u|sdw24|sdw24|7481|/data/primary481 +484|482|p|p|s|u|sdw24|sdw24|7482|/data/primary482 +485|483|p|p|s|u|sdw24|sdw24|7483|/data/primary483 +1486|484|m|m|s|u|sdw24|sdw24|8484|/data/mirror484 +1487|485|m|m|s|u|sdw24|sdw24|8485|/data/mirror485 +1488|486|m|m|s|u|sdw24|sdw24|8486|/data/mirror486 +1489|487|m|m|s|u|sdw24|sdw24|8487|/data/mirror487 +1490|488|m|m|s|u|sdw24|sdw24|8488|/data/mirror488 +1491|489|m|m|s|u|sdw24|sdw24|8489|/data/mirror489 +1492|490|m|m|s|u|sdw24|sdw24|8490|/data/mirror490 +1493|491|m|m|s|u|sdw24|sdw24|8491|/data/mirror491 +1494|492|m|m|s|u|sdw24|sdw24|8492|/data/mirror492 +1495|493|m|m|s|u|sdw24|sdw24|8493|/data/mirror493 +1496|494|m|m|s|u|sdw24|sdw24|8494|/data/mirror494 +1497|495|m|m|s|u|sdw24|sdw24|8495|/data/mirror495 +1498|496|m|m|s|u|sdw24|sdw24|8496|/data/mirror496 +1499|497|m|m|s|u|sdw24|sdw24|8497|/data/mirror497 +1500|498|m|m|s|u|sdw24|sdw24|8498|/data/mirror498 +1501|499|m|m|s|u|sdw24|sdw24|8499|/data/mirror499 +1502|500|m|m|s|u|sdw24|sdw24|8500|/data/mirror500 +1503|501|m|m|s|u|sdw24|sdw24|8501|/data/mirror501 +1504|502|m|m|s|u|sdw24|sdw24|8502|/data/mirror502 +1505|503|m|m|s|u|sdw24|sdw24|8503|/data/mirror503 +1506|504|m|m|s|u|sdw24|sdw24|8504|/data/mirror504 +1507|505|m|m|s|u|sdw24|sdw24|8505|/data/mirror505 +1508|506|m|m|s|u|sdw24|sdw24|8506|/data/mirror506 +1509|507|m|m|s|u|sdw24|sdw24|8507|/data/mirror507 +1510|508|m|m|s|u|sdw24|sdw24|8508|/data/mirror508 +# SDW25 +486|484|p|p|s|u|sdw25|sdw25|7484|/data/primary484 +487|485|p|p|s|u|sdw25|sdw25|7485|/data/primary485 +488|486|p|p|s|u|sdw25|sdw25|7486|/data/primary486 +489|487|p|p|s|u|sdw25|sdw25|7487|/data/primary487 +490|488|p|p|s|u|sdw25|sdw25|7488|/data/primary488 +491|489|p|p|s|u|sdw25|sdw25|7489|/data/primary489 +492|490|p|p|s|u|sdw25|sdw25|7490|/data/primary490 +493|491|p|p|s|u|sdw25|sdw25|7491|/data/primary491 +494|492|p|p|s|u|sdw25|sdw25|7492|/data/primary492 +495|493|p|p|s|u|sdw25|sdw25|7493|/data/primary493 +496|494|p|p|s|u|sdw25|sdw25|7494|/data/primary494 +497|495|p|p|s|u|sdw25|sdw25|7495|/data/primary495 +498|496|p|p|s|u|sdw25|sdw25|7496|/data/primary496 +499|497|p|p|s|u|sdw25|sdw25|7497|/data/primary497 +500|498|p|p|s|u|sdw25|sdw25|7498|/data/primary498 +501|499|p|p|s|u|sdw25|sdw25|7499|/data/primary499 +502|500|p|p|s|u|sdw25|sdw25|7500|/data/primary500 +503|501|p|p|s|u|sdw25|sdw25|7501|/data/primary501 +504|502|p|p|s|u|sdw25|sdw25|7502|/data/primary502 +505|503|p|p|s|u|sdw25|sdw25|7503|/data/primary503 +506|504|p|p|s|u|sdw25|sdw25|7504|/data/primary504 +507|505|p|p|s|u|sdw25|sdw25|7505|/data/primary505 +508|506|p|p|s|u|sdw25|sdw25|7506|/data/primary506 +509|507|p|p|s|u|sdw25|sdw25|7507|/data/primary507 +510|508|p|p|s|u|sdw25|sdw25|7508|/data/primary508 +1573|571|m|m|s|u|sdw25|sdw25|8571|/data/mirror571 +1574|572|m|m|s|u|sdw25|sdw25|8572|/data/mirror572 +1575|573|m|m|s|u|sdw25|sdw25|8573|/data/mirror573 +1576|574|m|m|s|u|sdw25|sdw25|8574|/data/mirror574 +1577|575|m|m|s|u|sdw25|sdw25|8575|/data/mirror575 +1578|576|m|m|s|u|sdw25|sdw25|8576|/data/mirror576 +1579|577|m|m|s|u|sdw25|sdw25|8577|/data/mirror577 +1580|578|m|m|s|u|sdw25|sdw25|8578|/data/mirror578 +1581|579|m|m|s|u|sdw25|sdw25|8579|/data/mirror579 +1582|580|m|m|s|u|sdw25|sdw25|8580|/data/mirror580 +1583|581|m|m|s|u|sdw25|sdw25|8581|/data/mirror581 +1584|582|m|m|s|u|sdw25|sdw25|8582|/data/mirror582 +1585|583|m|m|s|u|sdw25|sdw25|8583|/data/mirror583 +1586|584|m|m|s|u|sdw25|sdw25|8584|/data/mirror584 +1587|585|m|m|s|u|sdw25|sdw25|8585|/data/mirror585 +1588|586|m|m|s|u|sdw25|sdw25|8586|/data/mirror586 +1589|587|m|m|s|u|sdw25|sdw25|8587|/data/mirror587 +1590|588|m|m|s|u|sdw25|sdw25|8588|/data/mirror588 +1591|589|m|m|s|u|sdw25|sdw25|8589|/data/mirror589 +1592|590|m|m|s|u|sdw25|sdw25|8590|/data/mirror590 +1593|591|m|m|s|u|sdw25|sdw25|8591|/data/mirror591 +# SDW26 +511|509|p|p|s|u|sdw26|sdw26|7509|/data/primary509 +512|510|p|p|s|u|sdw26|sdw26|7510|/data/primary510 +513|511|p|p|s|u|sdw26|sdw26|7511|/data/primary511 +514|512|p|p|s|u|sdw26|sdw26|7512|/data/primary512 +515|513|p|p|s|u|sdw26|sdw26|7513|/data/primary513 +516|514|p|p|s|u|sdw26|sdw26|7514|/data/primary514 +517|515|p|p|s|u|sdw26|sdw26|7515|/data/primary515 +518|516|p|p|s|u|sdw26|sdw26|7516|/data/primary516 +519|517|p|p|s|u|sdw26|sdw26|7517|/data/primary517 +520|518|p|p|s|u|sdw26|sdw26|7518|/data/primary518 +521|519|p|p|s|u|sdw26|sdw26|7519|/data/primary519 +522|520|p|p|s|u|sdw26|sdw26|7520|/data/primary520 +523|521|p|p|s|u|sdw26|sdw26|7521|/data/primary521 +524|522|p|p|s|u|sdw26|sdw26|7522|/data/primary522 +525|523|p|p|s|u|sdw26|sdw26|7523|/data/primary523 +526|524|p|p|s|u|sdw26|sdw26|7524|/data/primary524 +527|525|p|p|s|u|sdw26|sdw26|7525|/data/primary525 +528|526|p|p|s|u|sdw26|sdw26|7526|/data/primary526 +529|527|p|p|s|u|sdw26|sdw26|7527|/data/primary527 +530|528|p|p|s|u|sdw26|sdw26|7528|/data/primary528 +1531|529|m|m|s|u|sdw26|sdw26|8529|/data/mirror529 +1532|530|m|m|s|u|sdw26|sdw26|8530|/data/mirror530 +1533|531|m|m|s|u|sdw26|sdw26|8531|/data/mirror531 +1534|532|m|m|s|u|sdw26|sdw26|8532|/data/mirror532 +1535|533|m|m|s|u|sdw26|sdw26|8533|/data/mirror533 +1536|534|m|m|s|u|sdw26|sdw26|8534|/data/mirror534 +1537|535|m|m|s|u|sdw26|sdw26|8535|/data/mirror535 +1538|536|m|m|s|u|sdw26|sdw26|8536|/data/mirror536 +1539|537|m|m|s|u|sdw26|sdw26|8537|/data/mirror537 +1540|538|m|m|s|u|sdw26|sdw26|8538|/data/mirror538 +1541|539|m|m|s|u|sdw26|sdw26|8539|/data/mirror539 +1542|540|m|m|s|u|sdw26|sdw26|8540|/data/mirror540 +1543|541|m|m|s|u|sdw26|sdw26|8541|/data/mirror541 +1544|542|m|m|s|u|sdw26|sdw26|8542|/data/mirror542 +1545|543|m|m|s|u|sdw26|sdw26|8543|/data/mirror543 +1546|544|m|m|s|u|sdw26|sdw26|8544|/data/mirror544 +1547|545|m|m|s|u|sdw26|sdw26|8545|/data/mirror545 +1548|546|m|m|s|u|sdw26|sdw26|8546|/data/mirror546 +# SDW27 +531|529|p|p|s|u|sdw27|sdw27|7529|/data/primary529 +532|530|p|p|s|u|sdw27|sdw27|7530|/data/primary530 +533|531|p|p|s|u|sdw27|sdw27|7531|/data/primary531 +534|532|p|p|s|u|sdw27|sdw27|7532|/data/primary532 +535|533|p|p|s|u|sdw27|sdw27|7533|/data/primary533 +536|534|p|p|s|u|sdw27|sdw27|7534|/data/primary534 +537|535|p|p|s|u|sdw27|sdw27|7535|/data/primary535 +538|536|p|p|s|u|sdw27|sdw27|7536|/data/primary536 +539|537|p|p|s|u|sdw27|sdw27|7537|/data/primary537 +540|538|p|p|s|u|sdw27|sdw27|7538|/data/primary538 +541|539|p|p|s|u|sdw27|sdw27|7539|/data/primary539 +542|540|p|p|s|u|sdw27|sdw27|7540|/data/primary540 +543|541|p|p|s|u|sdw27|sdw27|7541|/data/primary541 +544|542|p|p|s|u|sdw27|sdw27|7542|/data/primary542 +545|543|p|p|s|u|sdw27|sdw27|7543|/data/primary543 +546|544|p|p|s|u|sdw27|sdw27|7544|/data/primary544 +547|545|p|p|s|u|sdw27|sdw27|7545|/data/primary545 +548|546|p|p|s|u|sdw27|sdw27|7546|/data/primary546 +1463|461|m|m|s|u|sdw27|sdw27|8461|/data/mirror461 +1464|462|m|m|s|u|sdw27|sdw27|8462|/data/mirror462 +1465|463|m|m|s|u|sdw27|sdw27|8463|/data/mirror463 +1466|464|m|m|s|u|sdw27|sdw27|8464|/data/mirror464 +1467|465|m|m|s|u|sdw27|sdw27|8465|/data/mirror465 +1468|466|m|m|s|u|sdw27|sdw27|8466|/data/mirror466 +1469|467|m|m|s|u|sdw27|sdw27|8467|/data/mirror467 +1470|468|m|m|s|u|sdw27|sdw27|8468|/data/mirror468 +1471|469|m|m|s|u|sdw27|sdw27|8469|/data/mirror469 +1472|470|m|m|s|u|sdw27|sdw27|8470|/data/mirror470 +1473|471|m|m|s|u|sdw27|sdw27|8471|/data/mirror471 +1474|472|m|m|s|u|sdw27|sdw27|8472|/data/mirror472 +1475|473|m|m|s|u|sdw27|sdw27|8473|/data/mirror473 +1476|474|m|m|s|u|sdw27|sdw27|8474|/data/mirror474 +1477|475|m|m|s|u|sdw27|sdw27|8475|/data/mirror475 +1478|476|m|m|s|u|sdw27|sdw27|8476|/data/mirror476 +1479|477|m|m|s|u|sdw27|sdw27|8477|/data/mirror477 +1480|478|m|m|s|u|sdw27|sdw27|8478|/data/mirror478 +1481|479|m|m|s|u|sdw27|sdw27|8479|/data/mirror479 +1482|480|m|m|s|u|sdw27|sdw27|8480|/data/mirror480 +1483|481|m|m|s|u|sdw27|sdw27|8481|/data/mirror481 +1484|482|m|m|s|u|sdw27|sdw27|8482|/data/mirror482 +1485|483|m|m|s|u|sdw27|sdw27|8483|/data/mirror483 +# SDW28 +549|547|p|p|s|u|sdw28|sdw28|7547|/data/primary547 +550|548|p|p|s|u|sdw28|sdw28|7548|/data/primary548 +551|549|p|p|s|u|sdw28|sdw28|7549|/data/primary549 +552|550|p|p|s|u|sdw28|sdw28|7550|/data/primary550 +553|551|p|p|s|u|sdw28|sdw28|7551|/data/primary551 +554|552|p|p|s|u|sdw28|sdw28|7552|/data/primary552 +555|553|p|p|s|u|sdw28|sdw28|7553|/data/primary553 +556|554|p|p|s|u|sdw28|sdw28|7554|/data/primary554 +557|555|p|p|s|u|sdw28|sdw28|7555|/data/primary555 +558|556|p|p|s|u|sdw28|sdw28|7556|/data/primary556 +559|557|p|p|s|u|sdw28|sdw28|7557|/data/primary557 +560|558|p|p|s|u|sdw28|sdw28|7558|/data/primary558 +561|559|p|p|s|u|sdw28|sdw28|7559|/data/primary559 +562|560|p|p|s|u|sdw28|sdw28|7560|/data/primary560 +563|561|p|p|s|u|sdw28|sdw28|7561|/data/primary561 +564|562|p|p|s|u|sdw28|sdw28|7562|/data/primary562 +565|563|p|p|s|u|sdw28|sdw28|7563|/data/primary563 +566|564|p|p|s|u|sdw28|sdw28|7564|/data/primary564 +567|565|p|p|s|u|sdw28|sdw28|7565|/data/primary565 +568|566|p|p|s|u|sdw28|sdw28|7566|/data/primary566 +569|567|p|p|s|u|sdw28|sdw28|7567|/data/primary567 +570|568|p|p|s|u|sdw28|sdw28|7568|/data/primary568 +571|569|p|p|s|u|sdw28|sdw28|7569|/data/primary569 +572|570|p|p|s|u|sdw28|sdw28|7570|/data/primary570 +1511|509|m|m|s|u|sdw28|sdw28|8509|/data/mirror509 +1512|510|m|m|s|u|sdw28|sdw28|8510|/data/mirror510 +1513|511|m|m|s|u|sdw28|sdw28|8511|/data/mirror511 +1514|512|m|m|s|u|sdw28|sdw28|8512|/data/mirror512 +1515|513|m|m|s|u|sdw28|sdw28|8513|/data/mirror513 +1516|514|m|m|s|u|sdw28|sdw28|8514|/data/mirror514 +1517|515|m|m|s|u|sdw28|sdw28|8515|/data/mirror515 +1518|516|m|m|s|u|sdw28|sdw28|8516|/data/mirror516 +1519|517|m|m|s|u|sdw28|sdw28|8517|/data/mirror517 +1520|518|m|m|s|u|sdw28|sdw28|8518|/data/mirror518 +1521|519|m|m|s|u|sdw28|sdw28|8519|/data/mirror519 +1522|520|m|m|s|u|sdw28|sdw28|8520|/data/mirror520 +1523|521|m|m|s|u|sdw28|sdw28|8521|/data/mirror521 +1524|522|m|m|s|u|sdw28|sdw28|8522|/data/mirror522 +1525|523|m|m|s|u|sdw28|sdw28|8523|/data/mirror523 +1526|524|m|m|s|u|sdw28|sdw28|8524|/data/mirror524 +1527|525|m|m|s|u|sdw28|sdw28|8525|/data/mirror525 +1528|526|m|m|s|u|sdw28|sdw28|8526|/data/mirror526 +1529|527|m|m|s|u|sdw28|sdw28|8527|/data/mirror527 +1530|528|m|m|s|u|sdw28|sdw28|8528|/data/mirror528 +# SDW29 +573|571|p|p|s|u|sdw29|sdw29|7571|/data/primary571 +574|572|p|p|s|u|sdw29|sdw29|7572|/data/primary572 +575|573|p|p|s|u|sdw29|sdw29|7573|/data/primary573 +576|574|p|p|s|u|sdw29|sdw29|7574|/data/primary574 +577|575|p|p|s|u|sdw29|sdw29|7575|/data/primary575 +578|576|p|p|s|u|sdw29|sdw29|7576|/data/primary576 +579|577|p|p|s|u|sdw29|sdw29|7577|/data/primary577 +580|578|p|p|s|u|sdw29|sdw29|7578|/data/primary578 +581|579|p|p|s|u|sdw29|sdw29|7579|/data/primary579 +582|580|p|p|s|u|sdw29|sdw29|7580|/data/primary580 +583|581|p|p|s|u|sdw29|sdw29|7581|/data/primary581 +584|582|p|p|s|u|sdw29|sdw29|7582|/data/primary582 +585|583|p|p|s|u|sdw29|sdw29|7583|/data/primary583 +586|584|p|p|s|u|sdw29|sdw29|7584|/data/primary584 +587|585|p|p|s|u|sdw29|sdw29|7585|/data/primary585 +588|586|p|p|s|u|sdw29|sdw29|7586|/data/primary586 +589|587|p|p|s|u|sdw29|sdw29|7587|/data/primary587 +590|588|p|p|s|u|sdw29|sdw29|7588|/data/primary588 +591|589|p|p|s|u|sdw29|sdw29|7589|/data/primary589 +592|590|p|p|s|u|sdw29|sdw29|7590|/data/primary590 +593|591|p|p|s|u|sdw29|sdw29|7591|/data/primary591 +1677|675|m|m|s|u|sdw29|sdw29|8675|/data/mirror675 +1678|676|m|m|s|u|sdw29|sdw29|8676|/data/mirror676 +1679|677|m|m|s|u|sdw29|sdw29|8677|/data/mirror677 +1680|678|m|m|s|u|sdw29|sdw29|8678|/data/mirror678 +1681|679|m|m|s|u|sdw29|sdw29|8679|/data/mirror679 +1682|680|m|m|s|u|sdw29|sdw29|8680|/data/mirror680 +1683|681|m|m|s|u|sdw29|sdw29|8681|/data/mirror681 +1684|682|m|m|s|u|sdw29|sdw29|8682|/data/mirror682 +1685|683|m|m|s|u|sdw29|sdw29|8683|/data/mirror683 +1686|684|m|m|s|u|sdw29|sdw29|8684|/data/mirror684 +1687|685|m|m|s|u|sdw29|sdw29|8685|/data/mirror685 +1688|686|m|m|s|u|sdw29|sdw29|8686|/data/mirror686 +1689|687|m|m|s|u|sdw29|sdw29|8687|/data/mirror687 +1690|688|m|m|s|u|sdw29|sdw29|8688|/data/mirror688 +1691|689|m|m|s|u|sdw29|sdw29|8689|/data/mirror689 +1692|690|m|m|s|u|sdw29|sdw29|8690|/data/mirror690 +1693|691|m|m|s|u|sdw29|sdw29|8691|/data/mirror691 +1694|692|m|m|s|u|sdw29|sdw29|8692|/data/mirror692 +1695|693|m|m|s|u|sdw29|sdw29|8693|/data/mirror693 +1696|694|m|m|s|u|sdw29|sdw29|8694|/data/mirror694 +1697|695|m|m|s|u|sdw29|sdw29|8695|/data/mirror695 +1698|696|m|m|s|u|sdw29|sdw29|8696|/data/mirror696 +1699|697|m|m|s|u|sdw29|sdw29|8697|/data/mirror697 +1700|698|m|m|s|u|sdw29|sdw29|8698|/data/mirror698 +1701|699|m|m|s|u|sdw29|sdw29|8699|/data/mirror699 +1702|700|m|m|s|u|sdw29|sdw29|8700|/data/mirror700 +# SDW3 +42|40|p|p|s|u|sdw3|sdw3|7040|/data/primary40 +43|41|p|p|s|u|sdw3|sdw3|7041|/data/primary41 +44|42|p|p|s|u|sdw3|sdw3|7042|/data/primary42 +45|43|p|p|s|u|sdw3|sdw3|7043|/data/primary43 +46|44|p|p|s|u|sdw3|sdw3|7044|/data/primary44 +47|45|p|p|s|u|sdw3|sdw3|7045|/data/primary45 +48|46|p|p|s|u|sdw3|sdw3|7046|/data/primary46 +49|47|p|p|s|u|sdw3|sdw3|7047|/data/primary47 +50|48|p|p|s|u|sdw3|sdw3|7048|/data/primary48 +51|49|p|p|s|u|sdw3|sdw3|7049|/data/primary49 +52|50|p|p|s|u|sdw3|sdw3|7050|/data/primary50 +53|51|p|p|s|u|sdw3|sdw3|7051|/data/primary51 +54|52|p|p|s|u|sdw3|sdw3|7052|/data/primary52 +55|53|p|p|s|u|sdw3|sdw3|7053|/data/primary53 +1002|0|m|m|s|u|sdw3|sdw3|8000|/data/mirror0 +1003|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +1004|2|m|m|s|u|sdw3|sdw3|8002|/data/mirror2 +1005|3|m|m|s|u|sdw3|sdw3|8003|/data/mirror3 +1006|4|m|m|s|u|sdw3|sdw3|8004|/data/mirror4 +1007|5|m|m|s|u|sdw3|sdw3|8005|/data/mirror5 +1008|6|m|m|s|u|sdw3|sdw3|8006|/data/mirror6 +1009|7|m|m|s|u|sdw3|sdw3|8007|/data/mirror7 +1010|8|m|m|s|u|sdw3|sdw3|8008|/data/mirror8 +1011|9|m|m|s|u|sdw3|sdw3|8009|/data/mirror9 +1012|10|m|m|s|u|sdw3|sdw3|8010|/data/mirror10 +1013|11|m|m|s|u|sdw3|sdw3|8011|/data/mirror11 +1014|12|m|m|s|u|sdw3|sdw3|8012|/data/mirror12 +1015|13|m|m|s|u|sdw3|sdw3|8013|/data/mirror13 +1016|14|m|m|s|u|sdw3|sdw3|8014|/data/mirror14 +1017|15|m|m|s|u|sdw3|sdw3|8015|/data/mirror15 +1018|16|m|m|s|u|sdw3|sdw3|8016|/data/mirror16 +1019|17|m|m|s|u|sdw3|sdw3|8017|/data/mirror17 +1020|18|m|m|s|u|sdw3|sdw3|8018|/data/mirror18 +# SDW30 +594|592|p|p|s|u|sdw30|sdw30|7592|/data/primary592 +595|593|p|p|s|u|sdw30|sdw30|7593|/data/primary593 +596|594|p|p|s|u|sdw30|sdw30|7594|/data/primary594 +597|595|p|p|s|u|sdw30|sdw30|7595|/data/primary595 +598|596|p|p|s|u|sdw30|sdw30|7596|/data/primary596 +599|597|p|p|s|u|sdw30|sdw30|7597|/data/primary597 +600|598|p|p|s|u|sdw30|sdw30|7598|/data/primary598 +601|599|p|p|s|u|sdw30|sdw30|7599|/data/primary599 +602|600|p|p|s|u|sdw30|sdw30|7600|/data/primary600 +603|601|p|p|s|u|sdw30|sdw30|7601|/data/primary601 +604|602|p|p|s|u|sdw30|sdw30|7602|/data/primary602 +605|603|p|p|s|u|sdw30|sdw30|7603|/data/primary603 +606|604|p|p|s|u|sdw30|sdw30|7604|/data/primary604 +607|605|p|p|s|u|sdw30|sdw30|7605|/data/primary605 +608|606|p|p|s|u|sdw30|sdw30|7606|/data/primary606 +609|607|p|p|s|u|sdw30|sdw30|7607|/data/primary607 +610|608|p|p|s|u|sdw30|sdw30|7608|/data/primary608 +611|609|p|p|s|u|sdw30|sdw30|7609|/data/primary609 +612|610|p|p|s|u|sdw30|sdw30|7610|/data/primary610 +1549|547|m|m|s|u|sdw30|sdw30|8547|/data/mirror547 +1550|548|m|m|s|u|sdw30|sdw30|8548|/data/mirror548 +1551|549|m|m|s|u|sdw30|sdw30|8549|/data/mirror549 +1552|550|m|m|s|u|sdw30|sdw30|8550|/data/mirror550 +1553|551|m|m|s|u|sdw30|sdw30|8551|/data/mirror551 +1554|552|m|m|s|u|sdw30|sdw30|8552|/data/mirror552 +1555|553|m|m|s|u|sdw30|sdw30|8553|/data/mirror553 +1556|554|m|m|s|u|sdw30|sdw30|8554|/data/mirror554 +1557|555|m|m|s|u|sdw30|sdw30|8555|/data/mirror555 +1558|556|m|m|s|u|sdw30|sdw30|8556|/data/mirror556 +1559|557|m|m|s|u|sdw30|sdw30|8557|/data/mirror557 +1560|558|m|m|s|u|sdw30|sdw30|8558|/data/mirror558 +1561|559|m|m|s|u|sdw30|sdw30|8559|/data/mirror559 +1562|560|m|m|s|u|sdw30|sdw30|8560|/data/mirror560 +1563|561|m|m|s|u|sdw30|sdw30|8561|/data/mirror561 +1564|562|m|m|s|u|sdw30|sdw30|8562|/data/mirror562 +1565|563|m|m|s|u|sdw30|sdw30|8563|/data/mirror563 +1566|564|m|m|s|u|sdw30|sdw30|8564|/data/mirror564 +1567|565|m|m|s|u|sdw30|sdw30|8565|/data/mirror565 +1568|566|m|m|s|u|sdw30|sdw30|8566|/data/mirror566 +1569|567|m|m|s|u|sdw30|sdw30|8567|/data/mirror567 +1570|568|m|m|s|u|sdw30|sdw30|8568|/data/mirror568 +1571|569|m|m|s|u|sdw30|sdw30|8569|/data/mirror569 +1572|570|m|m|s|u|sdw30|sdw30|8570|/data/mirror570 +# SDW31 +613|611|p|p|s|u|sdw31|sdw31|7611|/data/primary611 +614|612|p|p|s|u|sdw31|sdw31|7612|/data/primary612 +615|613|p|p|s|u|sdw31|sdw31|7613|/data/primary613 +616|614|p|p|s|u|sdw31|sdw31|7614|/data/primary614 +617|615|p|p|s|u|sdw31|sdw31|7615|/data/primary615 +618|616|p|p|s|u|sdw31|sdw31|7616|/data/primary616 +619|617|p|p|s|u|sdw31|sdw31|7617|/data/primary617 +620|618|p|p|s|u|sdw31|sdw31|7618|/data/primary618 +621|619|p|p|s|u|sdw31|sdw31|7619|/data/primary619 +622|620|p|p|s|u|sdw31|sdw31|7620|/data/primary620 +623|621|p|p|s|u|sdw31|sdw31|7621|/data/primary621 +624|622|p|p|s|u|sdw31|sdw31|7622|/data/primary622 +625|623|p|p|s|u|sdw31|sdw31|7623|/data/primary623 +626|624|p|p|s|u|sdw31|sdw31|7624|/data/primary624 +627|625|p|p|s|u|sdw31|sdw31|7625|/data/primary625 +1628|626|m|m|s|u|sdw31|sdw31|8626|/data/mirror626 +1629|627|m|m|s|u|sdw31|sdw31|8627|/data/mirror627 +1630|628|m|m|s|u|sdw31|sdw31|8628|/data/mirror628 +1631|629|m|m|s|u|sdw31|sdw31|8629|/data/mirror629 +1632|630|m|m|s|u|sdw31|sdw31|8630|/data/mirror630 +1633|631|m|m|s|u|sdw31|sdw31|8631|/data/mirror631 +1634|632|m|m|s|u|sdw31|sdw31|8632|/data/mirror632 +1635|633|m|m|s|u|sdw31|sdw31|8633|/data/mirror633 +1636|634|m|m|s|u|sdw31|sdw31|8634|/data/mirror634 +1637|635|m|m|s|u|sdw31|sdw31|8635|/data/mirror635 +1638|636|m|m|s|u|sdw31|sdw31|8636|/data/mirror636 +1639|637|m|m|s|u|sdw31|sdw31|8637|/data/mirror637 +1640|638|m|m|s|u|sdw31|sdw31|8638|/data/mirror638 +1641|639|m|m|s|u|sdw31|sdw31|8639|/data/mirror639 +1642|640|m|m|s|u|sdw31|sdw31|8640|/data/mirror640 +1643|641|m|m|s|u|sdw31|sdw31|8641|/data/mirror641 +1644|642|m|m|s|u|sdw31|sdw31|8642|/data/mirror642 +1645|643|m|m|s|u|sdw31|sdw31|8643|/data/mirror643 +# SDW32 +628|626|p|p|s|u|sdw32|sdw32|7626|/data/primary626 +629|627|p|p|s|u|sdw32|sdw32|7627|/data/primary627 +630|628|p|p|s|u|sdw32|sdw32|7628|/data/primary628 +631|629|p|p|s|u|sdw32|sdw32|7629|/data/primary629 +632|630|p|p|s|u|sdw32|sdw32|7630|/data/primary630 +633|631|p|p|s|u|sdw32|sdw32|7631|/data/primary631 +634|632|p|p|s|u|sdw32|sdw32|7632|/data/primary632 +635|633|p|p|s|u|sdw32|sdw32|7633|/data/primary633 +636|634|p|p|s|u|sdw32|sdw32|7634|/data/primary634 +637|635|p|p|s|u|sdw32|sdw32|7635|/data/primary635 +638|636|p|p|s|u|sdw32|sdw32|7636|/data/primary636 +639|637|p|p|s|u|sdw32|sdw32|7637|/data/primary637 +640|638|p|p|s|u|sdw32|sdw32|7638|/data/primary638 +641|639|p|p|s|u|sdw32|sdw32|7639|/data/primary639 +642|640|p|p|s|u|sdw32|sdw32|7640|/data/primary640 +643|641|p|p|s|u|sdw32|sdw32|7641|/data/primary641 +644|642|p|p|s|u|sdw32|sdw32|7642|/data/primary642 +645|643|p|p|s|u|sdw32|sdw32|7643|/data/primary643 +1594|592|m|m|s|u|sdw32|sdw32|8592|/data/mirror592 +1595|593|m|m|s|u|sdw32|sdw32|8593|/data/mirror593 +1596|594|m|m|s|u|sdw32|sdw32|8594|/data/mirror594 +1597|595|m|m|s|u|sdw32|sdw32|8595|/data/mirror595 +1598|596|m|m|s|u|sdw32|sdw32|8596|/data/mirror596 +1599|597|m|m|s|u|sdw32|sdw32|8597|/data/mirror597 +1600|598|m|m|s|u|sdw32|sdw32|8598|/data/mirror598 +1601|599|m|m|s|u|sdw32|sdw32|8599|/data/mirror599 +1602|600|m|m|s|u|sdw32|sdw32|8600|/data/mirror600 +1603|601|m|m|s|u|sdw32|sdw32|8601|/data/mirror601 +1604|602|m|m|s|u|sdw32|sdw32|8602|/data/mirror602 +1605|603|m|m|s|u|sdw32|sdw32|8603|/data/mirror603 +1606|604|m|m|s|u|sdw32|sdw32|8604|/data/mirror604 +1607|605|m|m|s|u|sdw32|sdw32|8605|/data/mirror605 +1608|606|m|m|s|u|sdw32|sdw32|8606|/data/mirror606 +1609|607|m|m|s|u|sdw32|sdw32|8607|/data/mirror607 +1610|608|m|m|s|u|sdw32|sdw32|8608|/data/mirror608 +1611|609|m|m|s|u|sdw32|sdw32|8609|/data/mirror609 +1612|610|m|m|s|u|sdw32|sdw32|8610|/data/mirror610 +# SDW33 +646|644|p|p|s|u|sdw33|sdw33|7644|/data/primary644 +647|645|p|p|s|u|sdw33|sdw33|7645|/data/primary645 +648|646|p|p|s|u|sdw33|sdw33|7646|/data/primary646 +649|647|p|p|s|u|sdw33|sdw33|7647|/data/primary647 +650|648|p|p|s|u|sdw33|sdw33|7648|/data/primary648 +651|649|p|p|s|u|sdw33|sdw33|7649|/data/primary649 +652|650|p|p|s|u|sdw33|sdw33|7650|/data/primary650 +653|651|p|p|s|u|sdw33|sdw33|7651|/data/primary651 +654|652|p|p|s|u|sdw33|sdw33|7652|/data/primary652 +655|653|p|p|s|u|sdw33|sdw33|7653|/data/primary653 +656|654|p|p|s|u|sdw33|sdw33|7654|/data/primary654 +657|655|p|p|s|u|sdw33|sdw33|7655|/data/primary655 +658|656|p|p|s|u|sdw33|sdw33|7656|/data/primary656 +659|657|p|p|s|u|sdw33|sdw33|7657|/data/primary657 +660|658|p|p|s|u|sdw33|sdw33|7658|/data/primary658 +1613|611|m|m|s|u|sdw33|sdw33|8611|/data/mirror611 +1614|612|m|m|s|u|sdw33|sdw33|8612|/data/mirror612 +1615|613|m|m|s|u|sdw33|sdw33|8613|/data/mirror613 +1616|614|m|m|s|u|sdw33|sdw33|8614|/data/mirror614 +1617|615|m|m|s|u|sdw33|sdw33|8615|/data/mirror615 +1618|616|m|m|s|u|sdw33|sdw33|8616|/data/mirror616 +1619|617|m|m|s|u|sdw33|sdw33|8617|/data/mirror617 +1620|618|m|m|s|u|sdw33|sdw33|8618|/data/mirror618 +1621|619|m|m|s|u|sdw33|sdw33|8619|/data/mirror619 +1622|620|m|m|s|u|sdw33|sdw33|8620|/data/mirror620 +1623|621|m|m|s|u|sdw33|sdw33|8621|/data/mirror621 +1624|622|m|m|s|u|sdw33|sdw33|8622|/data/mirror622 +1625|623|m|m|s|u|sdw33|sdw33|8623|/data/mirror623 +1626|624|m|m|s|u|sdw33|sdw33|8624|/data/mirror624 +1627|625|m|m|s|u|sdw33|sdw33|8625|/data/mirror625 +# SDW34 +661|659|p|p|s|u|sdw34|sdw34|7659|/data/primary659 +662|660|p|p|s|u|sdw34|sdw34|7660|/data/primary660 +663|661|p|p|s|u|sdw34|sdw34|7661|/data/primary661 +664|662|p|p|s|u|sdw34|sdw34|7662|/data/primary662 +665|663|p|p|s|u|sdw34|sdw34|7663|/data/primary663 +666|664|p|p|s|u|sdw34|sdw34|7664|/data/primary664 +667|665|p|p|s|u|sdw34|sdw34|7665|/data/primary665 +668|666|p|p|s|u|sdw34|sdw34|7666|/data/primary666 +669|667|p|p|s|u|sdw34|sdw34|7667|/data/primary667 +670|668|p|p|s|u|sdw34|sdw34|7668|/data/primary668 +671|669|p|p|s|u|sdw34|sdw34|7669|/data/primary669 +672|670|p|p|s|u|sdw34|sdw34|7670|/data/primary670 +673|671|p|p|s|u|sdw34|sdw34|7671|/data/primary671 +674|672|p|p|s|u|sdw34|sdw34|7672|/data/primary672 +675|673|p|p|s|u|sdw34|sdw34|7673|/data/primary673 +676|674|p|p|s|u|sdw34|sdw34|7674|/data/primary674 +1646|644|m|m|s|u|sdw34|sdw34|8644|/data/mirror644 +1647|645|m|m|s|u|sdw34|sdw34|8645|/data/mirror645 +1648|646|m|m|s|u|sdw34|sdw34|8646|/data/mirror646 +1649|647|m|m|s|u|sdw34|sdw34|8647|/data/mirror647 +1650|648|m|m|s|u|sdw34|sdw34|8648|/data/mirror648 +1651|649|m|m|s|u|sdw34|sdw34|8649|/data/mirror649 +1652|650|m|m|s|u|sdw34|sdw34|8650|/data/mirror650 +1653|651|m|m|s|u|sdw34|sdw34|8651|/data/mirror651 +1654|652|m|m|s|u|sdw34|sdw34|8652|/data/mirror652 +1655|653|m|m|s|u|sdw34|sdw34|8653|/data/mirror653 +1656|654|m|m|s|u|sdw34|sdw34|8654|/data/mirror654 +1657|655|m|m|s|u|sdw34|sdw34|8655|/data/mirror655 +1658|656|m|m|s|u|sdw34|sdw34|8656|/data/mirror656 +1659|657|m|m|s|u|sdw34|sdw34|8657|/data/mirror657 +1660|658|m|m|s|u|sdw34|sdw34|8658|/data/mirror658 +# SDW35 +677|675|p|p|s|u|sdw35|sdw35|7675|/data/primary675 +678|676|p|p|s|u|sdw35|sdw35|7676|/data/primary676 +679|677|p|p|s|u|sdw35|sdw35|7677|/data/primary677 +680|678|p|p|s|u|sdw35|sdw35|7678|/data/primary678 +681|679|p|p|s|u|sdw35|sdw35|7679|/data/primary679 +682|680|p|p|s|u|sdw35|sdw35|7680|/data/primary680 +683|681|p|p|s|u|sdw35|sdw35|7681|/data/primary681 +684|682|p|p|s|u|sdw35|sdw35|7682|/data/primary682 +685|683|p|p|s|u|sdw35|sdw35|7683|/data/primary683 +686|684|p|p|s|u|sdw35|sdw35|7684|/data/primary684 +687|685|p|p|s|u|sdw35|sdw35|7685|/data/primary685 +688|686|p|p|s|u|sdw35|sdw35|7686|/data/primary686 +689|687|p|p|s|u|sdw35|sdw35|7687|/data/primary687 +690|688|p|p|s|u|sdw35|sdw35|7688|/data/primary688 +691|689|p|p|s|u|sdw35|sdw35|7689|/data/primary689 +692|690|p|p|s|u|sdw35|sdw35|7690|/data/primary690 +693|691|p|p|s|u|sdw35|sdw35|7691|/data/primary691 +694|692|p|p|s|u|sdw35|sdw35|7692|/data/primary692 +695|693|p|p|s|u|sdw35|sdw35|7693|/data/primary693 +696|694|p|p|s|u|sdw35|sdw35|7694|/data/primary694 +697|695|p|p|s|u|sdw35|sdw35|7695|/data/primary695 +698|696|p|p|s|u|sdw35|sdw35|7696|/data/primary696 +699|697|p|p|s|u|sdw35|sdw35|7697|/data/primary697 +700|698|p|p|s|u|sdw35|sdw35|7698|/data/primary698 +701|699|p|p|s|u|sdw35|sdw35|7699|/data/primary699 +702|700|p|p|s|u|sdw35|sdw35|7700|/data/primary700 +1661|659|m|m|s|u|sdw35|sdw35|8659|/data/mirror659 +1662|660|m|m|s|u|sdw35|sdw35|8660|/data/mirror660 +1663|661|m|m|s|u|sdw35|sdw35|8661|/data/mirror661 +1664|662|m|m|s|u|sdw35|sdw35|8662|/data/mirror662 +1665|663|m|m|s|u|sdw35|sdw35|8663|/data/mirror663 +1666|664|m|m|s|u|sdw35|sdw35|8664|/data/mirror664 +1667|665|m|m|s|u|sdw35|sdw35|8665|/data/mirror665 +1668|666|m|m|s|u|sdw35|sdw35|8666|/data/mirror666 +1669|667|m|m|s|u|sdw35|sdw35|8667|/data/mirror667 +1670|668|m|m|s|u|sdw35|sdw35|8668|/data/mirror668 +1671|669|m|m|s|u|sdw35|sdw35|8669|/data/mirror669 +1672|670|m|m|s|u|sdw35|sdw35|8670|/data/mirror670 +1673|671|m|m|s|u|sdw35|sdw35|8671|/data/mirror671 +1674|672|m|m|s|u|sdw35|sdw35|8672|/data/mirror672 +1675|673|m|m|s|u|sdw35|sdw35|8673|/data/mirror673 +1676|674|m|m|s|u|sdw35|sdw35|8674|/data/mirror674 +# SDW36 +703|701|p|p|s|u|sdw36|sdw36|7701|/data/primary701 +704|702|p|p|s|u|sdw36|sdw36|7702|/data/primary702 +705|703|p|p|s|u|sdw36|sdw36|7703|/data/primary703 +706|704|p|p|s|u|sdw36|sdw36|7704|/data/primary704 +707|705|p|p|s|u|sdw36|sdw36|7705|/data/primary705 +708|706|p|p|s|u|sdw36|sdw36|7706|/data/primary706 +709|707|p|p|s|u|sdw36|sdw36|7707|/data/primary707 +710|708|p|p|s|u|sdw36|sdw36|7708|/data/primary708 +711|709|p|p|s|u|sdw36|sdw36|7709|/data/primary709 +712|710|p|p|s|u|sdw36|sdw36|7710|/data/primary710 +713|711|p|p|s|u|sdw36|sdw36|7711|/data/primary711 +714|712|p|p|s|u|sdw36|sdw36|7712|/data/primary712 +715|713|p|p|s|u|sdw36|sdw36|7713|/data/primary713 +716|714|p|p|s|u|sdw36|sdw36|7714|/data/primary714 +717|715|p|p|s|u|sdw36|sdw36|7715|/data/primary715 +718|716|p|p|s|u|sdw36|sdw36|7716|/data/primary716 +719|717|p|p|s|u|sdw36|sdw36|7717|/data/primary717 +1746|744|m|m|s|u|sdw36|sdw36|8744|/data/mirror744 +1747|745|m|m|s|u|sdw36|sdw36|8745|/data/mirror745 +1748|746|m|m|s|u|sdw36|sdw36|8746|/data/mirror746 +1749|747|m|m|s|u|sdw36|sdw36|8747|/data/mirror747 +1750|748|m|m|s|u|sdw36|sdw36|8748|/data/mirror748 +1751|749|m|m|s|u|sdw36|sdw36|8749|/data/mirror749 +1752|750|m|m|s|u|sdw36|sdw36|8750|/data/mirror750 +1753|751|m|m|s|u|sdw36|sdw36|8751|/data/mirror751 +1754|752|m|m|s|u|sdw36|sdw36|8752|/data/mirror752 +1755|753|m|m|s|u|sdw36|sdw36|8753|/data/mirror753 +1756|754|m|m|s|u|sdw36|sdw36|8754|/data/mirror754 +1757|755|m|m|s|u|sdw36|sdw36|8755|/data/mirror755 +1758|756|m|m|s|u|sdw36|sdw36|8756|/data/mirror756 +1759|757|m|m|s|u|sdw36|sdw36|8757|/data/mirror757 +1760|758|m|m|s|u|sdw36|sdw36|8758|/data/mirror758 +1761|759|m|m|s|u|sdw36|sdw36|8759|/data/mirror759 +1762|760|m|m|s|u|sdw36|sdw36|8760|/data/mirror760 +1763|761|m|m|s|u|sdw36|sdw36|8761|/data/mirror761 +1764|762|m|m|s|u|sdw36|sdw36|8762|/data/mirror762 +1765|763|m|m|s|u|sdw36|sdw36|8763|/data/mirror763 +1766|764|m|m|s|u|sdw36|sdw36|8764|/data/mirror764 +1767|765|m|m|s|u|sdw36|sdw36|8765|/data/mirror765 +1768|766|m|m|s|u|sdw36|sdw36|8766|/data/mirror766 +# SDW37 +720|718|p|p|s|u|sdw37|sdw37|7718|/data/primary718 +721|719|p|p|s|u|sdw37|sdw37|7719|/data/primary719 +722|720|p|p|s|u|sdw37|sdw37|7720|/data/primary720 +723|721|p|p|s|u|sdw37|sdw37|7721|/data/primary721 +724|722|p|p|s|u|sdw37|sdw37|7722|/data/primary722 +725|723|p|p|s|u|sdw37|sdw37|7723|/data/primary723 +726|724|p|p|s|u|sdw37|sdw37|7724|/data/primary724 +727|725|p|p|s|u|sdw37|sdw37|7725|/data/primary725 +728|726|p|p|s|u|sdw37|sdw37|7726|/data/primary726 +729|727|p|p|s|u|sdw37|sdw37|7727|/data/primary727 +730|728|p|p|s|u|sdw37|sdw37|7728|/data/primary728 +731|729|p|p|s|u|sdw37|sdw37|7729|/data/primary729 +732|730|p|p|s|u|sdw37|sdw37|7730|/data/primary730 +733|731|p|p|s|u|sdw37|sdw37|7731|/data/primary731 +734|732|p|p|s|u|sdw37|sdw37|7732|/data/primary732 +735|733|p|p|s|u|sdw37|sdw37|7733|/data/primary733 +736|734|p|p|s|u|sdw37|sdw37|7734|/data/primary734 +737|735|p|p|s|u|sdw37|sdw37|7735|/data/primary735 +738|736|p|p|s|u|sdw37|sdw37|7736|/data/primary736 +739|737|p|p|s|u|sdw37|sdw37|7737|/data/primary737 +740|738|p|p|s|u|sdw37|sdw37|7738|/data/primary738 +741|739|p|p|s|u|sdw37|sdw37|7739|/data/primary739 +742|740|p|p|s|u|sdw37|sdw37|7740|/data/primary740 +743|741|p|p|s|u|sdw37|sdw37|7741|/data/primary741 +744|742|p|p|s|u|sdw37|sdw37|7742|/data/primary742 +745|743|p|p|s|u|sdw37|sdw37|7743|/data/primary743 +1814|812|m|m|s|u|sdw37|sdw37|8812|/data/mirror812 +1815|813|m|m|s|u|sdw37|sdw37|8813|/data/mirror813 +1816|814|m|m|s|u|sdw37|sdw37|8814|/data/mirror814 +1817|815|m|m|s|u|sdw37|sdw37|8815|/data/mirror815 +1818|816|m|m|s|u|sdw37|sdw37|8816|/data/mirror816 +1819|817|m|m|s|u|sdw37|sdw37|8817|/data/mirror817 +1820|818|m|m|s|u|sdw37|sdw37|8818|/data/mirror818 +1821|819|m|m|s|u|sdw37|sdw37|8819|/data/mirror819 +1822|820|m|m|s|u|sdw37|sdw37|8820|/data/mirror820 +1823|821|m|m|s|u|sdw37|sdw37|8821|/data/mirror821 +1824|822|m|m|s|u|sdw37|sdw37|8822|/data/mirror822 +1825|823|m|m|s|u|sdw37|sdw37|8823|/data/mirror823 +1826|824|m|m|s|u|sdw37|sdw37|8824|/data/mirror824 +1827|825|m|m|s|u|sdw37|sdw37|8825|/data/mirror825 +1828|826|m|m|s|u|sdw37|sdw37|8826|/data/mirror826 +1829|827|m|m|s|u|sdw37|sdw37|8827|/data/mirror827 +1830|828|m|m|s|u|sdw37|sdw37|8828|/data/mirror828 +1831|829|m|m|s|u|sdw37|sdw37|8829|/data/mirror829 +1832|830|m|m|s|u|sdw37|sdw37|8830|/data/mirror830 +1833|831|m|m|s|u|sdw37|sdw37|8831|/data/mirror831 +1834|832|m|m|s|u|sdw37|sdw37|8832|/data/mirror832 +# SDW38 +746|744|p|p|s|u|sdw38|sdw38|7744|/data/primary744 +747|745|p|p|s|u|sdw38|sdw38|7745|/data/primary745 +748|746|p|p|s|u|sdw38|sdw38|7746|/data/primary746 +749|747|p|p|s|u|sdw38|sdw38|7747|/data/primary747 +750|748|p|p|s|u|sdw38|sdw38|7748|/data/primary748 +751|749|p|p|s|u|sdw38|sdw38|7749|/data/primary749 +752|750|p|p|s|u|sdw38|sdw38|7750|/data/primary750 +753|751|p|p|s|u|sdw38|sdw38|7751|/data/primary751 +754|752|p|p|s|u|sdw38|sdw38|7752|/data/primary752 +755|753|p|p|s|u|sdw38|sdw38|7753|/data/primary753 +756|754|p|p|s|u|sdw38|sdw38|7754|/data/primary754 +757|755|p|p|s|u|sdw38|sdw38|7755|/data/primary755 +758|756|p|p|s|u|sdw38|sdw38|7756|/data/primary756 +759|757|p|p|s|u|sdw38|sdw38|7757|/data/primary757 +760|758|p|p|s|u|sdw38|sdw38|7758|/data/primary758 +761|759|p|p|s|u|sdw38|sdw38|7759|/data/primary759 +762|760|p|p|s|u|sdw38|sdw38|7760|/data/primary760 +763|761|p|p|s|u|sdw38|sdw38|7761|/data/primary761 +764|762|p|p|s|u|sdw38|sdw38|7762|/data/primary762 +765|763|p|p|s|u|sdw38|sdw38|7763|/data/primary763 +766|764|p|p|s|u|sdw38|sdw38|7764|/data/primary764 +767|765|p|p|s|u|sdw38|sdw38|7765|/data/primary765 +768|766|p|p|s|u|sdw38|sdw38|7766|/data/primary766 +1703|701|m|m|s|u|sdw38|sdw38|8701|/data/mirror701 +1704|702|m|m|s|u|sdw38|sdw38|8702|/data/mirror702 +1705|703|m|m|s|u|sdw38|sdw38|8703|/data/mirror703 +1706|704|m|m|s|u|sdw38|sdw38|8704|/data/mirror704 +1707|705|m|m|s|u|sdw38|sdw38|8705|/data/mirror705 +1708|706|m|m|s|u|sdw38|sdw38|8706|/data/mirror706 +1709|707|m|m|s|u|sdw38|sdw38|8707|/data/mirror707 +1710|708|m|m|s|u|sdw38|sdw38|8708|/data/mirror708 +1711|709|m|m|s|u|sdw38|sdw38|8709|/data/mirror709 +1712|710|m|m|s|u|sdw38|sdw38|8710|/data/mirror710 +1713|711|m|m|s|u|sdw38|sdw38|8711|/data/mirror711 +1714|712|m|m|s|u|sdw38|sdw38|8712|/data/mirror712 +1715|713|m|m|s|u|sdw38|sdw38|8713|/data/mirror713 +1716|714|m|m|s|u|sdw38|sdw38|8714|/data/mirror714 +1717|715|m|m|s|u|sdw38|sdw38|8715|/data/mirror715 +1718|716|m|m|s|u|sdw38|sdw38|8716|/data/mirror716 +1719|717|m|m|s|u|sdw38|sdw38|8717|/data/mirror717 +# SDW39 +769|767|p|p|s|u|sdw39|sdw39|7767|/data/primary767 +770|768|p|p|s|u|sdw39|sdw39|7768|/data/primary768 +771|769|p|p|s|u|sdw39|sdw39|7769|/data/primary769 +772|770|p|p|s|u|sdw39|sdw39|7770|/data/primary770 +773|771|p|p|s|u|sdw39|sdw39|7771|/data/primary771 +774|772|p|p|s|u|sdw39|sdw39|7772|/data/primary772 +775|773|p|p|s|u|sdw39|sdw39|7773|/data/primary773 +776|774|p|p|s|u|sdw39|sdw39|7774|/data/primary774 +777|775|p|p|s|u|sdw39|sdw39|7775|/data/primary775 +778|776|p|p|s|u|sdw39|sdw39|7776|/data/primary776 +779|777|p|p|s|u|sdw39|sdw39|7777|/data/primary777 +780|778|p|p|s|u|sdw39|sdw39|7778|/data/primary778 +781|779|p|p|s|u|sdw39|sdw39|7779|/data/primary779 +782|780|p|p|s|u|sdw39|sdw39|7780|/data/primary780 +783|781|p|p|s|u|sdw39|sdw39|7781|/data/primary781 +784|782|p|p|s|u|sdw39|sdw39|7782|/data/primary782 +785|783|p|p|s|u|sdw39|sdw39|7783|/data/primary783 +786|784|p|p|s|u|sdw39|sdw39|7784|/data/primary784 +787|785|p|p|s|u|sdw39|sdw39|7785|/data/primary785 +788|786|p|p|s|u|sdw39|sdw39|7786|/data/primary786 +789|787|p|p|s|u|sdw39|sdw39|7787|/data/primary787 +1856|854|m|m|s|u|sdw39|sdw39|8854|/data/mirror854 +1857|855|m|m|s|u|sdw39|sdw39|8855|/data/mirror855 +1858|856|m|m|s|u|sdw39|sdw39|8856|/data/mirror856 +1859|857|m|m|s|u|sdw39|sdw39|8857|/data/mirror857 +1860|858|m|m|s|u|sdw39|sdw39|8858|/data/mirror858 +1861|859|m|m|s|u|sdw39|sdw39|8859|/data/mirror859 +1862|860|m|m|s|u|sdw39|sdw39|8860|/data/mirror860 +1863|861|m|m|s|u|sdw39|sdw39|8861|/data/mirror861 +1864|862|m|m|s|u|sdw39|sdw39|8862|/data/mirror862 +1865|863|m|m|s|u|sdw39|sdw39|8863|/data/mirror863 +1866|864|m|m|s|u|sdw39|sdw39|8864|/data/mirror864 +1867|865|m|m|s|u|sdw39|sdw39|8865|/data/mirror865 +1868|866|m|m|s|u|sdw39|sdw39|8866|/data/mirror866 +1869|867|m|m|s|u|sdw39|sdw39|8867|/data/mirror867 +1870|868|m|m|s|u|sdw39|sdw39|8868|/data/mirror868 +1871|869|m|m|s|u|sdw39|sdw39|8869|/data/mirror869 +1872|870|m|m|s|u|sdw39|sdw39|8870|/data/mirror870 +1873|871|m|m|s|u|sdw39|sdw39|8871|/data/mirror871 +# SDW4 +56|54|p|p|s|u|sdw4|sdw4|7054|/data/primary54 +57|55|p|p|s|u|sdw4|sdw4|7055|/data/primary55 +58|56|p|p|s|u|sdw4|sdw4|7056|/data/primary56 +59|57|p|p|s|u|sdw4|sdw4|7057|/data/primary57 +60|58|p|p|s|u|sdw4|sdw4|7058|/data/primary58 +61|59|p|p|s|u|sdw4|sdw4|7059|/data/primary59 +62|60|p|p|s|u|sdw4|sdw4|7060|/data/primary60 +63|61|p|p|s|u|sdw4|sdw4|7061|/data/primary61 +64|62|p|p|s|u|sdw4|sdw4|7062|/data/primary62 +65|63|p|p|s|u|sdw4|sdw4|7063|/data/primary63 +66|64|p|p|s|u|sdw4|sdw4|7064|/data/primary64 +67|65|p|p|s|u|sdw4|sdw4|7065|/data/primary65 +68|66|p|p|s|u|sdw4|sdw4|7066|/data/primary66 +69|67|p|p|s|u|sdw4|sdw4|7067|/data/primary67 +70|68|p|p|s|u|sdw4|sdw4|7068|/data/primary68 +71|69|p|p|s|u|sdw4|sdw4|7069|/data/primary69 +72|70|p|p|s|u|sdw4|sdw4|7070|/data/primary70 +73|71|p|p|s|u|sdw4|sdw4|7071|/data/primary71 +1021|19|m|m|s|u|sdw4|sdw4|8019|/data/mirror19 +1022|20|m|m|s|u|sdw4|sdw4|8020|/data/mirror20 +1023|21|m|m|s|u|sdw4|sdw4|8021|/data/mirror21 +1024|22|m|m|s|u|sdw4|sdw4|8022|/data/mirror22 +1025|23|m|m|s|u|sdw4|sdw4|8023|/data/mirror23 +1026|24|m|m|s|u|sdw4|sdw4|8024|/data/mirror24 +1027|25|m|m|s|u|sdw4|sdw4|8025|/data/mirror25 +1028|26|m|m|s|u|sdw4|sdw4|8026|/data/mirror26 +1029|27|m|m|s|u|sdw4|sdw4|8027|/data/mirror27 +1030|28|m|m|s|u|sdw4|sdw4|8028|/data/mirror28 +1031|29|m|m|s|u|sdw4|sdw4|8029|/data/mirror29 +1032|30|m|m|s|u|sdw4|sdw4|8030|/data/mirror30 +1033|31|m|m|s|u|sdw4|sdw4|8031|/data/mirror31 +1034|32|m|m|s|u|sdw4|sdw4|8032|/data/mirror32 +1035|33|m|m|s|u|sdw4|sdw4|8033|/data/mirror33 +1036|34|m|m|s|u|sdw4|sdw4|8034|/data/mirror34 +1037|35|m|m|s|u|sdw4|sdw4|8035|/data/mirror35 +1038|36|m|m|s|u|sdw4|sdw4|8036|/data/mirror36 +1039|37|m|m|s|u|sdw4|sdw4|8037|/data/mirror37 +1040|38|m|m|s|u|sdw4|sdw4|8038|/data/mirror38 +1041|39|m|m|s|u|sdw4|sdw4|8039|/data/mirror39 +# SDW40 +790|788|p|p|s|u|sdw40|sdw40|7788|/data/primary788 +791|789|p|p|s|u|sdw40|sdw40|7789|/data/primary789 +792|790|p|p|s|u|sdw40|sdw40|7790|/data/primary790 +793|791|p|p|s|u|sdw40|sdw40|7791|/data/primary791 +794|792|p|p|s|u|sdw40|sdw40|7792|/data/primary792 +795|793|p|p|s|u|sdw40|sdw40|7793|/data/primary793 +796|794|p|p|s|u|sdw40|sdw40|7794|/data/primary794 +797|795|p|p|s|u|sdw40|sdw40|7795|/data/primary795 +798|796|p|p|s|u|sdw40|sdw40|7796|/data/primary796 +799|797|p|p|s|u|sdw40|sdw40|7797|/data/primary797 +800|798|p|p|s|u|sdw40|sdw40|7798|/data/primary798 +801|799|p|p|s|u|sdw40|sdw40|7799|/data/primary799 +802|800|p|p|s|u|sdw40|sdw40|7800|/data/primary800 +803|801|p|p|s|u|sdw40|sdw40|7801|/data/primary801 +804|802|p|p|s|u|sdw40|sdw40|7802|/data/primary802 +805|803|p|p|s|u|sdw40|sdw40|7803|/data/primary803 +806|804|p|p|s|u|sdw40|sdw40|7804|/data/primary804 +807|805|p|p|s|u|sdw40|sdw40|7805|/data/primary805 +808|806|p|p|s|u|sdw40|sdw40|7806|/data/primary806 +809|807|p|p|s|u|sdw40|sdw40|7807|/data/primary807 +810|808|p|p|s|u|sdw40|sdw40|7808|/data/primary808 +811|809|p|p|s|u|sdw40|sdw40|7809|/data/primary809 +812|810|p|p|s|u|sdw40|sdw40|7810|/data/primary810 +813|811|p|p|s|u|sdw40|sdw40|7811|/data/primary811 +1720|718|m|m|s|u|sdw40|sdw40|8718|/data/mirror718 +1721|719|m|m|s|u|sdw40|sdw40|8719|/data/mirror719 +1722|720|m|m|s|u|sdw40|sdw40|8720|/data/mirror720 +1723|721|m|m|s|u|sdw40|sdw40|8721|/data/mirror721 +1724|722|m|m|s|u|sdw40|sdw40|8722|/data/mirror722 +1725|723|m|m|s|u|sdw40|sdw40|8723|/data/mirror723 +1726|724|m|m|s|u|sdw40|sdw40|8724|/data/mirror724 +1727|725|m|m|s|u|sdw40|sdw40|8725|/data/mirror725 +1728|726|m|m|s|u|sdw40|sdw40|8726|/data/mirror726 +1729|727|m|m|s|u|sdw40|sdw40|8727|/data/mirror727 +1730|728|m|m|s|u|sdw40|sdw40|8728|/data/mirror728 +1731|729|m|m|s|u|sdw40|sdw40|8729|/data/mirror729 +1732|730|m|m|s|u|sdw40|sdw40|8730|/data/mirror730 +1733|731|m|m|s|u|sdw40|sdw40|8731|/data/mirror731 +1734|732|m|m|s|u|sdw40|sdw40|8732|/data/mirror732 +1735|733|m|m|s|u|sdw40|sdw40|8733|/data/mirror733 +1736|734|m|m|s|u|sdw40|sdw40|8734|/data/mirror734 +1737|735|m|m|s|u|sdw40|sdw40|8735|/data/mirror735 +1738|736|m|m|s|u|sdw40|sdw40|8736|/data/mirror736 +1739|737|m|m|s|u|sdw40|sdw40|8737|/data/mirror737 +1740|738|m|m|s|u|sdw40|sdw40|8738|/data/mirror738 +1741|739|m|m|s|u|sdw40|sdw40|8739|/data/mirror739 +1742|740|m|m|s|u|sdw40|sdw40|8740|/data/mirror740 +1743|741|m|m|s|u|sdw40|sdw40|8741|/data/mirror741 +1744|742|m|m|s|u|sdw40|sdw40|8742|/data/mirror742 +1745|743|m|m|s|u|sdw40|sdw40|8743|/data/mirror743 +# SDW41 +814|812|p|p|s|u|sdw41|sdw41|7812|/data/primary812 +815|813|p|p|s|u|sdw41|sdw41|7813|/data/primary813 +816|814|p|p|s|u|sdw41|sdw41|7814|/data/primary814 +817|815|p|p|s|u|sdw41|sdw41|7815|/data/primary815 +818|816|p|p|s|u|sdw41|sdw41|7816|/data/primary816 +819|817|p|p|s|u|sdw41|sdw41|7817|/data/primary817 +820|818|p|p|s|u|sdw41|sdw41|7818|/data/primary818 +821|819|p|p|s|u|sdw41|sdw41|7819|/data/primary819 +822|820|p|p|s|u|sdw41|sdw41|7820|/data/primary820 +823|821|p|p|s|u|sdw41|sdw41|7821|/data/primary821 +824|822|p|p|s|u|sdw41|sdw41|7822|/data/primary822 +825|823|p|p|s|u|sdw41|sdw41|7823|/data/primary823 +826|824|p|p|s|u|sdw41|sdw41|7824|/data/primary824 +827|825|p|p|s|u|sdw41|sdw41|7825|/data/primary825 +828|826|p|p|s|u|sdw41|sdw41|7826|/data/primary826 +829|827|p|p|s|u|sdw41|sdw41|7827|/data/primary827 +830|828|p|p|s|u|sdw41|sdw41|7828|/data/primary828 +831|829|p|p|s|u|sdw41|sdw41|7829|/data/primary829 +832|830|p|p|s|u|sdw41|sdw41|7830|/data/primary830 +833|831|p|p|s|u|sdw41|sdw41|7831|/data/primary831 +834|832|p|p|s|u|sdw41|sdw41|7832|/data/primary832 +1769|767|m|m|s|u|sdw41|sdw41|8767|/data/mirror767 +1770|768|m|m|s|u|sdw41|sdw41|8768|/data/mirror768 +1771|769|m|m|s|u|sdw41|sdw41|8769|/data/mirror769 +1772|770|m|m|s|u|sdw41|sdw41|8770|/data/mirror770 +1773|771|m|m|s|u|sdw41|sdw41|8771|/data/mirror771 +1774|772|m|m|s|u|sdw41|sdw41|8772|/data/mirror772 +1775|773|m|m|s|u|sdw41|sdw41|8773|/data/mirror773 +1776|774|m|m|s|u|sdw41|sdw41|8774|/data/mirror774 +1777|775|m|m|s|u|sdw41|sdw41|8775|/data/mirror775 +1778|776|m|m|s|u|sdw41|sdw41|8776|/data/mirror776 +1779|777|m|m|s|u|sdw41|sdw41|8777|/data/mirror777 +1780|778|m|m|s|u|sdw41|sdw41|8778|/data/mirror778 +1781|779|m|m|s|u|sdw41|sdw41|8779|/data/mirror779 +1782|780|m|m|s|u|sdw41|sdw41|8780|/data/mirror780 +1783|781|m|m|s|u|sdw41|sdw41|8781|/data/mirror781 +1784|782|m|m|s|u|sdw41|sdw41|8782|/data/mirror782 +1785|783|m|m|s|u|sdw41|sdw41|8783|/data/mirror783 +1786|784|m|m|s|u|sdw41|sdw41|8784|/data/mirror784 +1787|785|m|m|s|u|sdw41|sdw41|8785|/data/mirror785 +1788|786|m|m|s|u|sdw41|sdw41|8786|/data/mirror786 +1789|787|m|m|s|u|sdw41|sdw41|8787|/data/mirror787 +# SDW42 +835|833|p|p|s|u|sdw42|sdw42|7833|/data/primary833 +836|834|p|p|s|u|sdw42|sdw42|7834|/data/primary834 +837|835|p|p|s|u|sdw42|sdw42|7835|/data/primary835 +838|836|p|p|s|u|sdw42|sdw42|7836|/data/primary836 +839|837|p|p|s|u|sdw42|sdw42|7837|/data/primary837 +840|838|p|p|s|u|sdw42|sdw42|7838|/data/primary838 +841|839|p|p|s|u|sdw42|sdw42|7839|/data/primary839 +842|840|p|p|s|u|sdw42|sdw42|7840|/data/primary840 +843|841|p|p|s|u|sdw42|sdw42|7841|/data/primary841 +844|842|p|p|s|u|sdw42|sdw42|7842|/data/primary842 +845|843|p|p|s|u|sdw42|sdw42|7843|/data/primary843 +846|844|p|p|s|u|sdw42|sdw42|7844|/data/primary844 +847|845|p|p|s|u|sdw42|sdw42|7845|/data/primary845 +848|846|p|p|s|u|sdw42|sdw42|7846|/data/primary846 +849|847|p|p|s|u|sdw42|sdw42|7847|/data/primary847 +850|848|p|p|s|u|sdw42|sdw42|7848|/data/primary848 +851|849|p|p|s|u|sdw42|sdw42|7849|/data/primary849 +852|850|p|p|s|u|sdw42|sdw42|7850|/data/primary850 +853|851|p|p|s|u|sdw42|sdw42|7851|/data/primary851 +854|852|p|p|s|u|sdw42|sdw42|7852|/data/primary852 +855|853|p|p|s|u|sdw42|sdw42|7853|/data/primary853 +1790|788|m|m|s|u|sdw42|sdw42|8788|/data/mirror788 +1791|789|m|m|s|u|sdw42|sdw42|8789|/data/mirror789 +1792|790|m|m|s|u|sdw42|sdw42|8790|/data/mirror790 +1793|791|m|m|s|u|sdw42|sdw42|8791|/data/mirror791 +1794|792|m|m|s|u|sdw42|sdw42|8792|/data/mirror792 +1795|793|m|m|s|u|sdw42|sdw42|8793|/data/mirror793 +1796|794|m|m|s|u|sdw42|sdw42|8794|/data/mirror794 +1797|795|m|m|s|u|sdw42|sdw42|8795|/data/mirror795 +1798|796|m|m|s|u|sdw42|sdw42|8796|/data/mirror796 +1799|797|m|m|s|u|sdw42|sdw42|8797|/data/mirror797 +1800|798|m|m|s|u|sdw42|sdw42|8798|/data/mirror798 +1801|799|m|m|s|u|sdw42|sdw42|8799|/data/mirror799 +1802|800|m|m|s|u|sdw42|sdw42|8800|/data/mirror800 +1803|801|m|m|s|u|sdw42|sdw42|8801|/data/mirror801 +1804|802|m|m|s|u|sdw42|sdw42|8802|/data/mirror802 +1805|803|m|m|s|u|sdw42|sdw42|8803|/data/mirror803 +1806|804|m|m|s|u|sdw42|sdw42|8804|/data/mirror804 +1807|805|m|m|s|u|sdw42|sdw42|8805|/data/mirror805 +1808|806|m|m|s|u|sdw42|sdw42|8806|/data/mirror806 +1809|807|m|m|s|u|sdw42|sdw42|8807|/data/mirror807 +1810|808|m|m|s|u|sdw42|sdw42|8808|/data/mirror808 +1811|809|m|m|s|u|sdw42|sdw42|8809|/data/mirror809 +1812|810|m|m|s|u|sdw42|sdw42|8810|/data/mirror810 +1813|811|m|m|s|u|sdw42|sdw42|8811|/data/mirror811 +# SDW43 +856|854|p|p|s|u|sdw43|sdw43|7854|/data/primary854 +857|855|p|p|s|u|sdw43|sdw43|7855|/data/primary855 +858|856|p|p|s|u|sdw43|sdw43|7856|/data/primary856 +859|857|p|p|s|u|sdw43|sdw43|7857|/data/primary857 +860|858|p|p|s|u|sdw43|sdw43|7858|/data/primary858 +861|859|p|p|s|u|sdw43|sdw43|7859|/data/primary859 +862|860|p|p|s|u|sdw43|sdw43|7860|/data/primary860 +863|861|p|p|s|u|sdw43|sdw43|7861|/data/primary861 +864|862|p|p|s|u|sdw43|sdw43|7862|/data/primary862 +865|863|p|p|s|u|sdw43|sdw43|7863|/data/primary863 +866|864|p|p|s|u|sdw43|sdw43|7864|/data/primary864 +867|865|p|p|s|u|sdw43|sdw43|7865|/data/primary865 +868|866|p|p|s|u|sdw43|sdw43|7866|/data/primary866 +869|867|p|p|s|u|sdw43|sdw43|7867|/data/primary867 +870|868|p|p|s|u|sdw43|sdw43|7868|/data/primary868 +871|869|p|p|s|u|sdw43|sdw43|7869|/data/primary869 +872|870|p|p|s|u|sdw43|sdw43|7870|/data/primary870 +873|871|p|p|s|u|sdw43|sdw43|7871|/data/primary871 +1835|833|m|m|s|u|sdw43|sdw43|8833|/data/mirror833 +1836|834|m|m|s|u|sdw43|sdw43|8834|/data/mirror834 +1837|835|m|m|s|u|sdw43|sdw43|8835|/data/mirror835 +1838|836|m|m|s|u|sdw43|sdw43|8836|/data/mirror836 +1839|837|m|m|s|u|sdw43|sdw43|8837|/data/mirror837 +1840|838|m|m|s|u|sdw43|sdw43|8838|/data/mirror838 +1841|839|m|m|s|u|sdw43|sdw43|8839|/data/mirror839 +1842|840|m|m|s|u|sdw43|sdw43|8840|/data/mirror840 +1843|841|m|m|s|u|sdw43|sdw43|8841|/data/mirror841 +1844|842|m|m|s|u|sdw43|sdw43|8842|/data/mirror842 +1845|843|m|m|s|u|sdw43|sdw43|8843|/data/mirror843 +1846|844|m|m|s|u|sdw43|sdw43|8844|/data/mirror844 +1847|845|m|m|s|u|sdw43|sdw43|8845|/data/mirror845 +1848|846|m|m|s|u|sdw43|sdw43|8846|/data/mirror846 +1849|847|m|m|s|u|sdw43|sdw43|8847|/data/mirror847 +1850|848|m|m|s|u|sdw43|sdw43|8848|/data/mirror848 +1851|849|m|m|s|u|sdw43|sdw43|8849|/data/mirror849 +1852|850|m|m|s|u|sdw43|sdw43|8850|/data/mirror850 +1853|851|m|m|s|u|sdw43|sdw43|8851|/data/mirror851 +1854|852|m|m|s|u|sdw43|sdw43|8852|/data/mirror852 +1855|853|m|m|s|u|sdw43|sdw43|8853|/data/mirror853 +# SDW44 +874|872|p|p|s|u|sdw44|sdw44|7872|/data/primary872 +875|873|p|p|s|u|sdw44|sdw44|7873|/data/primary873 +876|874|p|p|s|u|sdw44|sdw44|7874|/data/primary874 +877|875|p|p|s|u|sdw44|sdw44|7875|/data/primary875 +878|876|p|p|s|u|sdw44|sdw44|7876|/data/primary876 +879|877|p|p|s|u|sdw44|sdw44|7877|/data/primary877 +880|878|p|p|s|u|sdw44|sdw44|7878|/data/primary878 +881|879|p|p|s|u|sdw44|sdw44|7879|/data/primary879 +882|880|p|p|s|u|sdw44|sdw44|7880|/data/primary880 +883|881|p|p|s|u|sdw44|sdw44|7881|/data/primary881 +884|882|p|p|s|u|sdw44|sdw44|7882|/data/primary882 +885|883|p|p|s|u|sdw44|sdw44|7883|/data/primary883 +886|884|p|p|s|u|sdw44|sdw44|7884|/data/primary884 +887|885|p|p|s|u|sdw44|sdw44|7885|/data/primary885 +888|886|p|p|s|u|sdw44|sdw44|7886|/data/primary886 +889|887|p|p|s|u|sdw44|sdw44|7887|/data/primary887 +890|888|p|p|s|u|sdw44|sdw44|7888|/data/primary888 +891|889|p|p|s|u|sdw44|sdw44|7889|/data/primary889 +892|890|p|p|s|u|sdw44|sdw44|7890|/data/primary890 +893|891|p|p|s|u|sdw44|sdw44|7891|/data/primary891 +894|892|p|p|s|u|sdw44|sdw44|7892|/data/primary892 +1986|984|m|m|s|u|sdw44|sdw44|8984|/data/mirror984 +1987|985|m|m|s|u|sdw44|sdw44|8985|/data/mirror985 +1988|986|m|m|s|u|sdw44|sdw44|8986|/data/mirror986 +1989|987|m|m|s|u|sdw44|sdw44|8987|/data/mirror987 +1990|988|m|m|s|u|sdw44|sdw44|8988|/data/mirror988 +1991|989|m|m|s|u|sdw44|sdw44|8989|/data/mirror989 +1992|990|m|m|s|u|sdw44|sdw44|8990|/data/mirror990 +1993|991|m|m|s|u|sdw44|sdw44|8991|/data/mirror991 +1994|992|m|m|s|u|sdw44|sdw44|8992|/data/mirror992 +1995|993|m|m|s|u|sdw44|sdw44|8993|/data/mirror993 +1996|994|m|m|s|u|sdw44|sdw44|8994|/data/mirror994 +1997|995|m|m|s|u|sdw44|sdw44|8995|/data/mirror995 +1998|996|m|m|s|u|sdw44|sdw44|8996|/data/mirror996 +1999|997|m|m|s|u|sdw44|sdw44|8997|/data/mirror997 +2000|998|m|m|s|u|sdw44|sdw44|8998|/data/mirror998 +2001|999|m|m|s|u|sdw44|sdw44|8999|/data/mirror999 +# SDW45 +895|893|p|p|s|u|sdw45|sdw45|7893|/data/primary893 +896|894|p|p|s|u|sdw45|sdw45|7894|/data/primary894 +897|895|p|p|s|u|sdw45|sdw45|7895|/data/primary895 +898|896|p|p|s|u|sdw45|sdw45|7896|/data/primary896 +899|897|p|p|s|u|sdw45|sdw45|7897|/data/primary897 +900|898|p|p|s|u|sdw45|sdw45|7898|/data/primary898 +901|899|p|p|s|u|sdw45|sdw45|7899|/data/primary899 +902|900|p|p|s|u|sdw45|sdw45|7900|/data/primary900 +903|901|p|p|s|u|sdw45|sdw45|7901|/data/primary901 +904|902|p|p|s|u|sdw45|sdw45|7902|/data/primary902 +905|903|p|p|s|u|sdw45|sdw45|7903|/data/primary903 +906|904|p|p|s|u|sdw45|sdw45|7904|/data/primary904 +907|905|p|p|s|u|sdw45|sdw45|7905|/data/primary905 +908|906|p|p|s|u|sdw45|sdw45|7906|/data/primary906 +909|907|p|p|s|u|sdw45|sdw45|7907|/data/primary907 +910|908|p|p|s|u|sdw45|sdw45|7908|/data/primary908 +911|909|p|p|s|u|sdw45|sdw45|7909|/data/primary909 +912|910|p|p|s|u|sdw45|sdw45|7910|/data/primary910 +913|911|p|p|s|u|sdw45|sdw45|7911|/data/primary911 +1931|929|m|m|s|u|sdw45|sdw45|8929|/data/mirror929 +1932|930|m|m|s|u|sdw45|sdw45|8930|/data/mirror930 +1933|931|m|m|s|u|sdw45|sdw45|8931|/data/mirror931 +1934|932|m|m|s|u|sdw45|sdw45|8932|/data/mirror932 +1935|933|m|m|s|u|sdw45|sdw45|8933|/data/mirror933 +1936|934|m|m|s|u|sdw45|sdw45|8934|/data/mirror934 +1937|935|m|m|s|u|sdw45|sdw45|8935|/data/mirror935 +1938|936|m|m|s|u|sdw45|sdw45|8936|/data/mirror936 +1939|937|m|m|s|u|sdw45|sdw45|8937|/data/mirror937 +1940|938|m|m|s|u|sdw45|sdw45|8938|/data/mirror938 +1941|939|m|m|s|u|sdw45|sdw45|8939|/data/mirror939 +1942|940|m|m|s|u|sdw45|sdw45|8940|/data/mirror940 +1943|941|m|m|s|u|sdw45|sdw45|8941|/data/mirror941 +1944|942|m|m|s|u|sdw45|sdw45|8942|/data/mirror942 +1945|943|m|m|s|u|sdw45|sdw45|8943|/data/mirror943 +1946|944|m|m|s|u|sdw45|sdw45|8944|/data/mirror944 +1947|945|m|m|s|u|sdw45|sdw45|8945|/data/mirror945 +# SDW46 +914|912|p|p|s|u|sdw46|sdw46|7912|/data/primary912 +915|913|p|p|s|u|sdw46|sdw46|7913|/data/primary913 +916|914|p|p|s|u|sdw46|sdw46|7914|/data/primary914 +917|915|p|p|s|u|sdw46|sdw46|7915|/data/primary915 +918|916|p|p|s|u|sdw46|sdw46|7916|/data/primary916 +919|917|p|p|s|u|sdw46|sdw46|7917|/data/primary917 +920|918|p|p|s|u|sdw46|sdw46|7918|/data/primary918 +921|919|p|p|s|u|sdw46|sdw46|7919|/data/primary919 +922|920|p|p|s|u|sdw46|sdw46|7920|/data/primary920 +923|921|p|p|s|u|sdw46|sdw46|7921|/data/primary921 +924|922|p|p|s|u|sdw46|sdw46|7922|/data/primary922 +925|923|p|p|s|u|sdw46|sdw46|7923|/data/primary923 +926|924|p|p|s|u|sdw46|sdw46|7924|/data/primary924 +927|925|p|p|s|u|sdw46|sdw46|7925|/data/primary925 +928|926|p|p|s|u|sdw46|sdw46|7926|/data/primary926 +929|927|p|p|s|u|sdw46|sdw46|7927|/data/primary927 +930|928|p|p|s|u|sdw46|sdw46|7928|/data/primary928 +1874|872|m|m|s|u|sdw46|sdw46|8872|/data/mirror872 +1875|873|m|m|s|u|sdw46|sdw46|8873|/data/mirror873 +1876|874|m|m|s|u|sdw46|sdw46|8874|/data/mirror874 +1877|875|m|m|s|u|sdw46|sdw46|8875|/data/mirror875 +1878|876|m|m|s|u|sdw46|sdw46|8876|/data/mirror876 +1879|877|m|m|s|u|sdw46|sdw46|8877|/data/mirror877 +1880|878|m|m|s|u|sdw46|sdw46|8878|/data/mirror878 +1881|879|m|m|s|u|sdw46|sdw46|8879|/data/mirror879 +1882|880|m|m|s|u|sdw46|sdw46|8880|/data/mirror880 +1883|881|m|m|s|u|sdw46|sdw46|8881|/data/mirror881 +1884|882|m|m|s|u|sdw46|sdw46|8882|/data/mirror882 +1885|883|m|m|s|u|sdw46|sdw46|8883|/data/mirror883 +1886|884|m|m|s|u|sdw46|sdw46|8884|/data/mirror884 +1887|885|m|m|s|u|sdw46|sdw46|8885|/data/mirror885 +1888|886|m|m|s|u|sdw46|sdw46|8886|/data/mirror886 +1889|887|m|m|s|u|sdw46|sdw46|8887|/data/mirror887 +1890|888|m|m|s|u|sdw46|sdw46|8888|/data/mirror888 +1891|889|m|m|s|u|sdw46|sdw46|8889|/data/mirror889 +1892|890|m|m|s|u|sdw46|sdw46|8890|/data/mirror890 +1893|891|m|m|s|u|sdw46|sdw46|8891|/data/mirror891 +1894|892|m|m|s|u|sdw46|sdw46|8892|/data/mirror892 +# SDW47 +931|929|p|p|s|u|sdw47|sdw47|7929|/data/primary929 +932|930|p|p|s|u|sdw47|sdw47|7930|/data/primary930 +933|931|p|p|s|u|sdw47|sdw47|7931|/data/primary931 +934|932|p|p|s|u|sdw47|sdw47|7932|/data/primary932 +935|933|p|p|s|u|sdw47|sdw47|7933|/data/primary933 +936|934|p|p|s|u|sdw47|sdw47|7934|/data/primary934 +937|935|p|p|s|u|sdw47|sdw47|7935|/data/primary935 +938|936|p|p|s|u|sdw47|sdw47|7936|/data/primary936 +939|937|p|p|s|u|sdw47|sdw47|7937|/data/primary937 +940|938|p|p|s|u|sdw47|sdw47|7938|/data/primary938 +941|939|p|p|s|u|sdw47|sdw47|7939|/data/primary939 +942|940|p|p|s|u|sdw47|sdw47|7940|/data/primary940 +943|941|p|p|s|u|sdw47|sdw47|7941|/data/primary941 +944|942|p|p|s|u|sdw47|sdw47|7942|/data/primary942 +945|943|p|p|s|u|sdw47|sdw47|7943|/data/primary943 +946|944|p|p|s|u|sdw47|sdw47|7944|/data/primary944 +947|945|p|p|s|u|sdw47|sdw47|7945|/data/primary945 +1895|893|m|m|s|u|sdw47|sdw47|8893|/data/mirror893 +1896|894|m|m|s|u|sdw47|sdw47|8894|/data/mirror894 +1897|895|m|m|s|u|sdw47|sdw47|8895|/data/mirror895 +1898|896|m|m|s|u|sdw47|sdw47|8896|/data/mirror896 +1899|897|m|m|s|u|sdw47|sdw47|8897|/data/mirror897 +1900|898|m|m|s|u|sdw47|sdw47|8898|/data/mirror898 +1901|899|m|m|s|u|sdw47|sdw47|8899|/data/mirror899 +1902|900|m|m|s|u|sdw47|sdw47|8900|/data/mirror900 +1903|901|m|m|s|u|sdw47|sdw47|8901|/data/mirror901 +1904|902|m|m|s|u|sdw47|sdw47|8902|/data/mirror902 +1905|903|m|m|s|u|sdw47|sdw47|8903|/data/mirror903 +1906|904|m|m|s|u|sdw47|sdw47|8904|/data/mirror904 +1907|905|m|m|s|u|sdw47|sdw47|8905|/data/mirror905 +1908|906|m|m|s|u|sdw47|sdw47|8906|/data/mirror906 +1909|907|m|m|s|u|sdw47|sdw47|8907|/data/mirror907 +1910|908|m|m|s|u|sdw47|sdw47|8908|/data/mirror908 +1911|909|m|m|s|u|sdw47|sdw47|8909|/data/mirror909 +1912|910|m|m|s|u|sdw47|sdw47|8910|/data/mirror910 +1913|911|m|m|s|u|sdw47|sdw47|8911|/data/mirror911 +# SDW48 +948|946|p|p|s|u|sdw48|sdw48|7946|/data/primary946 +949|947|p|p|s|u|sdw48|sdw48|7947|/data/primary947 +950|948|p|p|s|u|sdw48|sdw48|7948|/data/primary948 +951|949|p|p|s|u|sdw48|sdw48|7949|/data/primary949 +952|950|p|p|s|u|sdw48|sdw48|7950|/data/primary950 +953|951|p|p|s|u|sdw48|sdw48|7951|/data/primary951 +954|952|p|p|s|u|sdw48|sdw48|7952|/data/primary952 +955|953|p|p|s|u|sdw48|sdw48|7953|/data/primary953 +956|954|p|p|s|u|sdw48|sdw48|7954|/data/primary954 +957|955|p|p|s|u|sdw48|sdw48|7955|/data/primary955 +958|956|p|p|s|u|sdw48|sdw48|7956|/data/primary956 +959|957|p|p|s|u|sdw48|sdw48|7957|/data/primary957 +960|958|p|p|s|u|sdw48|sdw48|7958|/data/primary958 +961|959|p|p|s|u|sdw48|sdw48|7959|/data/primary959 +962|960|p|p|s|u|sdw48|sdw48|7960|/data/primary960 +963|961|p|p|s|u|sdw48|sdw48|7961|/data/primary961 +964|962|p|p|s|u|sdw48|sdw48|7962|/data/primary962 +1914|912|m|m|s|u|sdw48|sdw48|8912|/data/mirror912 +1915|913|m|m|s|u|sdw48|sdw48|8913|/data/mirror913 +1916|914|m|m|s|u|sdw48|sdw48|8914|/data/mirror914 +1917|915|m|m|s|u|sdw48|sdw48|8915|/data/mirror915 +1918|916|m|m|s|u|sdw48|sdw48|8916|/data/mirror916 +1919|917|m|m|s|u|sdw48|sdw48|8917|/data/mirror917 +1920|918|m|m|s|u|sdw48|sdw48|8918|/data/mirror918 +1921|919|m|m|s|u|sdw48|sdw48|8919|/data/mirror919 +1922|920|m|m|s|u|sdw48|sdw48|8920|/data/mirror920 +1923|921|m|m|s|u|sdw48|sdw48|8921|/data/mirror921 +1924|922|m|m|s|u|sdw48|sdw48|8922|/data/mirror922 +1925|923|m|m|s|u|sdw48|sdw48|8923|/data/mirror923 +1926|924|m|m|s|u|sdw48|sdw48|8924|/data/mirror924 +1927|925|m|m|s|u|sdw48|sdw48|8925|/data/mirror925 +1928|926|m|m|s|u|sdw48|sdw48|8926|/data/mirror926 +1929|927|m|m|s|u|sdw48|sdw48|8927|/data/mirror927 +1930|928|m|m|s|u|sdw48|sdw48|8928|/data/mirror928 +# SDW49 +965|963|p|p|s|u|sdw49|sdw49|7963|/data/primary963 +966|964|p|p|s|u|sdw49|sdw49|7964|/data/primary964 +967|965|p|p|s|u|sdw49|sdw49|7965|/data/primary965 +968|966|p|p|s|u|sdw49|sdw49|7966|/data/primary966 +969|967|p|p|s|u|sdw49|sdw49|7967|/data/primary967 +970|968|p|p|s|u|sdw49|sdw49|7968|/data/primary968 +971|969|p|p|s|u|sdw49|sdw49|7969|/data/primary969 +972|970|p|p|s|u|sdw49|sdw49|7970|/data/primary970 +973|971|p|p|s|u|sdw49|sdw49|7971|/data/primary971 +974|972|p|p|s|u|sdw49|sdw49|7972|/data/primary972 +975|973|p|p|s|u|sdw49|sdw49|7973|/data/primary973 +976|974|p|p|s|u|sdw49|sdw49|7974|/data/primary974 +977|975|p|p|s|u|sdw49|sdw49|7975|/data/primary975 +978|976|p|p|s|u|sdw49|sdw49|7976|/data/primary976 +979|977|p|p|s|u|sdw49|sdw49|7977|/data/primary977 +980|978|p|p|s|u|sdw49|sdw49|7978|/data/primary978 +981|979|p|p|s|u|sdw49|sdw49|7979|/data/primary979 +982|980|p|p|s|u|sdw49|sdw49|7980|/data/primary980 +983|981|p|p|s|u|sdw49|sdw49|7981|/data/primary981 +984|982|p|p|s|u|sdw49|sdw49|7982|/data/primary982 +985|983|p|p|s|u|sdw49|sdw49|7983|/data/primary983 +1948|946|m|m|s|u|sdw49|sdw49|8946|/data/mirror946 +1949|947|m|m|s|u|sdw49|sdw49|8947|/data/mirror947 +1950|948|m|m|s|u|sdw49|sdw49|8948|/data/mirror948 +1951|949|m|m|s|u|sdw49|sdw49|8949|/data/mirror949 +1952|950|m|m|s|u|sdw49|sdw49|8950|/data/mirror950 +1953|951|m|m|s|u|sdw49|sdw49|8951|/data/mirror951 +1954|952|m|m|s|u|sdw49|sdw49|8952|/data/mirror952 +1955|953|m|m|s|u|sdw49|sdw49|8953|/data/mirror953 +1956|954|m|m|s|u|sdw49|sdw49|8954|/data/mirror954 +1957|955|m|m|s|u|sdw49|sdw49|8955|/data/mirror955 +1958|956|m|m|s|u|sdw49|sdw49|8956|/data/mirror956 +1959|957|m|m|s|u|sdw49|sdw49|8957|/data/mirror957 +1960|958|m|m|s|u|sdw49|sdw49|8958|/data/mirror958 +1961|959|m|m|s|u|sdw49|sdw49|8959|/data/mirror959 +1962|960|m|m|s|u|sdw49|sdw49|8960|/data/mirror960 +1963|961|m|m|s|u|sdw49|sdw49|8961|/data/mirror961 +1964|962|m|m|s|u|sdw49|sdw49|8962|/data/mirror962 +# SDW5 +74|72|p|p|s|u|sdw5|sdw5|7072|/data/primary72 +75|73|p|p|s|u|sdw5|sdw5|7073|/data/primary73 +76|74|p|p|s|u|sdw5|sdw5|7074|/data/primary74 +77|75|p|p|s|u|sdw5|sdw5|7075|/data/primary75 +78|76|p|p|s|u|sdw5|sdw5|7076|/data/primary76 +79|77|p|p|s|u|sdw5|sdw5|7077|/data/primary77 +80|78|p|p|s|u|sdw5|sdw5|7078|/data/primary78 +81|79|p|p|s|u|sdw5|sdw5|7079|/data/primary79 +82|80|p|p|s|u|sdw5|sdw5|7080|/data/primary80 +83|81|p|p|s|u|sdw5|sdw5|7081|/data/primary81 +84|82|p|p|s|u|sdw5|sdw5|7082|/data/primary82 +85|83|p|p|s|u|sdw5|sdw5|7083|/data/primary83 +86|84|p|p|s|u|sdw5|sdw5|7084|/data/primary84 +87|85|p|p|s|u|sdw5|sdw5|7085|/data/primary85 +88|86|p|p|s|u|sdw5|sdw5|7086|/data/primary86 +89|87|p|p|s|u|sdw5|sdw5|7087|/data/primary87 +90|88|p|p|s|u|sdw5|sdw5|7088|/data/primary88 +91|89|p|p|s|u|sdw5|sdw5|7089|/data/primary89 +92|90|p|p|s|u|sdw5|sdw5|7090|/data/primary90 +93|91|p|p|s|u|sdw5|sdw5|7091|/data/primary91 +94|92|p|p|s|u|sdw5|sdw5|7092|/data/primary92 +95|93|p|p|s|u|sdw5|sdw5|7093|/data/primary93 +96|94|p|p|s|u|sdw5|sdw5|7094|/data/primary94 +1056|54|m|m|s|u|sdw5|sdw5|8054|/data/mirror54 +1057|55|m|m|s|u|sdw5|sdw5|8055|/data/mirror55 +1058|56|m|m|s|u|sdw5|sdw5|8056|/data/mirror56 +1059|57|m|m|s|u|sdw5|sdw5|8057|/data/mirror57 +1060|58|m|m|s|u|sdw5|sdw5|8058|/data/mirror58 +1061|59|m|m|s|u|sdw5|sdw5|8059|/data/mirror59 +1062|60|m|m|s|u|sdw5|sdw5|8060|/data/mirror60 +1063|61|m|m|s|u|sdw5|sdw5|8061|/data/mirror61 +1064|62|m|m|s|u|sdw5|sdw5|8062|/data/mirror62 +1065|63|m|m|s|u|sdw5|sdw5|8063|/data/mirror63 +1066|64|m|m|s|u|sdw5|sdw5|8064|/data/mirror64 +1067|65|m|m|s|u|sdw5|sdw5|8065|/data/mirror65 +1068|66|m|m|s|u|sdw5|sdw5|8066|/data/mirror66 +1069|67|m|m|s|u|sdw5|sdw5|8067|/data/mirror67 +1070|68|m|m|s|u|sdw5|sdw5|8068|/data/mirror68 +1071|69|m|m|s|u|sdw5|sdw5|8069|/data/mirror69 +1072|70|m|m|s|u|sdw5|sdw5|8070|/data/mirror70 +1073|71|m|m|s|u|sdw5|sdw5|8071|/data/mirror71 +# SDW50 +986|984|p|p|s|u|sdw50|sdw50|7984|/data/primary984 +987|985|p|p|s|u|sdw50|sdw50|7985|/data/primary985 +988|986|p|p|s|u|sdw50|sdw50|7986|/data/primary986 +989|987|p|p|s|u|sdw50|sdw50|7987|/data/primary987 +990|988|p|p|s|u|sdw50|sdw50|7988|/data/primary988 +991|989|p|p|s|u|sdw50|sdw50|7989|/data/primary989 +992|990|p|p|s|u|sdw50|sdw50|7990|/data/primary990 +993|991|p|p|s|u|sdw50|sdw50|7991|/data/primary991 +994|992|p|p|s|u|sdw50|sdw50|7992|/data/primary992 +995|993|p|p|s|u|sdw50|sdw50|7993|/data/primary993 +996|994|p|p|s|u|sdw50|sdw50|7994|/data/primary994 +997|995|p|p|s|u|sdw50|sdw50|7995|/data/primary995 +998|996|p|p|s|u|sdw50|sdw50|7996|/data/primary996 +999|997|p|p|s|u|sdw50|sdw50|7997|/data/primary997 +1000|998|p|p|s|u|sdw50|sdw50|7998|/data/primary998 +1001|999|p|p|s|u|sdw50|sdw50|7999|/data/primary999 +# SDW6 +97|95|p|p|s|u|sdw6|sdw6|7095|/data/primary95 +98|96|p|p|s|u|sdw6|sdw6|7096|/data/primary96 +99|97|p|p|s|u|sdw6|sdw6|7097|/data/primary97 +100|98|p|p|s|u|sdw6|sdw6|7098|/data/primary98 +101|99|p|p|s|u|sdw6|sdw6|7099|/data/primary99 +102|100|p|p|s|u|sdw6|sdw6|7100|/data/primary100 +103|101|p|p|s|u|sdw6|sdw6|7101|/data/primary101 +104|102|p|p|s|u|sdw6|sdw6|7102|/data/primary102 +105|103|p|p|s|u|sdw6|sdw6|7103|/data/primary103 +106|104|p|p|s|u|sdw6|sdw6|7104|/data/primary104 +107|105|p|p|s|u|sdw6|sdw6|7105|/data/primary105 +108|106|p|p|s|u|sdw6|sdw6|7106|/data/primary106 +109|107|p|p|s|u|sdw6|sdw6|7107|/data/primary107 +110|108|p|p|s|u|sdw6|sdw6|7108|/data/primary108 +111|109|p|p|s|u|sdw6|sdw6|7109|/data/primary109 +112|110|p|p|s|u|sdw6|sdw6|7110|/data/primary110 +113|111|p|p|s|u|sdw6|sdw6|7111|/data/primary111 +1074|72|m|m|s|u|sdw6|sdw6|8072|/data/mirror72 +1075|73|m|m|s|u|sdw6|sdw6|8073|/data/mirror73 +1076|74|m|m|s|u|sdw6|sdw6|8074|/data/mirror74 +1077|75|m|m|s|u|sdw6|sdw6|8075|/data/mirror75 +1078|76|m|m|s|u|sdw6|sdw6|8076|/data/mirror76 +1079|77|m|m|s|u|sdw6|sdw6|8077|/data/mirror77 +1080|78|m|m|s|u|sdw6|sdw6|8078|/data/mirror78 +1081|79|m|m|s|u|sdw6|sdw6|8079|/data/mirror79 +1082|80|m|m|s|u|sdw6|sdw6|8080|/data/mirror80 +1083|81|m|m|s|u|sdw6|sdw6|8081|/data/mirror81 +1084|82|m|m|s|u|sdw6|sdw6|8082|/data/mirror82 +1085|83|m|m|s|u|sdw6|sdw6|8083|/data/mirror83 +1086|84|m|m|s|u|sdw6|sdw6|8084|/data/mirror84 +1087|85|m|m|s|u|sdw6|sdw6|8085|/data/mirror85 +1088|86|m|m|s|u|sdw6|sdw6|8086|/data/mirror86 +1089|87|m|m|s|u|sdw6|sdw6|8087|/data/mirror87 +1090|88|m|m|s|u|sdw6|sdw6|8088|/data/mirror88 +1091|89|m|m|s|u|sdw6|sdw6|8089|/data/mirror89 +1092|90|m|m|s|u|sdw6|sdw6|8090|/data/mirror90 +1093|91|m|m|s|u|sdw6|sdw6|8091|/data/mirror91 +1094|92|m|m|s|u|sdw6|sdw6|8092|/data/mirror92 +1095|93|m|m|s|u|sdw6|sdw6|8093|/data/mirror93 +1096|94|m|m|s|u|sdw6|sdw6|8094|/data/mirror94 +# SDW7 +114|112|p|p|s|u|sdw7|sdw7|7112|/data/primary112 +115|113|p|p|s|u|sdw7|sdw7|7113|/data/primary113 +116|114|p|p|s|u|sdw7|sdw7|7114|/data/primary114 +117|115|p|p|s|u|sdw7|sdw7|7115|/data/primary115 +118|116|p|p|s|u|sdw7|sdw7|7116|/data/primary116 +119|117|p|p|s|u|sdw7|sdw7|7117|/data/primary117 +120|118|p|p|s|u|sdw7|sdw7|7118|/data/primary118 +121|119|p|p|s|u|sdw7|sdw7|7119|/data/primary119 +122|120|p|p|s|u|sdw7|sdw7|7120|/data/primary120 +123|121|p|p|s|u|sdw7|sdw7|7121|/data/primary121 +124|122|p|p|s|u|sdw7|sdw7|7122|/data/primary122 +125|123|p|p|s|u|sdw7|sdw7|7123|/data/primary123 +126|124|p|p|s|u|sdw7|sdw7|7124|/data/primary124 +127|125|p|p|s|u|sdw7|sdw7|7125|/data/primary125 +128|126|p|p|s|u|sdw7|sdw7|7126|/data/primary126 +129|127|p|p|s|u|sdw7|sdw7|7127|/data/primary127 +130|128|p|p|s|u|sdw7|sdw7|7128|/data/primary128 +131|129|p|p|s|u|sdw7|sdw7|7129|/data/primary129 +1175|173|m|m|s|u|sdw7|sdw7|8173|/data/mirror173 +1176|174|m|m|s|u|sdw7|sdw7|8174|/data/mirror174 +1177|175|m|m|s|u|sdw7|sdw7|8175|/data/mirror175 +1178|176|m|m|s|u|sdw7|sdw7|8176|/data/mirror176 +1179|177|m|m|s|u|sdw7|sdw7|8177|/data/mirror177 +1180|178|m|m|s|u|sdw7|sdw7|8178|/data/mirror178 +1181|179|m|m|s|u|sdw7|sdw7|8179|/data/mirror179 +1182|180|m|m|s|u|sdw7|sdw7|8180|/data/mirror180 +1183|181|m|m|s|u|sdw7|sdw7|8181|/data/mirror181 +1184|182|m|m|s|u|sdw7|sdw7|8182|/data/mirror182 +1185|183|m|m|s|u|sdw7|sdw7|8183|/data/mirror183 +1186|184|m|m|s|u|sdw7|sdw7|8184|/data/mirror184 +1187|185|m|m|s|u|sdw7|sdw7|8185|/data/mirror185 +1188|186|m|m|s|u|sdw7|sdw7|8186|/data/mirror186 +1189|187|m|m|s|u|sdw7|sdw7|8187|/data/mirror187 +1190|188|m|m|s|u|sdw7|sdw7|8188|/data/mirror188 +1191|189|m|m|s|u|sdw7|sdw7|8189|/data/mirror189 +1192|190|m|m|s|u|sdw7|sdw7|8190|/data/mirror190 +1193|191|m|m|s|u|sdw7|sdw7|8191|/data/mirror191 +1194|192|m|m|s|u|sdw7|sdw7|8192|/data/mirror192 +1195|193|m|m|s|u|sdw7|sdw7|8193|/data/mirror193 +1196|194|m|m|s|u|sdw7|sdw7|8194|/data/mirror194 +1197|195|m|m|s|u|sdw7|sdw7|8195|/data/mirror195 +1198|196|m|m|s|u|sdw7|sdw7|8196|/data/mirror196 +# SDW8 +132|130|p|p|s|u|sdw8|sdw8|7130|/data/primary130 +133|131|p|p|s|u|sdw8|sdw8|7131|/data/primary131 +134|132|p|p|s|u|sdw8|sdw8|7132|/data/primary132 +135|133|p|p|s|u|sdw8|sdw8|7133|/data/primary133 +136|134|p|p|s|u|sdw8|sdw8|7134|/data/primary134 +137|135|p|p|s|u|sdw8|sdw8|7135|/data/primary135 +138|136|p|p|s|u|sdw8|sdw8|7136|/data/primary136 +139|137|p|p|s|u|sdw8|sdw8|7137|/data/primary137 +140|138|p|p|s|u|sdw8|sdw8|7138|/data/primary138 +141|139|p|p|s|u|sdw8|sdw8|7139|/data/primary139 +142|140|p|p|s|u|sdw8|sdw8|7140|/data/primary140 +143|141|p|p|s|u|sdw8|sdw8|7141|/data/primary141 +144|142|p|p|s|u|sdw8|sdw8|7142|/data/primary142 +145|143|p|p|s|u|sdw8|sdw8|7143|/data/primary143 +146|144|p|p|s|u|sdw8|sdw8|7144|/data/primary144 +147|145|p|p|s|u|sdw8|sdw8|7145|/data/primary145 +148|146|p|p|s|u|sdw8|sdw8|7146|/data/primary146 +149|147|p|p|s|u|sdw8|sdw8|7147|/data/primary147 +150|148|p|p|s|u|sdw8|sdw8|7148|/data/primary148 +151|149|p|p|s|u|sdw8|sdw8|7149|/data/primary149 +1219|217|m|m|s|u|sdw8|sdw8|8217|/data/mirror217 +1220|218|m|m|s|u|sdw8|sdw8|8218|/data/mirror218 +1221|219|m|m|s|u|sdw8|sdw8|8219|/data/mirror219 +1222|220|m|m|s|u|sdw8|sdw8|8220|/data/mirror220 +1223|221|m|m|s|u|sdw8|sdw8|8221|/data/mirror221 +1224|222|m|m|s|u|sdw8|sdw8|8222|/data/mirror222 +1225|223|m|m|s|u|sdw8|sdw8|8223|/data/mirror223 +1226|224|m|m|s|u|sdw8|sdw8|8224|/data/mirror224 +1227|225|m|m|s|u|sdw8|sdw8|8225|/data/mirror225 +1228|226|m|m|s|u|sdw8|sdw8|8226|/data/mirror226 +1229|227|m|m|s|u|sdw8|sdw8|8227|/data/mirror227 +1230|228|m|m|s|u|sdw8|sdw8|8228|/data/mirror228 +1231|229|m|m|s|u|sdw8|sdw8|8229|/data/mirror229 +1232|230|m|m|s|u|sdw8|sdw8|8230|/data/mirror230 +1233|231|m|m|s|u|sdw8|sdw8|8231|/data/mirror231 +1234|232|m|m|s|u|sdw8|sdw8|8232|/data/mirror232 +1235|233|m|m|s|u|sdw8|sdw8|8233|/data/mirror233 +1236|234|m|m|s|u|sdw8|sdw8|8234|/data/mirror234 +1237|235|m|m|s|u|sdw8|sdw8|8235|/data/mirror235 +1238|236|m|m|s|u|sdw8|sdw8|8236|/data/mirror236 +1239|237|m|m|s|u|sdw8|sdw8|8237|/data/mirror237 +1240|238|m|m|s|u|sdw8|sdw8|8238|/data/mirror238 +# SDW9 +152|150|p|p|s|u|sdw9|sdw9|7150|/data/primary150 +153|151|p|p|s|u|sdw9|sdw9|7151|/data/primary151 +154|152|p|p|s|u|sdw9|sdw9|7152|/data/primary152 +155|153|p|p|s|u|sdw9|sdw9|7153|/data/primary153 +156|154|p|p|s|u|sdw9|sdw9|7154|/data/primary154 +157|155|p|p|s|u|sdw9|sdw9|7155|/data/primary155 +158|156|p|p|s|u|sdw9|sdw9|7156|/data/primary156 +159|157|p|p|s|u|sdw9|sdw9|7157|/data/primary157 +160|158|p|p|s|u|sdw9|sdw9|7158|/data/primary158 +161|159|p|p|s|u|sdw9|sdw9|7159|/data/primary159 +162|160|p|p|s|u|sdw9|sdw9|7160|/data/primary160 +163|161|p|p|s|u|sdw9|sdw9|7161|/data/primary161 +164|162|p|p|s|u|sdw9|sdw9|7162|/data/primary162 +165|163|p|p|s|u|sdw9|sdw9|7163|/data/primary163 +166|164|p|p|s|u|sdw9|sdw9|7164|/data/primary164 +167|165|p|p|s|u|sdw9|sdw9|7165|/data/primary165 +168|166|p|p|s|u|sdw9|sdw9|7166|/data/primary166 +169|167|p|p|s|u|sdw9|sdw9|7167|/data/primary167 +170|168|p|p|s|u|sdw9|sdw9|7168|/data/primary168 +171|169|p|p|s|u|sdw9|sdw9|7169|/data/primary169 +172|170|p|p|s|u|sdw9|sdw9|7170|/data/primary170 +173|171|p|p|s|u|sdw9|sdw9|7171|/data/primary171 +174|172|p|p|s|u|sdw9|sdw9|7172|/data/primary172 +1114|112|m|m|s|u|sdw9|sdw9|8112|/data/mirror112 +1115|113|m|m|s|u|sdw9|sdw9|8113|/data/mirror113 +1116|114|m|m|s|u|sdw9|sdw9|8114|/data/mirror114 +1117|115|m|m|s|u|sdw9|sdw9|8115|/data/mirror115 +1118|116|m|m|s|u|sdw9|sdw9|8116|/data/mirror116 +1119|117|m|m|s|u|sdw9|sdw9|8117|/data/mirror117 +1120|118|m|m|s|u|sdw9|sdw9|8118|/data/mirror118 +1121|119|m|m|s|u|sdw9|sdw9|8119|/data/mirror119 +1122|120|m|m|s|u|sdw9|sdw9|8120|/data/mirror120 +1123|121|m|m|s|u|sdw9|sdw9|8121|/data/mirror121 +1124|122|m|m|s|u|sdw9|sdw9|8122|/data/mirror122 +1125|123|m|m|s|u|sdw9|sdw9|8123|/data/mirror123 +1126|124|m|m|s|u|sdw9|sdw9|8124|/data/mirror124 +1127|125|m|m|s|u|sdw9|sdw9|8125|/data/mirror125 +1128|126|m|m|s|u|sdw9|sdw9|8126|/data/mirror126 +1129|127|m|m|s|u|sdw9|sdw9|8127|/data/mirror127 +1130|128|m|m|s|u|sdw9|sdw9|8128|/data/mirror128 +1131|129|m|m|s|u|sdw9|sdw9|8129|/data/mirror129 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_spread.array new file mode 100644 index 000000000000..38e9b4295181 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_spread.array @@ -0,0 +1,2052 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +8|6|p|p|s|u|sdw1|sdw1|7006|/data/primary6 +9|7|p|p|s|u|sdw1|sdw1|7007|/data/primary7 +10|8|p|p|s|u|sdw1|sdw1|7008|/data/primary8 +11|9|p|p|s|u|sdw1|sdw1|7009|/data/primary9 +12|10|p|p|s|u|sdw1|sdw1|7010|/data/primary10 +13|11|p|p|s|u|sdw1|sdw1|7011|/data/primary11 +14|12|p|p|s|u|sdw1|sdw1|7012|/data/primary12 +15|13|p|p|s|u|sdw1|sdw1|7013|/data/primary13 +16|14|p|p|s|u|sdw1|sdw1|7014|/data/primary14 +17|15|p|p|s|u|sdw1|sdw1|7015|/data/primary15 +18|16|p|p|s|u|sdw1|sdw1|7016|/data/primary16 +19|17|p|p|s|u|sdw1|sdw1|7017|/data/primary17 +20|18|p|p|s|u|sdw1|sdw1|7018|/data/primary18 +21|19|p|p|s|u|sdw1|sdw1|7019|/data/primary19 +22|20|p|p|s|u|sdw1|sdw1|7020|/data/primary20 +23|21|p|p|s|u|sdw1|sdw1|7021|/data/primary21 +24|22|p|p|s|u|sdw1|sdw1|7022|/data/primary22 +25|23|p|p|s|u|sdw1|sdw1|7023|/data/primary23 +1026|24|m|m|s|u|sdw1|sdw1|8024|/data/mirror24 +1057|55|m|m|s|u|sdw1|sdw1|8055|/data/mirror55 +1099|97|m|m|s|u|sdw1|sdw1|8097|/data/mirror97 +1134|132|m|m|s|u|sdw1|sdw1|8132|/data/mirror132 +1208|206|m|m|s|u|sdw1|sdw1|8206|/data/mirror206 +1253|251|m|m|s|u|sdw1|sdw1|8251|/data/mirror251 +1288|286|m|m|s|u|sdw1|sdw1|8286|/data/mirror286 +1334|332|m|m|s|u|sdw1|sdw1|8332|/data/mirror332 +1403|401|m|m|s|u|sdw1|sdw1|8401|/data/mirror401 +1453|451|m|m|s|u|sdw1|sdw1|8451|/data/mirror451 +1508|506|m|m|s|u|sdw1|sdw1|8506|/data/mirror506 +1557|555|m|m|s|u|sdw1|sdw1|8555|/data/mirror555 +1591|589|m|m|s|u|sdw1|sdw1|8589|/data/mirror589 +1657|655|m|m|s|u|sdw1|sdw1|8655|/data/mirror655 +1710|708|m|m|s|u|sdw1|sdw1|8708|/data/mirror708 +1755|753|m|m|s|u|sdw1|sdw1|8753|/data/mirror753 +1805|803|m|m|s|u|sdw1|sdw1|8803|/data/mirror803 +1848|846|m|m|s|u|sdw1|sdw1|8846|/data/mirror846 +1906|904|m|m|s|u|sdw1|sdw1|8904|/data/mirror904 +1939|937|m|m|s|u|sdw1|sdw1|8937|/data/mirror937 +# SDW10 +193|191|p|p|s|u|sdw10|sdw10|7191|/data/primary191 +194|192|p|p|s|u|sdw10|sdw10|7192|/data/primary192 +195|193|p|p|s|u|sdw10|sdw10|7193|/data/primary193 +196|194|p|p|s|u|sdw10|sdw10|7194|/data/primary194 +197|195|p|p|s|u|sdw10|sdw10|7195|/data/primary195 +198|196|p|p|s|u|sdw10|sdw10|7196|/data/primary196 +199|197|p|p|s|u|sdw10|sdw10|7197|/data/primary197 +200|198|p|p|s|u|sdw10|sdw10|7198|/data/primary198 +201|199|p|p|s|u|sdw10|sdw10|7199|/data/primary199 +202|200|p|p|s|u|sdw10|sdw10|7200|/data/primary200 +203|201|p|p|s|u|sdw10|sdw10|7201|/data/primary201 +204|202|p|p|s|u|sdw10|sdw10|7202|/data/primary202 +205|203|p|p|s|u|sdw10|sdw10|7203|/data/primary203 +206|204|p|p|s|u|sdw10|sdw10|7204|/data/primary204 +207|205|p|p|s|u|sdw10|sdw10|7205|/data/primary205 +208|206|p|p|s|u|sdw10|sdw10|7206|/data/primary206 +209|207|p|p|s|u|sdw10|sdw10|7207|/data/primary207 +210|208|p|p|s|u|sdw10|sdw10|7208|/data/primary208 +211|209|p|p|s|u|sdw10|sdw10|7209|/data/primary209 +212|210|p|p|s|u|sdw10|sdw10|7210|/data/primary210 +213|211|p|p|s|u|sdw10|sdw10|7211|/data/primary211 +214|212|p|p|s|u|sdw10|sdw10|7212|/data/primary212 +1006|4|m|m|s|u|sdw10|sdw10|8004|/data/mirror4 +1043|41|m|m|s|u|sdw10|sdw10|8041|/data/mirror41 +1103|101|m|m|s|u|sdw10|sdw10|8101|/data/mirror101 +1155|153|m|m|s|u|sdw10|sdw10|8153|/data/mirror153 +1218|216|m|m|s|u|sdw10|sdw10|8216|/data/mirror216 +1259|257|m|m|s|u|sdw10|sdw10|8257|/data/mirror257 +1312|310|m|m|s|u|sdw10|sdw10|8310|/data/mirror310 +1365|363|m|m|s|u|sdw10|sdw10|8363|/data/mirror363 +1413|411|m|m|s|u|sdw10|sdw10|8411|/data/mirror411 +1454|452|m|m|s|u|sdw10|sdw10|8452|/data/mirror452 +1521|519|m|m|s|u|sdw10|sdw10|8519|/data/mirror519 +1564|562|m|m|s|u|sdw10|sdw10|8562|/data/mirror562 +1613|611|m|m|s|u|sdw10|sdw10|8611|/data/mirror611 +1668|666|m|m|s|u|sdw10|sdw10|8666|/data/mirror666 +1717|715|m|m|s|u|sdw10|sdw10|8715|/data/mirror715 +1757|755|m|m|s|u|sdw10|sdw10|8755|/data/mirror755 +1814|812|m|m|s|u|sdw10|sdw10|8812|/data/mirror812 +1865|863|m|m|s|u|sdw10|sdw10|8863|/data/mirror863 +1888|886|m|m|s|u|sdw10|sdw10|8886|/data/mirror886 +1966|964|m|m|s|u|sdw10|sdw10|8964|/data/mirror964 +# SDW11 +215|213|p|p|s|u|sdw11|sdw11|7213|/data/primary213 +216|214|p|p|s|u|sdw11|sdw11|7214|/data/primary214 +217|215|p|p|s|u|sdw11|sdw11|7215|/data/primary215 +218|216|p|p|s|u|sdw11|sdw11|7216|/data/primary216 +219|217|p|p|s|u|sdw11|sdw11|7217|/data/primary217 +220|218|p|p|s|u|sdw11|sdw11|7218|/data/primary218 +221|219|p|p|s|u|sdw11|sdw11|7219|/data/primary219 +222|220|p|p|s|u|sdw11|sdw11|7220|/data/primary220 +223|221|p|p|s|u|sdw11|sdw11|7221|/data/primary221 +224|222|p|p|s|u|sdw11|sdw11|7222|/data/primary222 +225|223|p|p|s|u|sdw11|sdw11|7223|/data/primary223 +226|224|p|p|s|u|sdw11|sdw11|7224|/data/primary224 +227|225|p|p|s|u|sdw11|sdw11|7225|/data/primary225 +228|226|p|p|s|u|sdw11|sdw11|7226|/data/primary226 +229|227|p|p|s|u|sdw11|sdw11|7227|/data/primary227 +230|228|p|p|s|u|sdw11|sdw11|7228|/data/primary228 +231|229|p|p|s|u|sdw11|sdw11|7229|/data/primary229 +232|230|p|p|s|u|sdw11|sdw11|7230|/data/primary230 +233|231|p|p|s|u|sdw11|sdw11|7231|/data/primary231 +1013|11|m|m|s|u|sdw11|sdw11|8011|/data/mirror11 +1065|63|m|m|s|u|sdw11|sdw11|8063|/data/mirror63 +1118|116|m|m|s|u|sdw11|sdw11|8116|/data/mirror116 +1163|161|m|m|s|u|sdw11|sdw11|8161|/data/mirror161 +1202|200|m|m|s|u|sdw11|sdw11|8200|/data/mirror200 +1263|261|m|m|s|u|sdw11|sdw11|8261|/data/mirror261 +1313|311|m|m|s|u|sdw11|sdw11|8311|/data/mirror311 +1339|337|m|m|s|u|sdw11|sdw11|8337|/data/mirror337 +1414|412|m|m|s|u|sdw11|sdw11|8412|/data/mirror412 +1470|468|m|m|s|u|sdw11|sdw11|8468|/data/mirror468 +1499|497|m|m|s|u|sdw11|sdw11|8497|/data/mirror497 +1565|563|m|m|s|u|sdw11|sdw11|8563|/data/mirror563 +1614|612|m|m|s|u|sdw11|sdw11|8612|/data/mirror612 +1654|652|m|m|s|u|sdw11|sdw11|8652|/data/mirror652 +1699|697|m|m|s|u|sdw11|sdw11|8697|/data/mirror697 +1765|763|m|m|s|u|sdw11|sdw11|8763|/data/mirror763 +1789|787|m|m|s|u|sdw11|sdw11|8787|/data/mirror787 +1867|865|m|m|s|u|sdw11|sdw11|8865|/data/mirror865 +1900|898|m|m|s|u|sdw11|sdw11|8898|/data/mirror898 +1967|965|m|m|s|u|sdw11|sdw11|8965|/data/mirror965 +# SDW12 +234|232|p|p|s|u|sdw12|sdw12|7232|/data/primary232 +235|233|p|p|s|u|sdw12|sdw12|7233|/data/primary233 +236|234|p|p|s|u|sdw12|sdw12|7234|/data/primary234 +237|235|p|p|s|u|sdw12|sdw12|7235|/data/primary235 +238|236|p|p|s|u|sdw12|sdw12|7236|/data/primary236 +239|237|p|p|s|u|sdw12|sdw12|7237|/data/primary237 +240|238|p|p|s|u|sdw12|sdw12|7238|/data/primary238 +241|239|p|p|s|u|sdw12|sdw12|7239|/data/primary239 +242|240|p|p|s|u|sdw12|sdw12|7240|/data/primary240 +243|241|p|p|s|u|sdw12|sdw12|7241|/data/primary241 +244|242|p|p|s|u|sdw12|sdw12|7242|/data/primary242 +245|243|p|p|s|u|sdw12|sdw12|7243|/data/primary243 +246|244|p|p|s|u|sdw12|sdw12|7244|/data/primary244 +247|245|p|p|s|u|sdw12|sdw12|7245|/data/primary245 +248|246|p|p|s|u|sdw12|sdw12|7246|/data/primary246 +249|247|p|p|s|u|sdw12|sdw12|7247|/data/primary247 +250|248|p|p|s|u|sdw12|sdw12|7248|/data/primary248 +251|249|p|p|s|u|sdw12|sdw12|7249|/data/primary249 +252|250|p|p|s|u|sdw12|sdw12|7250|/data/primary250 +253|251|p|p|s|u|sdw12|sdw12|7251|/data/primary251 +254|252|p|p|s|u|sdw12|sdw12|7252|/data/primary252 +255|253|p|p|s|u|sdw12|sdw12|7253|/data/primary253 +1007|5|m|m|s|u|sdw12|sdw12|8005|/data/mirror5 +1044|42|m|m|s|u|sdw12|sdw12|8042|/data/mirror42 +1119|117|m|m|s|u|sdw12|sdw12|8117|/data/mirror117 +1156|154|m|m|s|u|sdw12|sdw12|8154|/data/mirror154 +1220|218|m|m|s|u|sdw12|sdw12|8218|/data/mirror218 +1264|262|m|m|s|u|sdw12|sdw12|8262|/data/mirror262 +1315|313|m|m|s|u|sdw12|sdw12|8313|/data/mirror313 +1367|365|m|m|s|u|sdw12|sdw12|8365|/data/mirror365 +1415|413|m|m|s|u|sdw12|sdw12|8413|/data/mirror413 +1455|453|m|m|s|u|sdw12|sdw12|8453|/data/mirror453 +1522|520|m|m|s|u|sdw12|sdw12|8520|/data/mirror520 +1566|564|m|m|s|u|sdw12|sdw12|8564|/data/mirror564 +1615|613|m|m|s|u|sdw12|sdw12|8613|/data/mirror613 +1641|639|m|m|s|u|sdw12|sdw12|8639|/data/mirror639 +1718|716|m|m|s|u|sdw12|sdw12|8716|/data/mirror716 +1758|756|m|m|s|u|sdw12|sdw12|8756|/data/mirror756 +1815|813|m|m|s|u|sdw12|sdw12|8813|/data/mirror813 +1868|866|m|m|s|u|sdw12|sdw12|8866|/data/mirror866 +1916|914|m|m|s|u|sdw12|sdw12|8914|/data/mirror914 +1968|966|m|m|s|u|sdw12|sdw12|8966|/data/mirror966 +# SDW13 +256|254|p|p|s|u|sdw13|sdw13|7254|/data/primary254 +257|255|p|p|s|u|sdw13|sdw13|7255|/data/primary255 +258|256|p|p|s|u|sdw13|sdw13|7256|/data/primary256 +259|257|p|p|s|u|sdw13|sdw13|7257|/data/primary257 +260|258|p|p|s|u|sdw13|sdw13|7258|/data/primary258 +261|259|p|p|s|u|sdw13|sdw13|7259|/data/primary259 +262|260|p|p|s|u|sdw13|sdw13|7260|/data/primary260 +263|261|p|p|s|u|sdw13|sdw13|7261|/data/primary261 +264|262|p|p|s|u|sdw13|sdw13|7262|/data/primary262 +265|263|p|p|s|u|sdw13|sdw13|7263|/data/primary263 +266|264|p|p|s|u|sdw13|sdw13|7264|/data/primary264 +267|265|p|p|s|u|sdw13|sdw13|7265|/data/primary265 +268|266|p|p|s|u|sdw13|sdw13|7266|/data/primary266 +269|267|p|p|s|u|sdw13|sdw13|7267|/data/primary267 +270|268|p|p|s|u|sdw13|sdw13|7268|/data/primary268 +271|269|p|p|s|u|sdw13|sdw13|7269|/data/primary269 +272|270|p|p|s|u|sdw13|sdw13|7270|/data/primary270 +273|271|p|p|s|u|sdw13|sdw13|7271|/data/primary271 +1014|12|m|m|s|u|sdw13|sdw13|8012|/data/mirror12 +1066|64|m|m|s|u|sdw13|sdw13|8064|/data/mirror64 +1120|118|m|m|s|u|sdw13|sdw13|8118|/data/mirror118 +1165|163|m|m|s|u|sdw13|sdw13|8163|/data/mirror163 +1203|201|m|m|s|u|sdw13|sdw13|8201|/data/mirror201 +1274|272|m|m|s|u|sdw13|sdw13|8272|/data/mirror272 +1316|314|m|m|s|u|sdw13|sdw13|8314|/data/mirror314 +1340|338|m|m|s|u|sdw13|sdw13|8338|/data/mirror338 +1405|403|m|m|s|u|sdw13|sdw13|8403|/data/mirror403 +1471|469|m|m|s|u|sdw13|sdw13|8469|/data/mirror469 +1500|498|m|m|s|u|sdw13|sdw13|8498|/data/mirror498 +1567|565|m|m|s|u|sdw13|sdw13|8565|/data/mirror565 +1616|614|m|m|s|u|sdw13|sdw13|8614|/data/mirror614 +1671|669|m|m|s|u|sdw13|sdw13|8669|/data/mirror669 +1719|717|m|m|s|u|sdw13|sdw13|8717|/data/mirror717 +1749|747|m|m|s|u|sdw13|sdw13|8747|/data/mirror747 +1816|814|m|m|s|u|sdw13|sdw13|8814|/data/mirror814 +1842|840|m|m|s|u|sdw13|sdw13|8840|/data/mirror840 +1920|918|m|m|s|u|sdw13|sdw13|8918|/data/mirror918 +1969|967|m|m|s|u|sdw13|sdw13|8967|/data/mirror967 +# SDW14 +274|272|p|p|s|u|sdw14|sdw14|7272|/data/primary272 +275|273|p|p|s|u|sdw14|sdw14|7273|/data/primary273 +276|274|p|p|s|u|sdw14|sdw14|7274|/data/primary274 +277|275|p|p|s|u|sdw14|sdw14|7275|/data/primary275 +278|276|p|p|s|u|sdw14|sdw14|7276|/data/primary276 +279|277|p|p|s|u|sdw14|sdw14|7277|/data/primary277 +280|278|p|p|s|u|sdw14|sdw14|7278|/data/primary278 +281|279|p|p|s|u|sdw14|sdw14|7279|/data/primary279 +282|280|p|p|s|u|sdw14|sdw14|7280|/data/primary280 +283|281|p|p|s|u|sdw14|sdw14|7281|/data/primary281 +284|282|p|p|s|u|sdw14|sdw14|7282|/data/primary282 +285|283|p|p|s|u|sdw14|sdw14|7283|/data/primary283 +286|284|p|p|s|u|sdw14|sdw14|7284|/data/primary284 +287|285|p|p|s|u|sdw14|sdw14|7285|/data/primary285 +288|286|p|p|s|u|sdw14|sdw14|7286|/data/primary286 +289|287|p|p|s|u|sdw14|sdw14|7287|/data/primary287 +290|288|p|p|s|u|sdw14|sdw14|7288|/data/primary288 +291|289|p|p|s|u|sdw14|sdw14|7289|/data/primary289 +292|290|p|p|s|u|sdw14|sdw14|7290|/data/primary290 +293|291|p|p|s|u|sdw14|sdw14|7291|/data/primary291 +294|292|p|p|s|u|sdw14|sdw14|7292|/data/primary292 +295|293|p|p|s|u|sdw14|sdw14|7293|/data/primary293 +296|294|p|p|s|u|sdw14|sdw14|7294|/data/primary294 +297|295|p|p|s|u|sdw14|sdw14|7295|/data/primary295 +298|296|p|p|s|u|sdw14|sdw14|7296|/data/primary296 +299|297|p|p|s|u|sdw14|sdw14|7297|/data/primary297 +300|298|p|p|s|u|sdw14|sdw14|7298|/data/primary298 +1015|13|m|m|s|u|sdw14|sdw14|8013|/data/mirror13 +1045|43|m|m|s|u|sdw14|sdw14|8043|/data/mirror43 +1121|119|m|m|s|u|sdw14|sdw14|8119|/data/mirror119 +1166|164|m|m|s|u|sdw14|sdw14|8164|/data/mirror164 +1222|220|m|m|s|u|sdw14|sdw14|8220|/data/mirror220 +1265|263|m|m|s|u|sdw14|sdw14|8263|/data/mirror263 +1317|315|m|m|s|u|sdw14|sdw14|8315|/data/mirror315 +1366|364|m|m|s|u|sdw14|sdw14|8364|/data/mirror364 +1416|414|m|m|s|u|sdw14|sdw14|8414|/data/mirror414 +1456|454|m|m|s|u|sdw14|sdw14|8454|/data/mirror454 +1513|511|m|m|s|u|sdw14|sdw14|8511|/data/mirror511 +1571|569|m|m|s|u|sdw14|sdw14|8569|/data/mirror569 +1606|604|m|m|s|u|sdw14|sdw14|8604|/data/mirror604 +1642|640|m|m|s|u|sdw14|sdw14|8640|/data/mirror640 +1720|718|m|m|s|u|sdw14|sdw14|8718|/data/mirror718 +1767|765|m|m|s|u|sdw14|sdw14|8765|/data/mirror765 +1817|815|m|m|s|u|sdw14|sdw14|8815|/data/mirror815 +1870|868|m|m|s|u|sdw14|sdw14|8868|/data/mirror868 +1921|919|m|m|s|u|sdw14|sdw14|8919|/data/mirror919 +1970|968|m|m|s|u|sdw14|sdw14|8968|/data/mirror968 +# SDW15 +301|299|p|p|s|u|sdw15|sdw15|7299|/data/primary299 +302|300|p|p|s|u|sdw15|sdw15|7300|/data/primary300 +303|301|p|p|s|u|sdw15|sdw15|7301|/data/primary301 +304|302|p|p|s|u|sdw15|sdw15|7302|/data/primary302 +305|303|p|p|s|u|sdw15|sdw15|7303|/data/primary303 +306|304|p|p|s|u|sdw15|sdw15|7304|/data/primary304 +307|305|p|p|s|u|sdw15|sdw15|7305|/data/primary305 +308|306|p|p|s|u|sdw15|sdw15|7306|/data/primary306 +309|307|p|p|s|u|sdw15|sdw15|7307|/data/primary307 +310|308|p|p|s|u|sdw15|sdw15|7308|/data/primary308 +311|309|p|p|s|u|sdw15|sdw15|7309|/data/primary309 +312|310|p|p|s|u|sdw15|sdw15|7310|/data/primary310 +313|311|p|p|s|u|sdw15|sdw15|7311|/data/primary311 +314|312|p|p|s|u|sdw15|sdw15|7312|/data/primary312 +315|313|p|p|s|u|sdw15|sdw15|7313|/data/primary313 +316|314|p|p|s|u|sdw15|sdw15|7314|/data/primary314 +1016|14|m|m|s|u|sdw15|sdw15|8014|/data/mirror14 +1067|65|m|m|s|u|sdw15|sdw15|8065|/data/mirror65 +1122|120|m|m|s|u|sdw15|sdw15|8120|/data/mirror120 +1169|167|m|m|s|u|sdw15|sdw15|8167|/data/mirror167 +1223|221|m|m|s|u|sdw15|sdw15|8221|/data/mirror221 +1266|264|m|m|s|u|sdw15|sdw15|8264|/data/mirror264 +1319|317|m|m|s|u|sdw15|sdw15|8317|/data/mirror317 +1368|366|m|m|s|u|sdw15|sdw15|8366|/data/mirror366 +1418|416|m|m|s|u|sdw15|sdw15|8416|/data/mirror416 +1468|466|m|m|s|u|sdw15|sdw15|8466|/data/mirror466 +1524|522|m|m|s|u|sdw15|sdw15|8522|/data/mirror522 +1573|571|m|m|s|u|sdw15|sdw15|8571|/data/mirror571 +1617|615|m|m|s|u|sdw15|sdw15|8615|/data/mirror615 +1655|653|m|m|s|u|sdw15|sdw15|8653|/data/mirror653 +1701|699|m|m|s|u|sdw15|sdw15|8699|/data/mirror699 +1768|766|m|m|s|u|sdw15|sdw15|8766|/data/mirror766 +1804|802|m|m|s|u|sdw15|sdw15|8802|/data/mirror802 +1872|870|m|m|s|u|sdw15|sdw15|8870|/data/mirror870 +1912|910|m|m|s|u|sdw15|sdw15|8910|/data/mirror910 +1973|971|m|m|s|u|sdw15|sdw15|8971|/data/mirror971 +# SDW16 +317|315|p|p|s|u|sdw16|sdw16|7315|/data/primary315 +318|316|p|p|s|u|sdw16|sdw16|7316|/data/primary316 +319|317|p|p|s|u|sdw16|sdw16|7317|/data/primary317 +320|318|p|p|s|u|sdw16|sdw16|7318|/data/primary318 +321|319|p|p|s|u|sdw16|sdw16|7319|/data/primary319 +322|320|p|p|s|u|sdw16|sdw16|7320|/data/primary320 +323|321|p|p|s|u|sdw16|sdw16|7321|/data/primary321 +324|322|p|p|s|u|sdw16|sdw16|7322|/data/primary322 +325|323|p|p|s|u|sdw16|sdw16|7323|/data/primary323 +326|324|p|p|s|u|sdw16|sdw16|7324|/data/primary324 +327|325|p|p|s|u|sdw16|sdw16|7325|/data/primary325 +328|326|p|p|s|u|sdw16|sdw16|7326|/data/primary326 +329|327|p|p|s|u|sdw16|sdw16|7327|/data/primary327 +330|328|p|p|s|u|sdw16|sdw16|7328|/data/primary328 +331|329|p|p|s|u|sdw16|sdw16|7329|/data/primary329 +332|330|p|p|s|u|sdw16|sdw16|7330|/data/primary330 +333|331|p|p|s|u|sdw16|sdw16|7331|/data/primary331 +334|332|p|p|s|u|sdw16|sdw16|7332|/data/primary332 +335|333|p|p|s|u|sdw16|sdw16|7333|/data/primary333 +336|334|p|p|s|u|sdw16|sdw16|7334|/data/primary334 +337|335|p|p|s|u|sdw16|sdw16|7335|/data/primary335 +338|336|p|p|s|u|sdw16|sdw16|7336|/data/primary336 +339|337|p|p|s|u|sdw16|sdw16|7337|/data/primary337 +340|338|p|p|s|u|sdw16|sdw16|7338|/data/primary338 +1018|16|m|m|s|u|sdw16|sdw16|8016|/data/mirror16 +1069|67|m|m|s|u|sdw16|sdw16|8067|/data/mirror67 +1123|121|m|m|s|u|sdw16|sdw16|8121|/data/mirror121 +1170|168|m|m|s|u|sdw16|sdw16|8168|/data/mirror168 +1224|222|m|m|s|u|sdw16|sdw16|8222|/data/mirror222 +1267|265|m|m|s|u|sdw16|sdw16|8265|/data/mirror265 +1341|339|m|m|s|u|sdw16|sdw16|8339|/data/mirror339 +1369|367|m|m|s|u|sdw16|sdw16|8367|/data/mirror367 +1419|417|m|m|s|u|sdw16|sdw16|8417|/data/mirror417 +1472|470|m|m|s|u|sdw16|sdw16|8470|/data/mirror470 +1514|512|m|m|s|u|sdw16|sdw16|8512|/data/mirror512 +1553|551|m|m|s|u|sdw16|sdw16|8551|/data/mirror551 +1619|617|m|m|s|u|sdw16|sdw16|8617|/data/mirror617 +1673|671|m|m|s|u|sdw16|sdw16|8671|/data/mirror671 +1721|719|m|m|s|u|sdw16|sdw16|8719|/data/mirror719 +1769|767|m|m|s|u|sdw16|sdw16|8767|/data/mirror767 +1818|816|m|m|s|u|sdw16|sdw16|8816|/data/mirror816 +1873|871|m|m|s|u|sdw16|sdw16|8871|/data/mirror871 +1922|920|m|m|s|u|sdw16|sdw16|8920|/data/mirror920 +1974|972|m|m|s|u|sdw16|sdw16|8972|/data/mirror972 +# SDW17 +341|339|p|p|s|u|sdw17|sdw17|7339|/data/primary339 +342|340|p|p|s|u|sdw17|sdw17|7340|/data/primary340 +343|341|p|p|s|u|sdw17|sdw17|7341|/data/primary341 +344|342|p|p|s|u|sdw17|sdw17|7342|/data/primary342 +345|343|p|p|s|u|sdw17|sdw17|7343|/data/primary343 +346|344|p|p|s|u|sdw17|sdw17|7344|/data/primary344 +347|345|p|p|s|u|sdw17|sdw17|7345|/data/primary345 +348|346|p|p|s|u|sdw17|sdw17|7346|/data/primary346 +349|347|p|p|s|u|sdw17|sdw17|7347|/data/primary347 +350|348|p|p|s|u|sdw17|sdw17|7348|/data/primary348 +351|349|p|p|s|u|sdw17|sdw17|7349|/data/primary349 +352|350|p|p|s|u|sdw17|sdw17|7350|/data/primary350 +353|351|p|p|s|u|sdw17|sdw17|7351|/data/primary351 +354|352|p|p|s|u|sdw17|sdw17|7352|/data/primary352 +355|353|p|p|s|u|sdw17|sdw17|7353|/data/primary353 +356|354|p|p|s|u|sdw17|sdw17|7354|/data/primary354 +357|355|p|p|s|u|sdw17|sdw17|7355|/data/primary355 +358|356|p|p|s|u|sdw17|sdw17|7356|/data/primary356 +359|357|p|p|s|u|sdw17|sdw17|7357|/data/primary357 +360|358|p|p|s|u|sdw17|sdw17|7358|/data/primary358 +361|359|p|p|s|u|sdw17|sdw17|7359|/data/primary359 +362|360|p|p|s|u|sdw17|sdw17|7360|/data/primary360 +363|361|p|p|s|u|sdw17|sdw17|7361|/data/primary361 +364|362|p|p|s|u|sdw17|sdw17|7362|/data/primary362 +1019|17|m|m|s|u|sdw17|sdw17|8017|/data/mirror17 +1070|68|m|m|s|u|sdw17|sdw17|8068|/data/mirror68 +1125|123|m|m|s|u|sdw17|sdw17|8123|/data/mirror123 +1173|171|m|m|s|u|sdw17|sdw17|8171|/data/mirror171 +1205|203|m|m|s|u|sdw17|sdw17|8203|/data/mirror203 +1268|266|m|m|s|u|sdw17|sdw17|8266|/data/mirror266 +1318|316|m|m|s|u|sdw17|sdw17|8316|/data/mirror316 +1370|368|m|m|s|u|sdw17|sdw17|8368|/data/mirror368 +1420|418|m|m|s|u|sdw17|sdw17|8418|/data/mirror418 +1473|471|m|m|s|u|sdw17|sdw17|8471|/data/mirror471 +1502|500|m|m|s|u|sdw17|sdw17|8500|/data/mirror500 +1574|572|m|m|s|u|sdw17|sdw17|8572|/data/mirror572 +1620|618|m|m|s|u|sdw17|sdw17|8618|/data/mirror618 +1667|665|m|m|s|u|sdw17|sdw17|8665|/data/mirror665 +1702|700|m|m|s|u|sdw17|sdw17|8700|/data/mirror700 +1770|768|m|m|s|u|sdw17|sdw17|8768|/data/mirror768 +1819|817|m|m|s|u|sdw17|sdw17|8817|/data/mirror817 +1869|867|m|m|s|u|sdw17|sdw17|8867|/data/mirror867 +1903|901|m|m|s|u|sdw17|sdw17|8901|/data/mirror901 +1955|953|m|m|s|u|sdw17|sdw17|8953|/data/mirror953 +# SDW18 +365|363|p|p|s|u|sdw18|sdw18|7363|/data/primary363 +366|364|p|p|s|u|sdw18|sdw18|7364|/data/primary364 +367|365|p|p|s|u|sdw18|sdw18|7365|/data/primary365 +368|366|p|p|s|u|sdw18|sdw18|7366|/data/primary366 +369|367|p|p|s|u|sdw18|sdw18|7367|/data/primary367 +370|368|p|p|s|u|sdw18|sdw18|7368|/data/primary368 +371|369|p|p|s|u|sdw18|sdw18|7369|/data/primary369 +372|370|p|p|s|u|sdw18|sdw18|7370|/data/primary370 +373|371|p|p|s|u|sdw18|sdw18|7371|/data/primary371 +374|372|p|p|s|u|sdw18|sdw18|7372|/data/primary372 +375|373|p|p|s|u|sdw18|sdw18|7373|/data/primary373 +376|374|p|p|s|u|sdw18|sdw18|7374|/data/primary374 +377|375|p|p|s|u|sdw18|sdw18|7375|/data/primary375 +378|376|p|p|s|u|sdw18|sdw18|7376|/data/primary376 +379|377|p|p|s|u|sdw18|sdw18|7377|/data/primary377 +380|378|p|p|s|u|sdw18|sdw18|7378|/data/primary378 +381|379|p|p|s|u|sdw18|sdw18|7379|/data/primary379 +382|380|p|p|s|u|sdw18|sdw18|7380|/data/primary380 +1010|8|m|m|s|u|sdw18|sdw18|8008|/data/mirror8 +1073|71|m|m|s|u|sdw18|sdw18|8071|/data/mirror71 +1107|105|m|m|s|u|sdw18|sdw18|8105|/data/mirror105 +1172|170|m|m|s|u|sdw18|sdw18|8170|/data/mirror170 +1219|217|m|m|s|u|sdw18|sdw18|8217|/data/mirror217 +1271|269|m|m|s|u|sdw18|sdw18|8269|/data/mirror269 +1321|319|m|m|s|u|sdw18|sdw18|8319|/data/mirror319 +1383|381|m|m|s|u|sdw18|sdw18|8381|/data/mirror381 +1421|419|m|m|s|u|sdw18|sdw18|8419|/data/mirror419 +1475|473|m|m|s|u|sdw18|sdw18|8473|/data/mirror473 +1525|523|m|m|s|u|sdw18|sdw18|8523|/data/mirror523 +1554|552|m|m|s|u|sdw18|sdw18|8552|/data/mirror552 +1621|619|m|m|s|u|sdw18|sdw18|8619|/data/mirror619 +1674|672|m|m|s|u|sdw18|sdw18|8672|/data/mirror672 +1712|710|m|m|s|u|sdw18|sdw18|8710|/data/mirror710 +1771|769|m|m|s|u|sdw18|sdw18|8769|/data/mirror769 +1820|818|m|m|s|u|sdw18|sdw18|8818|/data/mirror818 +1876|874|m|m|s|u|sdw18|sdw18|8874|/data/mirror874 +1923|921|m|m|s|u|sdw18|sdw18|8921|/data/mirror921 +1975|973|m|m|s|u|sdw18|sdw18|8973|/data/mirror973 +# SDW19 +383|381|p|p|s|u|sdw19|sdw19|7381|/data/primary381 +384|382|p|p|s|u|sdw19|sdw19|7382|/data/primary382 +385|383|p|p|s|u|sdw19|sdw19|7383|/data/primary383 +386|384|p|p|s|u|sdw19|sdw19|7384|/data/primary384 +387|385|p|p|s|u|sdw19|sdw19|7385|/data/primary385 +388|386|p|p|s|u|sdw19|sdw19|7386|/data/primary386 +389|387|p|p|s|u|sdw19|sdw19|7387|/data/primary387 +390|388|p|p|s|u|sdw19|sdw19|7388|/data/primary388 +391|389|p|p|s|u|sdw19|sdw19|7389|/data/primary389 +392|390|p|p|s|u|sdw19|sdw19|7390|/data/primary390 +393|391|p|p|s|u|sdw19|sdw19|7391|/data/primary391 +394|392|p|p|s|u|sdw19|sdw19|7392|/data/primary392 +395|393|p|p|s|u|sdw19|sdw19|7393|/data/primary393 +1020|18|m|m|s|u|sdw19|sdw19|8018|/data/mirror18 +1077|75|m|m|s|u|sdw19|sdw19|8075|/data/mirror75 +1126|124|m|m|s|u|sdw19|sdw19|8124|/data/mirror124 +1174|172|m|m|s|u|sdw19|sdw19|8172|/data/mirror172 +1226|224|m|m|s|u|sdw19|sdw19|8224|/data/mirror224 +1272|270|m|m|s|u|sdw19|sdw19|8270|/data/mirror270 +1322|320|m|m|s|u|sdw19|sdw19|8320|/data/mirror320 +1371|369|m|m|s|u|sdw19|sdw19|8369|/data/mirror369 +1422|420|m|m|s|u|sdw19|sdw19|8420|/data/mirror420 +1445|443|m|m|s|u|sdw19|sdw19|8443|/data/mirror443 +1503|501|m|m|s|u|sdw19|sdw19|8501|/data/mirror501 +1575|573|m|m|s|u|sdw19|sdw19|8573|/data/mirror573 +1622|620|m|m|s|u|sdw19|sdw19|8620|/data/mirror620 +1675|673|m|m|s|u|sdw19|sdw19|8673|/data/mirror673 +1703|701|m|m|s|u|sdw19|sdw19|8701|/data/mirror701 +1761|759|m|m|s|u|sdw19|sdw19|8759|/data/mirror759 +1821|819|m|m|s|u|sdw19|sdw19|8819|/data/mirror819 +1877|875|m|m|s|u|sdw19|sdw19|8875|/data/mirror875 +1924|922|m|m|s|u|sdw19|sdw19|8922|/data/mirror922 +1976|974|m|m|s|u|sdw19|sdw19|8974|/data/mirror974 +# SDW2 +26|24|p|p|s|u|sdw2|sdw2|7024|/data/primary24 +27|25|p|p|s|u|sdw2|sdw2|7025|/data/primary25 +28|26|p|p|s|u|sdw2|sdw2|7026|/data/primary26 +29|27|p|p|s|u|sdw2|sdw2|7027|/data/primary27 +30|28|p|p|s|u|sdw2|sdw2|7028|/data/primary28 +31|29|p|p|s|u|sdw2|sdw2|7029|/data/primary29 +32|30|p|p|s|u|sdw2|sdw2|7030|/data/primary30 +33|31|p|p|s|u|sdw2|sdw2|7031|/data/primary31 +34|32|p|p|s|u|sdw2|sdw2|7032|/data/primary32 +35|33|p|p|s|u|sdw2|sdw2|7033|/data/primary33 +36|34|p|p|s|u|sdw2|sdw2|7034|/data/primary34 +37|35|p|p|s|u|sdw2|sdw2|7035|/data/primary35 +38|36|p|p|s|u|sdw2|sdw2|7036|/data/primary36 +39|37|p|p|s|u|sdw2|sdw2|7037|/data/primary37 +40|38|p|p|s|u|sdw2|sdw2|7038|/data/primary38 +41|39|p|p|s|u|sdw2|sdw2|7039|/data/primary39 +42|40|p|p|s|u|sdw2|sdw2|7040|/data/primary40 +43|41|p|p|s|u|sdw2|sdw2|7041|/data/primary41 +44|42|p|p|s|u|sdw2|sdw2|7042|/data/primary42 +45|43|p|p|s|u|sdw2|sdw2|7043|/data/primary43 +46|44|p|p|s|u|sdw2|sdw2|7044|/data/primary44 +47|45|p|p|s|u|sdw2|sdw2|7045|/data/primary45 +48|46|p|p|s|u|sdw2|sdw2|7046|/data/primary46 +49|47|p|p|s|u|sdw2|sdw2|7047|/data/primary47 +1002|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +1058|56|m|m|s|u|sdw2|sdw2|8056|/data/mirror56 +1104|102|m|m|s|u|sdw2|sdw2|8102|/data/mirror102 +1157|155|m|m|s|u|sdw2|sdw2|8155|/data/mirror155 +1187|185|m|m|s|u|sdw2|sdw2|8185|/data/mirror185 +1254|252|m|m|s|u|sdw2|sdw2|8252|/data/mirror252 +1303|301|m|m|s|u|sdw2|sdw2|8301|/data/mirror301 +1356|354|m|m|s|u|sdw2|sdw2|8354|/data/mirror354 +1404|402|m|m|s|u|sdw2|sdw2|8402|/data/mirror402 +1457|455|m|m|s|u|sdw2|sdw2|8455|/data/mirror455 +1510|508|m|m|s|u|sdw2|sdw2|8508|/data/mirror508 +1558|556|m|m|s|u|sdw2|sdw2|8556|/data/mirror556 +1604|602|m|m|s|u|sdw2|sdw2|8602|/data/mirror602 +1658|656|m|m|s|u|sdw2|sdw2|8656|/data/mirror656 +1711|709|m|m|s|u|sdw2|sdw2|8709|/data/mirror709 +1756|754|m|m|s|u|sdw2|sdw2|8754|/data/mirror754 +1806|804|m|m|s|u|sdw2|sdw2|8804|/data/mirror804 +1854|852|m|m|s|u|sdw2|sdw2|8852|/data/mirror852 +1909|907|m|m|s|u|sdw2|sdw2|8907|/data/mirror907 +1956|954|m|m|s|u|sdw2|sdw2|8954|/data/mirror954 +# SDW20 +396|394|p|p|s|u|sdw20|sdw20|7394|/data/primary394 +397|395|p|p|s|u|sdw20|sdw20|7395|/data/primary395 +398|396|p|p|s|u|sdw20|sdw20|7396|/data/primary396 +399|397|p|p|s|u|sdw20|sdw20|7397|/data/primary397 +400|398|p|p|s|u|sdw20|sdw20|7398|/data/primary398 +401|399|p|p|s|u|sdw20|sdw20|7399|/data/primary399 +402|400|p|p|s|u|sdw20|sdw20|7400|/data/primary400 +403|401|p|p|s|u|sdw20|sdw20|7401|/data/primary401 +404|402|p|p|s|u|sdw20|sdw20|7402|/data/primary402 +405|403|p|p|s|u|sdw20|sdw20|7403|/data/primary403 +406|404|p|p|s|u|sdw20|sdw20|7404|/data/primary404 +407|405|p|p|s|u|sdw20|sdw20|7405|/data/primary405 +408|406|p|p|s|u|sdw20|sdw20|7406|/data/primary406 +409|407|p|p|s|u|sdw20|sdw20|7407|/data/primary407 +410|408|p|p|s|u|sdw20|sdw20|7408|/data/primary408 +411|409|p|p|s|u|sdw20|sdw20|7409|/data/primary409 +412|410|p|p|s|u|sdw20|sdw20|7410|/data/primary410 +413|411|p|p|s|u|sdw20|sdw20|7411|/data/primary411 +414|412|p|p|s|u|sdw20|sdw20|7412|/data/primary412 +415|413|p|p|s|u|sdw20|sdw20|7413|/data/primary413 +416|414|p|p|s|u|sdw20|sdw20|7414|/data/primary414 +417|415|p|p|s|u|sdw20|sdw20|7415|/data/primary415 +418|416|p|p|s|u|sdw20|sdw20|7416|/data/primary416 +419|417|p|p|s|u|sdw20|sdw20|7417|/data/primary417 +420|418|p|p|s|u|sdw20|sdw20|7418|/data/primary418 +1021|19|m|m|s|u|sdw20|sdw20|8019|/data/mirror19 +1080|78|m|m|s|u|sdw20|sdw20|8078|/data/mirror78 +1108|106|m|m|s|u|sdw20|sdw20|8106|/data/mirror106 +1160|158|m|m|s|u|sdw20|sdw20|8158|/data/mirror158 +1227|225|m|m|s|u|sdw20|sdw20|8225|/data/mirror225 +1273|271|m|m|s|u|sdw20|sdw20|8271|/data/mirror271 +1323|321|m|m|s|u|sdw20|sdw20|8321|/data/mirror321 +1372|370|m|m|s|u|sdw20|sdw20|8370|/data/mirror370 +1423|421|m|m|s|u|sdw20|sdw20|8421|/data/mirror421 +1476|474|m|m|s|u|sdw20|sdw20|8474|/data/mirror474 +1526|524|m|m|s|u|sdw20|sdw20|8524|/data/mirror524 +1555|553|m|m|s|u|sdw20|sdw20|8553|/data/mirror553 +1623|621|m|m|s|u|sdw20|sdw20|8621|/data/mirror621 +1676|674|m|m|s|u|sdw20|sdw20|8674|/data/mirror674 +1724|722|m|m|s|u|sdw20|sdw20|8722|/data/mirror722 +1772|770|m|m|s|u|sdw20|sdw20|8770|/data/mirror770 +1822|820|m|m|s|u|sdw20|sdw20|8820|/data/mirror820 +1857|855|m|m|s|u|sdw20|sdw20|8855|/data/mirror855 +1914|912|m|m|s|u|sdw20|sdw20|8912|/data/mirror912 +1977|975|m|m|s|u|sdw20|sdw20|8975|/data/mirror975 +# SDW21 +421|419|p|p|s|u|sdw21|sdw21|7419|/data/primary419 +422|420|p|p|s|u|sdw21|sdw21|7420|/data/primary420 +423|421|p|p|s|u|sdw21|sdw21|7421|/data/primary421 +424|422|p|p|s|u|sdw21|sdw21|7422|/data/primary422 +425|423|p|p|s|u|sdw21|sdw21|7423|/data/primary423 +426|424|p|p|s|u|sdw21|sdw21|7424|/data/primary424 +427|425|p|p|s|u|sdw21|sdw21|7425|/data/primary425 +428|426|p|p|s|u|sdw21|sdw21|7426|/data/primary426 +429|427|p|p|s|u|sdw21|sdw21|7427|/data/primary427 +430|428|p|p|s|u|sdw21|sdw21|7428|/data/primary428 +431|429|p|p|s|u|sdw21|sdw21|7429|/data/primary429 +432|430|p|p|s|u|sdw21|sdw21|7430|/data/primary430 +433|431|p|p|s|u|sdw21|sdw21|7431|/data/primary431 +434|432|p|p|s|u|sdw21|sdw21|7432|/data/primary432 +435|433|p|p|s|u|sdw21|sdw21|7433|/data/primary433 +436|434|p|p|s|u|sdw21|sdw21|7434|/data/primary434 +437|435|p|p|s|u|sdw21|sdw21|7435|/data/primary435 +438|436|p|p|s|u|sdw21|sdw21|7436|/data/primary436 +439|437|p|p|s|u|sdw21|sdw21|7437|/data/primary437 +440|438|p|p|s|u|sdw21|sdw21|7438|/data/primary438 +441|439|p|p|s|u|sdw21|sdw21|7439|/data/primary439 +442|440|p|p|s|u|sdw21|sdw21|7440|/data/primary440 +443|441|p|p|s|u|sdw21|sdw21|7441|/data/primary441 +444|442|p|p|s|u|sdw21|sdw21|7442|/data/primary442 +445|443|p|p|s|u|sdw21|sdw21|7443|/data/primary443 +1022|20|m|m|s|u|sdw21|sdw21|8020|/data/mirror20 +1074|72|m|m|s|u|sdw21|sdw21|8072|/data/mirror72 +1127|125|m|m|s|u|sdw21|sdw21|8125|/data/mirror125 +1175|173|m|m|s|u|sdw21|sdw21|8173|/data/mirror173 +1207|205|m|m|s|u|sdw21|sdw21|8205|/data/mirror205 +1275|273|m|m|s|u|sdw21|sdw21|8273|/data/mirror273 +1311|309|m|m|s|u|sdw21|sdw21|8309|/data/mirror309 +1373|371|m|m|s|u|sdw21|sdw21|8371|/data/mirror371 +1446|444|m|m|s|u|sdw21|sdw21|8444|/data/mirror444 +1478|476|m|m|s|u|sdw21|sdw21|8476|/data/mirror476 +1527|525|m|m|s|u|sdw21|sdw21|8525|/data/mirror525 +1576|574|m|m|s|u|sdw21|sdw21|8574|/data/mirror574 +1624|622|m|m|s|u|sdw21|sdw21|8622|/data/mirror622 +1669|667|m|m|s|u|sdw21|sdw21|8667|/data/mirror667 +1725|723|m|m|s|u|sdw21|sdw21|8723|/data/mirror723 +1773|771|m|m|s|u|sdw21|sdw21|8771|/data/mirror771 +1807|805|m|m|s|u|sdw21|sdw21|8805|/data/mirror805 +1878|876|m|m|s|u|sdw21|sdw21|8876|/data/mirror876 +1925|923|m|m|s|u|sdw21|sdw21|8923|/data/mirror923 +1957|955|m|m|s|u|sdw21|sdw21|8955|/data/mirror955 +# SDW22 +446|444|p|p|s|u|sdw22|sdw22|7444|/data/primary444 +447|445|p|p|s|u|sdw22|sdw22|7445|/data/primary445 +448|446|p|p|s|u|sdw22|sdw22|7446|/data/primary446 +449|447|p|p|s|u|sdw22|sdw22|7447|/data/primary447 +450|448|p|p|s|u|sdw22|sdw22|7448|/data/primary448 +451|449|p|p|s|u|sdw22|sdw22|7449|/data/primary449 +452|450|p|p|s|u|sdw22|sdw22|7450|/data/primary450 +453|451|p|p|s|u|sdw22|sdw22|7451|/data/primary451 +454|452|p|p|s|u|sdw22|sdw22|7452|/data/primary452 +455|453|p|p|s|u|sdw22|sdw22|7453|/data/primary453 +456|454|p|p|s|u|sdw22|sdw22|7454|/data/primary454 +457|455|p|p|s|u|sdw22|sdw22|7455|/data/primary455 +458|456|p|p|s|u|sdw22|sdw22|7456|/data/primary456 +459|457|p|p|s|u|sdw22|sdw22|7457|/data/primary457 +460|458|p|p|s|u|sdw22|sdw22|7458|/data/primary458 +461|459|p|p|s|u|sdw22|sdw22|7459|/data/primary459 +462|460|p|p|s|u|sdw22|sdw22|7460|/data/primary460 +463|461|p|p|s|u|sdw22|sdw22|7461|/data/primary461 +464|462|p|p|s|u|sdw22|sdw22|7462|/data/primary462 +1024|22|m|m|s|u|sdw22|sdw22|8022|/data/mirror22 +1061|59|m|m|s|u|sdw22|sdw22|8059|/data/mirror59 +1129|127|m|m|s|u|sdw22|sdw22|8127|/data/mirror127 +1176|174|m|m|s|u|sdw22|sdw22|8174|/data/mirror174 +1229|227|m|m|s|u|sdw22|sdw22|8227|/data/mirror227 +1276|274|m|m|s|u|sdw22|sdw22|8274|/data/mirror274 +1320|318|m|m|s|u|sdw22|sdw22|8318|/data/mirror318 +1374|372|m|m|s|u|sdw22|sdw22|8372|/data/mirror372 +1409|407|m|m|s|u|sdw22|sdw22|8407|/data/mirror407 +1480|478|m|m|s|u|sdw22|sdw22|8478|/data/mirror478 +1516|514|m|m|s|u|sdw22|sdw22|8514|/data/mirror514 +1577|575|m|m|s|u|sdw22|sdw22|8575|/data/mirror575 +1625|623|m|m|s|u|sdw22|sdw22|8623|/data/mirror623 +1677|675|m|m|s|u|sdw22|sdw22|8675|/data/mirror675 +1722|720|m|m|s|u|sdw22|sdw22|8720|/data/mirror720 +1774|772|m|m|s|u|sdw22|sdw22|8772|/data/mirror772 +1823|821|m|m|s|u|sdw22|sdw22|8821|/data/mirror821 +1871|869|m|m|s|u|sdw22|sdw22|8869|/data/mirror869 +1926|924|m|m|s|u|sdw22|sdw22|8924|/data/mirror924 +1979|977|m|m|s|u|sdw22|sdw22|8977|/data/mirror977 +# SDW23 +465|463|p|p|s|u|sdw23|sdw23|7463|/data/primary463 +466|464|p|p|s|u|sdw23|sdw23|7464|/data/primary464 +467|465|p|p|s|u|sdw23|sdw23|7465|/data/primary465 +468|466|p|p|s|u|sdw23|sdw23|7466|/data/primary466 +469|467|p|p|s|u|sdw23|sdw23|7467|/data/primary467 +470|468|p|p|s|u|sdw23|sdw23|7468|/data/primary468 +471|469|p|p|s|u|sdw23|sdw23|7469|/data/primary469 +472|470|p|p|s|u|sdw23|sdw23|7470|/data/primary470 +473|471|p|p|s|u|sdw23|sdw23|7471|/data/primary471 +474|472|p|p|s|u|sdw23|sdw23|7472|/data/primary472 +475|473|p|p|s|u|sdw23|sdw23|7473|/data/primary473 +476|474|p|p|s|u|sdw23|sdw23|7474|/data/primary474 +477|475|p|p|s|u|sdw23|sdw23|7475|/data/primary475 +478|476|p|p|s|u|sdw23|sdw23|7476|/data/primary476 +479|477|p|p|s|u|sdw23|sdw23|7477|/data/primary477 +480|478|p|p|s|u|sdw23|sdw23|7478|/data/primary478 +481|479|p|p|s|u|sdw23|sdw23|7479|/data/primary479 +482|480|p|p|s|u|sdw23|sdw23|7480|/data/primary480 +483|481|p|p|s|u|sdw23|sdw23|7481|/data/primary481 +484|482|p|p|s|u|sdw23|sdw23|7482|/data/primary482 +485|483|p|p|s|u|sdw23|sdw23|7483|/data/primary483 +1025|23|m|m|s|u|sdw23|sdw23|8023|/data/mirror23 +1081|79|m|m|s|u|sdw23|sdw23|8079|/data/mirror79 +1132|130|m|m|s|u|sdw23|sdw23|8130|/data/mirror130 +1179|177|m|m|s|u|sdw23|sdw23|8177|/data/mirror177 +1221|219|m|m|s|u|sdw23|sdw23|8219|/data/mirror219 +1278|276|m|m|s|u|sdw23|sdw23|8276|/data/mirror276 +1324|322|m|m|s|u|sdw23|sdw23|8322|/data/mirror322 +1375|373|m|m|s|u|sdw23|sdw23|8373|/data/mirror373 +1424|422|m|m|s|u|sdw23|sdw23|8422|/data/mirror422 +1486|484|m|m|s|u|sdw23|sdw23|8484|/data/mirror484 +1505|503|m|m|s|u|sdw23|sdw23|8503|/data/mirror503 +1579|577|m|m|s|u|sdw23|sdw23|8577|/data/mirror577 +1626|624|m|m|s|u|sdw23|sdw23|8624|/data/mirror624 +1670|668|m|m|s|u|sdw23|sdw23|8668|/data/mirror668 +1714|712|m|m|s|u|sdw23|sdw23|8712|/data/mirror712 +1775|773|m|m|s|u|sdw23|sdw23|8773|/data/mirror773 +1824|822|m|m|s|u|sdw23|sdw23|8822|/data/mirror822 +1879|877|m|m|s|u|sdw23|sdw23|8877|/data/mirror877 +1927|925|m|m|s|u|sdw23|sdw23|8925|/data/mirror925 +1958|956|m|m|s|u|sdw23|sdw23|8956|/data/mirror956 +# SDW24 +486|484|p|p|s|u|sdw24|sdw24|7484|/data/primary484 +487|485|p|p|s|u|sdw24|sdw24|7485|/data/primary485 +488|486|p|p|s|u|sdw24|sdw24|7486|/data/primary486 +489|487|p|p|s|u|sdw24|sdw24|7487|/data/primary487 +490|488|p|p|s|u|sdw24|sdw24|7488|/data/primary488 +491|489|p|p|s|u|sdw24|sdw24|7489|/data/primary489 +492|490|p|p|s|u|sdw24|sdw24|7490|/data/primary490 +493|491|p|p|s|u|sdw24|sdw24|7491|/data/primary491 +494|492|p|p|s|u|sdw24|sdw24|7492|/data/primary492 +495|493|p|p|s|u|sdw24|sdw24|7493|/data/primary493 +496|494|p|p|s|u|sdw24|sdw24|7494|/data/primary494 +497|495|p|p|s|u|sdw24|sdw24|7495|/data/primary495 +498|496|p|p|s|u|sdw24|sdw24|7496|/data/primary496 +499|497|p|p|s|u|sdw24|sdw24|7497|/data/primary497 +500|498|p|p|s|u|sdw24|sdw24|7498|/data/primary498 +501|499|p|p|s|u|sdw24|sdw24|7499|/data/primary499 +502|500|p|p|s|u|sdw24|sdw24|7500|/data/primary500 +503|501|p|p|s|u|sdw24|sdw24|7501|/data/primary501 +504|502|p|p|s|u|sdw24|sdw24|7502|/data/primary502 +505|503|p|p|s|u|sdw24|sdw24|7503|/data/primary503 +506|504|p|p|s|u|sdw24|sdw24|7504|/data/primary504 +1027|25|m|m|s|u|sdw24|sdw24|8025|/data/mirror25 +1062|60|m|m|s|u|sdw24|sdw24|8060|/data/mirror60 +1110|108|m|m|s|u|sdw24|sdw24|8108|/data/mirror108 +1181|179|m|m|s|u|sdw24|sdw24|8179|/data/mirror179 +1231|229|m|m|s|u|sdw24|sdw24|8229|/data/mirror229 +1280|278|m|m|s|u|sdw24|sdw24|8278|/data/mirror278 +1325|323|m|m|s|u|sdw24|sdw24|8323|/data/mirror323 +1376|374|m|m|s|u|sdw24|sdw24|8374|/data/mirror374 +1425|423|m|m|s|u|sdw24|sdw24|8423|/data/mirror423 +1481|479|m|m|s|u|sdw24|sdw24|8479|/data/mirror479 +1528|526|m|m|s|u|sdw24|sdw24|8526|/data/mirror526 +1580|578|m|m|s|u|sdw24|sdw24|8578|/data/mirror578 +1611|609|m|m|s|u|sdw24|sdw24|8609|/data/mirror609 +1678|676|m|m|s|u|sdw24|sdw24|8676|/data/mirror676 +1726|724|m|m|s|u|sdw24|sdw24|8724|/data/mirror724 +1776|774|m|m|s|u|sdw24|sdw24|8774|/data/mirror774 +1825|823|m|m|s|u|sdw24|sdw24|8823|/data/mirror823 +1883|881|m|m|s|u|sdw24|sdw24|8881|/data/mirror881 +1929|927|m|m|s|u|sdw24|sdw24|8927|/data/mirror927 +1980|978|m|m|s|u|sdw24|sdw24|8978|/data/mirror978 +# SDW25 +507|505|p|p|s|u|sdw25|sdw25|7505|/data/primary505 +508|506|p|p|s|u|sdw25|sdw25|7506|/data/primary506 +509|507|p|p|s|u|sdw25|sdw25|7507|/data/primary507 +510|508|p|p|s|u|sdw25|sdw25|7508|/data/primary508 +511|509|p|p|s|u|sdw25|sdw25|7509|/data/primary509 +512|510|p|p|s|u|sdw25|sdw25|7510|/data/primary510 +513|511|p|p|s|u|sdw25|sdw25|7511|/data/primary511 +514|512|p|p|s|u|sdw25|sdw25|7512|/data/primary512 +515|513|p|p|s|u|sdw25|sdw25|7513|/data/primary513 +516|514|p|p|s|u|sdw25|sdw25|7514|/data/primary514 +517|515|p|p|s|u|sdw25|sdw25|7515|/data/primary515 +518|516|p|p|s|u|sdw25|sdw25|7516|/data/primary516 +519|517|p|p|s|u|sdw25|sdw25|7517|/data/primary517 +520|518|p|p|s|u|sdw25|sdw25|7518|/data/primary518 +521|519|p|p|s|u|sdw25|sdw25|7519|/data/primary519 +522|520|p|p|s|u|sdw25|sdw25|7520|/data/primary520 +523|521|p|p|s|u|sdw25|sdw25|7521|/data/primary521 +1028|26|m|m|s|u|sdw25|sdw25|8026|/data/mirror26 +1075|73|m|m|s|u|sdw25|sdw25|8073|/data/mirror73 +1133|131|m|m|s|u|sdw25|sdw25|8131|/data/mirror131 +1184|182|m|m|s|u|sdw25|sdw25|8182|/data/mirror182 +1209|207|m|m|s|u|sdw25|sdw25|8207|/data/mirror207 +1281|279|m|m|s|u|sdw25|sdw25|8279|/data/mirror279 +1327|325|m|m|s|u|sdw25|sdw25|8325|/data/mirror325 +1358|356|m|m|s|u|sdw25|sdw25|8356|/data/mirror356 +1426|424|m|m|s|u|sdw25|sdw25|8424|/data/mirror424 +1482|480|m|m|s|u|sdw25|sdw25|8480|/data/mirror480 +1529|527|m|m|s|u|sdw25|sdw25|8527|/data/mirror527 +1581|579|m|m|s|u|sdw25|sdw25|8579|/data/mirror579 +1628|626|m|m|s|u|sdw25|sdw25|8626|/data/mirror626 +1679|677|m|m|s|u|sdw25|sdw25|8677|/data/mirror677 +1723|721|m|m|s|u|sdw25|sdw25|8721|/data/mirror721 +1777|775|m|m|s|u|sdw25|sdw25|8775|/data/mirror775 +1826|824|m|m|s|u|sdw25|sdw25|8824|/data/mirror824 +1884|882|m|m|s|u|sdw25|sdw25|8882|/data/mirror882 +1928|926|m|m|s|u|sdw25|sdw25|8926|/data/mirror926 +1981|979|m|m|s|u|sdw25|sdw25|8979|/data/mirror979 +# SDW26 +524|522|p|p|s|u|sdw26|sdw26|7522|/data/primary522 +525|523|p|p|s|u|sdw26|sdw26|7523|/data/primary523 +526|524|p|p|s|u|sdw26|sdw26|7524|/data/primary524 +527|525|p|p|s|u|sdw26|sdw26|7525|/data/primary525 +528|526|p|p|s|u|sdw26|sdw26|7526|/data/primary526 +529|527|p|p|s|u|sdw26|sdw26|7527|/data/primary527 +530|528|p|p|s|u|sdw26|sdw26|7528|/data/primary528 +531|529|p|p|s|u|sdw26|sdw26|7529|/data/primary529 +532|530|p|p|s|u|sdw26|sdw26|7530|/data/primary530 +533|531|p|p|s|u|sdw26|sdw26|7531|/data/primary531 +534|532|p|p|s|u|sdw26|sdw26|7532|/data/primary532 +535|533|p|p|s|u|sdw26|sdw26|7533|/data/primary533 +536|534|p|p|s|u|sdw26|sdw26|7534|/data/primary534 +537|535|p|p|s|u|sdw26|sdw26|7535|/data/primary535 +538|536|p|p|s|u|sdw26|sdw26|7536|/data/primary536 +1029|27|m|m|s|u|sdw26|sdw26|8027|/data/mirror27 +1082|80|m|m|s|u|sdw26|sdw26|8080|/data/mirror80 +1111|109|m|m|s|u|sdw26|sdw26|8109|/data/mirror109 +1185|183|m|m|s|u|sdw26|sdw26|8183|/data/mirror183 +1233|231|m|m|s|u|sdw26|sdw26|8231|/data/mirror231 +1255|253|m|m|s|u|sdw26|sdw26|8253|/data/mirror253 +1328|326|m|m|s|u|sdw26|sdw26|8326|/data/mirror326 +1377|375|m|m|s|u|sdw26|sdw26|8375|/data/mirror375 +1428|426|m|m|s|u|sdw26|sdw26|8426|/data/mirror426 +1461|459|m|m|s|u|sdw26|sdw26|8459|/data/mirror459 +1506|504|m|m|s|u|sdw26|sdw26|8504|/data/mirror504 +1582|580|m|m|s|u|sdw26|sdw26|8580|/data/mirror580 +1629|627|m|m|s|u|sdw26|sdw26|8627|/data/mirror627 +1680|678|m|m|s|u|sdw26|sdw26|8678|/data/mirror678 +1727|725|m|m|s|u|sdw26|sdw26|8725|/data/mirror725 +1778|776|m|m|s|u|sdw26|sdw26|8776|/data/mirror776 +1827|825|m|m|s|u|sdw26|sdw26|8825|/data/mirror825 +1885|883|m|m|s|u|sdw26|sdw26|8883|/data/mirror883 +1917|915|m|m|s|u|sdw26|sdw26|8915|/data/mirror915 +1983|981|m|m|s|u|sdw26|sdw26|8981|/data/mirror981 +# SDW27 +539|537|p|p|s|u|sdw27|sdw27|7537|/data/primary537 +540|538|p|p|s|u|sdw27|sdw27|7538|/data/primary538 +541|539|p|p|s|u|sdw27|sdw27|7539|/data/primary539 +542|540|p|p|s|u|sdw27|sdw27|7540|/data/primary540 +543|541|p|p|s|u|sdw27|sdw27|7541|/data/primary541 +544|542|p|p|s|u|sdw27|sdw27|7542|/data/primary542 +545|543|p|p|s|u|sdw27|sdw27|7543|/data/primary543 +546|544|p|p|s|u|sdw27|sdw27|7544|/data/primary544 +547|545|p|p|s|u|sdw27|sdw27|7545|/data/primary545 +548|546|p|p|s|u|sdw27|sdw27|7546|/data/primary546 +549|547|p|p|s|u|sdw27|sdw27|7547|/data/primary547 +550|548|p|p|s|u|sdw27|sdw27|7548|/data/primary548 +551|549|p|p|s|u|sdw27|sdw27|7549|/data/primary549 +552|550|p|p|s|u|sdw27|sdw27|7550|/data/primary550 +553|551|p|p|s|u|sdw27|sdw27|7551|/data/primary551 +554|552|p|p|s|u|sdw27|sdw27|7552|/data/primary552 +555|553|p|p|s|u|sdw27|sdw27|7553|/data/primary553 +556|554|p|p|s|u|sdw27|sdw27|7554|/data/primary554 +557|555|p|p|s|u|sdw27|sdw27|7555|/data/primary555 +1030|28|m|m|s|u|sdw27|sdw27|8028|/data/mirror28 +1076|74|m|m|s|u|sdw27|sdw27|8074|/data/mirror74 +1136|134|m|m|s|u|sdw27|sdw27|8134|/data/mirror134 +1186|184|m|m|s|u|sdw27|sdw27|8184|/data/mirror184 +1234|232|m|m|s|u|sdw27|sdw27|8232|/data/mirror232 +1277|275|m|m|s|u|sdw27|sdw27|8275|/data/mirror275 +1314|312|m|m|s|u|sdw27|sdw27|8312|/data/mirror312 +1359|357|m|m|s|u|sdw27|sdw27|8357|/data/mirror357 +1429|427|m|m|s|u|sdw27|sdw27|8427|/data/mirror427 +1483|481|m|m|s|u|sdw27|sdw27|8481|/data/mirror481 +1530|528|m|m|s|u|sdw27|sdw27|8528|/data/mirror528 +1568|566|m|m|s|u|sdw27|sdw27|8566|/data/mirror566 +1630|628|m|m|s|u|sdw27|sdw27|8628|/data/mirror628 +1661|659|m|m|s|u|sdw27|sdw27|8659|/data/mirror659 +1728|726|m|m|s|u|sdw27|sdw27|8726|/data/mirror726 +1780|778|m|m|s|u|sdw27|sdw27|8778|/data/mirror778 +1828|826|m|m|s|u|sdw27|sdw27|8826|/data/mirror826 +1886|884|m|m|s|u|sdw27|sdw27|8884|/data/mirror884 +1930|928|m|m|s|u|sdw27|sdw27|8928|/data/mirror928 +1960|958|m|m|s|u|sdw27|sdw27|8958|/data/mirror958 +# SDW28 +558|556|p|p|s|u|sdw28|sdw28|7556|/data/primary556 +559|557|p|p|s|u|sdw28|sdw28|7557|/data/primary557 +560|558|p|p|s|u|sdw28|sdw28|7558|/data/primary558 +561|559|p|p|s|u|sdw28|sdw28|7559|/data/primary559 +562|560|p|p|s|u|sdw28|sdw28|7560|/data/primary560 +563|561|p|p|s|u|sdw28|sdw28|7561|/data/primary561 +564|562|p|p|s|u|sdw28|sdw28|7562|/data/primary562 +565|563|p|p|s|u|sdw28|sdw28|7563|/data/primary563 +566|564|p|p|s|u|sdw28|sdw28|7564|/data/primary564 +567|565|p|p|s|u|sdw28|sdw28|7565|/data/primary565 +568|566|p|p|s|u|sdw28|sdw28|7566|/data/primary566 +569|567|p|p|s|u|sdw28|sdw28|7567|/data/primary567 +570|568|p|p|s|u|sdw28|sdw28|7568|/data/primary568 +571|569|p|p|s|u|sdw28|sdw28|7569|/data/primary569 +572|570|p|p|s|u|sdw28|sdw28|7570|/data/primary570 +573|571|p|p|s|u|sdw28|sdw28|7571|/data/primary571 +574|572|p|p|s|u|sdw28|sdw28|7572|/data/primary572 +575|573|p|p|s|u|sdw28|sdw28|7573|/data/primary573 +576|574|p|p|s|u|sdw28|sdw28|7574|/data/primary574 +577|575|p|p|s|u|sdw28|sdw28|7575|/data/primary575 +578|576|p|p|s|u|sdw28|sdw28|7576|/data/primary576 +579|577|p|p|s|u|sdw28|sdw28|7577|/data/primary577 +580|578|p|p|s|u|sdw28|sdw28|7578|/data/primary578 +1034|32|m|m|s|u|sdw28|sdw28|8032|/data/mirror32 +1083|81|m|m|s|u|sdw28|sdw28|8081|/data/mirror81 +1112|110|m|m|s|u|sdw28|sdw28|8110|/data/mirror110 +1164|162|m|m|s|u|sdw28|sdw28|8162|/data/mirror162 +1236|234|m|m|s|u|sdw28|sdw28|8234|/data/mirror234 +1282|280|m|m|s|u|sdw28|sdw28|8280|/data/mirror280 +1329|327|m|m|s|u|sdw28|sdw28|8327|/data/mirror327 +1378|376|m|m|s|u|sdw28|sdw28|8376|/data/mirror376 +1430|428|m|m|s|u|sdw28|sdw28|8428|/data/mirror428 +1462|460|m|m|s|u|sdw28|sdw28|8460|/data/mirror460 +1532|530|m|m|s|u|sdw28|sdw28|8530|/data/mirror530 +1583|581|m|m|s|u|sdw28|sdw28|8581|/data/mirror581 +1632|630|m|m|s|u|sdw28|sdw28|8630|/data/mirror630 +1672|670|m|m|s|u|sdw28|sdw28|8670|/data/mirror670 +1729|727|m|m|s|u|sdw28|sdw28|8727|/data/mirror727 +1782|780|m|m|s|u|sdw28|sdw28|8780|/data/mirror780 +1830|828|m|m|s|u|sdw28|sdw28|8828|/data/mirror828 +1874|872|m|m|s|u|sdw28|sdw28|8872|/data/mirror872 +1918|916|m|m|s|u|sdw28|sdw28|8916|/data/mirror916 +1984|982|m|m|s|u|sdw28|sdw28|8982|/data/mirror982 +# SDW29 +581|579|p|p|s|u|sdw29|sdw29|7579|/data/primary579 +582|580|p|p|s|u|sdw29|sdw29|7580|/data/primary580 +583|581|p|p|s|u|sdw29|sdw29|7581|/data/primary581 +584|582|p|p|s|u|sdw29|sdw29|7582|/data/primary582 +585|583|p|p|s|u|sdw29|sdw29|7583|/data/primary583 +586|584|p|p|s|u|sdw29|sdw29|7584|/data/primary584 +587|585|p|p|s|u|sdw29|sdw29|7585|/data/primary585 +588|586|p|p|s|u|sdw29|sdw29|7586|/data/primary586 +589|587|p|p|s|u|sdw29|sdw29|7587|/data/primary587 +590|588|p|p|s|u|sdw29|sdw29|7588|/data/primary588 +591|589|p|p|s|u|sdw29|sdw29|7589|/data/primary589 +592|590|p|p|s|u|sdw29|sdw29|7590|/data/primary590 +593|591|p|p|s|u|sdw29|sdw29|7591|/data/primary591 +594|592|p|p|s|u|sdw29|sdw29|7592|/data/primary592 +595|593|p|p|s|u|sdw29|sdw29|7593|/data/primary593 +596|594|p|p|s|u|sdw29|sdw29|7594|/data/primary594 +1036|34|m|m|s|u|sdw29|sdw29|8034|/data/mirror34 +1084|82|m|m|s|u|sdw29|sdw29|8082|/data/mirror82 +1137|135|m|m|s|u|sdw29|sdw29|8135|/data/mirror135 +1188|186|m|m|s|u|sdw29|sdw29|8186|/data/mirror186 +1235|233|m|m|s|u|sdw29|sdw29|8233|/data/mirror233 +1283|281|m|m|s|u|sdw29|sdw29|8281|/data/mirror281 +1330|328|m|m|s|u|sdw29|sdw29|8328|/data/mirror328 +1379|377|m|m|s|u|sdw29|sdw29|8377|/data/mirror377 +1431|429|m|m|s|u|sdw29|sdw29|8429|/data/mirror429 +1485|483|m|m|s|u|sdw29|sdw29|8483|/data/mirror483 +1533|531|m|m|s|u|sdw29|sdw29|8531|/data/mirror531 +1597|595|m|m|s|u|sdw29|sdw29|8595|/data/mirror595 +1633|631|m|m|s|u|sdw29|sdw29|8631|/data/mirror631 +1683|681|m|m|s|u|sdw29|sdw29|8681|/data/mirror681 +1731|729|m|m|s|u|sdw29|sdw29|8729|/data/mirror729 +1766|764|m|m|s|u|sdw29|sdw29|8764|/data/mirror764 +1831|829|m|m|s|u|sdw29|sdw29|8829|/data/mirror829 +1889|887|m|m|s|u|sdw29|sdw29|8887|/data/mirror887 +1931|929|m|m|s|u|sdw29|sdw29|8929|/data/mirror929 +1961|959|m|m|s|u|sdw29|sdw29|8959|/data/mirror959 +# SDW3 +50|48|p|p|s|u|sdw3|sdw3|7048|/data/primary48 +51|49|p|p|s|u|sdw3|sdw3|7049|/data/primary49 +52|50|p|p|s|u|sdw3|sdw3|7050|/data/primary50 +53|51|p|p|s|u|sdw3|sdw3|7051|/data/primary51 +54|52|p|p|s|u|sdw3|sdw3|7052|/data/primary52 +55|53|p|p|s|u|sdw3|sdw3|7053|/data/primary53 +56|54|p|p|s|u|sdw3|sdw3|7054|/data/primary54 +57|55|p|p|s|u|sdw3|sdw3|7055|/data/primary55 +58|56|p|p|s|u|sdw3|sdw3|7056|/data/primary56 +59|57|p|p|s|u|sdw3|sdw3|7057|/data/primary57 +60|58|p|p|s|u|sdw3|sdw3|7058|/data/primary58 +61|59|p|p|s|u|sdw3|sdw3|7059|/data/primary59 +62|60|p|p|s|u|sdw3|sdw3|7060|/data/primary60 +63|61|p|p|s|u|sdw3|sdw3|7061|/data/primary61 +64|62|p|p|s|u|sdw3|sdw3|7062|/data/primary62 +65|63|p|p|s|u|sdw3|sdw3|7063|/data/primary63 +66|64|p|p|s|u|sdw3|sdw3|7064|/data/primary64 +67|65|p|p|s|u|sdw3|sdw3|7065|/data/primary65 +68|66|p|p|s|u|sdw3|sdw3|7066|/data/primary66 +69|67|p|p|s|u|sdw3|sdw3|7067|/data/primary67 +70|68|p|p|s|u|sdw3|sdw3|7068|/data/primary68 +71|69|p|p|s|u|sdw3|sdw3|7069|/data/primary69 +1003|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +1072|70|m|m|s|u|sdw3|sdw3|8070|/data/mirror70 +1105|103|m|m|s|u|sdw3|sdw3|8103|/data/mirror103 +1135|133|m|m|s|u|sdw3|sdw3|8133|/data/mirror133 +1210|208|m|m|s|u|sdw3|sdw3|8208|/data/mirror208 +1256|254|m|m|s|u|sdw3|sdw3|8254|/data/mirror254 +1304|302|m|m|s|u|sdw3|sdw3|8302|/data/mirror302 +1357|355|m|m|s|u|sdw3|sdw3|8355|/data/mirror355 +1406|404|m|m|s|u|sdw3|sdw3|8404|/data/mirror404 +1458|456|m|m|s|u|sdw3|sdw3|8456|/data/mirror456 +1484|482|m|m|s|u|sdw3|sdw3|8482|/data/mirror482 +1559|557|m|m|s|u|sdw3|sdw3|8557|/data/mirror557 +1605|603|m|m|s|u|sdw3|sdw3|8603|/data/mirror603 +1650|648|m|m|s|u|sdw3|sdw3|8648|/data/mirror648 +1713|711|m|m|s|u|sdw3|sdw3|8711|/data/mirror711 +1759|757|m|m|s|u|sdw3|sdw3|8757|/data/mirror757 +1810|808|m|m|s|u|sdw3|sdw3|8808|/data/mirror808 +1855|853|m|m|s|u|sdw3|sdw3|8853|/data/mirror853 +1908|906|m|m|s|u|sdw3|sdw3|8906|/data/mirror906 +1940|938|m|m|s|u|sdw3|sdw3|8938|/data/mirror938 +1998|996|m|m|s|u|sdw3|sdw3|8996|/data/mirror996 +# SDW30 +597|595|p|p|s|u|sdw30|sdw30|7595|/data/primary595 +598|596|p|p|s|u|sdw30|sdw30|7596|/data/primary596 +599|597|p|p|s|u|sdw30|sdw30|7597|/data/primary597 +600|598|p|p|s|u|sdw30|sdw30|7598|/data/primary598 +601|599|p|p|s|u|sdw30|sdw30|7599|/data/primary599 +602|600|p|p|s|u|sdw30|sdw30|7600|/data/primary600 +603|601|p|p|s|u|sdw30|sdw30|7601|/data/primary601 +604|602|p|p|s|u|sdw30|sdw30|7602|/data/primary602 +605|603|p|p|s|u|sdw30|sdw30|7603|/data/primary603 +606|604|p|p|s|u|sdw30|sdw30|7604|/data/primary604 +607|605|p|p|s|u|sdw30|sdw30|7605|/data/primary605 +608|606|p|p|s|u|sdw30|sdw30|7606|/data/primary606 +609|607|p|p|s|u|sdw30|sdw30|7607|/data/primary607 +610|608|p|p|s|u|sdw30|sdw30|7608|/data/primary608 +611|609|p|p|s|u|sdw30|sdw30|7609|/data/primary609 +612|610|p|p|s|u|sdw30|sdw30|7610|/data/primary610 +613|611|p|p|s|u|sdw30|sdw30|7611|/data/primary611 +614|612|p|p|s|u|sdw30|sdw30|7612|/data/primary612 +615|613|p|p|s|u|sdw30|sdw30|7613|/data/primary613 +616|614|p|p|s|u|sdw30|sdw30|7614|/data/primary614 +617|615|p|p|s|u|sdw30|sdw30|7615|/data/primary615 +618|616|p|p|s|u|sdw30|sdw30|7616|/data/primary616 +619|617|p|p|s|u|sdw30|sdw30|7617|/data/primary617 +1037|35|m|m|s|u|sdw30|sdw30|8035|/data/mirror35 +1085|83|m|m|s|u|sdw30|sdw30|8083|/data/mirror83 +1113|111|m|m|s|u|sdw30|sdw30|8111|/data/mirror111 +1177|175|m|m|s|u|sdw30|sdw30|8175|/data/mirror175 +1237|235|m|m|s|u|sdw30|sdw30|8235|/data/mirror235 +1286|284|m|m|s|u|sdw30|sdw30|8284|/data/mirror284 +1332|330|m|m|s|u|sdw30|sdw30|8330|/data/mirror330 +1380|378|m|m|s|u|sdw30|sdw30|8378|/data/mirror378 +1432|430|m|m|s|u|sdw30|sdw30|8430|/data/mirror430 +1463|461|m|m|s|u|sdw30|sdw30|8461|/data/mirror461 +1534|532|m|m|s|u|sdw30|sdw30|8532|/data/mirror532 +1569|567|m|m|s|u|sdw30|sdw30|8567|/data/mirror567 +1635|633|m|m|s|u|sdw30|sdw30|8633|/data/mirror633 +1684|682|m|m|s|u|sdw30|sdw30|8682|/data/mirror682 +1732|730|m|m|s|u|sdw30|sdw30|8730|/data/mirror730 +1783|781|m|m|s|u|sdw30|sdw30|8781|/data/mirror781 +1832|830|m|m|s|u|sdw30|sdw30|8830|/data/mirror830 +1862|860|m|m|s|u|sdw30|sdw30|8860|/data/mirror860 +1919|917|m|m|s|u|sdw30|sdw30|8917|/data/mirror917 +1985|983|m|m|s|u|sdw30|sdw30|8983|/data/mirror983 +# SDW31 +620|618|p|p|s|u|sdw31|sdw31|7618|/data/primary618 +621|619|p|p|s|u|sdw31|sdw31|7619|/data/primary619 +622|620|p|p|s|u|sdw31|sdw31|7620|/data/primary620 +623|621|p|p|s|u|sdw31|sdw31|7621|/data/primary621 +624|622|p|p|s|u|sdw31|sdw31|7622|/data/primary622 +625|623|p|p|s|u|sdw31|sdw31|7623|/data/primary623 +626|624|p|p|s|u|sdw31|sdw31|7624|/data/primary624 +627|625|p|p|s|u|sdw31|sdw31|7625|/data/primary625 +628|626|p|p|s|u|sdw31|sdw31|7626|/data/primary626 +629|627|p|p|s|u|sdw31|sdw31|7627|/data/primary627 +630|628|p|p|s|u|sdw31|sdw31|7628|/data/primary628 +631|629|p|p|s|u|sdw31|sdw31|7629|/data/primary629 +632|630|p|p|s|u|sdw31|sdw31|7630|/data/primary630 +633|631|p|p|s|u|sdw31|sdw31|7631|/data/primary631 +634|632|p|p|s|u|sdw31|sdw31|7632|/data/primary632 +635|633|p|p|s|u|sdw31|sdw31|7633|/data/primary633 +636|634|p|p|s|u|sdw31|sdw31|7634|/data/primary634 +637|635|p|p|s|u|sdw31|sdw31|7635|/data/primary635 +638|636|p|p|s|u|sdw31|sdw31|7636|/data/primary636 +639|637|p|p|s|u|sdw31|sdw31|7637|/data/primary637 +640|638|p|p|s|u|sdw31|sdw31|7638|/data/primary638 +641|639|p|p|s|u|sdw31|sdw31|7639|/data/primary639 +642|640|p|p|s|u|sdw31|sdw31|7640|/data/primary640 +643|641|p|p|s|u|sdw31|sdw31|7641|/data/primary641 +1038|36|m|m|s|u|sdw31|sdw31|8036|/data/mirror36 +1078|76|m|m|s|u|sdw31|sdw31|8076|/data/mirror76 +1124|122|m|m|s|u|sdw31|sdw31|8122|/data/mirror122 +1190|188|m|m|s|u|sdw31|sdw31|8188|/data/mirror188 +1238|236|m|m|s|u|sdw31|sdw31|8236|/data/mirror236 +1279|277|m|m|s|u|sdw31|sdw31|8277|/data/mirror277 +1333|331|m|m|s|u|sdw31|sdw31|8331|/data/mirror331 +1381|379|m|m|s|u|sdw31|sdw31|8379|/data/mirror379 +1433|431|m|m|s|u|sdw31|sdw31|8431|/data/mirror431 +1489|487|m|m|s|u|sdw31|sdw31|8487|/data/mirror487 +1536|534|m|m|s|u|sdw31|sdw31|8534|/data/mirror534 +1584|582|m|m|s|u|sdw31|sdw31|8582|/data/mirror582 +1644|642|m|m|s|u|sdw31|sdw31|8642|/data/mirror642 +1663|661|m|m|s|u|sdw31|sdw31|8661|/data/mirror661 +1733|731|m|m|s|u|sdw31|sdw31|8731|/data/mirror731 +1784|782|m|m|s|u|sdw31|sdw31|8782|/data/mirror782 +1835|833|m|m|s|u|sdw31|sdw31|8833|/data/mirror833 +1875|873|m|m|s|u|sdw31|sdw31|8873|/data/mirror873 +1933|931|m|m|s|u|sdw31|sdw31|8931|/data/mirror931 +1978|976|m|m|s|u|sdw31|sdw31|8976|/data/mirror976 +# SDW32 +644|642|p|p|s|u|sdw32|sdw32|7642|/data/primary642 +645|643|p|p|s|u|sdw32|sdw32|7643|/data/primary643 +646|644|p|p|s|u|sdw32|sdw32|7644|/data/primary644 +647|645|p|p|s|u|sdw32|sdw32|7645|/data/primary645 +648|646|p|p|s|u|sdw32|sdw32|7646|/data/primary646 +649|647|p|p|s|u|sdw32|sdw32|7647|/data/primary647 +650|648|p|p|s|u|sdw32|sdw32|7648|/data/primary648 +651|649|p|p|s|u|sdw32|sdw32|7649|/data/primary649 +652|650|p|p|s|u|sdw32|sdw32|7650|/data/primary650 +653|651|p|p|s|u|sdw32|sdw32|7651|/data/primary651 +654|652|p|p|s|u|sdw32|sdw32|7652|/data/primary652 +655|653|p|p|s|u|sdw32|sdw32|7653|/data/primary653 +656|654|p|p|s|u|sdw32|sdw32|7654|/data/primary654 +657|655|p|p|s|u|sdw32|sdw32|7655|/data/primary655 +658|656|p|p|s|u|sdw32|sdw32|7656|/data/primary656 +659|657|p|p|s|u|sdw32|sdw32|7657|/data/primary657 +660|658|p|p|s|u|sdw32|sdw32|7658|/data/primary658 +661|659|p|p|s|u|sdw32|sdw32|7659|/data/primary659 +662|660|p|p|s|u|sdw32|sdw32|7660|/data/primary660 +663|661|p|p|s|u|sdw32|sdw32|7661|/data/primary661 +1017|15|m|m|s|u|sdw32|sdw32|8015|/data/mirror15 +1086|84|m|m|s|u|sdw32|sdw32|8084|/data/mirror84 +1138|136|m|m|s|u|sdw32|sdw32|8136|/data/mirror136 +1178|176|m|m|s|u|sdw32|sdw32|8176|/data/mirror176 +1225|223|m|m|s|u|sdw32|sdw32|8223|/data/mirror223 +1269|267|m|m|s|u|sdw32|sdw32|8267|/data/mirror267 +1335|333|m|m|s|u|sdw32|sdw32|8333|/data/mirror333 +1382|380|m|m|s|u|sdw32|sdw32|8380|/data/mirror380 +1427|425|m|m|s|u|sdw32|sdw32|8425|/data/mirror425 +1464|462|m|m|s|u|sdw32|sdw32|8462|/data/mirror462 +1537|535|m|m|s|u|sdw32|sdw32|8535|/data/mirror535 +1570|568|m|m|s|u|sdw32|sdw32|8568|/data/mirror568 +1636|634|m|m|s|u|sdw32|sdw32|8634|/data/mirror634 +1685|683|m|m|s|u|sdw32|sdw32|8683|/data/mirror683 +1735|733|m|m|s|u|sdw32|sdw32|8733|/data/mirror733 +1785|783|m|m|s|u|sdw32|sdw32|8783|/data/mirror783 +1836|834|m|m|s|u|sdw32|sdw32|8834|/data/mirror834 +1863|861|m|m|s|u|sdw32|sdw32|8861|/data/mirror861 +1934|932|m|m|s|u|sdw32|sdw32|8932|/data/mirror932 +1986|984|m|m|s|u|sdw32|sdw32|8984|/data/mirror984 +# SDW33 +664|662|p|p|s|u|sdw33|sdw33|7662|/data/primary662 +665|663|p|p|s|u|sdw33|sdw33|7663|/data/primary663 +666|664|p|p|s|u|sdw33|sdw33|7664|/data/primary664 +667|665|p|p|s|u|sdw33|sdw33|7665|/data/primary665 +668|666|p|p|s|u|sdw33|sdw33|7666|/data/primary666 +669|667|p|p|s|u|sdw33|sdw33|7667|/data/primary667 +670|668|p|p|s|u|sdw33|sdw33|7668|/data/primary668 +671|669|p|p|s|u|sdw33|sdw33|7669|/data/primary669 +672|670|p|p|s|u|sdw33|sdw33|7670|/data/primary670 +673|671|p|p|s|u|sdw33|sdw33|7671|/data/primary671 +674|672|p|p|s|u|sdw33|sdw33|7672|/data/primary672 +675|673|p|p|s|u|sdw33|sdw33|7673|/data/primary673 +676|674|p|p|s|u|sdw33|sdw33|7674|/data/primary674 +677|675|p|p|s|u|sdw33|sdw33|7675|/data/primary675 +678|676|p|p|s|u|sdw33|sdw33|7676|/data/primary676 +679|677|p|p|s|u|sdw33|sdw33|7677|/data/primary677 +680|678|p|p|s|u|sdw33|sdw33|7678|/data/primary678 +681|679|p|p|s|u|sdw33|sdw33|7679|/data/primary679 +682|680|p|p|s|u|sdw33|sdw33|7680|/data/primary680 +683|681|p|p|s|u|sdw33|sdw33|7681|/data/primary681 +684|682|p|p|s|u|sdw33|sdw33|7682|/data/primary682 +685|683|p|p|s|u|sdw33|sdw33|7683|/data/primary683 +686|684|p|p|s|u|sdw33|sdw33|7684|/data/primary684 +1039|37|m|m|s|u|sdw33|sdw33|8037|/data/mirror37 +1079|77|m|m|s|u|sdw33|sdw33|8077|/data/mirror77 +1140|138|m|m|s|u|sdw33|sdw33|8138|/data/mirror138 +1191|189|m|m|s|u|sdw33|sdw33|8189|/data/mirror189 +1213|211|m|m|s|u|sdw33|sdw33|8211|/data/mirror211 +1287|285|m|m|s|u|sdw33|sdw33|8285|/data/mirror285 +1336|334|m|m|s|u|sdw33|sdw33|8334|/data/mirror334 +1385|383|m|m|s|u|sdw33|sdw33|8383|/data/mirror383 +1435|433|m|m|s|u|sdw33|sdw33|8433|/data/mirror433 +1474|472|m|m|s|u|sdw33|sdw33|8472|/data/mirror472 +1520|518|m|m|s|u|sdw33|sdw33|8518|/data/mirror518 +1585|583|m|m|s|u|sdw33|sdw33|8583|/data/mirror583 +1627|625|m|m|s|u|sdw33|sdw33|8625|/data/mirror625 +1687|685|m|m|s|u|sdw33|sdw33|8685|/data/mirror685 +1736|734|m|m|s|u|sdw33|sdw33|8734|/data/mirror734 +1786|784|m|m|s|u|sdw33|sdw33|8784|/data/mirror784 +1837|835|m|m|s|u|sdw33|sdw33|8835|/data/mirror835 +1890|888|m|m|s|u|sdw33|sdw33|8888|/data/mirror888 +1941|939|m|m|s|u|sdw33|sdw33|8939|/data/mirror939 +1987|985|m|m|s|u|sdw33|sdw33|8985|/data/mirror985 +# SDW34 +687|685|p|p|s|u|sdw34|sdw34|7685|/data/primary685 +688|686|p|p|s|u|sdw34|sdw34|7686|/data/primary686 +689|687|p|p|s|u|sdw34|sdw34|7687|/data/primary687 +690|688|p|p|s|u|sdw34|sdw34|7688|/data/primary688 +691|689|p|p|s|u|sdw34|sdw34|7689|/data/primary689 +692|690|p|p|s|u|sdw34|sdw34|7690|/data/primary690 +693|691|p|p|s|u|sdw34|sdw34|7691|/data/primary691 +694|692|p|p|s|u|sdw34|sdw34|7692|/data/primary692 +695|693|p|p|s|u|sdw34|sdw34|7693|/data/primary693 +696|694|p|p|s|u|sdw34|sdw34|7694|/data/primary694 +697|695|p|p|s|u|sdw34|sdw34|7695|/data/primary695 +698|696|p|p|s|u|sdw34|sdw34|7696|/data/primary696 +699|697|p|p|s|u|sdw34|sdw34|7697|/data/primary697 +700|698|p|p|s|u|sdw34|sdw34|7698|/data/primary698 +701|699|p|p|s|u|sdw34|sdw34|7699|/data/primary699 +702|700|p|p|s|u|sdw34|sdw34|7700|/data/primary700 +703|701|p|p|s|u|sdw34|sdw34|7701|/data/primary701 +1031|29|m|m|s|u|sdw34|sdw34|8029|/data/mirror29 +1087|85|m|m|s|u|sdw34|sdw34|8085|/data/mirror85 +1139|137|m|m|s|u|sdw34|sdw34|8137|/data/mirror137 +1167|165|m|m|s|u|sdw34|sdw34|8165|/data/mirror165 +1240|238|m|m|s|u|sdw34|sdw34|8238|/data/mirror238 +1270|268|m|m|s|u|sdw34|sdw34|8268|/data/mirror268 +1338|336|m|m|s|u|sdw34|sdw34|8336|/data/mirror336 +1384|382|m|m|s|u|sdw34|sdw34|8382|/data/mirror382 +1436|434|m|m|s|u|sdw34|sdw34|8434|/data/mirror434 +1487|485|m|m|s|u|sdw34|sdw34|8485|/data/mirror485 +1538|536|m|m|s|u|sdw34|sdw34|8536|/data/mirror536 +1587|585|m|m|s|u|sdw34|sdw34|8585|/data/mirror585 +1637|635|m|m|s|u|sdw34|sdw34|8635|/data/mirror635 +1704|702|m|m|s|u|sdw34|sdw34|8702|/data/mirror702 +1737|735|m|m|s|u|sdw34|sdw34|8735|/data/mirror735 +1787|785|m|m|s|u|sdw34|sdw34|8785|/data/mirror785 +1838|836|m|m|s|u|sdw34|sdw34|8836|/data/mirror836 +1864|862|m|m|s|u|sdw34|sdw34|8862|/data/mirror862 +1942|940|m|m|s|u|sdw34|sdw34|8940|/data/mirror940 +1988|986|m|m|s|u|sdw34|sdw34|8986|/data/mirror986 +# SDW35 +704|702|p|p|s|u|sdw35|sdw35|7702|/data/primary702 +705|703|p|p|s|u|sdw35|sdw35|7703|/data/primary703 +706|704|p|p|s|u|sdw35|sdw35|7704|/data/primary704 +707|705|p|p|s|u|sdw35|sdw35|7705|/data/primary705 +708|706|p|p|s|u|sdw35|sdw35|7706|/data/primary706 +709|707|p|p|s|u|sdw35|sdw35|7707|/data/primary707 +710|708|p|p|s|u|sdw35|sdw35|7708|/data/primary708 +711|709|p|p|s|u|sdw35|sdw35|7709|/data/primary709 +712|710|p|p|s|u|sdw35|sdw35|7710|/data/primary710 +713|711|p|p|s|u|sdw35|sdw35|7711|/data/primary711 +714|712|p|p|s|u|sdw35|sdw35|7712|/data/primary712 +715|713|p|p|s|u|sdw35|sdw35|7713|/data/primary713 +716|714|p|p|s|u|sdw35|sdw35|7714|/data/primary714 +717|715|p|p|s|u|sdw35|sdw35|7715|/data/primary715 +718|716|p|p|s|u|sdw35|sdw35|7716|/data/primary716 +719|717|p|p|s|u|sdw35|sdw35|7717|/data/primary717 +1040|38|m|m|s|u|sdw35|sdw35|8038|/data/mirror38 +1088|86|m|m|s|u|sdw35|sdw35|8086|/data/mirror86 +1142|140|m|m|s|u|sdw35|sdw35|8140|/data/mirror140 +1192|190|m|m|s|u|sdw35|sdw35|8190|/data/mirror190 +1214|212|m|m|s|u|sdw35|sdw35|8212|/data/mirror212 +1289|287|m|m|s|u|sdw35|sdw35|8287|/data/mirror287 +1326|324|m|m|s|u|sdw35|sdw35|8324|/data/mirror324 +1386|384|m|m|s|u|sdw35|sdw35|8384|/data/mirror384 +1437|435|m|m|s|u|sdw35|sdw35|8435|/data/mirror435 +1490|488|m|m|s|u|sdw35|sdw35|8488|/data/mirror488 +1539|537|m|m|s|u|sdw35|sdw35|8537|/data/mirror537 +1588|586|m|m|s|u|sdw35|sdw35|8586|/data/mirror586 +1638|636|m|m|s|u|sdw35|sdw35|8636|/data/mirror636 +1690|688|m|m|s|u|sdw35|sdw35|8688|/data/mirror688 +1738|736|m|m|s|u|sdw35|sdw35|8736|/data/mirror736 +1790|788|m|m|s|u|sdw35|sdw35|8788|/data/mirror788 +1839|837|m|m|s|u|sdw35|sdw35|8837|/data/mirror837 +1891|889|m|m|s|u|sdw35|sdw35|8889|/data/mirror889 +1943|941|m|m|s|u|sdw35|sdw35|8941|/data/mirror941 +1990|988|m|m|s|u|sdw35|sdw35|8988|/data/mirror988 +# SDW36 +720|718|p|p|s|u|sdw36|sdw36|7718|/data/primary718 +721|719|p|p|s|u|sdw36|sdw36|7719|/data/primary719 +722|720|p|p|s|u|sdw36|sdw36|7720|/data/primary720 +723|721|p|p|s|u|sdw36|sdw36|7721|/data/primary721 +724|722|p|p|s|u|sdw36|sdw36|7722|/data/primary722 +725|723|p|p|s|u|sdw36|sdw36|7723|/data/primary723 +726|724|p|p|s|u|sdw36|sdw36|7724|/data/primary724 +727|725|p|p|s|u|sdw36|sdw36|7725|/data/primary725 +728|726|p|p|s|u|sdw36|sdw36|7726|/data/primary726 +729|727|p|p|s|u|sdw36|sdw36|7727|/data/primary727 +730|728|p|p|s|u|sdw36|sdw36|7728|/data/primary728 +731|729|p|p|s|u|sdw36|sdw36|7729|/data/primary729 +732|730|p|p|s|u|sdw36|sdw36|7730|/data/primary730 +733|731|p|p|s|u|sdw36|sdw36|7731|/data/primary731 +734|732|p|p|s|u|sdw36|sdw36|7732|/data/primary732 +1032|30|m|m|s|u|sdw36|sdw36|8030|/data/mirror30 +1068|66|m|m|s|u|sdw36|sdw36|8066|/data/mirror66 +1143|141|m|m|s|u|sdw36|sdw36|8141|/data/mirror141 +1168|166|m|m|s|u|sdw36|sdw36|8166|/data/mirror166 +1242|240|m|m|s|u|sdw36|sdw36|8240|/data/mirror240 +1290|288|m|m|s|u|sdw36|sdw36|8288|/data/mirror288 +1342|340|m|m|s|u|sdw36|sdw36|8340|/data/mirror340 +1387|385|m|m|s|u|sdw36|sdw36|8385|/data/mirror385 +1438|436|m|m|s|u|sdw36|sdw36|8436|/data/mirror436 +1488|486|m|m|s|u|sdw36|sdw36|8486|/data/mirror486 +1531|529|m|m|s|u|sdw36|sdw36|8529|/data/mirror529 +1572|570|m|m|s|u|sdw36|sdw36|8570|/data/mirror570 +1639|637|m|m|s|u|sdw36|sdw36|8637|/data/mirror637 +1688|686|m|m|s|u|sdw36|sdw36|8686|/data/mirror686 +1742|740|m|m|s|u|sdw36|sdw36|8740|/data/mirror740 +1791|789|m|m|s|u|sdw36|sdw36|8789|/data/mirror789 +1829|827|m|m|s|u|sdw36|sdw36|8827|/data/mirror827 +1892|890|m|m|s|u|sdw36|sdw36|8890|/data/mirror890 +1932|930|m|m|s|u|sdw36|sdw36|8930|/data/mirror930 +1991|989|m|m|s|u|sdw36|sdw36|8989|/data/mirror989 +# SDW37 +735|733|p|p|s|u|sdw37|sdw37|7733|/data/primary733 +736|734|p|p|s|u|sdw37|sdw37|7734|/data/primary734 +737|735|p|p|s|u|sdw37|sdw37|7735|/data/primary735 +738|736|p|p|s|u|sdw37|sdw37|7736|/data/primary736 +739|737|p|p|s|u|sdw37|sdw37|7737|/data/primary737 +740|738|p|p|s|u|sdw37|sdw37|7738|/data/primary738 +741|739|p|p|s|u|sdw37|sdw37|7739|/data/primary739 +742|740|p|p|s|u|sdw37|sdw37|7740|/data/primary740 +743|741|p|p|s|u|sdw37|sdw37|7741|/data/primary741 +744|742|p|p|s|u|sdw37|sdw37|7742|/data/primary742 +745|743|p|p|s|u|sdw37|sdw37|7743|/data/primary743 +746|744|p|p|s|u|sdw37|sdw37|7744|/data/primary744 +747|745|p|p|s|u|sdw37|sdw37|7745|/data/primary745 +748|746|p|p|s|u|sdw37|sdw37|7746|/data/primary746 +749|747|p|p|s|u|sdw37|sdw37|7747|/data/primary747 +750|748|p|p|s|u|sdw37|sdw37|7748|/data/primary748 +751|749|p|p|s|u|sdw37|sdw37|7749|/data/primary749 +1042|40|m|m|s|u|sdw37|sdw37|8040|/data/mirror40 +1089|87|m|m|s|u|sdw37|sdw37|8087|/data/mirror87 +1144|142|m|m|s|u|sdw37|sdw37|8142|/data/mirror142 +1193|191|m|m|s|u|sdw37|sdw37|8191|/data/mirror191 +1243|241|m|m|s|u|sdw37|sdw37|8241|/data/mirror241 +1291|289|m|m|s|u|sdw37|sdw37|8289|/data/mirror289 +1343|341|m|m|s|u|sdw37|sdw37|8341|/data/mirror341 +1389|387|m|m|s|u|sdw37|sdw37|8387|/data/mirror387 +1439|437|m|m|s|u|sdw37|sdw37|8437|/data/mirror437 +1493|491|m|m|s|u|sdw37|sdw37|8491|/data/mirror491 +1540|538|m|m|s|u|sdw37|sdw37|8538|/data/mirror538 +1589|587|m|m|s|u|sdw37|sdw37|8587|/data/mirror587 +1640|638|m|m|s|u|sdw37|sdw37|8638|/data/mirror638 +1693|691|m|m|s|u|sdw37|sdw37|8691|/data/mirror691 +1752|750|m|m|s|u|sdw37|sdw37|8750|/data/mirror750 +1792|790|m|m|s|u|sdw37|sdw37|8790|/data/mirror790 +1840|838|m|m|s|u|sdw37|sdw37|8838|/data/mirror838 +1894|892|m|m|s|u|sdw37|sdw37|8892|/data/mirror892 +1944|942|m|m|s|u|sdw37|sdw37|8942|/data/mirror942 +1965|963|m|m|s|u|sdw37|sdw37|8963|/data/mirror963 +# SDW38 +752|750|p|p|s|u|sdw38|sdw38|7750|/data/primary750 +753|751|p|p|s|u|sdw38|sdw38|7751|/data/primary751 +754|752|p|p|s|u|sdw38|sdw38|7752|/data/primary752 +755|753|p|p|s|u|sdw38|sdw38|7753|/data/primary753 +756|754|p|p|s|u|sdw38|sdw38|7754|/data/primary754 +757|755|p|p|s|u|sdw38|sdw38|7755|/data/primary755 +758|756|p|p|s|u|sdw38|sdw38|7756|/data/primary756 +759|757|p|p|s|u|sdw38|sdw38|7757|/data/primary757 +760|758|p|p|s|u|sdw38|sdw38|7758|/data/primary758 +761|759|p|p|s|u|sdw38|sdw38|7759|/data/primary759 +762|760|p|p|s|u|sdw38|sdw38|7760|/data/primary760 +763|761|p|p|s|u|sdw38|sdw38|7761|/data/primary761 +764|762|p|p|s|u|sdw38|sdw38|7762|/data/primary762 +765|763|p|p|s|u|sdw38|sdw38|7763|/data/primary763 +766|764|p|p|s|u|sdw38|sdw38|7764|/data/primary764 +1033|31|m|m|s|u|sdw38|sdw38|8031|/data/mirror31 +1090|88|m|m|s|u|sdw38|sdw38|8088|/data/mirror88 +1141|139|m|m|s|u|sdw38|sdw38|8139|/data/mirror139 +1180|178|m|m|s|u|sdw38|sdw38|8178|/data/mirror178 +1244|242|m|m|s|u|sdw38|sdw38|8242|/data/mirror242 +1292|290|m|m|s|u|sdw38|sdw38|8290|/data/mirror290 +1344|342|m|m|s|u|sdw38|sdw38|8342|/data/mirror342 +1390|388|m|m|s|u|sdw38|sdw38|8388|/data/mirror388 +1417|415|m|m|s|u|sdw38|sdw38|8415|/data/mirror415 +1495|493|m|m|s|u|sdw38|sdw38|8493|/data/mirror493 +1541|539|m|m|s|u|sdw38|sdw38|8539|/data/mirror539 +1590|588|m|m|s|u|sdw38|sdw38|8588|/data/mirror588 +1643|641|m|m|s|u|sdw38|sdw38|8641|/data/mirror641 +1689|687|m|m|s|u|sdw38|sdw38|8687|/data/mirror687 +1743|741|m|m|s|u|sdw38|sdw38|8741|/data/mirror741 +1793|791|m|m|s|u|sdw38|sdw38|8791|/data/mirror791 +1841|839|m|m|s|u|sdw38|sdw38|8839|/data/mirror839 +1895|893|m|m|s|u|sdw38|sdw38|8893|/data/mirror893 +1945|943|m|m|s|u|sdw38|sdw38|8943|/data/mirror943 +1992|990|m|m|s|u|sdw38|sdw38|8990|/data/mirror990 +# SDW39 +767|765|p|p|s|u|sdw39|sdw39|7765|/data/primary765 +768|766|p|p|s|u|sdw39|sdw39|7766|/data/primary766 +769|767|p|p|s|u|sdw39|sdw39|7767|/data/primary767 +770|768|p|p|s|u|sdw39|sdw39|7768|/data/primary768 +771|769|p|p|s|u|sdw39|sdw39|7769|/data/primary769 +772|770|p|p|s|u|sdw39|sdw39|7770|/data/primary770 +773|771|p|p|s|u|sdw39|sdw39|7771|/data/primary771 +774|772|p|p|s|u|sdw39|sdw39|7772|/data/primary772 +775|773|p|p|s|u|sdw39|sdw39|7773|/data/primary773 +776|774|p|p|s|u|sdw39|sdw39|7774|/data/primary774 +777|775|p|p|s|u|sdw39|sdw39|7775|/data/primary775 +778|776|p|p|s|u|sdw39|sdw39|7776|/data/primary776 +779|777|p|p|s|u|sdw39|sdw39|7777|/data/primary777 +780|778|p|p|s|u|sdw39|sdw39|7778|/data/primary778 +781|779|p|p|s|u|sdw39|sdw39|7779|/data/primary779 +782|780|p|p|s|u|sdw39|sdw39|7780|/data/primary780 +783|781|p|p|s|u|sdw39|sdw39|7781|/data/primary781 +784|782|p|p|s|u|sdw39|sdw39|7782|/data/primary782 +785|783|p|p|s|u|sdw39|sdw39|7783|/data/primary783 +786|784|p|p|s|u|sdw39|sdw39|7784|/data/primary784 +787|785|p|p|s|u|sdw39|sdw39|7785|/data/primary785 +788|786|p|p|s|u|sdw39|sdw39|7786|/data/primary786 +789|787|p|p|s|u|sdw39|sdw39|7787|/data/primary787 +790|788|p|p|s|u|sdw39|sdw39|7788|/data/primary788 +791|789|p|p|s|u|sdw39|sdw39|7789|/data/primary789 +792|790|p|p|s|u|sdw39|sdw39|7790|/data/primary790 +1046|44|m|m|s|u|sdw39|sdw39|8044|/data/mirror44 +1091|89|m|m|s|u|sdw39|sdw39|8089|/data/mirror89 +1128|126|m|m|s|u|sdw39|sdw39|8126|/data/mirror126 +1194|192|m|m|s|u|sdw39|sdw39|8192|/data/mirror192 +1245|243|m|m|s|u|sdw39|sdw39|8243|/data/mirror243 +1293|291|m|m|s|u|sdw39|sdw39|8291|/data/mirror291 +1345|343|m|m|s|u|sdw39|sdw39|8343|/data/mirror343 +1391|389|m|m|s|u|sdw39|sdw39|8389|/data/mirror389 +1440|438|m|m|s|u|sdw39|sdw39|8438|/data/mirror438 +1477|475|m|m|s|u|sdw39|sdw39|8475|/data/mirror475 +1523|521|m|m|s|u|sdw39|sdw39|8521|/data/mirror521 +1592|590|m|m|s|u|sdw39|sdw39|8590|/data/mirror590 +1618|616|m|m|s|u|sdw39|sdw39|8616|/data/mirror616 +1695|693|m|m|s|u|sdw39|sdw39|8693|/data/mirror693 +1744|742|m|m|s|u|sdw39|sdw39|8742|/data/mirror742 +1794|792|m|m|s|u|sdw39|sdw39|8792|/data/mirror792 +1843|841|m|m|s|u|sdw39|sdw39|8841|/data/mirror841 +1896|894|m|m|s|u|sdw39|sdw39|8894|/data/mirror894 +1946|944|m|m|s|u|sdw39|sdw39|8944|/data/mirror944 +1993|991|m|m|s|u|sdw39|sdw39|8991|/data/mirror991 +# SDW4 +72|70|p|p|s|u|sdw4|sdw4|7070|/data/primary70 +73|71|p|p|s|u|sdw4|sdw4|7071|/data/primary71 +74|72|p|p|s|u|sdw4|sdw4|7072|/data/primary72 +75|73|p|p|s|u|sdw4|sdw4|7073|/data/primary73 +76|74|p|p|s|u|sdw4|sdw4|7074|/data/primary74 +77|75|p|p|s|u|sdw4|sdw4|7075|/data/primary75 +78|76|p|p|s|u|sdw4|sdw4|7076|/data/primary76 +79|77|p|p|s|u|sdw4|sdw4|7077|/data/primary77 +80|78|p|p|s|u|sdw4|sdw4|7078|/data/primary78 +81|79|p|p|s|u|sdw4|sdw4|7079|/data/primary79 +82|80|p|p|s|u|sdw4|sdw4|7080|/data/primary80 +83|81|p|p|s|u|sdw4|sdw4|7081|/data/primary81 +84|82|p|p|s|u|sdw4|sdw4|7082|/data/primary82 +85|83|p|p|s|u|sdw4|sdw4|7083|/data/primary83 +86|84|p|p|s|u|sdw4|sdw4|7084|/data/primary84 +87|85|p|p|s|u|sdw4|sdw4|7085|/data/primary85 +88|86|p|p|s|u|sdw4|sdw4|7086|/data/primary86 +89|87|p|p|s|u|sdw4|sdw4|7087|/data/primary87 +90|88|p|p|s|u|sdw4|sdw4|7088|/data/primary88 +91|89|p|p|s|u|sdw4|sdw4|7089|/data/primary89 +92|90|p|p|s|u|sdw4|sdw4|7090|/data/primary90 +93|91|p|p|s|u|sdw4|sdw4|7091|/data/primary91 +94|92|p|p|s|u|sdw4|sdw4|7092|/data/primary92 +95|93|p|p|s|u|sdw4|sdw4|7093|/data/primary93 +1004|2|m|m|s|u|sdw4|sdw4|8002|/data/mirror2 +1059|57|m|m|s|u|sdw4|sdw4|8057|/data/mirror57 +1106|104|m|m|s|u|sdw4|sdw4|8104|/data/mirror104 +1158|156|m|m|s|u|sdw4|sdw4|8156|/data/mirror156 +1211|209|m|m|s|u|sdw4|sdw4|8209|/data/mirror209 +1257|255|m|m|s|u|sdw4|sdw4|8255|/data/mirror255 +1306|304|m|m|s|u|sdw4|sdw4|8304|/data/mirror304 +1360|358|m|m|s|u|sdw4|sdw4|8358|/data/mirror358 +1407|405|m|m|s|u|sdw4|sdw4|8405|/data/mirror405 +1459|457|m|m|s|u|sdw4|sdw4|8457|/data/mirror457 +1511|509|m|m|s|u|sdw4|sdw4|8509|/data/mirror509 +1547|545|m|m|s|u|sdw4|sdw4|8545|/data/mirror545 +1607|605|m|m|s|u|sdw4|sdw4|8605|/data/mirror605 +1659|657|m|m|s|u|sdw4|sdw4|8657|/data/mirror657 +1696|694|m|m|s|u|sdw4|sdw4|8694|/data/mirror694 +1760|758|m|m|s|u|sdw4|sdw4|8758|/data/mirror758 +1809|807|m|m|s|u|sdw4|sdw4|8807|/data/mirror807 +1856|854|m|m|s|u|sdw4|sdw4|8854|/data/mirror854 +1910|908|m|m|s|u|sdw4|sdw4|8908|/data/mirror908 +1959|957|m|m|s|u|sdw4|sdw4|8957|/data/mirror957 +# SDW40 +793|791|p|p|s|u|sdw40|sdw40|7791|/data/primary791 +794|792|p|p|s|u|sdw40|sdw40|7792|/data/primary792 +795|793|p|p|s|u|sdw40|sdw40|7793|/data/primary793 +796|794|p|p|s|u|sdw40|sdw40|7794|/data/primary794 +797|795|p|p|s|u|sdw40|sdw40|7795|/data/primary795 +798|796|p|p|s|u|sdw40|sdw40|7796|/data/primary796 +799|797|p|p|s|u|sdw40|sdw40|7797|/data/primary797 +800|798|p|p|s|u|sdw40|sdw40|7798|/data/primary798 +801|799|p|p|s|u|sdw40|sdw40|7799|/data/primary799 +802|800|p|p|s|u|sdw40|sdw40|7800|/data/primary800 +803|801|p|p|s|u|sdw40|sdw40|7801|/data/primary801 +804|802|p|p|s|u|sdw40|sdw40|7802|/data/primary802 +805|803|p|p|s|u|sdw40|sdw40|7803|/data/primary803 +806|804|p|p|s|u|sdw40|sdw40|7804|/data/primary804 +807|805|p|p|s|u|sdw40|sdw40|7805|/data/primary805 +1047|45|m|m|s|u|sdw40|sdw40|8045|/data/mirror45 +1092|90|m|m|s|u|sdw40|sdw40|8090|/data/mirror90 +1145|143|m|m|s|u|sdw40|sdw40|8143|/data/mirror143 +1195|193|m|m|s|u|sdw40|sdw40|8193|/data/mirror193 +1228|226|m|m|s|u|sdw40|sdw40|8226|/data/mirror226 +1294|292|m|m|s|u|sdw40|sdw40|8292|/data/mirror292 +1346|344|m|m|s|u|sdw40|sdw40|8344|/data/mirror344 +1392|390|m|m|s|u|sdw40|sdw40|8390|/data/mirror390 +1441|439|m|m|s|u|sdw40|sdw40|8439|/data/mirror439 +1496|494|m|m|s|u|sdw40|sdw40|8494|/data/mirror494 +1543|541|m|m|s|u|sdw40|sdw40|8541|/data/mirror541 +1586|584|m|m|s|u|sdw40|sdw40|8584|/data/mirror584 +1645|643|m|m|s|u|sdw40|sdw40|8643|/data/mirror643 +1697|695|m|m|s|u|sdw40|sdw40|8695|/data/mirror695 +1730|728|m|m|s|u|sdw40|sdw40|8728|/data/mirror728 +1808|806|m|m|s|u|sdw40|sdw40|8806|/data/mirror806 +1844|842|m|m|s|u|sdw40|sdw40|8842|/data/mirror842 +1897|895|m|m|s|u|sdw40|sdw40|8895|/data/mirror895 +1948|946|m|m|s|u|sdw40|sdw40|8946|/data/mirror946 +1982|980|m|m|s|u|sdw40|sdw40|8980|/data/mirror980 +# SDW41 +808|806|p|p|s|u|sdw41|sdw41|7806|/data/primary806 +809|807|p|p|s|u|sdw41|sdw41|7807|/data/primary807 +810|808|p|p|s|u|sdw41|sdw41|7808|/data/primary808 +811|809|p|p|s|u|sdw41|sdw41|7809|/data/primary809 +812|810|p|p|s|u|sdw41|sdw41|7810|/data/primary810 +813|811|p|p|s|u|sdw41|sdw41|7811|/data/primary811 +814|812|p|p|s|u|sdw41|sdw41|7812|/data/primary812 +815|813|p|p|s|u|sdw41|sdw41|7813|/data/primary813 +816|814|p|p|s|u|sdw41|sdw41|7814|/data/primary814 +817|815|p|p|s|u|sdw41|sdw41|7815|/data/primary815 +818|816|p|p|s|u|sdw41|sdw41|7816|/data/primary816 +819|817|p|p|s|u|sdw41|sdw41|7817|/data/primary817 +820|818|p|p|s|u|sdw41|sdw41|7818|/data/primary818 +1048|46|m|m|s|u|sdw41|sdw41|8046|/data/mirror46 +1093|91|m|m|s|u|sdw41|sdw41|8091|/data/mirror91 +1146|144|m|m|s|u|sdw41|sdw41|8144|/data/mirror144 +1196|194|m|m|s|u|sdw41|sdw41|8194|/data/mirror194 +1239|237|m|m|s|u|sdw41|sdw41|8237|/data/mirror237 +1295|293|m|m|s|u|sdw41|sdw41|8293|/data/mirror293 +1348|346|m|m|s|u|sdw41|sdw41|8346|/data/mirror346 +1393|391|m|m|s|u|sdw41|sdw41|8391|/data/mirror391 +1442|440|m|m|s|u|sdw41|sdw41|8440|/data/mirror440 +1497|495|m|m|s|u|sdw41|sdw41|8495|/data/mirror495 +1544|542|m|m|s|u|sdw41|sdw41|8542|/data/mirror542 +1593|591|m|m|s|u|sdw41|sdw41|8591|/data/mirror591 +1647|645|m|m|s|u|sdw41|sdw41|8645|/data/mirror645 +1698|696|m|m|s|u|sdw41|sdw41|8696|/data/mirror696 +1747|745|m|m|s|u|sdw41|sdw41|8745|/data/mirror745 +1779|777|m|m|s|u|sdw41|sdw41|8777|/data/mirror777 +1845|843|m|m|s|u|sdw41|sdw41|8843|/data/mirror843 +1893|891|m|m|s|u|sdw41|sdw41|8891|/data/mirror891 +1950|948|m|m|s|u|sdw41|sdw41|8948|/data/mirror948 +1994|992|m|m|s|u|sdw41|sdw41|8992|/data/mirror992 +# SDW42 +821|819|p|p|s|u|sdw42|sdw42|7819|/data/primary819 +822|820|p|p|s|u|sdw42|sdw42|7820|/data/primary820 +823|821|p|p|s|u|sdw42|sdw42|7821|/data/primary821 +824|822|p|p|s|u|sdw42|sdw42|7822|/data/primary822 +825|823|p|p|s|u|sdw42|sdw42|7823|/data/primary823 +826|824|p|p|s|u|sdw42|sdw42|7824|/data/primary824 +827|825|p|p|s|u|sdw42|sdw42|7825|/data/primary825 +828|826|p|p|s|u|sdw42|sdw42|7826|/data/primary826 +829|827|p|p|s|u|sdw42|sdw42|7827|/data/primary827 +830|828|p|p|s|u|sdw42|sdw42|7828|/data/primary828 +831|829|p|p|s|u|sdw42|sdw42|7829|/data/primary829 +832|830|p|p|s|u|sdw42|sdw42|7830|/data/primary830 +833|831|p|p|s|u|sdw42|sdw42|7831|/data/primary831 +834|832|p|p|s|u|sdw42|sdw42|7832|/data/primary832 +835|833|p|p|s|u|sdw42|sdw42|7833|/data/primary833 +836|834|p|p|s|u|sdw42|sdw42|7834|/data/primary834 +837|835|p|p|s|u|sdw42|sdw42|7835|/data/primary835 +838|836|p|p|s|u|sdw42|sdw42|7836|/data/primary836 +839|837|p|p|s|u|sdw42|sdw42|7837|/data/primary837 +840|838|p|p|s|u|sdw42|sdw42|7838|/data/primary838 +841|839|p|p|s|u|sdw42|sdw42|7839|/data/primary839 +842|840|p|p|s|u|sdw42|sdw42|7840|/data/primary840 +843|841|p|p|s|u|sdw42|sdw42|7841|/data/primary841 +1035|33|m|m|s|u|sdw42|sdw42|8033|/data/mirror33 +1071|69|m|m|s|u|sdw42|sdw42|8069|/data/mirror69 +1147|145|m|m|s|u|sdw42|sdw42|8145|/data/mirror145 +1182|180|m|m|s|u|sdw42|sdw42|8180|/data/mirror180 +1247|245|m|m|s|u|sdw42|sdw42|8245|/data/mirror245 +1296|294|m|m|s|u|sdw42|sdw42|8294|/data/mirror294 +1349|347|m|m|s|u|sdw42|sdw42|8347|/data/mirror347 +1388|386|m|m|s|u|sdw42|sdw42|8386|/data/mirror386 +1443|441|m|m|s|u|sdw42|sdw42|8441|/data/mirror441 +1498|496|m|m|s|u|sdw42|sdw42|8496|/data/mirror496 +1545|543|m|m|s|u|sdw42|sdw42|8543|/data/mirror543 +1594|592|m|m|s|u|sdw42|sdw42|8592|/data/mirror592 +1631|629|m|m|s|u|sdw42|sdw42|8629|/data/mirror629 +1691|689|m|m|s|u|sdw42|sdw42|8689|/data/mirror689 +1739|737|m|m|s|u|sdw42|sdw42|8737|/data/mirror737 +1795|793|m|m|s|u|sdw42|sdw42|8793|/data/mirror793 +1846|844|m|m|s|u|sdw42|sdw42|8844|/data/mirror844 +1898|896|m|m|s|u|sdw42|sdw42|8896|/data/mirror896 +1935|933|m|m|s|u|sdw42|sdw42|8933|/data/mirror933 +1996|994|m|m|s|u|sdw42|sdw42|8994|/data/mirror994 +# SDW43 +844|842|p|p|s|u|sdw43|sdw43|7842|/data/primary842 +845|843|p|p|s|u|sdw43|sdw43|7843|/data/primary843 +846|844|p|p|s|u|sdw43|sdw43|7844|/data/primary844 +847|845|p|p|s|u|sdw43|sdw43|7845|/data/primary845 +848|846|p|p|s|u|sdw43|sdw43|7846|/data/primary846 +849|847|p|p|s|u|sdw43|sdw43|7847|/data/primary847 +850|848|p|p|s|u|sdw43|sdw43|7848|/data/primary848 +851|849|p|p|s|u|sdw43|sdw43|7849|/data/primary849 +852|850|p|p|s|u|sdw43|sdw43|7850|/data/primary850 +853|851|p|p|s|u|sdw43|sdw43|7851|/data/primary851 +854|852|p|p|s|u|sdw43|sdw43|7852|/data/primary852 +855|853|p|p|s|u|sdw43|sdw43|7853|/data/primary853 +856|854|p|p|s|u|sdw43|sdw43|7854|/data/primary854 +857|855|p|p|s|u|sdw43|sdw43|7855|/data/primary855 +858|856|p|p|s|u|sdw43|sdw43|7856|/data/primary856 +859|857|p|p|s|u|sdw43|sdw43|7857|/data/primary857 +860|858|p|p|s|u|sdw43|sdw43|7858|/data/primary858 +861|859|p|p|s|u|sdw43|sdw43|7859|/data/primary859 +862|860|p|p|s|u|sdw43|sdw43|7860|/data/primary860 +863|861|p|p|s|u|sdw43|sdw43|7861|/data/primary861 +864|862|p|p|s|u|sdw43|sdw43|7862|/data/primary862 +865|863|p|p|s|u|sdw43|sdw43|7863|/data/primary863 +1049|47|m|m|s|u|sdw43|sdw43|8047|/data/mirror47 +1094|92|m|m|s|u|sdw43|sdw43|8092|/data/mirror92 +1130|128|m|m|s|u|sdw43|sdw43|8128|/data/mirror128 +1197|195|m|m|s|u|sdw43|sdw43|8195|/data/mirror195 +1248|246|m|m|s|u|sdw43|sdw43|8246|/data/mirror246 +1284|282|m|m|s|u|sdw43|sdw43|8282|/data/mirror282 +1350|348|m|m|s|u|sdw43|sdw43|8348|/data/mirror348 +1394|392|m|m|s|u|sdw43|sdw43|8392|/data/mirror392 +1444|442|m|m|s|u|sdw43|sdw43|8442|/data/mirror442 +1479|477|m|m|s|u|sdw43|sdw43|8477|/data/mirror477 +1542|540|m|m|s|u|sdw43|sdw43|8540|/data/mirror540 +1596|594|m|m|s|u|sdw43|sdw43|8594|/data/mirror594 +1648|646|m|m|s|u|sdw43|sdw43|8646|/data/mirror646 +1700|698|m|m|s|u|sdw43|sdw43|8698|/data/mirror698 +1748|746|m|m|s|u|sdw43|sdw43|8746|/data/mirror746 +1796|794|m|m|s|u|sdw43|sdw43|8794|/data/mirror794 +1866|864|m|m|s|u|sdw43|sdw43|8864|/data/mirror864 +1880|878|m|m|s|u|sdw43|sdw43|8878|/data/mirror878 +1952|950|m|m|s|u|sdw43|sdw43|8950|/data/mirror950 +1997|995|m|m|s|u|sdw43|sdw43|8995|/data/mirror995 +# SDW44 +866|864|p|p|s|u|sdw44|sdw44|7864|/data/primary864 +867|865|p|p|s|u|sdw44|sdw44|7865|/data/primary865 +868|866|p|p|s|u|sdw44|sdw44|7866|/data/primary866 +869|867|p|p|s|u|sdw44|sdw44|7867|/data/primary867 +870|868|p|p|s|u|sdw44|sdw44|7868|/data/primary868 +871|869|p|p|s|u|sdw44|sdw44|7869|/data/primary869 +872|870|p|p|s|u|sdw44|sdw44|7870|/data/primary870 +873|871|p|p|s|u|sdw44|sdw44|7871|/data/primary871 +874|872|p|p|s|u|sdw44|sdw44|7872|/data/primary872 +875|873|p|p|s|u|sdw44|sdw44|7873|/data/primary873 +876|874|p|p|s|u|sdw44|sdw44|7874|/data/primary874 +877|875|p|p|s|u|sdw44|sdw44|7875|/data/primary875 +878|876|p|p|s|u|sdw44|sdw44|7876|/data/primary876 +879|877|p|p|s|u|sdw44|sdw44|7877|/data/primary877 +880|878|p|p|s|u|sdw44|sdw44|7878|/data/primary878 +881|879|p|p|s|u|sdw44|sdw44|7879|/data/primary879 +882|880|p|p|s|u|sdw44|sdw44|7880|/data/primary880 +883|881|p|p|s|u|sdw44|sdw44|7881|/data/primary881 +884|882|p|p|s|u|sdw44|sdw44|7882|/data/primary882 +885|883|p|p|s|u|sdw44|sdw44|7883|/data/primary883 +886|884|p|p|s|u|sdw44|sdw44|7884|/data/primary884 +887|885|p|p|s|u|sdw44|sdw44|7885|/data/primary885 +888|886|p|p|s|u|sdw44|sdw44|7886|/data/primary886 +1023|21|m|m|s|u|sdw44|sdw44|8021|/data/mirror21 +1095|93|m|m|s|u|sdw44|sdw44|8093|/data/mirror93 +1148|146|m|m|s|u|sdw44|sdw44|8146|/data/mirror146 +1183|181|m|m|s|u|sdw44|sdw44|8181|/data/mirror181 +1230|228|m|m|s|u|sdw44|sdw44|8228|/data/mirror228 +1297|295|m|m|s|u|sdw44|sdw44|8295|/data/mirror295 +1351|349|m|m|s|u|sdw44|sdw44|8349|/data/mirror349 +1396|394|m|m|s|u|sdw44|sdw44|8394|/data/mirror394 +1447|445|m|m|s|u|sdw44|sdw44|8445|/data/mirror445 +1491|489|m|m|s|u|sdw44|sdw44|8489|/data/mirror489 +1546|544|m|m|s|u|sdw44|sdw44|8544|/data/mirror544 +1599|597|m|m|s|u|sdw44|sdw44|8597|/data/mirror597 +1646|644|m|m|s|u|sdw44|sdw44|8644|/data/mirror644 +1692|690|m|m|s|u|sdw44|sdw44|8690|/data/mirror690 +1740|738|m|m|s|u|sdw44|sdw44|8738|/data/mirror738 +1797|795|m|m|s|u|sdw44|sdw44|8795|/data/mirror795 +1847|845|m|m|s|u|sdw44|sdw44|8845|/data/mirror845 +1901|899|m|m|s|u|sdw44|sdw44|8899|/data/mirror899 +1936|934|m|m|s|u|sdw44|sdw44|8934|/data/mirror934 +1999|997|m|m|s|u|sdw44|sdw44|8997|/data/mirror997 +# SDW45 +889|887|p|p|s|u|sdw45|sdw45|7887|/data/primary887 +890|888|p|p|s|u|sdw45|sdw45|7888|/data/primary888 +891|889|p|p|s|u|sdw45|sdw45|7889|/data/primary889 +892|890|p|p|s|u|sdw45|sdw45|7890|/data/primary890 +893|891|p|p|s|u|sdw45|sdw45|7891|/data/primary891 +894|892|p|p|s|u|sdw45|sdw45|7892|/data/primary892 +895|893|p|p|s|u|sdw45|sdw45|7893|/data/primary893 +896|894|p|p|s|u|sdw45|sdw45|7894|/data/primary894 +897|895|p|p|s|u|sdw45|sdw45|7895|/data/primary895 +898|896|p|p|s|u|sdw45|sdw45|7896|/data/primary896 +899|897|p|p|s|u|sdw45|sdw45|7897|/data/primary897 +900|898|p|p|s|u|sdw45|sdw45|7898|/data/primary898 +901|899|p|p|s|u|sdw45|sdw45|7899|/data/primary899 +902|900|p|p|s|u|sdw45|sdw45|7900|/data/primary900 +903|901|p|p|s|u|sdw45|sdw45|7901|/data/primary901 +904|902|p|p|s|u|sdw45|sdw45|7902|/data/primary902 +905|903|p|p|s|u|sdw45|sdw45|7903|/data/primary903 +906|904|p|p|s|u|sdw45|sdw45|7904|/data/primary904 +1050|48|m|m|s|u|sdw45|sdw45|8048|/data/mirror48 +1096|94|m|m|s|u|sdw45|sdw45|8094|/data/mirror94 +1131|129|m|m|s|u|sdw45|sdw45|8129|/data/mirror129 +1198|196|m|m|s|u|sdw45|sdw45|8196|/data/mirror196 +1249|247|m|m|s|u|sdw45|sdw45|8247|/data/mirror247 +1285|283|m|m|s|u|sdw45|sdw45|8283|/data/mirror283 +1331|329|m|m|s|u|sdw45|sdw45|8329|/data/mirror329 +1397|395|m|m|s|u|sdw45|sdw45|8395|/data/mirror395 +1448|446|m|m|s|u|sdw45|sdw45|8446|/data/mirror446 +1501|499|m|m|s|u|sdw45|sdw45|8499|/data/mirror499 +1535|533|m|m|s|u|sdw45|sdw45|8533|/data/mirror533 +1598|596|m|m|s|u|sdw45|sdw45|8596|/data/mirror596 +1649|647|m|m|s|u|sdw45|sdw45|8647|/data/mirror647 +1705|703|m|m|s|u|sdw45|sdw45|8703|/data/mirror703 +1750|748|m|m|s|u|sdw45|sdw45|8748|/data/mirror748 +1781|779|m|m|s|u|sdw45|sdw45|8779|/data/mirror779 +1833|831|m|m|s|u|sdw45|sdw45|8831|/data/mirror831 +1907|905|m|m|s|u|sdw45|sdw45|8905|/data/mirror905 +1953|951|m|m|s|u|sdw45|sdw45|8951|/data/mirror951 +2001|999|m|m|s|u|sdw45|sdw45|8999|/data/mirror999 +# SDW46 +907|905|p|p|s|u|sdw46|sdw46|7905|/data/primary905 +908|906|p|p|s|u|sdw46|sdw46|7906|/data/primary906 +909|907|p|p|s|u|sdw46|sdw46|7907|/data/primary907 +910|908|p|p|s|u|sdw46|sdw46|7908|/data/primary908 +911|909|p|p|s|u|sdw46|sdw46|7909|/data/primary909 +912|910|p|p|s|u|sdw46|sdw46|7910|/data/primary910 +913|911|p|p|s|u|sdw46|sdw46|7911|/data/primary911 +914|912|p|p|s|u|sdw46|sdw46|7912|/data/primary912 +915|913|p|p|s|u|sdw46|sdw46|7913|/data/primary913 +916|914|p|p|s|u|sdw46|sdw46|7914|/data/primary914 +917|915|p|p|s|u|sdw46|sdw46|7915|/data/primary915 +918|916|p|p|s|u|sdw46|sdw46|7916|/data/primary916 +919|917|p|p|s|u|sdw46|sdw46|7917|/data/primary917 +920|918|p|p|s|u|sdw46|sdw46|7918|/data/primary918 +921|919|p|p|s|u|sdw46|sdw46|7919|/data/primary919 +922|920|p|p|s|u|sdw46|sdw46|7920|/data/primary920 +923|921|p|p|s|u|sdw46|sdw46|7921|/data/primary921 +924|922|p|p|s|u|sdw46|sdw46|7922|/data/primary922 +925|923|p|p|s|u|sdw46|sdw46|7923|/data/primary923 +926|924|p|p|s|u|sdw46|sdw46|7924|/data/primary924 +1052|50|m|m|s|u|sdw46|sdw46|8050|/data/mirror50 +1097|95|m|m|s|u|sdw46|sdw46|8095|/data/mirror95 +1149|147|m|m|s|u|sdw46|sdw46|8147|/data/mirror147 +1199|197|m|m|s|u|sdw46|sdw46|8197|/data/mirror197 +1241|239|m|m|s|u|sdw46|sdw46|8239|/data/mirror239 +1298|296|m|m|s|u|sdw46|sdw46|8296|/data/mirror296 +1352|350|m|m|s|u|sdw46|sdw46|8350|/data/mirror350 +1398|396|m|m|s|u|sdw46|sdw46|8396|/data/mirror396 +1449|447|m|m|s|u|sdw46|sdw46|8447|/data/mirror447 +1492|490|m|m|s|u|sdw46|sdw46|8490|/data/mirror490 +1548|546|m|m|s|u|sdw46|sdw46|8546|/data/mirror546 +1600|598|m|m|s|u|sdw46|sdw46|8598|/data/mirror598 +1651|649|m|m|s|u|sdw46|sdw46|8649|/data/mirror649 +1706|704|m|m|s|u|sdw46|sdw46|8704|/data/mirror704 +1741|739|m|m|s|u|sdw46|sdw46|8739|/data/mirror739 +1798|796|m|m|s|u|sdw46|sdw46|8796|/data/mirror796 +1849|847|m|m|s|u|sdw46|sdw46|8847|/data/mirror847 +1881|879|m|m|s|u|sdw46|sdw46|8879|/data/mirror879 +1937|935|m|m|s|u|sdw46|sdw46|8935|/data/mirror935 +1995|993|m|m|s|u|sdw46|sdw46|8993|/data/mirror993 +# SDW47 +927|925|p|p|s|u|sdw47|sdw47|7925|/data/primary925 +928|926|p|p|s|u|sdw47|sdw47|7926|/data/primary926 +929|927|p|p|s|u|sdw47|sdw47|7927|/data/primary927 +930|928|p|p|s|u|sdw47|sdw47|7928|/data/primary928 +931|929|p|p|s|u|sdw47|sdw47|7929|/data/primary929 +932|930|p|p|s|u|sdw47|sdw47|7930|/data/primary930 +933|931|p|p|s|u|sdw47|sdw47|7931|/data/primary931 +934|932|p|p|s|u|sdw47|sdw47|7932|/data/primary932 +935|933|p|p|s|u|sdw47|sdw47|7933|/data/primary933 +936|934|p|p|s|u|sdw47|sdw47|7934|/data/primary934 +937|935|p|p|s|u|sdw47|sdw47|7935|/data/primary935 +938|936|p|p|s|u|sdw47|sdw47|7936|/data/primary936 +939|937|p|p|s|u|sdw47|sdw47|7937|/data/primary937 +940|938|p|p|s|u|sdw47|sdw47|7938|/data/primary938 +941|939|p|p|s|u|sdw47|sdw47|7939|/data/primary939 +942|940|p|p|s|u|sdw47|sdw47|7940|/data/primary940 +1051|49|m|m|s|u|sdw47|sdw47|8049|/data/mirror49 +1098|96|m|m|s|u|sdw47|sdw47|8096|/data/mirror96 +1150|148|m|m|s|u|sdw47|sdw47|8148|/data/mirror148 +1200|198|m|m|s|u|sdw47|sdw47|8198|/data/mirror198 +1250|248|m|m|s|u|sdw47|sdw47|8248|/data/mirror248 +1299|297|m|m|s|u|sdw47|sdw47|8297|/data/mirror297 +1353|351|m|m|s|u|sdw47|sdw47|8351|/data/mirror351 +1399|397|m|m|s|u|sdw47|sdw47|8397|/data/mirror397 +1434|432|m|m|s|u|sdw47|sdw47|8432|/data/mirror432 +1504|502|m|m|s|u|sdw47|sdw47|8502|/data/mirror502 +1550|548|m|m|s|u|sdw47|sdw47|8548|/data/mirror548 +1601|599|m|m|s|u|sdw47|sdw47|8599|/data/mirror599 +1652|650|m|m|s|u|sdw47|sdw47|8650|/data/mirror650 +1707|705|m|m|s|u|sdw47|sdw47|8705|/data/mirror705 +1751|749|m|m|s|u|sdw47|sdw47|8749|/data/mirror749 +1799|797|m|m|s|u|sdw47|sdw47|8797|/data/mirror797 +1834|832|m|m|s|u|sdw47|sdw47|8832|/data/mirror832 +1902|900|m|m|s|u|sdw47|sdw47|8900|/data/mirror900 +1947|945|m|m|s|u|sdw47|sdw47|8945|/data/mirror945 +# SDW48 +943|941|p|p|s|u|sdw48|sdw48|7941|/data/primary941 +944|942|p|p|s|u|sdw48|sdw48|7942|/data/primary942 +945|943|p|p|s|u|sdw48|sdw48|7943|/data/primary943 +946|944|p|p|s|u|sdw48|sdw48|7944|/data/primary944 +947|945|p|p|s|u|sdw48|sdw48|7945|/data/primary945 +948|946|p|p|s|u|sdw48|sdw48|7946|/data/primary946 +949|947|p|p|s|u|sdw48|sdw48|7947|/data/primary947 +950|948|p|p|s|u|sdw48|sdw48|7948|/data/primary948 +951|949|p|p|s|u|sdw48|sdw48|7949|/data/primary949 +952|950|p|p|s|u|sdw48|sdw48|7950|/data/primary950 +953|951|p|p|s|u|sdw48|sdw48|7951|/data/primary951 +954|952|p|p|s|u|sdw48|sdw48|7952|/data/primary952 +955|953|p|p|s|u|sdw48|sdw48|7953|/data/primary953 +956|954|p|p|s|u|sdw48|sdw48|7954|/data/primary954 +957|955|p|p|s|u|sdw48|sdw48|7955|/data/primary955 +958|956|p|p|s|u|sdw48|sdw48|7956|/data/primary956 +959|957|p|p|s|u|sdw48|sdw48|7957|/data/primary957 +960|958|p|p|s|u|sdw48|sdw48|7958|/data/primary958 +961|959|p|p|s|u|sdw48|sdw48|7959|/data/primary959 +962|960|p|p|s|u|sdw48|sdw48|7960|/data/primary960 +963|961|p|p|s|u|sdw48|sdw48|7961|/data/primary961 +964|962|p|p|s|u|sdw48|sdw48|7962|/data/primary962 +965|963|p|p|s|u|sdw48|sdw48|7963|/data/primary963 +966|964|p|p|s|u|sdw48|sdw48|7964|/data/primary964 +967|965|p|p|s|u|sdw48|sdw48|7965|/data/primary965 +968|966|p|p|s|u|sdw48|sdw48|7966|/data/primary966 +969|967|p|p|s|u|sdw48|sdw48|7967|/data/primary967 +970|968|p|p|s|u|sdw48|sdw48|7968|/data/primary968 +971|969|p|p|s|u|sdw48|sdw48|7969|/data/primary969 +1053|51|m|m|s|u|sdw48|sdw48|8051|/data/mirror51 +1100|98|m|m|s|u|sdw48|sdw48|8098|/data/mirror98 +1151|149|m|m|s|u|sdw48|sdw48|8149|/data/mirror149 +1201|199|m|m|s|u|sdw48|sdw48|8199|/data/mirror199 +1232|230|m|m|s|u|sdw48|sdw48|8230|/data/mirror230 +1300|298|m|m|s|u|sdw48|sdw48|8298|/data/mirror298 +1347|345|m|m|s|u|sdw48|sdw48|8345|/data/mirror345 +1400|398|m|m|s|u|sdw48|sdw48|8398|/data/mirror398 +1450|448|m|m|s|u|sdw48|sdw48|8448|/data/mirror448 +1507|505|m|m|s|u|sdw48|sdw48|8505|/data/mirror505 +1551|549|m|m|s|u|sdw48|sdw48|8549|/data/mirror549 +1578|576|m|m|s|u|sdw48|sdw48|8576|/data/mirror576 +1634|632|m|m|s|u|sdw48|sdw48|8632|/data/mirror632 +1681|679|m|m|s|u|sdw48|sdw48|8679|/data/mirror679 +1734|732|m|m|s|u|sdw48|sdw48|8732|/data/mirror732 +1800|798|m|m|s|u|sdw48|sdw48|8798|/data/mirror798 +1851|849|m|m|s|u|sdw48|sdw48|8849|/data/mirror849 +1882|880|m|m|s|u|sdw48|sdw48|8880|/data/mirror880 +1972|970|m|m|s|u|sdw48|sdw48|8970|/data/mirror970 +# SDW49 +972|970|p|p|s|u|sdw49|sdw49|7970|/data/primary970 +973|971|p|p|s|u|sdw49|sdw49|7971|/data/primary971 +974|972|p|p|s|u|sdw49|sdw49|7972|/data/primary972 +975|973|p|p|s|u|sdw49|sdw49|7973|/data/primary973 +976|974|p|p|s|u|sdw49|sdw49|7974|/data/primary974 +977|975|p|p|s|u|sdw49|sdw49|7975|/data/primary975 +978|976|p|p|s|u|sdw49|sdw49|7976|/data/primary976 +979|977|p|p|s|u|sdw49|sdw49|7977|/data/primary977 +980|978|p|p|s|u|sdw49|sdw49|7978|/data/primary978 +981|979|p|p|s|u|sdw49|sdw49|7979|/data/primary979 +982|980|p|p|s|u|sdw49|sdw49|7980|/data/primary980 +983|981|p|p|s|u|sdw49|sdw49|7981|/data/primary981 +984|982|p|p|s|u|sdw49|sdw49|7982|/data/primary982 +985|983|p|p|s|u|sdw49|sdw49|7983|/data/primary983 +986|984|p|p|s|u|sdw49|sdw49|7984|/data/primary984 +987|985|p|p|s|u|sdw49|sdw49|7985|/data/primary985 +988|986|p|p|s|u|sdw49|sdw49|7986|/data/primary986 +989|987|p|p|s|u|sdw49|sdw49|7987|/data/primary987 +990|988|p|p|s|u|sdw49|sdw49|7988|/data/primary988 +1054|52|m|m|s|u|sdw49|sdw49|8052|/data/mirror52 +1101|99|m|m|s|u|sdw49|sdw49|8099|/data/mirror99 +1152|150|m|m|s|u|sdw49|sdw49|8150|/data/mirror150 +1204|202|m|m|s|u|sdw49|sdw49|8202|/data/mirror202 +1251|249|m|m|s|u|sdw49|sdw49|8249|/data/mirror249 +1301|299|m|m|s|u|sdw49|sdw49|8299|/data/mirror299 +1354|352|m|m|s|u|sdw49|sdw49|8352|/data/mirror352 +1401|399|m|m|s|u|sdw49|sdw49|8399|/data/mirror399 +1451|449|m|m|s|u|sdw49|sdw49|8449|/data/mirror449 +1509|507|m|m|s|u|sdw49|sdw49|8507|/data/mirror507 +1552|550|m|m|s|u|sdw49|sdw49|8550|/data/mirror550 +1602|600|m|m|s|u|sdw49|sdw49|8600|/data/mirror600 +1653|651|m|m|s|u|sdw49|sdw49|8651|/data/mirror651 +1694|692|m|m|s|u|sdw49|sdw49|8692|/data/mirror692 +1753|751|m|m|s|u|sdw49|sdw49|8751|/data/mirror751 +1802|800|m|m|s|u|sdw49|sdw49|8800|/data/mirror800 +1852|850|m|m|s|u|sdw49|sdw49|8850|/data/mirror850 +1904|902|m|m|s|u|sdw49|sdw49|8902|/data/mirror902 +1938|936|m|m|s|u|sdw49|sdw49|8936|/data/mirror936 +# SDW5 +96|94|p|p|s|u|sdw5|sdw5|7094|/data/primary94 +97|95|p|p|s|u|sdw5|sdw5|7095|/data/primary95 +98|96|p|p|s|u|sdw5|sdw5|7096|/data/primary96 +99|97|p|p|s|u|sdw5|sdw5|7097|/data/primary97 +100|98|p|p|s|u|sdw5|sdw5|7098|/data/primary98 +101|99|p|p|s|u|sdw5|sdw5|7099|/data/primary99 +102|100|p|p|s|u|sdw5|sdw5|7100|/data/primary100 +103|101|p|p|s|u|sdw5|sdw5|7101|/data/primary101 +104|102|p|p|s|u|sdw5|sdw5|7102|/data/primary102 +105|103|p|p|s|u|sdw5|sdw5|7103|/data/primary103 +106|104|p|p|s|u|sdw5|sdw5|7104|/data/primary104 +107|105|p|p|s|u|sdw5|sdw5|7105|/data/primary105 +108|106|p|p|s|u|sdw5|sdw5|7106|/data/primary106 +109|107|p|p|s|u|sdw5|sdw5|7107|/data/primary107 +110|108|p|p|s|u|sdw5|sdw5|7108|/data/primary108 +111|109|p|p|s|u|sdw5|sdw5|7109|/data/primary109 +112|110|p|p|s|u|sdw5|sdw5|7110|/data/primary110 +113|111|p|p|s|u|sdw5|sdw5|7111|/data/primary111 +114|112|p|p|s|u|sdw5|sdw5|7112|/data/primary112 +115|113|p|p|s|u|sdw5|sdw5|7113|/data/primary113 +1005|3|m|m|s|u|sdw5|sdw5|8003|/data/mirror3 +1060|58|m|m|s|u|sdw5|sdw5|8058|/data/mirror58 +1116|114|m|m|s|u|sdw5|sdw5|8114|/data/mirror114 +1153|151|m|m|s|u|sdw5|sdw5|8151|/data/mirror151 +1212|210|m|m|s|u|sdw5|sdw5|8210|/data/mirror210 +1260|258|m|m|s|u|sdw5|sdw5|8258|/data/mirror258 +1307|305|m|m|s|u|sdw5|sdw5|8305|/data/mirror305 +1361|359|m|m|s|u|sdw5|sdw5|8359|/data/mirror359 +1408|406|m|m|s|u|sdw5|sdw5|8406|/data/mirror406 +1460|458|m|m|s|u|sdw5|sdw5|8458|/data/mirror458 +1512|510|m|m|s|u|sdw5|sdw5|8510|/data/mirror510 +1560|558|m|m|s|u|sdw5|sdw5|8558|/data/mirror558 +1608|606|m|m|s|u|sdw5|sdw5|8606|/data/mirror606 +1660|658|m|m|s|u|sdw5|sdw5|8658|/data/mirror658 +1715|713|m|m|s|u|sdw5|sdw5|8713|/data/mirror713 +1745|743|m|m|s|u|sdw5|sdw5|8743|/data/mirror743 +1811|809|m|m|s|u|sdw5|sdw5|8809|/data/mirror809 +1850|848|m|m|s|u|sdw5|sdw5|8848|/data/mirror848 +1911|909|m|m|s|u|sdw5|sdw5|8909|/data/mirror909 +1949|947|m|m|s|u|sdw5|sdw5|8947|/data/mirror947 +# SDW50 +991|989|p|p|s|u|sdw50|sdw50|7989|/data/primary989 +992|990|p|p|s|u|sdw50|sdw50|7990|/data/primary990 +993|991|p|p|s|u|sdw50|sdw50|7991|/data/primary991 +994|992|p|p|s|u|sdw50|sdw50|7992|/data/primary992 +995|993|p|p|s|u|sdw50|sdw50|7993|/data/primary993 +996|994|p|p|s|u|sdw50|sdw50|7994|/data/primary994 +997|995|p|p|s|u|sdw50|sdw50|7995|/data/primary995 +998|996|p|p|s|u|sdw50|sdw50|7996|/data/primary996 +999|997|p|p|s|u|sdw50|sdw50|7997|/data/primary997 +1000|998|p|p|s|u|sdw50|sdw50|7998|/data/primary998 +1001|999|p|p|s|u|sdw50|sdw50|7999|/data/primary999 +1055|53|m|m|s|u|sdw50|sdw50|8053|/data/mirror53 +1102|100|m|m|s|u|sdw50|sdw50|8100|/data/mirror100 +1154|152|m|m|s|u|sdw50|sdw50|8152|/data/mirror152 +1206|204|m|m|s|u|sdw50|sdw50|8204|/data/mirror204 +1252|250|m|m|s|u|sdw50|sdw50|8250|/data/mirror250 +1302|300|m|m|s|u|sdw50|sdw50|8300|/data/mirror300 +1355|353|m|m|s|u|sdw50|sdw50|8353|/data/mirror353 +1402|400|m|m|s|u|sdw50|sdw50|8400|/data/mirror400 +1452|450|m|m|s|u|sdw50|sdw50|8450|/data/mirror450 +1494|492|m|m|s|u|sdw50|sdw50|8492|/data/mirror492 +1556|554|m|m|s|u|sdw50|sdw50|8554|/data/mirror554 +1603|601|m|m|s|u|sdw50|sdw50|8601|/data/mirror601 +1656|654|m|m|s|u|sdw50|sdw50|8654|/data/mirror654 +1682|680|m|m|s|u|sdw50|sdw50|8680|/data/mirror680 +1754|752|m|m|s|u|sdw50|sdw50|8752|/data/mirror752 +1803|801|m|m|s|u|sdw50|sdw50|8801|/data/mirror801 +1853|851|m|m|s|u|sdw50|sdw50|8851|/data/mirror851 +1905|903|m|m|s|u|sdw50|sdw50|8903|/data/mirror903 +1954|952|m|m|s|u|sdw50|sdw50|8952|/data/mirror952 +1971|969|m|m|s|u|sdw50|sdw50|8969|/data/mirror969 +# SDW6 +116|114|p|p|s|u|sdw6|sdw6|7114|/data/primary114 +117|115|p|p|s|u|sdw6|sdw6|7115|/data/primary115 +118|116|p|p|s|u|sdw6|sdw6|7116|/data/primary116 +119|117|p|p|s|u|sdw6|sdw6|7117|/data/primary117 +120|118|p|p|s|u|sdw6|sdw6|7118|/data/primary118 +121|119|p|p|s|u|sdw6|sdw6|7119|/data/primary119 +122|120|p|p|s|u|sdw6|sdw6|7120|/data/primary120 +123|121|p|p|s|u|sdw6|sdw6|7121|/data/primary121 +124|122|p|p|s|u|sdw6|sdw6|7122|/data/primary122 +125|123|p|p|s|u|sdw6|sdw6|7123|/data/primary123 +126|124|p|p|s|u|sdw6|sdw6|7124|/data/primary124 +127|125|p|p|s|u|sdw6|sdw6|7125|/data/primary125 +128|126|p|p|s|u|sdw6|sdw6|7126|/data/primary126 +129|127|p|p|s|u|sdw6|sdw6|7127|/data/primary127 +130|128|p|p|s|u|sdw6|sdw6|7128|/data/primary128 +131|129|p|p|s|u|sdw6|sdw6|7129|/data/primary129 +132|130|p|p|s|u|sdw6|sdw6|7130|/data/primary130 +133|131|p|p|s|u|sdw6|sdw6|7131|/data/primary131 +134|132|p|p|s|u|sdw6|sdw6|7132|/data/primary132 +135|133|p|p|s|u|sdw6|sdw6|7133|/data/primary133 +136|134|p|p|s|u|sdw6|sdw6|7134|/data/primary134 +137|135|p|p|s|u|sdw6|sdw6|7135|/data/primary135 +1008|6|m|m|s|u|sdw6|sdw6|8006|/data/mirror6 +1041|39|m|m|s|u|sdw6|sdw6|8039|/data/mirror39 +1109|107|m|m|s|u|sdw6|sdw6|8107|/data/mirror107 +1159|157|m|m|s|u|sdw6|sdw6|8157|/data/mirror157 +1189|187|m|m|s|u|sdw6|sdw6|8187|/data/mirror187 +1261|259|m|m|s|u|sdw6|sdw6|8259|/data/mirror259 +1308|306|m|m|s|u|sdw6|sdw6|8306|/data/mirror306 +1362|360|m|m|s|u|sdw6|sdw6|8360|/data/mirror360 +1395|393|m|m|s|u|sdw6|sdw6|8393|/data/mirror393 +1465|463|m|m|s|u|sdw6|sdw6|8463|/data/mirror463 +1515|513|m|m|s|u|sdw6|sdw6|8513|/data/mirror513 +1561|559|m|m|s|u|sdw6|sdw6|8559|/data/mirror559 +1609|607|m|m|s|u|sdw6|sdw6|8607|/data/mirror607 +1662|660|m|m|s|u|sdw6|sdw6|8660|/data/mirror660 +1708|706|m|m|s|u|sdw6|sdw6|8706|/data/mirror706 +1762|760|m|m|s|u|sdw6|sdw6|8760|/data/mirror760 +1812|810|m|m|s|u|sdw6|sdw6|8810|/data/mirror810 +1858|856|m|m|s|u|sdw6|sdw6|8856|/data/mirror856 +1913|911|m|m|s|u|sdw6|sdw6|8911|/data/mirror911 +1962|960|m|m|s|u|sdw6|sdw6|8960|/data/mirror960 +1989|987|m|m|s|u|sdw6|sdw6|8987|/data/mirror987 +# SDW7 +138|136|p|p|s|u|sdw7|sdw7|7136|/data/primary136 +139|137|p|p|s|u|sdw7|sdw7|7137|/data/primary137 +140|138|p|p|s|u|sdw7|sdw7|7138|/data/primary138 +141|139|p|p|s|u|sdw7|sdw7|7139|/data/primary139 +142|140|p|p|s|u|sdw7|sdw7|7140|/data/primary140 +143|141|p|p|s|u|sdw7|sdw7|7141|/data/primary141 +144|142|p|p|s|u|sdw7|sdw7|7142|/data/primary142 +145|143|p|p|s|u|sdw7|sdw7|7143|/data/primary143 +146|144|p|p|s|u|sdw7|sdw7|7144|/data/primary144 +147|145|p|p|s|u|sdw7|sdw7|7145|/data/primary145 +148|146|p|p|s|u|sdw7|sdw7|7146|/data/primary146 +149|147|p|p|s|u|sdw7|sdw7|7147|/data/primary147 +1009|7|m|m|s|u|sdw7|sdw7|8007|/data/mirror7 +1063|61|m|m|s|u|sdw7|sdw7|8061|/data/mirror61 +1114|112|m|m|s|u|sdw7|sdw7|8112|/data/mirror112 +1161|159|m|m|s|u|sdw7|sdw7|8159|/data/mirror159 +1215|213|m|m|s|u|sdw7|sdw7|8213|/data/mirror213 +1246|244|m|m|s|u|sdw7|sdw7|8244|/data/mirror244 +1309|307|m|m|s|u|sdw7|sdw7|8307|/data/mirror307 +1337|335|m|m|s|u|sdw7|sdw7|8335|/data/mirror335 +1410|408|m|m|s|u|sdw7|sdw7|8408|/data/mirror408 +1466|464|m|m|s|u|sdw7|sdw7|8464|/data/mirror464 +1517|515|m|m|s|u|sdw7|sdw7|8515|/data/mirror515 +1562|560|m|m|s|u|sdw7|sdw7|8560|/data/mirror560 +1610|608|m|m|s|u|sdw7|sdw7|8608|/data/mirror608 +1664|662|m|m|s|u|sdw7|sdw7|8662|/data/mirror662 +1716|714|m|m|s|u|sdw7|sdw7|8714|/data/mirror714 +1746|744|m|m|s|u|sdw7|sdw7|8744|/data/mirror744 +1801|799|m|m|s|u|sdw7|sdw7|8799|/data/mirror799 +1859|857|m|m|s|u|sdw7|sdw7|8857|/data/mirror857 +1899|897|m|m|s|u|sdw7|sdw7|8897|/data/mirror897 +1963|961|m|m|s|u|sdw7|sdw7|8961|/data/mirror961 +# SDW8 +150|148|p|p|s|u|sdw8|sdw8|7148|/data/primary148 +151|149|p|p|s|u|sdw8|sdw8|7149|/data/primary149 +152|150|p|p|s|u|sdw8|sdw8|7150|/data/primary150 +153|151|p|p|s|u|sdw8|sdw8|7151|/data/primary151 +154|152|p|p|s|u|sdw8|sdw8|7152|/data/primary152 +155|153|p|p|s|u|sdw8|sdw8|7153|/data/primary153 +156|154|p|p|s|u|sdw8|sdw8|7154|/data/primary154 +157|155|p|p|s|u|sdw8|sdw8|7155|/data/primary155 +158|156|p|p|s|u|sdw8|sdw8|7156|/data/primary156 +159|157|p|p|s|u|sdw8|sdw8|7157|/data/primary157 +160|158|p|p|s|u|sdw8|sdw8|7158|/data/primary158 +161|159|p|p|s|u|sdw8|sdw8|7159|/data/primary159 +162|160|p|p|s|u|sdw8|sdw8|7160|/data/primary160 +163|161|p|p|s|u|sdw8|sdw8|7161|/data/primary161 +164|162|p|p|s|u|sdw8|sdw8|7162|/data/primary162 +165|163|p|p|s|u|sdw8|sdw8|7163|/data/primary163 +166|164|p|p|s|u|sdw8|sdw8|7164|/data/primary164 +167|165|p|p|s|u|sdw8|sdw8|7165|/data/primary165 +168|166|p|p|s|u|sdw8|sdw8|7166|/data/primary166 +169|167|p|p|s|u|sdw8|sdw8|7167|/data/primary167 +170|168|p|p|s|u|sdw8|sdw8|7168|/data/primary168 +1011|9|m|m|s|u|sdw8|sdw8|8009|/data/mirror9 +1064|62|m|m|s|u|sdw8|sdw8|8062|/data/mirror62 +1115|113|m|m|s|u|sdw8|sdw8|8113|/data/mirror113 +1171|169|m|m|s|u|sdw8|sdw8|8169|/data/mirror169 +1216|214|m|m|s|u|sdw8|sdw8|8214|/data/mirror214 +1258|256|m|m|s|u|sdw8|sdw8|8256|/data/mirror256 +1305|303|m|m|s|u|sdw8|sdw8|8303|/data/mirror303 +1363|361|m|m|s|u|sdw8|sdw8|8361|/data/mirror361 +1411|409|m|m|s|u|sdw8|sdw8|8409|/data/mirror409 +1467|465|m|m|s|u|sdw8|sdw8|8465|/data/mirror465 +1518|516|m|m|s|u|sdw8|sdw8|8516|/data/mirror516 +1549|547|m|m|s|u|sdw8|sdw8|8547|/data/mirror547 +1612|610|m|m|s|u|sdw8|sdw8|8610|/data/mirror610 +1665|663|m|m|s|u|sdw8|sdw8|8663|/data/mirror663 +1686|684|m|m|s|u|sdw8|sdw8|8684|/data/mirror684 +1763|761|m|m|s|u|sdw8|sdw8|8761|/data/mirror761 +1813|811|m|m|s|u|sdw8|sdw8|8811|/data/mirror811 +1860|858|m|m|s|u|sdw8|sdw8|8858|/data/mirror858 +1887|885|m|m|s|u|sdw8|sdw8|8885|/data/mirror885 +1964|962|m|m|s|u|sdw8|sdw8|8962|/data/mirror962 +2000|998|m|m|s|u|sdw8|sdw8|8998|/data/mirror998 +# SDW9 +171|169|p|p|s|u|sdw9|sdw9|7169|/data/primary169 +172|170|p|p|s|u|sdw9|sdw9|7170|/data/primary170 +173|171|p|p|s|u|sdw9|sdw9|7171|/data/primary171 +174|172|p|p|s|u|sdw9|sdw9|7172|/data/primary172 +175|173|p|p|s|u|sdw9|sdw9|7173|/data/primary173 +176|174|p|p|s|u|sdw9|sdw9|7174|/data/primary174 +177|175|p|p|s|u|sdw9|sdw9|7175|/data/primary175 +178|176|p|p|s|u|sdw9|sdw9|7176|/data/primary176 +179|177|p|p|s|u|sdw9|sdw9|7177|/data/primary177 +180|178|p|p|s|u|sdw9|sdw9|7178|/data/primary178 +181|179|p|p|s|u|sdw9|sdw9|7179|/data/primary179 +182|180|p|p|s|u|sdw9|sdw9|7180|/data/primary180 +183|181|p|p|s|u|sdw9|sdw9|7181|/data/primary181 +184|182|p|p|s|u|sdw9|sdw9|7182|/data/primary182 +185|183|p|p|s|u|sdw9|sdw9|7183|/data/primary183 +186|184|p|p|s|u|sdw9|sdw9|7184|/data/primary184 +187|185|p|p|s|u|sdw9|sdw9|7185|/data/primary185 +188|186|p|p|s|u|sdw9|sdw9|7186|/data/primary186 +189|187|p|p|s|u|sdw9|sdw9|7187|/data/primary187 +190|188|p|p|s|u|sdw9|sdw9|7188|/data/primary188 +191|189|p|p|s|u|sdw9|sdw9|7189|/data/primary189 +192|190|p|p|s|u|sdw9|sdw9|7190|/data/primary190 +1012|10|m|m|s|u|sdw9|sdw9|8010|/data/mirror10 +1056|54|m|m|s|u|sdw9|sdw9|8054|/data/mirror54 +1117|115|m|m|s|u|sdw9|sdw9|8115|/data/mirror115 +1162|160|m|m|s|u|sdw9|sdw9|8160|/data/mirror160 +1217|215|m|m|s|u|sdw9|sdw9|8215|/data/mirror215 +1262|260|m|m|s|u|sdw9|sdw9|8260|/data/mirror260 +1310|308|m|m|s|u|sdw9|sdw9|8308|/data/mirror308 +1364|362|m|m|s|u|sdw9|sdw9|8362|/data/mirror362 +1412|410|m|m|s|u|sdw9|sdw9|8410|/data/mirror410 +1469|467|m|m|s|u|sdw9|sdw9|8467|/data/mirror467 +1519|517|m|m|s|u|sdw9|sdw9|8517|/data/mirror517 +1563|561|m|m|s|u|sdw9|sdw9|8561|/data/mirror561 +1595|593|m|m|s|u|sdw9|sdw9|8593|/data/mirror593 +1666|664|m|m|s|u|sdw9|sdw9|8664|/data/mirror664 +1709|707|m|m|s|u|sdw9|sdw9|8707|/data/mirror707 +1764|762|m|m|s|u|sdw9|sdw9|8762|/data/mirror762 +1788|786|m|m|s|u|sdw9|sdw9|8786|/data/mirror786 +1861|859|m|m|s|u|sdw9|sdw9|8859|/data/mirror859 +1915|913|m|m|s|u|sdw9|sdw9|8913|/data/mirror913 +1951|949|m|m|s|u|sdw9|sdw9|8949|/data/mirror949 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_grouped.array b/gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_grouped.array new file mode 100644 index 000000000000..ae9d07953137 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_grouped.array @@ -0,0 +1,262 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +236|114|m|m|s|u|sdw1|sdw1|8114|/data/mirror114 +237|115|m|m|s|u|sdw1|sdw1|8115|/data/mirror115 +238|116|m|m|s|u|sdw1|sdw1|8116|/data/mirror116 +239|117|m|m|s|u|sdw1|sdw1|8117|/data/mirror117 +240|118|m|m|s|u|sdw1|sdw1|8118|/data/mirror118 +241|119|m|m|s|u|sdw1|sdw1|8119|/data/mirror119 +# SDW10 +56|54|p|p|s|u|sdw10|sdw10|7054|/data/primary54 +57|55|p|p|s|u|sdw10|sdw10|7055|/data/primary55 +58|56|p|p|s|u|sdw10|sdw10|7056|/data/primary56 +59|57|p|p|s|u|sdw10|sdw10|7057|/data/primary57 +60|58|p|p|s|u|sdw10|sdw10|7058|/data/primary58 +61|59|p|p|s|u|sdw10|sdw10|7059|/data/primary59 +170|48|m|m|s|u|sdw10|sdw10|8048|/data/mirror48 +171|49|m|m|s|u|sdw10|sdw10|8049|/data/mirror49 +172|50|m|m|s|u|sdw10|sdw10|8050|/data/mirror50 +173|51|m|m|s|u|sdw10|sdw10|8051|/data/mirror51 +174|52|m|m|s|u|sdw10|sdw10|8052|/data/mirror52 +175|53|m|m|s|u|sdw10|sdw10|8053|/data/mirror53 +# SDW11 +62|60|p|p|s|u|sdw11|sdw11|7060|/data/primary60 +63|61|p|p|s|u|sdw11|sdw11|7061|/data/primary61 +64|62|p|p|s|u|sdw11|sdw11|7062|/data/primary62 +65|63|p|p|s|u|sdw11|sdw11|7063|/data/primary63 +66|64|p|p|s|u|sdw11|sdw11|7064|/data/primary64 +67|65|p|p|s|u|sdw11|sdw11|7065|/data/primary65 +176|54|m|m|s|u|sdw11|sdw11|8054|/data/mirror54 +177|55|m|m|s|u|sdw11|sdw11|8055|/data/mirror55 +178|56|m|m|s|u|sdw11|sdw11|8056|/data/mirror56 +179|57|m|m|s|u|sdw11|sdw11|8057|/data/mirror57 +180|58|m|m|s|u|sdw11|sdw11|8058|/data/mirror58 +181|59|m|m|s|u|sdw11|sdw11|8059|/data/mirror59 +# SDW12 +68|66|p|p|s|u|sdw12|sdw12|7066|/data/primary66 +69|67|p|p|s|u|sdw12|sdw12|7067|/data/primary67 +70|68|p|p|s|u|sdw12|sdw12|7068|/data/primary68 +71|69|p|p|s|u|sdw12|sdw12|7069|/data/primary69 +72|70|p|p|s|u|sdw12|sdw12|7070|/data/primary70 +73|71|p|p|s|u|sdw12|sdw12|7071|/data/primary71 +182|60|m|m|s|u|sdw12|sdw12|8060|/data/mirror60 +183|61|m|m|s|u|sdw12|sdw12|8061|/data/mirror61 +184|62|m|m|s|u|sdw12|sdw12|8062|/data/mirror62 +185|63|m|m|s|u|sdw12|sdw12|8063|/data/mirror63 +186|64|m|m|s|u|sdw12|sdw12|8064|/data/mirror64 +187|65|m|m|s|u|sdw12|sdw12|8065|/data/mirror65 +# SDW13 +74|72|p|p|s|u|sdw13|sdw13|7072|/data/primary72 +75|73|p|p|s|u|sdw13|sdw13|7073|/data/primary73 +76|74|p|p|s|u|sdw13|sdw13|7074|/data/primary74 +77|75|p|p|s|u|sdw13|sdw13|7075|/data/primary75 +78|76|p|p|s|u|sdw13|sdw13|7076|/data/primary76 +79|77|p|p|s|u|sdw13|sdw13|7077|/data/primary77 +188|66|m|m|s|u|sdw13|sdw13|8066|/data/mirror66 +189|67|m|m|s|u|sdw13|sdw13|8067|/data/mirror67 +190|68|m|m|s|u|sdw13|sdw13|8068|/data/mirror68 +191|69|m|m|s|u|sdw13|sdw13|8069|/data/mirror69 +192|70|m|m|s|u|sdw13|sdw13|8070|/data/mirror70 +193|71|m|m|s|u|sdw13|sdw13|8071|/data/mirror71 +# SDW14 +80|78|p|p|s|u|sdw14|sdw14|7078|/data/primary78 +81|79|p|p|s|u|sdw14|sdw14|7079|/data/primary79 +82|80|p|p|s|u|sdw14|sdw14|7080|/data/primary80 +83|81|p|p|s|u|sdw14|sdw14|7081|/data/primary81 +84|82|p|p|s|u|sdw14|sdw14|7082|/data/primary82 +85|83|p|p|s|u|sdw14|sdw14|7083|/data/primary83 +194|72|m|m|s|u|sdw14|sdw14|8072|/data/mirror72 +195|73|m|m|s|u|sdw14|sdw14|8073|/data/mirror73 +196|74|m|m|s|u|sdw14|sdw14|8074|/data/mirror74 +197|75|m|m|s|u|sdw14|sdw14|8075|/data/mirror75 +198|76|m|m|s|u|sdw14|sdw14|8076|/data/mirror76 +199|77|m|m|s|u|sdw14|sdw14|8077|/data/mirror77 +# SDW15 +86|84|p|p|s|u|sdw15|sdw15|7084|/data/primary84 +87|85|p|p|s|u|sdw15|sdw15|7085|/data/primary85 +88|86|p|p|s|u|sdw15|sdw15|7086|/data/primary86 +89|87|p|p|s|u|sdw15|sdw15|7087|/data/primary87 +90|88|p|p|s|u|sdw15|sdw15|7088|/data/primary88 +91|89|p|p|s|u|sdw15|sdw15|7089|/data/primary89 +200|78|m|m|s|u|sdw15|sdw15|8078|/data/mirror78 +201|79|m|m|s|u|sdw15|sdw15|8079|/data/mirror79 +202|80|m|m|s|u|sdw15|sdw15|8080|/data/mirror80 +203|81|m|m|s|u|sdw15|sdw15|8081|/data/mirror81 +204|82|m|m|s|u|sdw15|sdw15|8082|/data/mirror82 +205|83|m|m|s|u|sdw15|sdw15|8083|/data/mirror83 +# SDW16 +92|90|p|p|s|u|sdw16|sdw16|7090|/data/primary90 +93|91|p|p|s|u|sdw16|sdw16|7091|/data/primary91 +94|92|p|p|s|u|sdw16|sdw16|7092|/data/primary92 +95|93|p|p|s|u|sdw16|sdw16|7093|/data/primary93 +96|94|p|p|s|u|sdw16|sdw16|7094|/data/primary94 +97|95|p|p|s|u|sdw16|sdw16|7095|/data/primary95 +206|84|m|m|s|u|sdw16|sdw16|8084|/data/mirror84 +207|85|m|m|s|u|sdw16|sdw16|8085|/data/mirror85 +208|86|m|m|s|u|sdw16|sdw16|8086|/data/mirror86 +209|87|m|m|s|u|sdw16|sdw16|8087|/data/mirror87 +210|88|m|m|s|u|sdw16|sdw16|8088|/data/mirror88 +211|89|m|m|s|u|sdw16|sdw16|8089|/data/mirror89 +# SDW17 +98|96|p|p|s|u|sdw17|sdw17|7096|/data/primary96 +99|97|p|p|s|u|sdw17|sdw17|7097|/data/primary97 +100|98|p|p|s|u|sdw17|sdw17|7098|/data/primary98 +101|99|p|p|s|u|sdw17|sdw17|7099|/data/primary99 +102|100|p|p|s|u|sdw17|sdw17|7100|/data/primary100 +103|101|p|p|s|u|sdw17|sdw17|7101|/data/primary101 +212|90|m|m|s|u|sdw17|sdw17|8090|/data/mirror90 +213|91|m|m|s|u|sdw17|sdw17|8091|/data/mirror91 +214|92|m|m|s|u|sdw17|sdw17|8092|/data/mirror92 +215|93|m|m|s|u|sdw17|sdw17|8093|/data/mirror93 +216|94|m|m|s|u|sdw17|sdw17|8094|/data/mirror94 +217|95|m|m|s|u|sdw17|sdw17|8095|/data/mirror95 +# SDW18 +104|102|p|p|s|u|sdw18|sdw18|7102|/data/primary102 +105|103|p|p|s|u|sdw18|sdw18|7103|/data/primary103 +106|104|p|p|s|u|sdw18|sdw18|7104|/data/primary104 +107|105|p|p|s|u|sdw18|sdw18|7105|/data/primary105 +108|106|p|p|s|u|sdw18|sdw18|7106|/data/primary106 +109|107|p|p|s|u|sdw18|sdw18|7107|/data/primary107 +218|96|m|m|s|u|sdw18|sdw18|8096|/data/mirror96 +219|97|m|m|s|u|sdw18|sdw18|8097|/data/mirror97 +220|98|m|m|s|u|sdw18|sdw18|8098|/data/mirror98 +221|99|m|m|s|u|sdw18|sdw18|8099|/data/mirror99 +222|100|m|m|s|u|sdw18|sdw18|8100|/data/mirror100 +223|101|m|m|s|u|sdw18|sdw18|8101|/data/mirror101 +# SDW19 +110|108|p|p|s|u|sdw19|sdw19|7108|/data/primary108 +111|109|p|p|s|u|sdw19|sdw19|7109|/data/primary109 +112|110|p|p|s|u|sdw19|sdw19|7110|/data/primary110 +113|111|p|p|s|u|sdw19|sdw19|7111|/data/primary111 +114|112|p|p|s|u|sdw19|sdw19|7112|/data/primary112 +115|113|p|p|s|u|sdw19|sdw19|7113|/data/primary113 +224|102|m|m|s|u|sdw19|sdw19|8102|/data/mirror102 +225|103|m|m|s|u|sdw19|sdw19|8103|/data/mirror103 +226|104|m|m|s|u|sdw19|sdw19|8104|/data/mirror104 +227|105|m|m|s|u|sdw19|sdw19|8105|/data/mirror105 +228|106|m|m|s|u|sdw19|sdw19|8106|/data/mirror106 +229|107|m|m|s|u|sdw19|sdw19|8107|/data/mirror107 +# SDW2 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +13|11|p|p|s|u|sdw2|sdw2|7011|/data/primary11 +122|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +123|1|m|m|s|u|sdw2|sdw2|8001|/data/mirror1 +124|2|m|m|s|u|sdw2|sdw2|8002|/data/mirror2 +125|3|m|m|s|u|sdw2|sdw2|8003|/data/mirror3 +126|4|m|m|s|u|sdw2|sdw2|8004|/data/mirror4 +127|5|m|m|s|u|sdw2|sdw2|8005|/data/mirror5 +# SDW20 +116|114|p|p|s|u|sdw20|sdw20|7114|/data/primary114 +117|115|p|p|s|u|sdw20|sdw20|7115|/data/primary115 +118|116|p|p|s|u|sdw20|sdw20|7116|/data/primary116 +119|117|p|p|s|u|sdw20|sdw20|7117|/data/primary117 +120|118|p|p|s|u|sdw20|sdw20|7118|/data/primary118 +121|119|p|p|s|u|sdw20|sdw20|7119|/data/primary119 +230|108|m|m|s|u|sdw20|sdw20|8108|/data/mirror108 +231|109|m|m|s|u|sdw20|sdw20|8109|/data/mirror109 +232|110|m|m|s|u|sdw20|sdw20|8110|/data/mirror110 +233|111|m|m|s|u|sdw20|sdw20|8111|/data/mirror111 +234|112|m|m|s|u|sdw20|sdw20|8112|/data/mirror112 +235|113|m|m|s|u|sdw20|sdw20|8113|/data/mirror113 +# SDW3 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +17|15|p|p|s|u|sdw3|sdw3|7015|/data/primary15 +18|16|p|p|s|u|sdw3|sdw3|7016|/data/primary16 +19|17|p|p|s|u|sdw3|sdw3|7017|/data/primary17 +128|6|m|m|s|u|sdw3|sdw3|8006|/data/mirror6 +129|7|m|m|s|u|sdw3|sdw3|8007|/data/mirror7 +130|8|m|m|s|u|sdw3|sdw3|8008|/data/mirror8 +131|9|m|m|s|u|sdw3|sdw3|8009|/data/mirror9 +132|10|m|m|s|u|sdw3|sdw3|8010|/data/mirror10 +133|11|m|m|s|u|sdw3|sdw3|8011|/data/mirror11 +# SDW4 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +22|20|p|p|s|u|sdw4|sdw4|7020|/data/primary20 +23|21|p|p|s|u|sdw4|sdw4|7021|/data/primary21 +24|22|p|p|s|u|sdw4|sdw4|7022|/data/primary22 +25|23|p|p|s|u|sdw4|sdw4|7023|/data/primary23 +134|12|m|m|s|u|sdw4|sdw4|8012|/data/mirror12 +135|13|m|m|s|u|sdw4|sdw4|8013|/data/mirror13 +136|14|m|m|s|u|sdw4|sdw4|8014|/data/mirror14 +137|15|m|m|s|u|sdw4|sdw4|8015|/data/mirror15 +138|16|m|m|s|u|sdw4|sdw4|8016|/data/mirror16 +139|17|m|m|s|u|sdw4|sdw4|8017|/data/mirror17 +# SDW5 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +27|25|p|p|s|u|sdw5|sdw5|7025|/data/primary25 +28|26|p|p|s|u|sdw5|sdw5|7026|/data/primary26 +29|27|p|p|s|u|sdw5|sdw5|7027|/data/primary27 +30|28|p|p|s|u|sdw5|sdw5|7028|/data/primary28 +31|29|p|p|s|u|sdw5|sdw5|7029|/data/primary29 +140|18|m|m|s|u|sdw5|sdw5|8018|/data/mirror18 +141|19|m|m|s|u|sdw5|sdw5|8019|/data/mirror19 +142|20|m|m|s|u|sdw5|sdw5|8020|/data/mirror20 +143|21|m|m|s|u|sdw5|sdw5|8021|/data/mirror21 +144|22|m|m|s|u|sdw5|sdw5|8022|/data/mirror22 +145|23|m|m|s|u|sdw5|sdw5|8023|/data/mirror23 +# SDW6 +32|30|p|p|s|u|sdw6|sdw6|7030|/data/primary30 +33|31|p|p|s|u|sdw6|sdw6|7031|/data/primary31 +34|32|p|p|s|u|sdw6|sdw6|7032|/data/primary32 +35|33|p|p|s|u|sdw6|sdw6|7033|/data/primary33 +36|34|p|p|s|u|sdw6|sdw6|7034|/data/primary34 +37|35|p|p|s|u|sdw6|sdw6|7035|/data/primary35 +146|24|m|m|s|u|sdw6|sdw6|8024|/data/mirror24 +147|25|m|m|s|u|sdw6|sdw6|8025|/data/mirror25 +148|26|m|m|s|u|sdw6|sdw6|8026|/data/mirror26 +149|27|m|m|s|u|sdw6|sdw6|8027|/data/mirror27 +150|28|m|m|s|u|sdw6|sdw6|8028|/data/mirror28 +151|29|m|m|s|u|sdw6|sdw6|8029|/data/mirror29 +# SDW7 +38|36|p|p|s|u|sdw7|sdw7|7036|/data/primary36 +39|37|p|p|s|u|sdw7|sdw7|7037|/data/primary37 +40|38|p|p|s|u|sdw7|sdw7|7038|/data/primary38 +41|39|p|p|s|u|sdw7|sdw7|7039|/data/primary39 +42|40|p|p|s|u|sdw7|sdw7|7040|/data/primary40 +43|41|p|p|s|u|sdw7|sdw7|7041|/data/primary41 +152|30|m|m|s|u|sdw7|sdw7|8030|/data/mirror30 +153|31|m|m|s|u|sdw7|sdw7|8031|/data/mirror31 +154|32|m|m|s|u|sdw7|sdw7|8032|/data/mirror32 +155|33|m|m|s|u|sdw7|sdw7|8033|/data/mirror33 +156|34|m|m|s|u|sdw7|sdw7|8034|/data/mirror34 +157|35|m|m|s|u|sdw7|sdw7|8035|/data/mirror35 +# SDW8 +44|42|p|p|s|u|sdw8|sdw8|7042|/data/primary42 +45|43|p|p|s|u|sdw8|sdw8|7043|/data/primary43 +46|44|p|p|s|u|sdw8|sdw8|7044|/data/primary44 +47|45|p|p|s|u|sdw8|sdw8|7045|/data/primary45 +48|46|p|p|s|u|sdw8|sdw8|7046|/data/primary46 +49|47|p|p|s|u|sdw8|sdw8|7047|/data/primary47 +158|36|m|m|s|u|sdw8|sdw8|8036|/data/mirror36 +159|37|m|m|s|u|sdw8|sdw8|8037|/data/mirror37 +160|38|m|m|s|u|sdw8|sdw8|8038|/data/mirror38 +161|39|m|m|s|u|sdw8|sdw8|8039|/data/mirror39 +162|40|m|m|s|u|sdw8|sdw8|8040|/data/mirror40 +163|41|m|m|s|u|sdw8|sdw8|8041|/data/mirror41 +# SDW9 +50|48|p|p|s|u|sdw9|sdw9|7048|/data/primary48 +51|49|p|p|s|u|sdw9|sdw9|7049|/data/primary49 +52|50|p|p|s|u|sdw9|sdw9|7050|/data/primary50 +53|51|p|p|s|u|sdw9|sdw9|7051|/data/primary51 +54|52|p|p|s|u|sdw9|sdw9|7052|/data/primary52 +55|53|p|p|s|u|sdw9|sdw9|7053|/data/primary53 +164|42|m|m|s|u|sdw9|sdw9|8042|/data/mirror42 +165|43|m|m|s|u|sdw9|sdw9|8043|/data/mirror43 +166|44|m|m|s|u|sdw9|sdw9|8044|/data/mirror44 +167|45|m|m|s|u|sdw9|sdw9|8045|/data/mirror45 +168|46|m|m|s|u|sdw9|sdw9|8046|/data/mirror46 +169|47|m|m|s|u|sdw9|sdw9|8047|/data/mirror47 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_spread.array new file mode 100644 index 000000000000..1148fa76016b --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_spread.array @@ -0,0 +1,262 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +211|89|m|m|s|u|sdw1|sdw1|8089|/data/mirror89 +216|94|m|m|s|u|sdw1|sdw1|8094|/data/mirror94 +221|99|m|m|s|u|sdw1|sdw1|8099|/data/mirror99 +226|104|m|m|s|u|sdw1|sdw1|8104|/data/mirror104 +231|109|m|m|s|u|sdw1|sdw1|8109|/data/mirror109 +236|114|m|m|s|u|sdw1|sdw1|8114|/data/mirror114 +# SDW10 +56|54|p|p|s|u|sdw10|sdw10|7054|/data/primary54 +57|55|p|p|s|u|sdw10|sdw10|7055|/data/primary55 +58|56|p|p|s|u|sdw10|sdw10|7056|/data/primary56 +59|57|p|p|s|u|sdw10|sdw10|7057|/data/primary57 +60|58|p|p|s|u|sdw10|sdw10|7058|/data/primary58 +61|59|p|p|s|u|sdw10|sdw10|7059|/data/primary59 +145|23|m|m|s|u|sdw10|sdw10|8023|/data/mirror23 +150|28|m|m|s|u|sdw10|sdw10|8028|/data/mirror28 +155|33|m|m|s|u|sdw10|sdw10|8033|/data/mirror33 +160|38|m|m|s|u|sdw10|sdw10|8038|/data/mirror38 +165|43|m|m|s|u|sdw10|sdw10|8043|/data/mirror43 +170|48|m|m|s|u|sdw10|sdw10|8048|/data/mirror48 +# SDW11 +62|60|p|p|s|u|sdw11|sdw11|7060|/data/primary60 +63|61|p|p|s|u|sdw11|sdw11|7061|/data/primary61 +64|62|p|p|s|u|sdw11|sdw11|7062|/data/primary62 +65|63|p|p|s|u|sdw11|sdw11|7063|/data/primary63 +66|64|p|p|s|u|sdw11|sdw11|7064|/data/primary64 +67|65|p|p|s|u|sdw11|sdw11|7065|/data/primary65 +151|29|m|m|s|u|sdw11|sdw11|8029|/data/mirror29 +156|34|m|m|s|u|sdw11|sdw11|8034|/data/mirror34 +161|39|m|m|s|u|sdw11|sdw11|8039|/data/mirror39 +166|44|m|m|s|u|sdw11|sdw11|8044|/data/mirror44 +171|49|m|m|s|u|sdw11|sdw11|8049|/data/mirror49 +176|54|m|m|s|u|sdw11|sdw11|8054|/data/mirror54 +# SDW12 +68|66|p|p|s|u|sdw12|sdw12|7066|/data/primary66 +69|67|p|p|s|u|sdw12|sdw12|7067|/data/primary67 +70|68|p|p|s|u|sdw12|sdw12|7068|/data/primary68 +71|69|p|p|s|u|sdw12|sdw12|7069|/data/primary69 +72|70|p|p|s|u|sdw12|sdw12|7070|/data/primary70 +73|71|p|p|s|u|sdw12|sdw12|7071|/data/primary71 +157|35|m|m|s|u|sdw12|sdw12|8035|/data/mirror35 +162|40|m|m|s|u|sdw12|sdw12|8040|/data/mirror40 +167|45|m|m|s|u|sdw12|sdw12|8045|/data/mirror45 +172|50|m|m|s|u|sdw12|sdw12|8050|/data/mirror50 +177|55|m|m|s|u|sdw12|sdw12|8055|/data/mirror55 +182|60|m|m|s|u|sdw12|sdw12|8060|/data/mirror60 +# SDW13 +74|72|p|p|s|u|sdw13|sdw13|7072|/data/primary72 +75|73|p|p|s|u|sdw13|sdw13|7073|/data/primary73 +76|74|p|p|s|u|sdw13|sdw13|7074|/data/primary74 +77|75|p|p|s|u|sdw13|sdw13|7075|/data/primary75 +78|76|p|p|s|u|sdw13|sdw13|7076|/data/primary76 +79|77|p|p|s|u|sdw13|sdw13|7077|/data/primary77 +163|41|m|m|s|u|sdw13|sdw13|8041|/data/mirror41 +168|46|m|m|s|u|sdw13|sdw13|8046|/data/mirror46 +173|51|m|m|s|u|sdw13|sdw13|8051|/data/mirror51 +178|56|m|m|s|u|sdw13|sdw13|8056|/data/mirror56 +183|61|m|m|s|u|sdw13|sdw13|8061|/data/mirror61 +188|66|m|m|s|u|sdw13|sdw13|8066|/data/mirror66 +# SDW14 +80|78|p|p|s|u|sdw14|sdw14|7078|/data/primary78 +81|79|p|p|s|u|sdw14|sdw14|7079|/data/primary79 +82|80|p|p|s|u|sdw14|sdw14|7080|/data/primary80 +83|81|p|p|s|u|sdw14|sdw14|7081|/data/primary81 +84|82|p|p|s|u|sdw14|sdw14|7082|/data/primary82 +85|83|p|p|s|u|sdw14|sdw14|7083|/data/primary83 +169|47|m|m|s|u|sdw14|sdw14|8047|/data/mirror47 +174|52|m|m|s|u|sdw14|sdw14|8052|/data/mirror52 +179|57|m|m|s|u|sdw14|sdw14|8057|/data/mirror57 +184|62|m|m|s|u|sdw14|sdw14|8062|/data/mirror62 +189|67|m|m|s|u|sdw14|sdw14|8067|/data/mirror67 +194|72|m|m|s|u|sdw14|sdw14|8072|/data/mirror72 +# SDW15 +86|84|p|p|s|u|sdw15|sdw15|7084|/data/primary84 +87|85|p|p|s|u|sdw15|sdw15|7085|/data/primary85 +88|86|p|p|s|u|sdw15|sdw15|7086|/data/primary86 +89|87|p|p|s|u|sdw15|sdw15|7087|/data/primary87 +90|88|p|p|s|u|sdw15|sdw15|7088|/data/primary88 +91|89|p|p|s|u|sdw15|sdw15|7089|/data/primary89 +175|53|m|m|s|u|sdw15|sdw15|8053|/data/mirror53 +180|58|m|m|s|u|sdw15|sdw15|8058|/data/mirror58 +185|63|m|m|s|u|sdw15|sdw15|8063|/data/mirror63 +190|68|m|m|s|u|sdw15|sdw15|8068|/data/mirror68 +195|73|m|m|s|u|sdw15|sdw15|8073|/data/mirror73 +200|78|m|m|s|u|sdw15|sdw15|8078|/data/mirror78 +# SDW16 +92|90|p|p|s|u|sdw16|sdw16|7090|/data/primary90 +93|91|p|p|s|u|sdw16|sdw16|7091|/data/primary91 +94|92|p|p|s|u|sdw16|sdw16|7092|/data/primary92 +95|93|p|p|s|u|sdw16|sdw16|7093|/data/primary93 +96|94|p|p|s|u|sdw16|sdw16|7094|/data/primary94 +97|95|p|p|s|u|sdw16|sdw16|7095|/data/primary95 +181|59|m|m|s|u|sdw16|sdw16|8059|/data/mirror59 +186|64|m|m|s|u|sdw16|sdw16|8064|/data/mirror64 +191|69|m|m|s|u|sdw16|sdw16|8069|/data/mirror69 +196|74|m|m|s|u|sdw16|sdw16|8074|/data/mirror74 +201|79|m|m|s|u|sdw16|sdw16|8079|/data/mirror79 +206|84|m|m|s|u|sdw16|sdw16|8084|/data/mirror84 +# SDW17 +98|96|p|p|s|u|sdw17|sdw17|7096|/data/primary96 +99|97|p|p|s|u|sdw17|sdw17|7097|/data/primary97 +100|98|p|p|s|u|sdw17|sdw17|7098|/data/primary98 +101|99|p|p|s|u|sdw17|sdw17|7099|/data/primary99 +102|100|p|p|s|u|sdw17|sdw17|7100|/data/primary100 +103|101|p|p|s|u|sdw17|sdw17|7101|/data/primary101 +187|65|m|m|s|u|sdw17|sdw17|8065|/data/mirror65 +192|70|m|m|s|u|sdw17|sdw17|8070|/data/mirror70 +197|75|m|m|s|u|sdw17|sdw17|8075|/data/mirror75 +202|80|m|m|s|u|sdw17|sdw17|8080|/data/mirror80 +207|85|m|m|s|u|sdw17|sdw17|8085|/data/mirror85 +212|90|m|m|s|u|sdw17|sdw17|8090|/data/mirror90 +# SDW18 +104|102|p|p|s|u|sdw18|sdw18|7102|/data/primary102 +105|103|p|p|s|u|sdw18|sdw18|7103|/data/primary103 +106|104|p|p|s|u|sdw18|sdw18|7104|/data/primary104 +107|105|p|p|s|u|sdw18|sdw18|7105|/data/primary105 +108|106|p|p|s|u|sdw18|sdw18|7106|/data/primary106 +109|107|p|p|s|u|sdw18|sdw18|7107|/data/primary107 +193|71|m|m|s|u|sdw18|sdw18|8071|/data/mirror71 +198|76|m|m|s|u|sdw18|sdw18|8076|/data/mirror76 +203|81|m|m|s|u|sdw18|sdw18|8081|/data/mirror81 +208|86|m|m|s|u|sdw18|sdw18|8086|/data/mirror86 +213|91|m|m|s|u|sdw18|sdw18|8091|/data/mirror91 +218|96|m|m|s|u|sdw18|sdw18|8096|/data/mirror96 +# SDW19 +110|108|p|p|s|u|sdw19|sdw19|7108|/data/primary108 +111|109|p|p|s|u|sdw19|sdw19|7109|/data/primary109 +112|110|p|p|s|u|sdw19|sdw19|7110|/data/primary110 +113|111|p|p|s|u|sdw19|sdw19|7111|/data/primary111 +114|112|p|p|s|u|sdw19|sdw19|7112|/data/primary112 +115|113|p|p|s|u|sdw19|sdw19|7113|/data/primary113 +199|77|m|m|s|u|sdw19|sdw19|8077|/data/mirror77 +204|82|m|m|s|u|sdw19|sdw19|8082|/data/mirror82 +209|87|m|m|s|u|sdw19|sdw19|8087|/data/mirror87 +214|92|m|m|s|u|sdw19|sdw19|8092|/data/mirror92 +219|97|m|m|s|u|sdw19|sdw19|8097|/data/mirror97 +224|102|m|m|s|u|sdw19|sdw19|8102|/data/mirror102 +# SDW2 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +13|11|p|p|s|u|sdw2|sdw2|7011|/data/primary11 +122|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +217|95|m|m|s|u|sdw2|sdw2|8095|/data/mirror95 +222|100|m|m|s|u|sdw2|sdw2|8100|/data/mirror100 +227|105|m|m|s|u|sdw2|sdw2|8105|/data/mirror105 +232|110|m|m|s|u|sdw2|sdw2|8110|/data/mirror110 +237|115|m|m|s|u|sdw2|sdw2|8115|/data/mirror115 +# SDW20 +116|114|p|p|s|u|sdw20|sdw20|7114|/data/primary114 +117|115|p|p|s|u|sdw20|sdw20|7115|/data/primary115 +118|116|p|p|s|u|sdw20|sdw20|7116|/data/primary116 +119|117|p|p|s|u|sdw20|sdw20|7117|/data/primary117 +120|118|p|p|s|u|sdw20|sdw20|7118|/data/primary118 +121|119|p|p|s|u|sdw20|sdw20|7119|/data/primary119 +205|83|m|m|s|u|sdw20|sdw20|8083|/data/mirror83 +210|88|m|m|s|u|sdw20|sdw20|8088|/data/mirror88 +215|93|m|m|s|u|sdw20|sdw20|8093|/data/mirror93 +220|98|m|m|s|u|sdw20|sdw20|8098|/data/mirror98 +225|103|m|m|s|u|sdw20|sdw20|8103|/data/mirror103 +230|108|m|m|s|u|sdw20|sdw20|8108|/data/mirror108 +# SDW3 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +17|15|p|p|s|u|sdw3|sdw3|7015|/data/primary15 +18|16|p|p|s|u|sdw3|sdw3|7016|/data/primary16 +19|17|p|p|s|u|sdw3|sdw3|7017|/data/primary17 +123|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +128|6|m|m|s|u|sdw3|sdw3|8006|/data/mirror6 +223|101|m|m|s|u|sdw3|sdw3|8101|/data/mirror101 +228|106|m|m|s|u|sdw3|sdw3|8106|/data/mirror106 +233|111|m|m|s|u|sdw3|sdw3|8111|/data/mirror111 +238|116|m|m|s|u|sdw3|sdw3|8116|/data/mirror116 +# SDW4 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +22|20|p|p|s|u|sdw4|sdw4|7020|/data/primary20 +23|21|p|p|s|u|sdw4|sdw4|7021|/data/primary21 +24|22|p|p|s|u|sdw4|sdw4|7022|/data/primary22 +25|23|p|p|s|u|sdw4|sdw4|7023|/data/primary23 +124|2|m|m|s|u|sdw4|sdw4|8002|/data/mirror2 +129|7|m|m|s|u|sdw4|sdw4|8007|/data/mirror7 +134|12|m|m|s|u|sdw4|sdw4|8012|/data/mirror12 +229|107|m|m|s|u|sdw4|sdw4|8107|/data/mirror107 +234|112|m|m|s|u|sdw4|sdw4|8112|/data/mirror112 +239|117|m|m|s|u|sdw4|sdw4|8117|/data/mirror117 +# SDW5 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +27|25|p|p|s|u|sdw5|sdw5|7025|/data/primary25 +28|26|p|p|s|u|sdw5|sdw5|7026|/data/primary26 +29|27|p|p|s|u|sdw5|sdw5|7027|/data/primary27 +30|28|p|p|s|u|sdw5|sdw5|7028|/data/primary28 +31|29|p|p|s|u|sdw5|sdw5|7029|/data/primary29 +125|3|m|m|s|u|sdw5|sdw5|8003|/data/mirror3 +130|8|m|m|s|u|sdw5|sdw5|8008|/data/mirror8 +135|13|m|m|s|u|sdw5|sdw5|8013|/data/mirror13 +140|18|m|m|s|u|sdw5|sdw5|8018|/data/mirror18 +235|113|m|m|s|u|sdw5|sdw5|8113|/data/mirror113 +240|118|m|m|s|u|sdw5|sdw5|8118|/data/mirror118 +# SDW6 +32|30|p|p|s|u|sdw6|sdw6|7030|/data/primary30 +33|31|p|p|s|u|sdw6|sdw6|7031|/data/primary31 +34|32|p|p|s|u|sdw6|sdw6|7032|/data/primary32 +35|33|p|p|s|u|sdw6|sdw6|7033|/data/primary33 +36|34|p|p|s|u|sdw6|sdw6|7034|/data/primary34 +37|35|p|p|s|u|sdw6|sdw6|7035|/data/primary35 +126|4|m|m|s|u|sdw6|sdw6|8004|/data/mirror4 +131|9|m|m|s|u|sdw6|sdw6|8009|/data/mirror9 +136|14|m|m|s|u|sdw6|sdw6|8014|/data/mirror14 +141|19|m|m|s|u|sdw6|sdw6|8019|/data/mirror19 +146|24|m|m|s|u|sdw6|sdw6|8024|/data/mirror24 +241|119|m|m|s|u|sdw6|sdw6|8119|/data/mirror119 +# SDW7 +38|36|p|p|s|u|sdw7|sdw7|7036|/data/primary36 +39|37|p|p|s|u|sdw7|sdw7|7037|/data/primary37 +40|38|p|p|s|u|sdw7|sdw7|7038|/data/primary38 +41|39|p|p|s|u|sdw7|sdw7|7039|/data/primary39 +42|40|p|p|s|u|sdw7|sdw7|7040|/data/primary40 +43|41|p|p|s|u|sdw7|sdw7|7041|/data/primary41 +127|5|m|m|s|u|sdw7|sdw7|8005|/data/mirror5 +132|10|m|m|s|u|sdw7|sdw7|8010|/data/mirror10 +137|15|m|m|s|u|sdw7|sdw7|8015|/data/mirror15 +142|20|m|m|s|u|sdw7|sdw7|8020|/data/mirror20 +147|25|m|m|s|u|sdw7|sdw7|8025|/data/mirror25 +152|30|m|m|s|u|sdw7|sdw7|8030|/data/mirror30 +# SDW8 +44|42|p|p|s|u|sdw8|sdw8|7042|/data/primary42 +45|43|p|p|s|u|sdw8|sdw8|7043|/data/primary43 +46|44|p|p|s|u|sdw8|sdw8|7044|/data/primary44 +47|45|p|p|s|u|sdw8|sdw8|7045|/data/primary45 +48|46|p|p|s|u|sdw8|sdw8|7046|/data/primary46 +49|47|p|p|s|u|sdw8|sdw8|7047|/data/primary47 +133|11|m|m|s|u|sdw8|sdw8|8011|/data/mirror11 +138|16|m|m|s|u|sdw8|sdw8|8016|/data/mirror16 +143|21|m|m|s|u|sdw8|sdw8|8021|/data/mirror21 +148|26|m|m|s|u|sdw8|sdw8|8026|/data/mirror26 +153|31|m|m|s|u|sdw8|sdw8|8031|/data/mirror31 +158|36|m|m|s|u|sdw8|sdw8|8036|/data/mirror36 +# SDW9 +50|48|p|p|s|u|sdw9|sdw9|7048|/data/primary48 +51|49|p|p|s|u|sdw9|sdw9|7049|/data/primary49 +52|50|p|p|s|u|sdw9|sdw9|7050|/data/primary50 +53|51|p|p|s|u|sdw9|sdw9|7051|/data/primary51 +54|52|p|p|s|u|sdw9|sdw9|7052|/data/primary52 +55|53|p|p|s|u|sdw9|sdw9|7053|/data/primary53 +139|17|m|m|s|u|sdw9|sdw9|8017|/data/mirror17 +144|22|m|m|s|u|sdw9|sdw9|8022|/data/mirror22 +149|27|m|m|s|u|sdw9|sdw9|8027|/data/mirror27 +154|32|m|m|s|u|sdw9|sdw9|8032|/data/mirror32 +159|37|m|m|s|u|sdw9|sdw9|8037|/data/mirror37 +164|42|m|m|s|u|sdw9|sdw9|8042|/data/mirror42 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_grouped.array b/gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_grouped.array new file mode 100644 index 000000000000..9a807e00dc9b --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_grouped.array @@ -0,0 +1,262 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +139|17|m|m|s|u|sdw1|sdw1|8017|/data/mirror17 +140|18|m|m|s|u|sdw1|sdw1|8018|/data/mirror18 +141|19|m|m|s|u|sdw1|sdw1|8019|/data/mirror19 +142|20|m|m|s|u|sdw1|sdw1|8020|/data/mirror20 +143|21|m|m|s|u|sdw1|sdw1|8021|/data/mirror21 +144|22|m|m|s|u|sdw1|sdw1|8022|/data/mirror22 +# SDW10 +52|50|p|p|s|u|sdw10|sdw10|7050|/data/primary50 +53|51|p|p|s|u|sdw10|sdw10|7051|/data/primary51 +54|52|p|p|s|u|sdw10|sdw10|7052|/data/primary52 +55|53|p|p|s|u|sdw10|sdw10|7053|/data/primary53 +56|54|p|p|s|u|sdw10|sdw10|7054|/data/primary54 +57|55|p|p|s|u|sdw10|sdw10|7055|/data/primary55 +58|56|p|p|s|u|sdw10|sdw10|7056|/data/primary56 +# SDW11 +59|57|p|p|s|u|sdw11|sdw11|7057|/data/primary57 +60|58|p|p|s|u|sdw11|sdw11|7058|/data/primary58 +61|59|p|p|s|u|sdw11|sdw11|7059|/data/primary59 +62|60|p|p|s|u|sdw11|sdw11|7060|/data/primary60 +63|61|p|p|s|u|sdw11|sdw11|7061|/data/primary61 +64|62|p|p|s|u|sdw11|sdw11|7062|/data/primary62 +167|45|m|m|s|u|sdw11|sdw11|8045|/data/mirror45 +168|46|m|m|s|u|sdw11|sdw11|8046|/data/mirror46 +169|47|m|m|s|u|sdw11|sdw11|8047|/data/mirror47 +170|48|m|m|s|u|sdw11|sdw11|8048|/data/mirror48 +171|49|m|m|s|u|sdw11|sdw11|8049|/data/mirror49 +# SDW12 +65|63|p|p|s|u|sdw12|sdw12|7063|/data/primary63 +66|64|p|p|s|u|sdw12|sdw12|7064|/data/primary64 +67|65|p|p|s|u|sdw12|sdw12|7065|/data/primary65 +68|66|p|p|s|u|sdw12|sdw12|7066|/data/primary66 +189|67|m|m|s|u|sdw12|sdw12|8067|/data/mirror67 +190|68|m|m|s|u|sdw12|sdw12|8068|/data/mirror68 +191|69|m|m|s|u|sdw12|sdw12|8069|/data/mirror69 +192|70|m|m|s|u|sdw12|sdw12|8070|/data/mirror70 +193|71|m|m|s|u|sdw12|sdw12|8071|/data/mirror71 +# SDW13 +69|67|p|p|s|u|sdw13|sdw13|7067|/data/primary67 +70|68|p|p|s|u|sdw13|sdw13|7068|/data/primary68 +71|69|p|p|s|u|sdw13|sdw13|7069|/data/primary69 +72|70|p|p|s|u|sdw13|sdw13|7070|/data/primary70 +73|71|p|p|s|u|sdw13|sdw13|7071|/data/primary71 +172|50|m|m|s|u|sdw13|sdw13|8050|/data/mirror50 +173|51|m|m|s|u|sdw13|sdw13|8051|/data/mirror51 +174|52|m|m|s|u|sdw13|sdw13|8052|/data/mirror52 +175|53|m|m|s|u|sdw13|sdw13|8053|/data/mirror53 +176|54|m|m|s|u|sdw13|sdw13|8054|/data/mirror54 +177|55|m|m|s|u|sdw13|sdw13|8055|/data/mirror55 +178|56|m|m|s|u|sdw13|sdw13|8056|/data/mirror56 +# SDW14 +74|72|p|p|s|u|sdw14|sdw14|7072|/data/primary72 +75|73|p|p|s|u|sdw14|sdw14|7073|/data/primary73 +76|74|p|p|s|u|sdw14|sdw14|7074|/data/primary74 +77|75|p|p|s|u|sdw14|sdw14|7075|/data/primary75 +78|76|p|p|s|u|sdw14|sdw14|7076|/data/primary76 +79|77|p|p|s|u|sdw14|sdw14|7077|/data/primary77 +80|78|p|p|s|u|sdw14|sdw14|7078|/data/primary78 +185|63|m|m|s|u|sdw14|sdw14|8063|/data/mirror63 +186|64|m|m|s|u|sdw14|sdw14|8064|/data/mirror64 +187|65|m|m|s|u|sdw14|sdw14|8065|/data/mirror65 +188|66|m|m|s|u|sdw14|sdw14|8066|/data/mirror66 +# SDW15 +81|79|p|p|s|u|sdw15|sdw15|7079|/data/primary79 +82|80|p|p|s|u|sdw15|sdw15|7080|/data/primary80 +83|81|p|p|s|u|sdw15|sdw15|7081|/data/primary81 +84|82|p|p|s|u|sdw15|sdw15|7082|/data/primary82 +85|83|p|p|s|u|sdw15|sdw15|7083|/data/primary83 +86|84|p|p|s|u|sdw15|sdw15|7084|/data/primary84 +194|72|m|m|s|u|sdw15|sdw15|8072|/data/mirror72 +195|73|m|m|s|u|sdw15|sdw15|8073|/data/mirror73 +196|74|m|m|s|u|sdw15|sdw15|8074|/data/mirror74 +197|75|m|m|s|u|sdw15|sdw15|8075|/data/mirror75 +198|76|m|m|s|u|sdw15|sdw15|8076|/data/mirror76 +199|77|m|m|s|u|sdw15|sdw15|8077|/data/mirror77 +200|78|m|m|s|u|sdw15|sdw15|8078|/data/mirror78 +# SDW16 +87|85|p|p|s|u|sdw16|sdw16|7085|/data/primary85 +88|86|p|p|s|u|sdw16|sdw16|7086|/data/primary86 +89|87|p|p|s|u|sdw16|sdw16|7087|/data/primary87 +90|88|p|p|s|u|sdw16|sdw16|7088|/data/primary88 +91|89|p|p|s|u|sdw16|sdw16|7089|/data/primary89 +92|90|p|p|s|u|sdw16|sdw16|7090|/data/primary90 +93|91|p|p|s|u|sdw16|sdw16|7091|/data/primary91 +94|92|p|p|s|u|sdw16|sdw16|7092|/data/primary92 +201|79|m|m|s|u|sdw16|sdw16|8079|/data/mirror79 +202|80|m|m|s|u|sdw16|sdw16|8080|/data/mirror80 +203|81|m|m|s|u|sdw16|sdw16|8081|/data/mirror81 +204|82|m|m|s|u|sdw16|sdw16|8082|/data/mirror82 +205|83|m|m|s|u|sdw16|sdw16|8083|/data/mirror83 +206|84|m|m|s|u|sdw16|sdw16|8084|/data/mirror84 +# SDW17 +95|93|p|p|s|u|sdw17|sdw17|7093|/data/primary93 +96|94|p|p|s|u|sdw17|sdw17|7094|/data/primary94 +97|95|p|p|s|u|sdw17|sdw17|7095|/data/primary95 +98|96|p|p|s|u|sdw17|sdw17|7096|/data/primary96 +99|97|p|p|s|u|sdw17|sdw17|7097|/data/primary97 +100|98|p|p|s|u|sdw17|sdw17|7098|/data/primary98 +101|99|p|p|s|u|sdw17|sdw17|7099|/data/primary99 +207|85|m|m|s|u|sdw17|sdw17|8085|/data/mirror85 +208|86|m|m|s|u|sdw17|sdw17|8086|/data/mirror86 +209|87|m|m|s|u|sdw17|sdw17|8087|/data/mirror87 +210|88|m|m|s|u|sdw17|sdw17|8088|/data/mirror88 +211|89|m|m|s|u|sdw17|sdw17|8089|/data/mirror89 +212|90|m|m|s|u|sdw17|sdw17|8090|/data/mirror90 +213|91|m|m|s|u|sdw17|sdw17|8091|/data/mirror91 +214|92|m|m|s|u|sdw17|sdw17|8092|/data/mirror92 +# SDW18 +102|100|p|p|s|u|sdw18|sdw18|7100|/data/primary100 +103|101|p|p|s|u|sdw18|sdw18|7101|/data/primary101 +104|102|p|p|s|u|sdw18|sdw18|7102|/data/primary102 +105|103|p|p|s|u|sdw18|sdw18|7103|/data/primary103 +106|104|p|p|s|u|sdw18|sdw18|7104|/data/primary104 +107|105|p|p|s|u|sdw18|sdw18|7105|/data/primary105 +215|93|m|m|s|u|sdw18|sdw18|8093|/data/mirror93 +216|94|m|m|s|u|sdw18|sdw18|8094|/data/mirror94 +217|95|m|m|s|u|sdw18|sdw18|8095|/data/mirror95 +218|96|m|m|s|u|sdw18|sdw18|8096|/data/mirror96 +219|97|m|m|s|u|sdw18|sdw18|8097|/data/mirror97 +220|98|m|m|s|u|sdw18|sdw18|8098|/data/mirror98 +221|99|m|m|s|u|sdw18|sdw18|8099|/data/mirror99 +# SDW19 +108|106|p|p|s|u|sdw19|sdw19|7106|/data/primary106 +109|107|p|p|s|u|sdw19|sdw19|7107|/data/primary107 +110|108|p|p|s|u|sdw19|sdw19|7108|/data/primary108 +111|109|p|p|s|u|sdw19|sdw19|7109|/data/primary109 +112|110|p|p|s|u|sdw19|sdw19|7110|/data/primary110 +113|111|p|p|s|u|sdw19|sdw19|7111|/data/primary111 +114|112|p|p|s|u|sdw19|sdw19|7112|/data/primary112 +235|113|m|m|s|u|sdw19|sdw19|8113|/data/mirror113 +236|114|m|m|s|u|sdw19|sdw19|8114|/data/mirror114 +237|115|m|m|s|u|sdw19|sdw19|8115|/data/mirror115 +238|116|m|m|s|u|sdw19|sdw19|8116|/data/mirror116 +239|117|m|m|s|u|sdw19|sdw19|8117|/data/mirror117 +240|118|m|m|s|u|sdw19|sdw19|8118|/data/mirror118 +241|119|m|m|s|u|sdw19|sdw19|8119|/data/mirror119 +# SDW2 +7|5|p|p|s|u|sdw2|sdw2|7005|/data/primary5 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +122|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +123|1|m|m|s|u|sdw2|sdw2|8001|/data/mirror1 +124|2|m|m|s|u|sdw2|sdw2|8002|/data/mirror2 +125|3|m|m|s|u|sdw2|sdw2|8003|/data/mirror3 +126|4|m|m|s|u|sdw2|sdw2|8004|/data/mirror4 +228|106|m|m|s|u|sdw2|sdw2|8106|/data/mirror106 +229|107|m|m|s|u|sdw2|sdw2|8107|/data/mirror107 +230|108|m|m|s|u|sdw2|sdw2|8108|/data/mirror108 +231|109|m|m|s|u|sdw2|sdw2|8109|/data/mirror109 +232|110|m|m|s|u|sdw2|sdw2|8110|/data/mirror110 +233|111|m|m|s|u|sdw2|sdw2|8111|/data/mirror111 +234|112|m|m|s|u|sdw2|sdw2|8112|/data/mirror112 +# SDW20 +115|113|p|p|s|u|sdw20|sdw20|7113|/data/primary113 +116|114|p|p|s|u|sdw20|sdw20|7114|/data/primary114 +117|115|p|p|s|u|sdw20|sdw20|7115|/data/primary115 +118|116|p|p|s|u|sdw20|sdw20|7116|/data/primary116 +119|117|p|p|s|u|sdw20|sdw20|7117|/data/primary117 +120|118|p|p|s|u|sdw20|sdw20|7118|/data/primary118 +121|119|p|p|s|u|sdw20|sdw20|7119|/data/primary119 +222|100|m|m|s|u|sdw20|sdw20|8100|/data/mirror100 +223|101|m|m|s|u|sdw20|sdw20|8101|/data/mirror101 +224|102|m|m|s|u|sdw20|sdw20|8102|/data/mirror102 +225|103|m|m|s|u|sdw20|sdw20|8103|/data/mirror103 +226|104|m|m|s|u|sdw20|sdw20|8104|/data/mirror104 +227|105|m|m|s|u|sdw20|sdw20|8105|/data/mirror105 +# SDW3 +13|11|p|p|s|u|sdw3|sdw3|7011|/data/primary11 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +17|15|p|p|s|u|sdw3|sdw3|7015|/data/primary15 +18|16|p|p|s|u|sdw3|sdw3|7016|/data/primary16 +151|29|m|m|s|u|sdw3|sdw3|8029|/data/mirror29 +152|30|m|m|s|u|sdw3|sdw3|8030|/data/mirror30 +153|31|m|m|s|u|sdw3|sdw3|8031|/data/mirror31 +154|32|m|m|s|u|sdw3|sdw3|8032|/data/mirror32 +155|33|m|m|s|u|sdw3|sdw3|8033|/data/mirror33 +156|34|m|m|s|u|sdw3|sdw3|8034|/data/mirror34 +# SDW4 +19|17|p|p|s|u|sdw4|sdw4|7017|/data/primary17 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +22|20|p|p|s|u|sdw4|sdw4|7020|/data/primary20 +23|21|p|p|s|u|sdw4|sdw4|7021|/data/primary21 +24|22|p|p|s|u|sdw4|sdw4|7022|/data/primary22 +127|5|m|m|s|u|sdw4|sdw4|8005|/data/mirror5 +128|6|m|m|s|u|sdw4|sdw4|8006|/data/mirror6 +129|7|m|m|s|u|sdw4|sdw4|8007|/data/mirror7 +130|8|m|m|s|u|sdw4|sdw4|8008|/data/mirror8 +131|9|m|m|s|u|sdw4|sdw4|8009|/data/mirror9 +132|10|m|m|s|u|sdw4|sdw4|8010|/data/mirror10 +# SDW5 +25|23|p|p|s|u|sdw5|sdw5|7023|/data/primary23 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +27|25|p|p|s|u|sdw5|sdw5|7025|/data/primary25 +28|26|p|p|s|u|sdw5|sdw5|7026|/data/primary26 +29|27|p|p|s|u|sdw5|sdw5|7027|/data/primary27 +30|28|p|p|s|u|sdw5|sdw5|7028|/data/primary28 +162|40|m|m|s|u|sdw5|sdw5|8040|/data/mirror40 +163|41|m|m|s|u|sdw5|sdw5|8041|/data/mirror41 +164|42|m|m|s|u|sdw5|sdw5|8042|/data/mirror42 +165|43|m|m|s|u|sdw5|sdw5|8043|/data/mirror43 +166|44|m|m|s|u|sdw5|sdw5|8044|/data/mirror44 +# SDW6 +31|29|p|p|s|u|sdw6|sdw6|7029|/data/primary29 +32|30|p|p|s|u|sdw6|sdw6|7030|/data/primary30 +33|31|p|p|s|u|sdw6|sdw6|7031|/data/primary31 +34|32|p|p|s|u|sdw6|sdw6|7032|/data/primary32 +35|33|p|p|s|u|sdw6|sdw6|7033|/data/primary33 +36|34|p|p|s|u|sdw6|sdw6|7034|/data/primary34 +133|11|m|m|s|u|sdw6|sdw6|8011|/data/mirror11 +134|12|m|m|s|u|sdw6|sdw6|8012|/data/mirror12 +135|13|m|m|s|u|sdw6|sdw6|8013|/data/mirror13 +136|14|m|m|s|u|sdw6|sdw6|8014|/data/mirror14 +137|15|m|m|s|u|sdw6|sdw6|8015|/data/mirror15 +138|16|m|m|s|u|sdw6|sdw6|8016|/data/mirror16 +# SDW7 +37|35|p|p|s|u|sdw7|sdw7|7035|/data/primary35 +38|36|p|p|s|u|sdw7|sdw7|7036|/data/primary36 +39|37|p|p|s|u|sdw7|sdw7|7037|/data/primary37 +40|38|p|p|s|u|sdw7|sdw7|7038|/data/primary38 +41|39|p|p|s|u|sdw7|sdw7|7039|/data/primary39 +145|23|m|m|s|u|sdw7|sdw7|8023|/data/mirror23 +146|24|m|m|s|u|sdw7|sdw7|8024|/data/mirror24 +147|25|m|m|s|u|sdw7|sdw7|8025|/data/mirror25 +148|26|m|m|s|u|sdw7|sdw7|8026|/data/mirror26 +149|27|m|m|s|u|sdw7|sdw7|8027|/data/mirror27 +150|28|m|m|s|u|sdw7|sdw7|8028|/data/mirror28 +# SDW8 +42|40|p|p|s|u|sdw8|sdw8|7040|/data/primary40 +43|41|p|p|s|u|sdw8|sdw8|7041|/data/primary41 +44|42|p|p|s|u|sdw8|sdw8|7042|/data/primary42 +45|43|p|p|s|u|sdw8|sdw8|7043|/data/primary43 +46|44|p|p|s|u|sdw8|sdw8|7044|/data/primary44 +179|57|m|m|s|u|sdw8|sdw8|8057|/data/mirror57 +180|58|m|m|s|u|sdw8|sdw8|8058|/data/mirror58 +181|59|m|m|s|u|sdw8|sdw8|8059|/data/mirror59 +182|60|m|m|s|u|sdw8|sdw8|8060|/data/mirror60 +183|61|m|m|s|u|sdw8|sdw8|8061|/data/mirror61 +184|62|m|m|s|u|sdw8|sdw8|8062|/data/mirror62 +# SDW9 +47|45|p|p|s|u|sdw9|sdw9|7045|/data/primary45 +48|46|p|p|s|u|sdw9|sdw9|7046|/data/primary46 +49|47|p|p|s|u|sdw9|sdw9|7047|/data/primary47 +50|48|p|p|s|u|sdw9|sdw9|7048|/data/primary48 +51|49|p|p|s|u|sdw9|sdw9|7049|/data/primary49 +157|35|m|m|s|u|sdw9|sdw9|8035|/data/mirror35 +158|36|m|m|s|u|sdw9|sdw9|8036|/data/mirror36 +159|37|m|m|s|u|sdw9|sdw9|8037|/data/mirror37 +160|38|m|m|s|u|sdw9|sdw9|8038|/data/mirror38 +161|39|m|m|s|u|sdw9|sdw9|8039|/data/mirror39 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_spread.array new file mode 100644 index 000000000000..16faef4b141c --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_spread.array @@ -0,0 +1,262 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +128|6|m|m|s|u|sdw1|sdw1|8006|/data/mirror6 +142|20|m|m|s|u|sdw1|sdw1|8020|/data/mirror20 +163|41|m|m|s|u|sdw1|sdw1|8041|/data/mirror41 +182|60|m|m|s|u|sdw1|sdw1|8060|/data/mirror60 +201|79|m|m|s|u|sdw1|sdw1|8079|/data/mirror79 +223|101|m|m|s|u|sdw1|sdw1|8101|/data/mirror101 +236|114|m|m|s|u|sdw1|sdw1|8114|/data/mirror114 +# SDW10 +56|54|p|p|s|u|sdw10|sdw10|7054|/data/primary54 +57|55|p|p|s|u|sdw10|sdw10|7055|/data/primary55 +58|56|p|p|s|u|sdw10|sdw10|7056|/data/primary56 +59|57|p|p|s|u|sdw10|sdw10|7057|/data/primary57 +60|58|p|p|s|u|sdw10|sdw10|7058|/data/primary58 +61|59|p|p|s|u|sdw10|sdw10|7059|/data/primary59 +131|9|m|m|s|u|sdw10|sdw10|8009|/data/mirror9 +152|30|m|m|s|u|sdw10|sdw10|8030|/data/mirror30 +171|49|m|m|s|u|sdw10|sdw10|8049|/data/mirror49 +192|70|m|m|s|u|sdw10|sdw10|8070|/data/mirror70 +208|86|m|m|s|u|sdw10|sdw10|8086|/data/mirror86 +231|109|m|m|s|u|sdw10|sdw10|8109|/data/mirror109 +# SDW11 +62|60|p|p|s|u|sdw11|sdw11|7060|/data/primary60 +63|61|p|p|s|u|sdw11|sdw11|7061|/data/primary61 +64|62|p|p|s|u|sdw11|sdw11|7062|/data/primary62 +65|63|p|p|s|u|sdw11|sdw11|7063|/data/primary63 +133|11|m|m|s|u|sdw11|sdw11|8011|/data/mirror11 +148|26|m|m|s|u|sdw11|sdw11|8026|/data/mirror26 +172|50|m|m|s|u|sdw11|sdw11|8050|/data/mirror50 +193|71|m|m|s|u|sdw11|sdw11|8071|/data/mirror71 +212|90|m|m|s|u|sdw11|sdw11|8090|/data/mirror90 +232|110|m|m|s|u|sdw11|sdw11|8110|/data/mirror110 +# SDW12 +66|64|p|p|s|u|sdw12|sdw12|7064|/data/primary64 +67|65|p|p|s|u|sdw12|sdw12|7065|/data/primary65 +68|66|p|p|s|u|sdw12|sdw12|7066|/data/primary66 +69|67|p|p|s|u|sdw12|sdw12|7067|/data/primary67 +70|68|p|p|s|u|sdw12|sdw12|7068|/data/primary68 +134|12|m|m|s|u|sdw12|sdw12|8012|/data/mirror12 +153|31|m|m|s|u|sdw12|sdw12|8031|/data/mirror31 +173|51|m|m|s|u|sdw12|sdw12|8051|/data/mirror51 +194|72|m|m|s|u|sdw12|sdw12|8072|/data/mirror72 +213|91|m|m|s|u|sdw12|sdw12|8091|/data/mirror91 +233|111|m|m|s|u|sdw12|sdw12|8111|/data/mirror111 +# SDW13 +71|69|p|p|s|u|sdw13|sdw13|7069|/data/primary69 +72|70|p|p|s|u|sdw13|sdw13|7070|/data/primary70 +73|71|p|p|s|u|sdw13|sdw13|7071|/data/primary71 +74|72|p|p|s|u|sdw13|sdw13|7072|/data/primary72 +75|73|p|p|s|u|sdw13|sdw13|7073|/data/primary73 +76|74|p|p|s|u|sdw13|sdw13|7074|/data/primary74 +77|75|p|p|s|u|sdw13|sdw13|7075|/data/primary75 +78|76|p|p|s|u|sdw13|sdw13|7076|/data/primary76 +136|14|m|m|s|u|sdw13|sdw13|8014|/data/mirror14 +154|32|m|m|s|u|sdw13|sdw13|8032|/data/mirror32 +168|46|m|m|s|u|sdw13|sdw13|8046|/data/mirror46 +199|77|m|m|s|u|sdw13|sdw13|8077|/data/mirror77 +214|92|m|m|s|u|sdw13|sdw13|8092|/data/mirror92 +234|112|m|m|s|u|sdw13|sdw13|8112|/data/mirror112 +# SDW14 +79|77|p|p|s|u|sdw14|sdw14|7077|/data/primary77 +80|78|p|p|s|u|sdw14|sdw14|7078|/data/primary78 +81|79|p|p|s|u|sdw14|sdw14|7079|/data/primary79 +82|80|p|p|s|u|sdw14|sdw14|7080|/data/primary80 +83|81|p|p|s|u|sdw14|sdw14|7081|/data/primary81 +84|82|p|p|s|u|sdw14|sdw14|7082|/data/primary82 +138|16|m|m|s|u|sdw14|sdw14|8016|/data/mirror16 +155|33|m|m|s|u|sdw14|sdw14|8033|/data/mirror33 +174|52|m|m|s|u|sdw14|sdw14|8052|/data/mirror52 +190|68|m|m|s|u|sdw14|sdw14|8068|/data/mirror68 +216|94|m|m|s|u|sdw14|sdw14|8094|/data/mirror94 +235|113|m|m|s|u|sdw14|sdw14|8113|/data/mirror113 +# SDW15 +85|83|p|p|s|u|sdw15|sdw15|7083|/data/primary83 +86|84|p|p|s|u|sdw15|sdw15|7084|/data/primary84 +87|85|p|p|s|u|sdw15|sdw15|7085|/data/primary85 +88|86|p|p|s|u|sdw15|sdw15|7086|/data/primary86 +89|87|p|p|s|u|sdw15|sdw15|7087|/data/primary87 +90|88|p|p|s|u|sdw15|sdw15|7088|/data/primary88 +91|89|p|p|s|u|sdw15|sdw15|7089|/data/primary89 +132|10|m|m|s|u|sdw15|sdw15|8010|/data/mirror10 +158|36|m|m|s|u|sdw15|sdw15|8036|/data/mirror36 +177|55|m|m|s|u|sdw15|sdw15|8055|/data/mirror55 +195|73|m|m|s|u|sdw15|sdw15|8073|/data/mirror73 +217|95|m|m|s|u|sdw15|sdw15|8095|/data/mirror95 +237|115|m|m|s|u|sdw15|sdw15|8115|/data/mirror115 +# SDW16 +92|90|p|p|s|u|sdw16|sdw16|7090|/data/primary90 +93|91|p|p|s|u|sdw16|sdw16|7091|/data/primary91 +94|92|p|p|s|u|sdw16|sdw16|7092|/data/primary92 +95|93|p|p|s|u|sdw16|sdw16|7093|/data/primary93 +96|94|p|p|s|u|sdw16|sdw16|7094|/data/primary94 +97|95|p|p|s|u|sdw16|sdw16|7095|/data/primary95 +98|96|p|p|s|u|sdw16|sdw16|7096|/data/primary96 +135|13|m|m|s|u|sdw16|sdw16|8013|/data/mirror13 +159|37|m|m|s|u|sdw16|sdw16|8037|/data/mirror37 +179|57|m|m|s|u|sdw16|sdw16|8057|/data/mirror57 +196|74|m|m|s|u|sdw16|sdw16|8074|/data/mirror74 +219|97|m|m|s|u|sdw16|sdw16|8097|/data/mirror97 +238|116|m|m|s|u|sdw16|sdw16|8116|/data/mirror116 +# SDW17 +99|97|p|p|s|u|sdw17|sdw17|7097|/data/primary97 +100|98|p|p|s|u|sdw17|sdw17|7098|/data/primary98 +101|99|p|p|s|u|sdw17|sdw17|7099|/data/primary99 +102|100|p|p|s|u|sdw17|sdw17|7100|/data/primary100 +103|101|p|p|s|u|sdw17|sdw17|7101|/data/primary101 +104|102|p|p|s|u|sdw17|sdw17|7102|/data/primary102 +139|17|m|m|s|u|sdw17|sdw17|8017|/data/mirror17 +161|39|m|m|s|u|sdw17|sdw17|8039|/data/mirror39 +180|58|m|m|s|u|sdw17|sdw17|8058|/data/mirror58 +198|76|m|m|s|u|sdw17|sdw17|8076|/data/mirror76 +218|96|m|m|s|u|sdw17|sdw17|8096|/data/mirror96 +240|118|m|m|s|u|sdw17|sdw17|8118|/data/mirror118 +# SDW18 +105|103|p|p|s|u|sdw18|sdw18|7103|/data/primary103 +106|104|p|p|s|u|sdw18|sdw18|7104|/data/primary104 +107|105|p|p|s|u|sdw18|sdw18|7105|/data/primary105 +108|106|p|p|s|u|sdw18|sdw18|7106|/data/primary106 +109|107|p|p|s|u|sdw18|sdw18|7107|/data/primary107 +110|108|p|p|s|u|sdw18|sdw18|7108|/data/primary108 +140|18|m|m|s|u|sdw18|sdw18|8018|/data/mirror18 +162|40|m|m|s|u|sdw18|sdw18|8040|/data/mirror40 +178|56|m|m|s|u|sdw18|sdw18|8056|/data/mirror56 +200|78|m|m|s|u|sdw18|sdw18|8078|/data/mirror78 +215|93|m|m|s|u|sdw18|sdw18|8093|/data/mirror93 +241|119|m|m|s|u|sdw18|sdw18|8119|/data/mirror119 +# SDW19 +111|109|p|p|s|u|sdw19|sdw19|7109|/data/primary109 +112|110|p|p|s|u|sdw19|sdw19|7110|/data/primary110 +113|111|p|p|s|u|sdw19|sdw19|7111|/data/primary111 +114|112|p|p|s|u|sdw19|sdw19|7112|/data/primary112 +115|113|p|p|s|u|sdw19|sdw19|7113|/data/primary113 +116|114|p|p|s|u|sdw19|sdw19|7114|/data/primary114 +141|19|m|m|s|u|sdw19|sdw19|8019|/data/mirror19 +160|38|m|m|s|u|sdw19|sdw19|8038|/data/mirror38 +181|59|m|m|s|u|sdw19|sdw19|8059|/data/mirror59 +202|80|m|m|s|u|sdw19|sdw19|8080|/data/mirror80 +220|98|m|m|s|u|sdw19|sdw19|8098|/data/mirror98 +239|117|m|m|s|u|sdw19|sdw19|8117|/data/mirror117 +# SDW2 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +122|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +143|21|m|m|s|u|sdw2|sdw2|8021|/data/mirror21 +157|35|m|m|s|u|sdw2|sdw2|8035|/data/mirror35 +183|61|m|m|s|u|sdw2|sdw2|8061|/data/mirror61 +205|83|m|m|s|u|sdw2|sdw2|8083|/data/mirror83 +221|99|m|m|s|u|sdw2|sdw2|8099|/data/mirror99 +# SDW20 +117|115|p|p|s|u|sdw20|sdw20|7115|/data/primary115 +118|116|p|p|s|u|sdw20|sdw20|7116|/data/primary116 +119|117|p|p|s|u|sdw20|sdw20|7117|/data/primary117 +120|118|p|p|s|u|sdw20|sdw20|7118|/data/primary118 +121|119|p|p|s|u|sdw20|sdw20|7119|/data/primary119 +137|15|m|m|s|u|sdw20|sdw20|8015|/data/mirror15 +156|34|m|m|s|u|sdw20|sdw20|8034|/data/mirror34 +175|53|m|m|s|u|sdw20|sdw20|8053|/data/mirror53 +203|81|m|m|s|u|sdw20|sdw20|8081|/data/mirror81 +222|100|m|m|s|u|sdw20|sdw20|8100|/data/mirror100 +# SDW3 +13|11|p|p|s|u|sdw3|sdw3|7011|/data/primary11 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +17|15|p|p|s|u|sdw3|sdw3|7015|/data/primary15 +18|16|p|p|s|u|sdw3|sdw3|7016|/data/primary16 +123|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +144|22|m|m|s|u|sdw3|sdw3|8022|/data/mirror22 +165|43|m|m|s|u|sdw3|sdw3|8043|/data/mirror43 +184|62|m|m|s|u|sdw3|sdw3|8062|/data/mirror62 +197|75|m|m|s|u|sdw3|sdw3|8075|/data/mirror75 +225|103|m|m|s|u|sdw3|sdw3|8103|/data/mirror103 +# SDW4 +19|17|p|p|s|u|sdw4|sdw4|7017|/data/primary17 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +22|20|p|p|s|u|sdw4|sdw4|7020|/data/primary20 +23|21|p|p|s|u|sdw4|sdw4|7021|/data/primary21 +24|22|p|p|s|u|sdw4|sdw4|7022|/data/primary22 +124|2|m|m|s|u|sdw4|sdw4|8002|/data/mirror2 +145|23|m|m|s|u|sdw4|sdw4|8023|/data/mirror23 +164|42|m|m|s|u|sdw4|sdw4|8042|/data/mirror42 +185|63|m|m|s|u|sdw4|sdw4|8063|/data/mirror63 +207|85|m|m|s|u|sdw4|sdw4|8085|/data/mirror85 +226|104|m|m|s|u|sdw4|sdw4|8104|/data/mirror104 +# SDW5 +25|23|p|p|s|u|sdw5|sdw5|7023|/data/primary23 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +27|25|p|p|s|u|sdw5|sdw5|7025|/data/primary25 +28|26|p|p|s|u|sdw5|sdw5|7026|/data/primary26 +29|27|p|p|s|u|sdw5|sdw5|7027|/data/primary27 +30|28|p|p|s|u|sdw5|sdw5|7028|/data/primary28 +125|3|m|m|s|u|sdw5|sdw5|8003|/data/mirror3 +151|29|m|m|s|u|sdw5|sdw5|8029|/data/mirror29 +166|44|m|m|s|u|sdw5|sdw5|8044|/data/mirror44 +186|64|m|m|s|u|sdw5|sdw5|8064|/data/mirror64 +206|84|m|m|s|u|sdw5|sdw5|8084|/data/mirror84 +227|105|m|m|s|u|sdw5|sdw5|8105|/data/mirror105 +# SDW6 +31|29|p|p|s|u|sdw6|sdw6|7029|/data/primary29 +32|30|p|p|s|u|sdw6|sdw6|7030|/data/primary30 +33|31|p|p|s|u|sdw6|sdw6|7031|/data/primary31 +34|32|p|p|s|u|sdw6|sdw6|7032|/data/primary32 +35|33|p|p|s|u|sdw6|sdw6|7033|/data/primary33 +36|34|p|p|s|u|sdw6|sdw6|7034|/data/primary34 +37|35|p|p|s|u|sdw6|sdw6|7035|/data/primary35 +126|4|m|m|s|u|sdw6|sdw6|8004|/data/mirror4 +146|24|m|m|s|u|sdw6|sdw6|8024|/data/mirror24 +167|45|m|m|s|u|sdw6|sdw6|8045|/data/mirror45 +187|65|m|m|s|u|sdw6|sdw6|8065|/data/mirror65 +209|87|m|m|s|u|sdw6|sdw6|8087|/data/mirror87 +228|106|m|m|s|u|sdw6|sdw6|8106|/data/mirror106 +# SDW7 +38|36|p|p|s|u|sdw7|sdw7|7036|/data/primary36 +39|37|p|p|s|u|sdw7|sdw7|7037|/data/primary37 +40|38|p|p|s|u|sdw7|sdw7|7038|/data/primary38 +41|39|p|p|s|u|sdw7|sdw7|7039|/data/primary39 +42|40|p|p|s|u|sdw7|sdw7|7040|/data/primary40 +127|5|m|m|s|u|sdw7|sdw7|8005|/data/mirror5 +149|27|m|m|s|u|sdw7|sdw7|8027|/data/mirror27 +169|47|m|m|s|u|sdw7|sdw7|8047|/data/mirror47 +188|66|m|m|s|u|sdw7|sdw7|8066|/data/mirror66 +210|88|m|m|s|u|sdw7|sdw7|8088|/data/mirror88 +229|107|m|m|s|u|sdw7|sdw7|8107|/data/mirror107 +# SDW8 +43|41|p|p|s|u|sdw8|sdw8|7041|/data/primary41 +44|42|p|p|s|u|sdw8|sdw8|7042|/data/primary42 +45|43|p|p|s|u|sdw8|sdw8|7043|/data/primary43 +46|44|p|p|s|u|sdw8|sdw8|7044|/data/primary44 +47|45|p|p|s|u|sdw8|sdw8|7045|/data/primary45 +48|46|p|p|s|u|sdw8|sdw8|7046|/data/primary46 +49|47|p|p|s|u|sdw8|sdw8|7047|/data/primary47 +129|7|m|m|s|u|sdw8|sdw8|8007|/data/mirror7 +150|28|m|m|s|u|sdw8|sdw8|8028|/data/mirror28 +170|48|m|m|s|u|sdw8|sdw8|8048|/data/mirror48 +189|67|m|m|s|u|sdw8|sdw8|8067|/data/mirror67 +204|82|m|m|s|u|sdw8|sdw8|8082|/data/mirror82 +224|102|m|m|s|u|sdw8|sdw8|8102|/data/mirror102 +# SDW9 +50|48|p|p|s|u|sdw9|sdw9|7048|/data/primary48 +51|49|p|p|s|u|sdw9|sdw9|7049|/data/primary49 +52|50|p|p|s|u|sdw9|sdw9|7050|/data/primary50 +53|51|p|p|s|u|sdw9|sdw9|7051|/data/primary51 +54|52|p|p|s|u|sdw9|sdw9|7052|/data/primary52 +55|53|p|p|s|u|sdw9|sdw9|7053|/data/primary53 +130|8|m|m|s|u|sdw9|sdw9|8008|/data/mirror8 +147|25|m|m|s|u|sdw9|sdw9|8025|/data/mirror25 +176|54|m|m|s|u|sdw9|sdw9|8054|/data/mirror54 +191|69|m|m|s|u|sdw9|sdw9|8069|/data/mirror69 +211|89|m|m|s|u|sdw9|sdw9|8089|/data/mirror89 +230|108|m|m|s|u|sdw9|sdw9|8108|/data/mirror108 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_grouped.array b/gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_grouped.array new file mode 100644 index 000000000000..844aeb214fc9 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_grouped.array @@ -0,0 +1,79 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +67|30|m|m|s|u|sdw1|sdw1|8030|/data/mirror30 +68|31|m|m|s|u|sdw1|sdw1|8031|/data/mirror31 +69|32|m|m|s|u|sdw1|sdw1|8032|/data/mirror32 +70|33|m|m|s|u|sdw1|sdw1|8033|/data/mirror33 +71|34|m|m|s|u|sdw1|sdw1|8034|/data/mirror34 +# SDW2 +7|5|p|p|s|u|sdw2|sdw2|7005|/data/primary5 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +37|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +38|1|m|m|s|u|sdw2|sdw2|8001|/data/mirror1 +39|2|m|m|s|u|sdw2|sdw2|8002|/data/mirror2 +40|3|m|m|s|u|sdw2|sdw2|8003|/data/mirror3 +41|4|m|m|s|u|sdw2|sdw2|8004|/data/mirror4 +# SDW3 +12|10|p|p|s|u|sdw3|sdw3|7010|/data/primary10 +13|11|p|p|s|u|sdw3|sdw3|7011|/data/primary11 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +42|5|m|m|s|u|sdw3|sdw3|8005|/data/mirror5 +43|6|m|m|s|u|sdw3|sdw3|8006|/data/mirror6 +44|7|m|m|s|u|sdw3|sdw3|8007|/data/mirror7 +45|8|m|m|s|u|sdw3|sdw3|8008|/data/mirror8 +46|9|m|m|s|u|sdw3|sdw3|8009|/data/mirror9 +# SDW4 +17|15|p|p|s|u|sdw4|sdw4|7015|/data/primary15 +18|16|p|p|s|u|sdw4|sdw4|7016|/data/primary16 +19|17|p|p|s|u|sdw4|sdw4|7017|/data/primary17 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +47|10|m|m|s|u|sdw4|sdw4|8010|/data/mirror10 +48|11|m|m|s|u|sdw4|sdw4|8011|/data/mirror11 +49|12|m|m|s|u|sdw4|sdw4|8012|/data/mirror12 +50|13|m|m|s|u|sdw4|sdw4|8013|/data/mirror13 +51|14|m|m|s|u|sdw4|sdw4|8014|/data/mirror14 +# SDW5 +22|20|p|p|s|u|sdw5|sdw5|7020|/data/primary20 +23|21|p|p|s|u|sdw5|sdw5|7021|/data/primary21 +24|22|p|p|s|u|sdw5|sdw5|7022|/data/primary22 +25|23|p|p|s|u|sdw5|sdw5|7023|/data/primary23 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +52|15|m|m|s|u|sdw5|sdw5|8015|/data/mirror15 +53|16|m|m|s|u|sdw5|sdw5|8016|/data/mirror16 +54|17|m|m|s|u|sdw5|sdw5|8017|/data/mirror17 +55|18|m|m|s|u|sdw5|sdw5|8018|/data/mirror18 +56|19|m|m|s|u|sdw5|sdw5|8019|/data/mirror19 +# SDW6 +27|25|p|p|s|u|sdw6|sdw6|7025|/data/primary25 +28|26|p|p|s|u|sdw6|sdw6|7026|/data/primary26 +29|27|p|p|s|u|sdw6|sdw6|7027|/data/primary27 +30|28|p|p|s|u|sdw6|sdw6|7028|/data/primary28 +31|29|p|p|s|u|sdw6|sdw6|7029|/data/primary29 +57|20|m|m|s|u|sdw6|sdw6|8020|/data/mirror20 +58|21|m|m|s|u|sdw6|sdw6|8021|/data/mirror21 +59|22|m|m|s|u|sdw6|sdw6|8022|/data/mirror22 +60|23|m|m|s|u|sdw6|sdw6|8023|/data/mirror23 +61|24|m|m|s|u|sdw6|sdw6|8024|/data/mirror24 +# SDW7 +32|30|p|p|s|u|sdw7|sdw7|7030|/data/primary30 +33|31|p|p|s|u|sdw7|sdw7|7031|/data/primary31 +34|32|p|p|s|u|sdw7|sdw7|7032|/data/primary32 +35|33|p|p|s|u|sdw7|sdw7|7033|/data/primary33 +36|34|p|p|s|u|sdw7|sdw7|7034|/data/primary34 +62|25|m|m|s|u|sdw7|sdw7|8025|/data/mirror25 +63|26|m|m|s|u|sdw7|sdw7|8026|/data/mirror26 +64|27|m|m|s|u|sdw7|sdw7|8027|/data/mirror27 +65|28|m|m|s|u|sdw7|sdw7|8028|/data/mirror28 +66|29|m|m|s|u|sdw7|sdw7|8029|/data/mirror29 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_spread.array new file mode 100644 index 000000000000..8a09d0130228 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_spread.array @@ -0,0 +1,79 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +51|14|m|m|s|u|sdw1|sdw1|8014|/data/mirror14 +55|18|m|m|s|u|sdw1|sdw1|8018|/data/mirror18 +59|22|m|m|s|u|sdw1|sdw1|8022|/data/mirror22 +63|26|m|m|s|u|sdw1|sdw1|8026|/data/mirror26 +67|30|m|m|s|u|sdw1|sdw1|8030|/data/mirror30 +# SDW2 +7|5|p|p|s|u|sdw2|sdw2|7005|/data/primary5 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +37|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +56|19|m|m|s|u|sdw2|sdw2|8019|/data/mirror19 +60|23|m|m|s|u|sdw2|sdw2|8023|/data/mirror23 +64|27|m|m|s|u|sdw2|sdw2|8027|/data/mirror27 +68|31|m|m|s|u|sdw2|sdw2|8031|/data/mirror31 +# SDW3 +12|10|p|p|s|u|sdw3|sdw3|7010|/data/primary10 +13|11|p|p|s|u|sdw3|sdw3|7011|/data/primary11 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +38|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +42|5|m|m|s|u|sdw3|sdw3|8005|/data/mirror5 +61|24|m|m|s|u|sdw3|sdw3|8024|/data/mirror24 +65|28|m|m|s|u|sdw3|sdw3|8028|/data/mirror28 +69|32|m|m|s|u|sdw3|sdw3|8032|/data/mirror32 +# SDW4 +17|15|p|p|s|u|sdw4|sdw4|7015|/data/primary15 +18|16|p|p|s|u|sdw4|sdw4|7016|/data/primary16 +19|17|p|p|s|u|sdw4|sdw4|7017|/data/primary17 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +39|2|m|m|s|u|sdw4|sdw4|8002|/data/mirror2 +43|6|m|m|s|u|sdw4|sdw4|8006|/data/mirror6 +47|10|m|m|s|u|sdw4|sdw4|8010|/data/mirror10 +66|29|m|m|s|u|sdw4|sdw4|8029|/data/mirror29 +70|33|m|m|s|u|sdw4|sdw4|8033|/data/mirror33 +# SDW5 +22|20|p|p|s|u|sdw5|sdw5|7020|/data/primary20 +23|21|p|p|s|u|sdw5|sdw5|7021|/data/primary21 +24|22|p|p|s|u|sdw5|sdw5|7022|/data/primary22 +25|23|p|p|s|u|sdw5|sdw5|7023|/data/primary23 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +40|3|m|m|s|u|sdw5|sdw5|8003|/data/mirror3 +44|7|m|m|s|u|sdw5|sdw5|8007|/data/mirror7 +48|11|m|m|s|u|sdw5|sdw5|8011|/data/mirror11 +52|15|m|m|s|u|sdw5|sdw5|8015|/data/mirror15 +71|34|m|m|s|u|sdw5|sdw5|8034|/data/mirror34 +# SDW6 +27|25|p|p|s|u|sdw6|sdw6|7025|/data/primary25 +28|26|p|p|s|u|sdw6|sdw6|7026|/data/primary26 +29|27|p|p|s|u|sdw6|sdw6|7027|/data/primary27 +30|28|p|p|s|u|sdw6|sdw6|7028|/data/primary28 +31|29|p|p|s|u|sdw6|sdw6|7029|/data/primary29 +41|4|m|m|s|u|sdw6|sdw6|8004|/data/mirror4 +45|8|m|m|s|u|sdw6|sdw6|8008|/data/mirror8 +49|12|m|m|s|u|sdw6|sdw6|8012|/data/mirror12 +53|16|m|m|s|u|sdw6|sdw6|8016|/data/mirror16 +57|20|m|m|s|u|sdw6|sdw6|8020|/data/mirror20 +# SDW7 +32|30|p|p|s|u|sdw7|sdw7|7030|/data/primary30 +33|31|p|p|s|u|sdw7|sdw7|7031|/data/primary31 +34|32|p|p|s|u|sdw7|sdw7|7032|/data/primary32 +35|33|p|p|s|u|sdw7|sdw7|7033|/data/primary33 +36|34|p|p|s|u|sdw7|sdw7|7034|/data/primary34 +46|9|m|m|s|u|sdw7|sdw7|8009|/data/mirror9 +50|13|m|m|s|u|sdw7|sdw7|8013|/data/mirror13 +54|17|m|m|s|u|sdw7|sdw7|8017|/data/mirror17 +58|21|m|m|s|u|sdw7|sdw7|8021|/data/mirror21 +62|25|m|m|s|u|sdw7|sdw7|8025|/data/mirror25 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_grouped.array b/gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_grouped.array new file mode 100644 index 000000000000..9d21e2e84857 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_grouped.array @@ -0,0 +1,87 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +8|6|p|p|s|u|sdw1|sdw1|7006|/data/primary6 +49|7|m|m|s|u|sdw1|sdw1|8007|/data/mirror7 +50|8|m|m|s|u|sdw1|sdw1|8008|/data/mirror8 +51|9|m|m|s|u|sdw1|sdw1|8009|/data/mirror9 +52|10|m|m|s|u|sdw1|sdw1|8010|/data/mirror10 +53|11|m|m|s|u|sdw1|sdw1|8011|/data/mirror11 +54|12|m|m|s|u|sdw1|sdw1|8012|/data/mirror12 +55|13|m|m|s|u|sdw1|sdw1|8013|/data/mirror13 +56|14|m|m|s|u|sdw1|sdw1|8014|/data/mirror14 +57|15|m|m|s|u|sdw1|sdw1|8015|/data/mirror15 +75|33|m|m|s|u|sdw1|sdw1|8033|/data/mirror33 +76|34|m|m|s|u|sdw1|sdw1|8034|/data/mirror34 +77|35|m|m|s|u|sdw1|sdw1|8035|/data/mirror35 +78|36|m|m|s|u|sdw1|sdw1|8036|/data/mirror36 +79|37|m|m|s|u|sdw1|sdw1|8037|/data/mirror37 +80|38|m|m|s|u|sdw1|sdw1|8038|/data/mirror38 +81|39|m|m|s|u|sdw1|sdw1|8039|/data/mirror39 +# SDW2 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +13|11|p|p|s|u|sdw2|sdw2|7011|/data/primary11 +14|12|p|p|s|u|sdw2|sdw2|7012|/data/primary12 +15|13|p|p|s|u|sdw2|sdw2|7013|/data/primary13 +16|14|p|p|s|u|sdw2|sdw2|7014|/data/primary14 +17|15|p|p|s|u|sdw2|sdw2|7015|/data/primary15 +66|24|m|m|s|u|sdw2|sdw2|8024|/data/mirror24 +67|25|m|m|s|u|sdw2|sdw2|8025|/data/mirror25 +68|26|m|m|s|u|sdw2|sdw2|8026|/data/mirror26 +69|27|m|m|s|u|sdw2|sdw2|8027|/data/mirror27 +70|28|m|m|s|u|sdw2|sdw2|8028|/data/mirror28 +71|29|m|m|s|u|sdw2|sdw2|8029|/data/mirror29 +72|30|m|m|s|u|sdw2|sdw2|8030|/data/mirror30 +73|31|m|m|s|u|sdw2|sdw2|8031|/data/mirror31 +74|32|m|m|s|u|sdw2|sdw2|8032|/data/mirror32 +# SDW3 +18|16|p|p|s|u|sdw3|sdw3|7016|/data/primary16 +19|17|p|p|s|u|sdw3|sdw3|7017|/data/primary17 +20|18|p|p|s|u|sdw3|sdw3|7018|/data/primary18 +21|19|p|p|s|u|sdw3|sdw3|7019|/data/primary19 +22|20|p|p|s|u|sdw3|sdw3|7020|/data/primary20 +23|21|p|p|s|u|sdw3|sdw3|7021|/data/primary21 +24|22|p|p|s|u|sdw3|sdw3|7022|/data/primary22 +25|23|p|p|s|u|sdw3|sdw3|7023|/data/primary23 +# SDW4 +26|24|p|p|s|u|sdw4|sdw4|7024|/data/primary24 +27|25|p|p|s|u|sdw4|sdw4|7025|/data/primary25 +28|26|p|p|s|u|sdw4|sdw4|7026|/data/primary26 +29|27|p|p|s|u|sdw4|sdw4|7027|/data/primary27 +30|28|p|p|s|u|sdw4|sdw4|7028|/data/primary28 +31|29|p|p|s|u|sdw4|sdw4|7029|/data/primary29 +32|30|p|p|s|u|sdw4|sdw4|7030|/data/primary30 +33|31|p|p|s|u|sdw4|sdw4|7031|/data/primary31 +34|32|p|p|s|u|sdw4|sdw4|7032|/data/primary32 +42|0|m|m|s|u|sdw4|sdw4|8000|/data/mirror0 +43|1|m|m|s|u|sdw4|sdw4|8001|/data/mirror1 +44|2|m|m|s|u|sdw4|sdw4|8002|/data/mirror2 +45|3|m|m|s|u|sdw4|sdw4|8003|/data/mirror3 +46|4|m|m|s|u|sdw4|sdw4|8004|/data/mirror4 +47|5|m|m|s|u|sdw4|sdw4|8005|/data/mirror5 +48|6|m|m|s|u|sdw4|sdw4|8006|/data/mirror6 +58|16|m|m|s|u|sdw4|sdw4|8016|/data/mirror16 +59|17|m|m|s|u|sdw4|sdw4|8017|/data/mirror17 +60|18|m|m|s|u|sdw4|sdw4|8018|/data/mirror18 +61|19|m|m|s|u|sdw4|sdw4|8019|/data/mirror19 +62|20|m|m|s|u|sdw4|sdw4|8020|/data/mirror20 +63|21|m|m|s|u|sdw4|sdw4|8021|/data/mirror21 +64|22|m|m|s|u|sdw4|sdw4|8022|/data/mirror22 +65|23|m|m|s|u|sdw4|sdw4|8023|/data/mirror23 +# SDW5 +35|33|p|p|s|u|sdw5|sdw5|7033|/data/primary33 +36|34|p|p|s|u|sdw5|sdw5|7034|/data/primary34 +37|35|p|p|s|u|sdw5|sdw5|7035|/data/primary35 +38|36|p|p|s|u|sdw5|sdw5|7036|/data/primary36 +39|37|p|p|s|u|sdw5|sdw5|7037|/data/primary37 +40|38|p|p|s|u|sdw5|sdw5|7038|/data/primary38 +41|39|p|p|s|u|sdw5|sdw5|7039|/data/primary39 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_spread.array new file mode 100644 index 000000000000..fac7059b73a6 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_spread.array @@ -0,0 +1,87 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +8|6|p|p|s|u|sdw1|sdw1|7006|/data/primary6 +9|7|p|p|s|u|sdw1|sdw1|7007|/data/primary7 +50|8|m|m|s|u|sdw1|sdw1|8008|/data/mirror8 +51|9|m|m|s|u|sdw1|sdw1|8009|/data/mirror9 +53|11|m|m|s|u|sdw1|sdw1|8011|/data/mirror11 +57|15|m|m|s|u|sdw1|sdw1|8015|/data/mirror15 +62|20|m|m|s|u|sdw1|sdw1|8020|/data/mirror20 +72|30|m|m|s|u|sdw1|sdw1|8030|/data/mirror30 +75|33|m|m|s|u|sdw1|sdw1|8033|/data/mirror33 +78|36|m|m|s|u|sdw1|sdw1|8036|/data/mirror36 +# SDW2 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +13|11|p|p|s|u|sdw2|sdw2|7011|/data/primary11 +14|12|p|p|s|u|sdw2|sdw2|7012|/data/primary12 +15|13|p|p|s|u|sdw2|sdw2|7013|/data/primary13 +16|14|p|p|s|u|sdw2|sdw2|7014|/data/primary14 +17|15|p|p|s|u|sdw2|sdw2|7015|/data/primary15 +18|16|p|p|s|u|sdw2|sdw2|7016|/data/primary16 +42|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +44|2|m|m|s|u|sdw2|sdw2|8002|/data/mirror2 +59|17|m|m|s|u|sdw2|sdw2|8017|/data/mirror17 +60|18|m|m|s|u|sdw2|sdw2|8018|/data/mirror18 +63|21|m|m|s|u|sdw2|sdw2|8021|/data/mirror21 +65|23|m|m|s|u|sdw2|sdw2|8023|/data/mirror23 +66|24|m|m|s|u|sdw2|sdw2|8024|/data/mirror24 +79|37|m|m|s|u|sdw2|sdw2|8037|/data/mirror37 +80|38|m|m|s|u|sdw2|sdw2|8038|/data/mirror38 +# SDW3 +19|17|p|p|s|u|sdw3|sdw3|7017|/data/primary17 +20|18|p|p|s|u|sdw3|sdw3|7018|/data/primary18 +21|19|p|p|s|u|sdw3|sdw3|7019|/data/primary19 +22|20|p|p|s|u|sdw3|sdw3|7020|/data/primary20 +23|21|p|p|s|u|sdw3|sdw3|7021|/data/primary21 +24|22|p|p|s|u|sdw3|sdw3|7022|/data/primary22 +25|23|p|p|s|u|sdw3|sdw3|7023|/data/primary23 +26|24|p|p|s|u|sdw3|sdw3|7024|/data/primary24 +27|25|p|p|s|u|sdw3|sdw3|7025|/data/primary25 +43|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +48|6|m|m|s|u|sdw3|sdw3|8006|/data/mirror6 +54|12|m|m|s|u|sdw3|sdw3|8012|/data/mirror12 +58|16|m|m|s|u|sdw3|sdw3|8016|/data/mirror16 +68|26|m|m|s|u|sdw3|sdw3|8026|/data/mirror26 +69|27|m|m|s|u|sdw3|sdw3|8027|/data/mirror27 +70|28|m|m|s|u|sdw3|sdw3|8028|/data/mirror28 +71|29|m|m|s|u|sdw3|sdw3|8029|/data/mirror29 +77|35|m|m|s|u|sdw3|sdw3|8035|/data/mirror35 +81|39|m|m|s|u|sdw3|sdw3|8039|/data/mirror39 +# SDW4 +28|26|p|p|s|u|sdw4|sdw4|7026|/data/primary26 +29|27|p|p|s|u|sdw4|sdw4|7027|/data/primary27 +30|28|p|p|s|u|sdw4|sdw4|7028|/data/primary28 +31|29|p|p|s|u|sdw4|sdw4|7029|/data/primary29 +32|30|p|p|s|u|sdw4|sdw4|7030|/data/primary30 +33|31|p|p|s|u|sdw4|sdw4|7031|/data/primary31 +45|3|m|m|s|u|sdw4|sdw4|8003|/data/mirror3 +47|5|m|m|s|u|sdw4|sdw4|8005|/data/mirror5 +52|10|m|m|s|u|sdw4|sdw4|8010|/data/mirror10 +56|14|m|m|s|u|sdw4|sdw4|8014|/data/mirror14 +67|25|m|m|s|u|sdw4|sdw4|8025|/data/mirror25 +74|32|m|m|s|u|sdw4|sdw4|8032|/data/mirror32 +76|34|m|m|s|u|sdw4|sdw4|8034|/data/mirror34 +# SDW5 +34|32|p|p|s|u|sdw5|sdw5|7032|/data/primary32 +35|33|p|p|s|u|sdw5|sdw5|7033|/data/primary33 +36|34|p|p|s|u|sdw5|sdw5|7034|/data/primary34 +37|35|p|p|s|u|sdw5|sdw5|7035|/data/primary35 +38|36|p|p|s|u|sdw5|sdw5|7036|/data/primary36 +39|37|p|p|s|u|sdw5|sdw5|7037|/data/primary37 +40|38|p|p|s|u|sdw5|sdw5|7038|/data/primary38 +41|39|p|p|s|u|sdw5|sdw5|7039|/data/primary39 +46|4|m|m|s|u|sdw5|sdw5|8004|/data/mirror4 +49|7|m|m|s|u|sdw5|sdw5|8007|/data/mirror7 +55|13|m|m|s|u|sdw5|sdw5|8013|/data/mirror13 +61|19|m|m|s|u|sdw5|sdw5|8019|/data/mirror19 +64|22|m|m|s|u|sdw5|sdw5|8022|/data/mirror22 +73|31|m|m|s|u|sdw5|sdw5|8031|/data/mirror31 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/40_8_unbalanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/40_8_unbalanced_spread.array new file mode 100644 index 000000000000..950ee7dbed27 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/40_8_unbalanced_spread.array @@ -0,0 +1,90 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +47|5|m|m|s|u|sdw1|sdw1|8005|/data/mirror5 +50|8|m|m|s|u|sdw1|sdw1|8008|/data/mirror8 +60|18|m|m|s|u|sdw1|sdw1|8018|/data/mirror18 +69|27|m|m|s|u|sdw1|sdw1|8027|/data/mirror27 +75|33|m|m|s|u|sdw1|sdw1|8033|/data/mirror33 +# SDW2 +7|5|p|p|s|u|sdw2|sdw2|7005|/data/primary5 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +42|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +53|11|m|m|s|u|sdw2|sdw2|8011|/data/mirror11 +56|14|m|m|s|u|sdw2|sdw2|8014|/data/mirror14 +70|28|m|m|s|u|sdw2|sdw2|8028|/data/mirror28 +72|30|m|m|s|u|sdw2|sdw2|8030|/data/mirror30 +# SDW3 +13|11|p|p|s|u|sdw3|sdw3|7011|/data/primary11 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +17|15|p|p|s|u|sdw3|sdw3|7015|/data/primary15 +18|16|p|p|s|u|sdw3|sdw3|7016|/data/primary16 +43|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +51|9|m|m|s|u|sdw3|sdw3|8009|/data/mirror9 +62|20|m|m|s|u|sdw3|sdw3|8020|/data/mirror20 +71|29|m|m|s|u|sdw3|sdw3|8029|/data/mirror29 +78|36|m|m|s|u|sdw3|sdw3|8036|/data/mirror36 +# SDW4 +19|17|p|p|s|u|sdw4|sdw4|7017|/data/primary17 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +22|20|p|p|s|u|sdw4|sdw4|7020|/data/primary20 +46|4|m|m|s|u|sdw4|sdw4|8004|/data/mirror4 +52|10|m|m|s|u|sdw4|sdw4|8010|/data/mirror10 +63|21|m|m|s|u|sdw4|sdw4|8021|/data/mirror21 +66|24|m|m|s|u|sdw4|sdw4|8024|/data/mirror24 +68|26|m|m|s|u|sdw4|sdw4|8026|/data/mirror26 +81|39|m|m|s|u|sdw4|sdw4|8039|/data/mirror39 +# SDW5 +23|21|p|p|s|u|sdw5|sdw5|7021|/data/primary21 +24|22|p|p|s|u|sdw5|sdw5|7022|/data/primary22 +25|23|p|p|s|u|sdw5|sdw5|7023|/data/primary23 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +27|25|p|p|s|u|sdw5|sdw5|7025|/data/primary25 +28|26|p|p|s|u|sdw5|sdw5|7026|/data/primary26 +48|6|m|m|s|u|sdw5|sdw5|8006|/data/mirror6 +54|12|m|m|s|u|sdw5|sdw5|8012|/data/mirror12 +57|15|m|m|s|u|sdw5|sdw5|8015|/data/mirror15 +73|31|m|m|s|u|sdw5|sdw5|8031|/data/mirror31 +76|34|m|m|s|u|sdw5|sdw5|8034|/data/mirror34 +# SDW6 +29|27|p|p|s|u|sdw6|sdw6|7027|/data/primary27 +30|28|p|p|s|u|sdw6|sdw6|7028|/data/primary28 +31|29|p|p|s|u|sdw6|sdw6|7029|/data/primary29 +32|30|p|p|s|u|sdw6|sdw6|7030|/data/primary30 +33|31|p|p|s|u|sdw6|sdw6|7031|/data/primary31 +44|2|m|m|s|u|sdw6|sdw6|8002|/data/mirror2 +55|13|m|m|s|u|sdw6|sdw6|8013|/data/mirror13 +64|22|m|m|s|u|sdw6|sdw6|8022|/data/mirror22 +74|32|m|m|s|u|sdw6|sdw6|8032|/data/mirror32 +80|38|m|m|s|u|sdw6|sdw6|8038|/data/mirror38 +# SDW7 +34|32|p|p|s|u|sdw7|sdw7|7032|/data/primary32 +35|33|p|p|s|u|sdw7|sdw7|7033|/data/primary33 +36|34|p|p|s|u|sdw7|sdw7|7034|/data/primary34 +37|35|p|p|s|u|sdw7|sdw7|7035|/data/primary35 +49|7|m|m|s|u|sdw7|sdw7|8007|/data/mirror7 +58|16|m|m|s|u|sdw7|sdw7|8016|/data/mirror16 +61|19|m|m|s|u|sdw7|sdw7|8019|/data/mirror19 +67|25|m|m|s|u|sdw7|sdw7|8025|/data/mirror25 +79|37|m|m|s|u|sdw7|sdw7|8037|/data/mirror37 +# SDW8 +38|36|p|p|s|u|sdw8|sdw8|7036|/data/primary36 +39|37|p|p|s|u|sdw8|sdw8|7037|/data/primary37 +40|38|p|p|s|u|sdw8|sdw8|7038|/data/primary38 +41|39|p|p|s|u|sdw8|sdw8|7039|/data/primary39 +45|3|m|m|s|u|sdw8|sdw8|8003|/data/mirror3 +59|17|m|m|s|u|sdw8|sdw8|8017|/data/mirror17 +65|23|m|m|s|u|sdw8|sdw8|8023|/data/mirror23 +77|35|m|m|s|u|sdw8|sdw8|8035|/data/mirror35 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/balanced_grouped_6.array b/gpMgmt/bin/gprebalance_modules/test/data/balanced_grouped_6.array new file mode 100644 index 000000000000..7afb7c748c5a --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/balanced_grouped_6.array @@ -0,0 +1,16 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 +11|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 +12|4|m|m|s|u|sdw1|sdw1|50004|/data/mirror4 +13|5|m|m|s|u|sdw1|sdw1|50005|/data/mirror5 +# SDW2 +5|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 +6|4|p|p|s|u|sdw2|sdw2|40004|/data/primary4 +7|5|p|p|s|u|sdw2|sdw2|40005|/data/primary5 +8|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 +9|1|m|m|s|u|sdw2|sdw2|50001|/data/mirror1 +10|2|m|m|s|u|sdw2|sdw2|50002|/data/mirror2 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/balanced_spread_24.array b/gpMgmt/bin/gprebalance_modules/test/data/balanced_spread_24.array new file mode 100644 index 000000000000..32584e9be11f --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/balanced_spread_24.array @@ -0,0 +1,54 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|40003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|40004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|40005|/data/primary5 +32|6|m|m|s|u|sdw1|sdw1|50006|/data/mirror6 +35|9|m|m|s|u|sdw1|sdw1|50009|/data/mirror9 +38|12|m|m|s|u|sdw1|sdw1|50012|/data/mirror12 +41|15|m|m|s|u|sdw1|sdw1|50015|/data/mirror15 +44|18|m|m|s|u|sdw1|sdw1|50018|/data/mirror18 +47|21|m|m|s|u|sdw1|sdw1|50021|/data/mirror21 +# SDW2 +8|6|p|p|s|u|sdw2|sdw2|40006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|40007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|40008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|40009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|40010|/data/primary10 +13|11|p|p|s|u|sdw2|sdw2|40011|/data/primary11 +26|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 +29|3|m|m|s|u|sdw2|sdw2|50003|/data/mirror3 +39|13|m|m|s|u|sdw2|sdw2|50013|/data/mirror13 +42|16|m|m|s|u|sdw2|sdw2|50016|/data/mirror16 +45|19|m|m|s|u|sdw2|sdw2|50019|/data/mirror19 +48|22|m|m|s|u|sdw2|sdw2|50022|/data/mirror22 +# SDW3 +14|12|p|p|s|u|sdw3|sdw3|40012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|40013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|40014|/data/primary14 +17|15|p|p|s|u|sdw3|sdw3|40015|/data/primary15 +18|16|p|p|s|u|sdw3|sdw3|40016|/data/primary16 +19|17|p|p|s|u|sdw3|sdw3|40017|/data/primary17 +27|1|m|m|s|u|sdw3|sdw3|50001|/data/mirror1 +30|4|m|m|s|u|sdw3|sdw3|50004|/data/mirror4 +33|7|m|m|s|u|sdw3|sdw3|50007|/data/mirror7 +36|10|m|m|s|u|sdw3|sdw3|50010|/data/mirror10 +46|20|m|m|s|u|sdw3|sdw3|50020|/data/mirror20 +49|23|m|m|s|u|sdw3|sdw3|50023|/data/mirror23 +# SDW4 +20|18|p|p|s|u|sdw4|sdw4|40018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|40019|/data/primary19 +22|20|p|p|s|u|sdw4|sdw4|40020|/data/primary20 +23|21|p|p|s|u|sdw4|sdw4|40021|/data/primary21 +24|22|p|p|s|u|sdw4|sdw4|40022|/data/primary22 +25|23|p|p|s|u|sdw4|sdw4|40023|/data/primary23 +28|2|m|m|s|u|sdw4|sdw4|50002|/data/mirror2 +31|5|m|m|s|u|sdw4|sdw4|50005|/data/mirror5 +34|8|m|m|s|u|sdw4|sdw4|50008|/data/mirror8 +37|11|m|m|s|u|sdw4|sdw4|50011|/data/mirror11 +40|14|m|m|s|u|sdw4|sdw4|50014|/data/mirror14 +43|17|m|m|s|u|sdw4|sdw4|50017|/data/mirror17 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/config_generator.py b/gpMgmt/bin/gprebalance_modules/test/data/config_generator.py new file mode 100644 index 000000000000..8ca82d1fb7fc --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/config_generator.py @@ -0,0 +1,616 @@ +#!/usr/bin/env python3 +""" +Cluster Configuration Generator for Testing + +Generates various cluster configurations with different characteristics: +- Balanced vs Unbalanced load distribution +- Grouped vs Spread mirroring strategies +- Various cluster sizes (hosts, segments) +""" + +import random +from typing import List, Dict, Tuple, Optional +from dataclasses import dataclass +from enum import Enum +import argparse + +class MirroringStrategy(Enum): + GROUPED = "grouped" + SPREAD = "spread" + +@dataclass +class SegmentConfig: + """ + Single segment configuration + """ + dbid: int + content: int + role: str # 'p' or 'm' + preferred_role: str + mode: str = 's' + status: str = 'u' + hostname: str = '' + address: str = '' + port: int = 0 + datadir: str = '' + + def to_line(self) -> str: + """ + Convert to str + """ + return (f"{self.dbid}|{self.content}|{self.role}|{self.preferred_role}|" + f"{self.mode}|{self.status}|{self.hostname}|{self.address}|" + f"{self.port}|{self.datadir}") + +class ClusterConfigGenerator: + """ + Generate Greengage cluster configurations + """ + + def __init__(self, n_segments: int, n_hosts: int, strategy: MirroringStrategy, + base_primary_port: int = 7000, base_mirror_port: int = 8000): + self.n_segments = n_segments + self.n_hosts = n_hosts + self.strategy = strategy + self.base_primary_port = base_primary_port + self.base_mirror_port = base_mirror_port + self.next_dbid = 2 # Start after coordinator (dbid=1) + + def generate_balanced_grouped(self) -> List[SegmentConfig]: + """ + Generate balanced configuration with GROUPED mirroring. + + Pattern: Primaries and mirrors evenly distributed. + All mirrors for primaries on host_i go to host_((i+1) % n_hosts) + """ + configs = [] + segments_per_host = self.n_segments // self.n_hosts + + if self.n_segments % self.n_hosts != 0: + raise ValueError(f"Cannot evenly distribute {self.n_segments} segments " + f"across {self.n_hosts} hosts for balanced config") + + # Assign primaries + primary_assignment = {} + content_id = 0 + + for host_id in range(self.n_hosts): + for _ in range(segments_per_host): + primary_assignment[content_id] = host_id + content_id += 1 + + # Assign mirrors (grouped) + mirror_assignment = {} + for content_id, primary_host in primary_assignment.items(): + mirror_host = (primary_host + 1) % self.n_hosts + mirror_assignment[content_id] = mirror_host + + # Generate segment configs + configs.extend(self._create_segments(primary_assignment, mirror_assignment)) + + return configs + + def generate_balanced_spread(self) -> List[SegmentConfig]: + """ + Generate balanced configuration with SPREAD mirroring. + + Pattern: Primaries and mirrors evenly distributed. + Mirrors for primaries on one host are spread across different hosts. + """ + configs = [] + segments_per_host = self.n_segments // self.n_hosts + + if self.n_segments % self.n_hosts != 0: + raise ValueError(f"Cannot evenly distribute {self.n_segments} segments " + f"across {self.n_hosts} hosts for balanced config") + + # Assign primaries + primary_assignment = {} + content_id = 0 + + for host_id in range(self.n_hosts): + for _ in range(segments_per_host): + primary_assignment[content_id] = host_id + content_id += 1 + + # Assign mirrors using global offset pattern + mirror_assignment = {} + + for content_id in range(self.n_segments): + primary_host = primary_assignment[content_id] + + # Calculate local index within primary host's segments + local_idx = content_id % segments_per_host + + # Global offset ensures different mirrors for same local_idx across hosts + # Formula: mirror = (primary + local_idx + global_offset) % n_hosts + # where global_offset depends on which primary host we're on + + # Use: mirror = (primary + 1 + local_idx) % n_hosts + # If mirror == primary, shift by 1 again + mirror_host = (primary_host + 1 + local_idx) % self.n_hosts + + # Ensure no colocation + if mirror_host == primary_host: + mirror_host = (mirror_host + 1) % self.n_hosts + + mirror_assignment[content_id] = mirror_host + + configs.extend(self._create_segments(primary_assignment, mirror_assignment)) + + return configs + + def generate_unbalanced_grouped(self, skew_factor: float = 0.2) -> List[SegmentConfig]: + """ + Generate UNBALANCED configuration with GROUPED mirroring. + + Args: + skew_factor: 0.0 = perfectly balanced, 1.0 = maximum imbalance + Recommended: 0.1-0.3 for realistic scenarios + + Pattern: Some hosts have slightly more segments than others. + All mirrors for primaries on host_i still go to one host. + """ + configs = [] + + # Create SLIGHTLY skewed distribution (not extreme) + primary_assignment = self._create_realistic_skewed_distribution(skew_factor) + + # Assign mirrors (grouped) + mirror_assignment = {} + mirror_load = [0] * self.n_hosts + + for host_id in range(self.n_hosts): + # Get all primaries on this host + primary_contents = [c for c, h in primary_assignment.items() if h == host_id] + + if not primary_contents: + continue + + # Choose mirror host - prefer underloaded hosts for realism + available_hosts = [h for h in range(self.n_hosts) if h != host_id] + + # Bias towards least loaded mirrors (creates more realistic initial state) + available_hosts.sort(key=lambda h: mirror_load[h]) + + # Pick from top 3 least loaded (adds some randomness) + target_mirror_host = random.choice(available_hosts[:min(3, len(available_hosts))]) + + for content_id in primary_contents: + mirror_assignment[content_id] = target_mirror_host + mirror_load[target_mirror_host] += 1 + + configs.extend(self._create_segments(primary_assignment, mirror_assignment)) + + return configs + + def generate_unbalanced_spread(self, skew_factor: float = 0.2) -> List[SegmentConfig]: + """ + Generate UNBALANCED configuration with SPREAD mirroring. + + Args: + skew_factor: 0.0 = perfectly balanced, 1.0 = maximum imbalance + Recommended: 0.1-0.3 for realistic scenarios + """ + configs = [] + + # Create slightly skewed distribution + primary_assignment = self._create_realistic_skewed_distribution(skew_factor) + + # Assign mirrors (spread) with slight preference for balance + mirror_assignment = {} + mirror_load = [0] * self.n_hosts + + for host_id in range(self.n_hosts): + # Get primaries on this host + primary_contents = [c for c, h in primary_assignment.items() if h == host_id] + + if not primary_contents: + continue + + # Spread mirrors across OTHER hosts, biased towards less loaded + available_mirror_hosts = [h for h in range(self.n_hosts) if h != host_id] + + for idx, content_id in enumerate(sorted(primary_contents)): + # Sort by load each time (semi-balanced spread) + available_mirror_hosts.sort(key=lambda h: (mirror_load[h], h)) + + # Pick with some randomness: 70% least loaded, 30% random + if random.random() < 0.7: + mirror_host = available_mirror_hosts[0] # Least loaded + else: + mirror_host = available_mirror_hosts[idx % len(available_mirror_hosts)] # Round-robin + + mirror_assignment[content_id] = mirror_host + mirror_load[mirror_host] += 1 + + configs.extend(self._create_segments(primary_assignment, mirror_assignment)) + + return configs + + def _create_realistic_skewed_distribution(self, skew_factor: float) -> Dict[int, int]: + """ + Create realistic skewed distribution. + + Instead of extreme imbalance, this creates a distribution where: + - Most hosts are within 10-20% of target load + - A few hosts are overloaded/underloaded + - Skew_factor controls the magnitude + + Args: + skew_factor: 0.0 = perfectly balanced, 1.0 = severe imbalance + + Returns: + Dict[content_id] -> host_id + """ + target_per_host = self.n_segments / self.n_hosts + + # Calculate actual segments per host with skew + # Use normal distribution around target with controlled variance + segments_per_host = [] + + for host_id in range(self.n_hosts): + if skew_factor == 0.0: + # Perfectly balanced + count = int(target_per_host) + else: + # Add controlled noise + # skew_factor controls standard deviation + std_dev = skew_factor * target_per_host * 0.3 # Max 30% deviation + noise = random.gauss(0, std_dev) + count = int(target_per_host + noise) + + # Ensure minimum 50% of target (never empty unless skew=1.0) + min_count = int(target_per_host * max(0.5, 1.0 - skew_factor)) + max_count = int(target_per_host * (1.0 + skew_factor * 0.5)) + count = max(min_count, min(max_count, count)) + + segments_per_host.append(count) + + # Adjust to exact total + current_total = sum(segments_per_host) + diff = self.n_segments - current_total + + # Distribute difference to random hosts + while diff != 0: + host_id = random.randint(0, self.n_hosts - 1) + if diff > 0: + segments_per_host[host_id] += 1 + diff -= 1 + elif segments_per_host[host_id] > 1: # Never go to 0 + segments_per_host[host_id] -= 1 + diff += 1 + + # Create assignment + primary_assignment = {} + content_id = 0 + + for host_id in range(self.n_hosts): + for _ in range(segments_per_host[host_id]): + primary_assignment[content_id] = host_id + content_id += 1 + + return primary_assignment + + def _create_segments(self, primary_assignment: Dict[int, int], + mirror_assignment: Dict[int, int]) -> List[SegmentConfig]: + """Create SegmentConfig objects from assignments""" + configs = [] + + # Create primaries + for content_id in sorted(primary_assignment.keys()): + host_id = primary_assignment[content_id] + hostname = f"sdw{host_id + 1}" + + # Count how many primaries on this host before this one + port_offset = sum(1 for c, h in primary_assignment.items() + if h == host_id and c < content_id) + + config = SegmentConfig( + dbid=self.next_dbid, + content=content_id, + role='p', + preferred_role='p', + hostname=hostname, + address=hostname, + port=self.base_primary_port + content_id, + datadir=f"/data/primary{content_id}" + ) + self.next_dbid += 1 + configs.append(config) + + # Create mirrors + for content_id in sorted(mirror_assignment.keys()): + host_id = mirror_assignment[content_id] + hostname = f"sdw{host_id + 1}" + + config = SegmentConfig( + dbid=self.next_dbid, + content=content_id, + role='m', + preferred_role='m', + hostname=hostname, + address=hostname, + port=self.base_mirror_port + content_id, + datadir=f"/data/mirror{content_id}" + ) + self.next_dbid += 1 + configs.append(config) + + return configs + + def generate_coordinator(self) -> SegmentConfig: + """Generate coordinator segment""" + return SegmentConfig( + dbid=1, + content=-1, + role='p', + preferred_role='p', + hostname='cdw', + address='cdw', + port=5432, + datadir='/data/coordinator' + ) + + def generate_config(self, balanced: bool = True) -> List[SegmentConfig]: + """ + Generate complete configuration. + + Args: + balanced: If True, generate balanced config; otherwise unbalanced + """ + if balanced: + if self.strategy == MirroringStrategy.GROUPED: + return self.generate_balanced_grouped() + else: + return self.generate_balanced_spread() + else: + skew_factor = random.uniform(0.3, 0.8) + if self.strategy == MirroringStrategy.GROUPED: + return self.generate_unbalanced_grouped(skew_factor) + else: + return self.generate_unbalanced_spread(skew_factor) + +def write_config_file(filename: str, configs: List[SegmentConfig], + coordinator: SegmentConfig): + """Write configuration to file""" + with open(filename, 'w') as f: + # Write coordinator + f.write("# Coordinator\n") + f.write(coordinator.to_line() + "\n") + + # Group segments by host + segments_by_host = {} + for config in configs: + host = config.hostname + if host not in segments_by_host: + segments_by_host[host] = [] + segments_by_host[host].append(config) + + # Write segments by host + for hostname in sorted(segments_by_host.keys()): + f.write(f"# {hostname.upper()}\n") + + host_segments = segments_by_host[hostname] + # Sort by role (primaries first), then by content + host_segments.sort(key=lambda s: (s.role != 'p', s.content)) + + for seg in host_segments: + f.write(seg.to_line() + "\n") + +def print_statistics(configs: List[SegmentConfig], n_hosts: int): + """Print configuration statistics""" + print("\n" + "=" * 80) + print("CONFIGURATION STATISTICS".center(80)) + print("=" * 80) + + # Count segments per host + primaries_per_host = [0] * n_hosts + mirrors_per_host = [0] * n_hosts + + for config in configs: + host_id = int(config.hostname.replace('sdw', '')) - 1 + if config.role == 'p': + primaries_per_host[host_id] += 1 + else: + mirrors_per_host[host_id] += 1 + + print(f"\nTotal Segments: {len(configs) // 2}") + print(f"Total Hosts: {n_hosts}") + print(f"\nLoad Distribution:") + print(f"{'Host':<10} {'Primaries':<12} {'Mirrors':<12} {'Total':<10}") + print("-" * 50) + + for host_id in range(n_hosts): + hostname = f"sdw{host_id + 1}" + primaries = primaries_per_host[host_id] + mirrors = mirrors_per_host[host_id] + total = primaries + mirrors + print(f"{hostname:<10} {primaries:<12} {mirrors:<12} {total:<10}") + + print("-" * 50) + + # Calculate balance metrics + total_loads = [p + m for p, m in zip(primaries_per_host, mirrors_per_host)] + avg_load = sum(total_loads) / n_hosts + max_load = max(total_loads) + min_load = min(total_loads) + + print(f"\nBalance Metrics:") + print(f" Average load per host: {avg_load:.2f}") + print(f" Min load: {min_load}") + print(f" Max load: {max_load}") + print(f" Load deviation: {max_load - min_load}") + print(f" Balance ratio: {min_load / max_load * 100:.1f}%") + + # Check mirroring strategy + print(f"\nMirroring Analysis:") + + for host_id in range(n_hosts): + # Get primaries on this host + primary_contents = [c.content for c in configs + if c.role == 'p' and c.hostname == f"sdw{host_id + 1}"] + + if not primary_contents: + continue + + # Find where their mirrors are + mirror_hosts = [] + for content in primary_contents: + mirror = next(c for c in configs if c.role == 'm' and c.content == content) + mirror_hosts.append(mirror.hostname) + + unique_mirror_hosts = len(set(mirror_hosts)) + + if unique_mirror_hosts == 1: + strategy = "GROUPED" + elif unique_mirror_hosts == len(mirror_hosts): + strategy = "SPREAD (perfect)" + else: + strategy = f"SPREAD (partial - {unique_mirror_hosts} hosts)" + + print(f" sdw{host_id + 1}: {len(primary_contents)} primaries → " + f"mirrors on {unique_mirror_hosts} host(s) [{strategy}]") + + print("=" * 80 + "\n") + +def main(): + parser = argparse.ArgumentParser( + description='Generate Greengage cluster test configurations' + ) + parser.add_argument('-n', '--segments', type=int, required=True, + help='Number of segment pairs') + parser.add_argument('-m', '--hosts', type=int, required=True, + help='Number of hosts') + parser.add_argument('-s', '--strategy', choices=['grouped', 'spread'], + required=True, help='Mirroring strategy') + parser.add_argument('-b', '--balanced', action='store_true', + help='Generate balanced configuration') + parser.add_argument('-u', '--unbalanced', action='store_true', + help='Generate unbalanced configuration') + parser.add_argument('-o', '--output', type=str, default='cluster_config.txt', + help='Output filename') + parser.add_argument('--seed', type=int, default=None, + help='Random seed for reproducibility') + + args = parser.parse_args() + + # Set random seed + if args.seed is not None: + random.seed(args.seed) + + # Determine balance + if args.balanced and args.unbalanced: + print("Error: Cannot specify both --balanced and --unbalanced") + return + + balanced = args.balanced or not args.unbalanced + + # Create generator + strategy = MirroringStrategy.GROUPED if args.strategy == 'grouped' else MirroringStrategy.SPREAD + + generator = ClusterConfigGenerator( + n_segments=args.segments, + n_hosts=args.hosts, + strategy=strategy + ) + + # Generate configuration + print(f"Generating {'balanced' if balanced else 'unbalanced'} configuration...") + print(f" Segments: {args.segments}") + print(f" Hosts: {args.hosts}") + print(f" Strategy: {args.strategy.upper()}") + + try: + configs = generator.generate_config(balanced=balanced) + coordinator = generator.generate_coordinator() + + # Write to file + write_config_file(args.output, configs, coordinator) + print(f"\nConfiguration written to: {args.output}") + + # Print statistics + print_statistics(configs, args.hosts) + + except ValueError as e: + print(f"Error: {e}") + return + +# Batch generator for test suite +def generate_test_suite(): + """Generate a comprehensive test suite""" + + test_cases = [ + # Small balanced configs + {'n': 35, 'm': 7, 'strategy': 'grouped', 'balanced': True, 'name': '35_7_balanced_grouped'}, + {'n': 35, 'm': 7, 'strategy': 'spread', 'balanced': True, 'name': '35_7_balanced_spread'}, + + # Small unbalanced configs + {'n': 40, 'm': 5, 'strategy': 'grouped', 'balanced': False, 'name': '40_5_unbalanced_grouped'}, + {'n': 40, 'm': 5, 'strategy': 'spread', 'balanced': False, 'name': '40_5_unbalanced_spread'}, + + # Medium balanced configs + {'n': 120, 'm': 20, 'strategy': 'grouped', 'balanced': True, 'name': '120_20_balanced_grouped'}, + {'n': 120, 'm': 20, 'strategy': 'spread', 'balanced': True, 'name': '120_20_balanced_spread'}, + + # Medium unbalanced configs + {'n': 120, 'm': 20, 'strategy': 'grouped', 'balanced': False, 'name': '120_20_unbalanced_grouped'}, + {'n': 120, 'm': 20, 'strategy': 'spread', 'balanced': False, 'name': '120_20_unbalanced_spread'}, + + # Large balanced configs + {'n': 1000, 'm': 50, 'strategy': 'grouped', 'balanced': True, 'name': '1000_50_balanced_grouped'}, + {'n': 1000, 'm': 50, 'strategy': 'spread', 'balanced': True, 'name': '1000_50_balanced_spread'}, + + # Large unbalanced configs + {'n': 1000, 'm': 50, 'strategy': 'grouped', 'balanced': False, 'name': '1000_50_unbalanced_grouped'}, + {'n': 1000, 'm': 50, 'strategy': 'spread', 'balanced': False, 'name': '1000_50_unbalanced_spread'}, + ] + + print("=" * 80) + print("GENERATING TEST SUITE".center(80)) + print("=" * 80) + + for idx, test_case in enumerate(test_cases, 1): + print(f"\n[{idx}/{len(test_cases)}] Generating: {test_case['name']}") + + strategy = MirroringStrategy.GROUPED if test_case['strategy'] == 'grouped' else MirroringStrategy.SPREAD + + generator = ClusterConfigGenerator( + n_segments=test_case['n'], + n_hosts=test_case['m'], + strategy=strategy + ) + + try: + configs = generator.generate_config(balanced=test_case['balanced']) + coordinator = generator.generate_coordinator() + + filename = f"{test_case['name']}.array" + write_config_file(filename, configs, coordinator) + + print(f" ✓ Written to: {filename}") + print_statistics(configs, test_case['m']) + + except Exception as e: + print(f" ✗ Failed: {e}") + + print("\n" + "=" * 80) + print("TEST SUITE GENERATION COMPLETE".center(80)) + print("=" * 80) + +if __name__ == '__main__': + import sys + + if len(sys.argv) > 1 and sys.argv[1] == '--generate-suite': + generate_test_suite() + elif len(sys.argv) == 1: + # Interactive mode or show help + print("Usage:") + print(" Generate single config:") + print(" python config_generator.py -n 6 -m 3 -s grouped -b -o config.txt") + print("\n Generate test suite:") + print(" python config_generator.py --generate-suite") + print("\n For help:") + print(" python config_generator.py -h") + else: + main() diff --git a/gpMgmt/bin/gprebalance_modules/test/data/role_mismatch.array b/gpMgmt/bin/gprebalance_modules/test/data/role_mismatch.array new file mode 100644 index 000000000000..d082012d4e89 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/role_mismatch.array @@ -0,0 +1,12 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|m|s|u|sdw1|sdw1|40001|/data/primary1 +8|2|m|m|s|u|sdw1|sdw1|50002|/data/mirror2 +9|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 +# SDW2 +4|2|p|p|s|u|sdw2|sdw2|40002|/data/primary2 +5|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 +6|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 +7|1|m|p|s|u|sdw2|sdw2|50001|/data/mirror1 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/seg_down.array b/gpMgmt/bin/gprebalance_modules/test/data/seg_down.array new file mode 100644 index 000000000000..debb30bb58b4 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/seg_down.array @@ -0,0 +1,12 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|d|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +8|2|m|m|s|u|sdw1|sdw1|50002|/data/mirror2 +9|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 +# SDW2 +4|2|p|p|s|u|sdw2|sdw2|40002|/data/primary2 +5|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 +6|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 +7|1|m|m|s|u|sdw2|sdw2|50001|/data/mirror1 diff --git a/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py b/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py new file mode 100644 index 000000000000..275b2ca0de4f --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py @@ -0,0 +1,123 @@ +import os +import imp +import sys + +from gppylib.test.unit.gp_unittest import * +from mock import * +from gppylib.db.dbconn import DbURL +from gppylib.gplog import * +from gppylib.system.configurationInterface import GpConfigurationProvider +from gppylib.system.environment import GpCoordinatorEnvironment +from .config import initGparrayFromFile + +def rebalance_only(numsegs): + def inner(func): + def wrapper(self, *args, **kwargs): + self.options.target_segment_count = numsegs + ret = func(self, *args, **kwargs) + self.options.target_segment_count = None + return ret + return wrapper + return inner + +def check_query(conn, query): + if "SELECT COUNT(1) FROM pg_namespace WHERE nspname =" in query: + return [0] + return None + +#Test options +class GpTestRebalanceValidation(GpTestCase): + def setUp(self): + gprebalance_file = os.path.abspath( + os.path.dirname(__file__) + "/../../ggrebalance") + self.subject = imp.load_source('ggrebalance', gprebalance_file) + self.old_sys_argv = sys.argv + sys.argv = [] + self.options, self.args, self.parser = self.subject.parseargs() + + self.subject.logger = Mock( + spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal', 'exception']) + + self.subject.check_running_gputils = Mock(return_value=False) + + self.apply_patches([ + patch('builtins.open', mock_open(), create=True), + patch('builtins.input'), + patch('ggrebalance.dbconn.DbURL', return_value=Mock(), spec=DbURL), + patch('ggrebalance.dbconn.connect', return_value=Mock()), + patch('ggrebalance.GpCoordinatorEnvironment', + return_value=Mock(), spec=GpCoordinatorEnvironment), + patch('ggrebalance.configurationInterface.getConfigurationProvider'), + patch('os.path.exists', return_value=Mock()), + patch('ggrebalance.get_default_logger', + return_value=self.subject.logger), + ]) + + self.getConfigProviderFunctionMock = self.get_mock_from_apply_patch( + "getConfigurationProvider") + self.gpCoordinatorEnvironmentMock = self.get_mock_from_apply_patch( + "GpCoordinatorEnvironment") + self.previous_coordinator_data_directory = os.getenv( + 'COORDINATOR_DATA_DIRECTORY', '') + os.environ["COORDINATOR_DATA_DIRECTORY"] = '/tmp/dirdoesnotexist' + os.environ["GPHOME"] = '/tmp/dirdoesnotexist' + configProviderMock = Mock(spec=GpConfigurationProvider) + self.getConfigProviderFunctionMock.return_value = configProviderMock + configProviderMock.initializeProvider.return_value = configProviderMock + self.gpCoordinatorEnvironmentMock.return_value.getCoordinatorPort.return_value = 123456 + + def tearDown(self): + os.environ['COORDINATOR_DATA_DIRECTORY'] = self.previous_coordinator_data_directory + sys.argv = self.old_sys_argv + super(GpTestRebalanceValidation, self).tearDown() + + @patch('ggrebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("balanced_grouped_6")) + @patch('os.path.exists', side_effect=lambda path: path not in ['/tmp/dirdoesnotexist/gparraydump']) + @patch('gppylib.db.dbconn.queryRow', side_effect=check_query) + @rebalance_only(numsegs = 6) + def test_already_balanced_grouped(self, mockCatalog, mockOsPath, mockCursor): + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.info.assert_any_call( + "Cluster is already balanced, no segment moves will be held.") + + @patch('ggrebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("seg_down")) + @patch('os.path.exists', side_effect=lambda path: path not in ['/tmp/dirdoesnotexist/gparraydump']) + @patch('gppylib.db.dbconn.queryRow', side_effect=check_query) + @rebalance_only(numsegs = 4) + def test_segment_down(self, mockCatalog, mockOsPath, mockCursor): + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.error.assert_any_call( + "ggrebalance failed: Some segments in 'down' status. ggrebalance can't proceed further \n\nExiting...") + + @patch('ggrebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("balanced_spread_24")) + @patch('os.path.exists', side_effect=lambda path: path not in ['/tmp/dirdoesnotexist/gparraydump']) + @patch('gppylib.db.dbconn.queryRow', side_effect=check_query) + @rebalance_only(numsegs = 24) + def test_invalid_target_datadir(self, mockCatalog, mockOsPath, mockCursor): + self.options.target_hosts = "sdw1, sdw2, sdw3" + self.options.target_datadirs = '/data/primary/gpseg{content}, /data/mirror/gpseg' + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.error.assert_any_call( + 'ggrebalance failed: --target_datadirs options should have format like ' + '"/data/primary/gpseg{content} , /data/mirror/gpseg{content}" \n\nExiting...') + + @patch('ggrebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("role_mismatch")) + @patch('os.path.exists', side_effect=lambda path: path not in ['/tmp/dirdoesnotexist/gparraydump']) + @patch('gppylib.db.dbconn.queryRow', side_effect=check_query) + @rebalance_only(numsegs = 4) + def test_role_mistmatch(self, mockCatalog, mockOsPath, mockCursor): + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.error.assert_any_call( + "ggrebalance failed: Current role does not match preferred role for several segments. \n\nExiting...") + + +if __name__ == '__main__': + run_tests() diff --git a/gpMgmt/bin/gprebalance_modules/test/test_unit_solver.py b/gpMgmt/bin/gprebalance_modules/test/test_unit_solver.py new file mode 100644 index 000000000000..ce1e4eb18ae3 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/test_unit_solver.py @@ -0,0 +1,502 @@ +import random +from typing import List, Dict, Tuple +from collections import defaultdict + +from gppylib.test.unit.gp_unittest import * +from ..solver import GreedySolver, Solution, SolverConfig +from .config import getEncoding + + +class TestGreedySolver(GpTestCase): + + def setUp(self): + random.seed(42) + + def _validate_solution_allassign(self, solution: Solution, + solver: GreedySolver) -> bool: + if len(solution) != solver.n_segments: + return False + + return True + def _validate_solution_host_ids(self, solution: Solution, + solver: GreedySolver) -> bool: + for seg_id, (primary, mirror) in solution.items(): + if not (0 <= primary < solver.n_hosts_target): + return False + if not (0 <= mirror < solver.n_hosts_target): + return False + + return True + + def _validate_solution_nocolocation(self, solution: Solution, + solver: GreedySolver) -> bool: + for seg_id, (primary, mirror) in solution.items(): + if primary == mirror: + return False + return True + + def _validate_solution_balance(self, solution: Solution, + solver: GreedySolver) -> bool: + load = [0] * solver.n_hosts_target + for seg_id, (primary, mirror) in solution.items(): + load[primary] += 1 + load[mirror] += 1 + + for host, host_load in enumerate(load): + if host_load != solver.target_load: + return False + + return True + + def _validate_strategy(self, solution: Solution, + solver: GreedySolver): + primary_to_mirrors = defaultdict(list) + for seg_id, (primary, mirror) in solution.items(): + primary_to_mirrors[primary].append(mirror) + + for primary, mirrors in primary_to_mirrors.items(): + if solver.strategy == 'grouped': + unique_mirrors = set(mirrors) + if len(unique_mirrors) > 1: + return False + elif solver.strategy == 'spread': + unique_mirrors = set(mirrors) + if len(mirrors) != len(unique_mirrors): + return False + return True + + def _validate_solition(self, solution: Solution, + solver: GreedySolver): + self.assertTrue(self._validate_solution_allassign(solution, solver), + f"Missing segments. Expected {solver.n_segments}, got {len(solution)}") + + self.assertTrue(self._validate_solution_host_ids(solution, solver), + "Some segments has impossible host assignment") + + self.assertTrue(self._validate_solution_nocolocation(solution, solver), + "Some segment has primary==mirror assignment") + + self.assertTrue(self._validate_solution_balance(solution, solver), + f"One of hosts has load different from expected {solver.target_load}") + + self.assertTrue(self._validate_strategy(solution, solver), + "Grouped mirroring strategy is violated") + + def perform_run(self, run_improve:bool, cost:int): + conf = self.encoding[0] + solver = GreedySolver(conf, run_improve=run_improve) + solution, actual_cost = solver.solve() + self.assertEqual(actual_cost, cost) + self._validate_solition(solution, solver) + + @getEncoding('35_7_balanced_grouped', 'grouped', None, None, None) + def test_validity_small_grouped_balanced(self): + self.perform_run(False, 0) + + @getEncoding('35_7_balanced_spread', 'spread', None, None, None) + def test_validity_small_spread_balanced(self): + self.perform_run(False, 0) + + @getEncoding('40_5_unbalanced_grouped', 'grouped', None, None, None) + def test_validity_small_grouped_unbalanced(self): + self.perform_run(False, 19) + + @getEncoding('40_5_unbalanced_grouped', 'grouped', None, None, None) + def test_validity_small_grouped_unbalanced_with_improve(self): + self.perform_run(True, 18) + + @getEncoding('40_5_unbalanced_spread', 'spread', None, None, None) + def test_validity_small_spread_neg(self): + conf = self.encoding[0] + + with self.assertRaises(ValueError, msg='Cannot follow spread mirroring strategy') as cm: + solver = GreedySolver(conf, run_improve=False) + + @getEncoding('40_8_unbalanced_spread', 'spread', None, None, None) + def test_validity_small_spread_unbalanced(self): + self.perform_run(False, 10) + + @getEncoding('40_8_unbalanced_spread', 'spread', None, None, None) + def test_validity_small_spread_unbalanced_with_improve(self): + self.perform_run(True, 6) + + @getEncoding('120_20_unbalanced_spread', 'spread', None, None, None) + def test_validity_medium_spread_unbalanced(self): + self.perform_run(False, 10) + + @getEncoding('120_20_unbalanced_spread', 'spread', None, None, None) + def test_validity_medium_spread_unbalanced_with_improve(self): + self.perform_run(True, 7) + + @getEncoding('1000_50_unbalanced_spread', 'spread', None, None, None) + def test_validity_large_spread_unbalanced(self): + self.perform_run(False, 124) + + @getEncoding('1000_50_unbalanced_spread', 'spread', None, None, None) + def test_validity_large_spread_unbalanced_with_improve(self): + # in more or less standart configurations with lightly skewed + # distribution greedy initial solution is pretty-well generated. + # it's expected that ALNS may not bring any impovements. + self.perform_run(True, 124) + + @getEncoding('1000_50_unbalanced_grouped', 'grouped', None, None, None) + def test_validity_large_grouped_unbalanced(self): + self.perform_run(False, 140) + + @getEncoding('1000_50_unbalanced_grouped', 'grouped', None, None, None) + def test_validity_large_grouped_unbalanced_with_improve(self): + # in standart configurations with lightly skewed + # distribution greedy initial solution is pretty-well generated. + # it's expected that ALNS may not bring any impovements. + self.perform_run(True, 140) + + @getEncoding('120_20_unbalanced_grouped', 'spread', None, None, None) + def test_strategy_change_medium_grouped_unbalanced(self): + self.perform_run(False, 101) + + @getEncoding('120_20_unbalanced_grouped', 'spread', None, None, None) + def test_strategy_change_medium_grouped_unbalanced_with_improve(self): + self.perform_run(True, 100) + + @getEncoding('120_20_unbalanced_spread', 'grouped', None, None, None) + def test_strategy_change_medium_spread_unbalanced(self): + self.perform_run(False, 106) + + @getEncoding('120_20_unbalanced_spread', 'grouped', None, None, None) + def test_strategy_change_medium_spread_unbalanced_with_improve(self): + self.perform_run(True, 102) + + @getEncoding('120_20_unbalanced_grouped', 'grouped', target_hosts=None, + add_hosts=None, remove_hosts="sdw13, sdw14, sdw15, sdw16, sdw17, sdw18, sdw19, sdw20") + def test_decomission_hosts(self): + self.perform_run(False, 116) + + @getEncoding('120_20_unbalanced_grouped', 'grouped', target_hosts=None, + add_hosts=None, remove_hosts="sdw13, sdw14, sdw15, sdw16, sdw17, sdw18, sdw19, sdw20") + def test_decomission_hosts_with_improve(self): + self.perform_run(True, 109) + + @getEncoding('1000_50_unbalanced_grouped', 'grouped', target_hosts=None, + add_hosts=None, remove_hosts=",".join(["sdw" + str(i) for i in range(20, 30)])) + def test_decomission_hosts_large(self): + self.perform_run(False, 470) + + @getEncoding('1000_50_unbalanced_grouped', 'grouped', target_hosts=None, + add_hosts=None, remove_hosts=",".join(["sdw" + str(i) for i in range(20, 30)])) + def test_decomission_hosts_large_with_improve(self): + self.perform_run(True, 457) + + @getEncoding('1000_50_balanced_grouped', 'grouped', target_hosts=None, + add_hosts=",".join(["sdw" + str(i) for i in range(51, 101)]), + remove_hosts=None) + def test_new_hosts_balanced(self): + self.perform_run(False, 1000) + + @getEncoding('1000_50_balanced_grouped', 'grouped', target_hosts=None, + add_hosts=",".join(["sdw" + str(i) for i in range(51, 101)]), + remove_hosts=None) + def test_new_hosts_balanced_with_improve(self): + self.perform_run(True, 1000) + + @getEncoding('120_20_unbalanced_grouped', 'grouped', target_hosts= + "sdw1, sdw2, sdw3, sdw4, sdw5, sdw21, sdw22, sdw23, sdw24, sdw25, sdw12, sdw13", + add_hosts=None, remove_hosts=None) + def test_target_hosts(self): + self.perform_run(False, 175) + + @getEncoding('120_20_unbalanced_grouped', 'grouped', target_hosts= + "sdw1, sdw2, sdw3, sdw4, sdw5, sdw21, sdw22, sdw23, sdw24, sdw25, sdw12, sdw13", + add_hosts=None, remove_hosts=None) + def test_target_hosts_with_improve(self): + self.perform_run(True, 160) + +class TestGreedySolverFunc(GpTestCase): + """ + Test GreedySolver class. + """ + + def test_init_single_host_error(self): + config = SolverConfig( + n_segments=4, + n_hosts_target=1, # Invalid! + n_hosts_initial=2, + initial_primary_mapping=[0, 0, 1, 1], + initial_mirror_mapping=[1, 1, 0, 0], + strategy='grouped' + ) + with self.assertRaises(ValueError) as ctx: + GreedySolver(config, run_improve=False) + self.assertIn("Cannot balance to single host", str(ctx.exception)) + + def test_init_uneven_distribution_error(self): + config = SolverConfig( + n_segments=13, + n_hosts_target=3, + n_hosts_initial=3, + initial_primary_mapping=[0]*13, + initial_mirror_mapping=[1]*13, + strategy='grouped' + ) + with self.assertRaises(ValueError) as ctx: + GreedySolver(config, run_improve=False) + self.assertIn("Cannot evenly distribute", str(ctx.exception)) + + def test_init_spread_impossible_error(self): + config = SolverConfig( + n_segments=12, # 12/3 = 4 per host, but spread needs max 2 (3-1) + n_hosts_target=3, + n_hosts_initial=3, + initial_primary_mapping=[0]*12, + initial_mirror_mapping=[1]*12, + strategy='spread' + ) + with self.assertRaises(ValueError) as ctx: + GreedySolver(config, run_improve=False) + self.assertIn("Cannot follow spread mirroring strategy", str(ctx.exception)) + + def test_balance_primaries_keep_on_original(self): + config = SolverConfig( + n_segments=6, + n_hosts_target=3, + n_hosts_initial=3, + initial_primary_mapping=[0, 0, 1, 1, 2, 2], # Already balanced + initial_mirror_mapping=[1, 1, 2, 2, 0, 0], + strategy='grouped' + ) + solver = GreedySolver(config, run_improve=False) + primary = solver._balance_primaries() + + # Should keep original placements + self.assertEqual(primary, [0, 0, 1, 1, 2, 2]) + + def test_balance_primaries_move_from_decommissioned(self): + """ + Test that _balance_primaries keeps valid placements. + """ + config_grouped = SolverConfig( + n_segments=12, + n_hosts_target=3, + n_hosts_initial=4, + initial_primary_mapping=[0, 1, 2, 0, 1, 2, 0, 1, 2, 3, 3, 3], + initial_mirror_mapping=[1, 2, 0, 1, 2, 0, 1, 2, 0, 0, 1, 2], + strategy='grouped' + ) + + solver = GreedySolver(config_grouped, run_improve=False) + primary_mapping = solver._balance_primaries() + + # Check all segments assigned + self.assertEqual(len(primary_mapping), 12) + self.assertTrue(all( p in [0, 1, 2] + for p in primary_mapping)) + + def test_balance_primaries_move_from_overloaded(self): + config = SolverConfig( + n_segments=12, + n_hosts_target=3, + n_hosts_initial=3, + initial_primary_mapping=[0]*8 + [1]*2 + [2]*2, # Host 0 overloaded + initial_mirror_mapping=[1]*8 + [2]*2 + [0]*2, + strategy='grouped' + ) + solver = GreedySolver(config, run_improve=False) + primary = solver._balance_primaries() + + # Should balance to 4 per host + primary_count = defaultdict(int) + for p in primary: + primary_count[p] += 1 + + for h in range(3): + self.assertEqual(primary_count[h], 4) + + def test_assign_mirrors_grouped_strategy_balanced(self): + config = SolverConfig( + n_segments=12, + n_hosts_target=3, + n_hosts_initial=3, + initial_primary_mapping=[0]*4 + [1]*4 + [2]*4, + initial_mirror_mapping=[1]*4 + [2]*4 + [0]*4, + strategy='grouped' + ) + solver = GreedySolver(config, run_improve=False) + primary = solver._balance_primaries() + mirror = solver._assign_mirror_hosts(primary) + + self.assertEqual(primary, config.initial_primary_mapping) + self.assertEqual(mirror, config.initial_mirror_mapping) + + def test_assign_mirrors_spread_strategy_balanced(self): + config = SolverConfig( + n_segments=8, + n_hosts_target=4, + n_hosts_initial=4, + initial_primary_mapping=[0, 0, 1, 1, 2, 2, 3, 3], + initial_mirror_mapping=[1, 2, 2, 3, 3, 0, 0, 1], + strategy='spread' + ) + solver = GreedySolver(config, run_improve=False) + primary = solver._balance_primaries() + mirror = solver._assign_mirror_hosts(primary) + + self.assertEqual(primary, config.initial_primary_mapping) + self.assertEqual(mirror, config.initial_mirror_mapping) + + def test_select_group_mirror_priority1_most_used(self): + """ most-used original mirror with capacity""" + config = SolverConfig( + n_segments=12, + n_hosts_target=3, + n_hosts_initial=3, + initial_primary_mapping=[0, 0, 0, 0, 0, 2, 2, 2, 2, 1, 1, 1], + initial_mirror_mapping=[1, 1, 1, 2, 2, 1, 1, 2, 2, 0, 0, 0], + strategy='grouped' + ) + solver = GreedySolver(config, run_improve=False) + + mirror_mapping = [-1] * 12 + mirror_load = [0] * 3 + groups = {0: [0, 1, 2, 3], 1: [4, 5, 6, 7], 2: [8, 9, 10, 11]} + phost_to_mhost = {} + + preferences = solver._compute_mirror_preferences(groups) + + best = solver._select_group_mirror( + p_host=0, + mirror_mapping=mirror_mapping, + mirror_load=mirror_load, + phost_to_mhost=phost_to_mhost, + groups=groups, + preferences=preferences + ) + + # Should select most-preferred mirror (host 1) + self.assertEqual(best, 1) + + def test_select_group_mirror_priority2_least_loaded(self): + config = SolverConfig( + n_segments=12, + n_hosts_target=3, + n_hosts_initial=4, # Host 3 decommissioned + initial_primary_mapping=[0]*4 + [1]*4 + [2]*4, + initial_mirror_mapping=[3]*12, # All on decommissioned host + strategy='grouped' + ) + solver = GreedySolver(config, run_improve=False) + + mirror_mapping = [-1] * 12 + mirror_load = [2, 0, 1] # Host 1 is least loaded + groups = {0: [0, 1, 2, 3], 1: [4, 5, 6, 7], 2: [8, 9, 10, 11]} + phost_to_mhost = {} + + preferences = solver._compute_mirror_preferences(groups) + + best = solver._select_group_mirror( + p_host=0, + mirror_mapping=mirror_mapping, + mirror_load=mirror_load, + phost_to_mhost=phost_to_mhost, + groups=groups, + preferences=preferences + ) + + # Should select least loaded (host 1) + self.assertEqual(best, 1) + + def test_select_group_mirror_priority3_deadlock_swap(self): + config = SolverConfig( + n_segments=8, + n_hosts_target=4, + n_hosts_initial=4, + initial_primary_mapping=[0]*2 + [1]*2 + [2]*2 + [3]*2,# does not matter + initial_mirror_mapping=[1]*4 + [0]*4,# does not matter + strategy='grouped' + ) + solver = GreedySolver(config, run_improve=False) + + # Simulate deadlock: all mirror hosts are full, except the host 0. + # Need swap in case of such mapping where primary group at host 0 + # has no options to put the mirrors + mirror_mapping = [2, 2, 3, 3, 1, 1, -1, -1] + mirror_load = [0, 2, 2, 2] # Both full + groups = {0: [6, 7], 1: [0, 1], 2: [2, 3], 3: [4, 5]} + phost_to_mhost = {1: 2, 2: 3, 3: 1} + + best = solver._swap_to_resolve_deadlock( + blocked_p_host=0, + mirror_load=mirror_load, + phost_to_mhost=phost_to_mhost, + groups=groups, + mirror_mapping=mirror_mapping + ) + + # This tests the deadlock branch is reached + self.assertIsNotNone(best) + + def test_spread_use_original_mirror(self): + config = SolverConfig( + n_segments=8, + n_hosts_target=4, + n_hosts_initial=4, + initial_primary_mapping=[0, 0, 1, 1, 2, 2, 3, 3], + initial_mirror_mapping=[1, 2, 2, 3, 3, 0, 0, 1], # Valid originals + strategy='spread' + ) + solver = GreedySolver(config, run_improve=False) + primary_mapping = solver._balance_primaries() + mirror_mapping = solver._assign_mirror_hosts(primary_mapping) + + self.assertEqual(primary_mapping, config.initial_primary_mapping) + self.assertEqual(mirror_mapping, config.initial_mirror_mapping) + + def test_spread_find_alternative(self): + config = SolverConfig( + n_segments=8, + n_hosts_target=4, + n_hosts_initial=5, + initial_primary_mapping=[0, 0, 1, 1, 2, 2, 3, 3], + initial_mirror_mapping=[4, 4, 4, 4, 4, 4, 4, 4], # All on decommissioned + strategy='spread' + ) + solver = GreedySolver(config, run_improve=False) + primary_mapping = solver._balance_primaries() + mirror_mapping = solver._assign_mirror_hosts(primary_mapping) + + # Should assign to valid hosts + for m in mirror_mapping: + self.assertLess(m, 4) + + def test_spread_deadlock_swap_found(self): + config = SolverConfig( + n_segments=8, + n_hosts_target=4, + n_hosts_initial=5, + initial_primary_mapping=[0, 0, 1, 1, 2, 2, 3, 3], + initial_mirror_mapping=[4, 4, 4, 4, 4, 4, 4, 4],# All on decommissioned + strategy='spread' + ) + solver = GreedySolver(config, run_improve=False) + + primary_mapping = [0, 0, 1, 1, 2, 2, 3, 3] + mirror_mapping = [1, 2, 2, 1, 3, 0, 0, -1] + mirror_load = [2, 2, 2, 1] + primary_host_to_mirror_hosts = { + 0: {1, 2}, + 1: {1, 2}, + 2: {3, 0}, + 3: {0} + } + + result = solver._resolve_spread_deadlock_for_segment( + seg=7, + primary_mapping=primary_mapping, + mirror_mapping=mirror_mapping, + mirror_load=mirror_load, + used_in_group=primary_host_to_mirror_hosts + ) + + # Must find a swap + self.assertEqual(1, result) + +if __name__ == '__main__': + run_tests() diff --git a/gpMgmt/test/behave/mgmt_utils/environment.py b/gpMgmt/test/behave/mgmt_utils/environment.py index 43acb7c40306..00fca9002b1b 100644 --- a/gpMgmt/test/behave/mgmt_utils/environment.py +++ b/gpMgmt/test/behave/mgmt_utils/environment.py @@ -22,7 +22,7 @@ def before_feature(context, feature): # we should be able to run gpexpand without having a cluster initialized tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors', 'gpconfig', 'gpssh-exkeys', 'gpstop', 'gpinitsystem', 'cross_subnet', - 'gplogfilter'] + 'gplogfilter', 'ggrebalance'] if set(context.feature.tags).intersection(tags_to_skip): return @@ -125,7 +125,7 @@ def before_scenario(context, scenario): tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors', 'gpconfig', 'gpssh-exkeys', 'gpstop', 'gpinitsystem', 'cross_subnet', - 'gplogfilter'] + 'gplogfilter', 'ggrebalance'] if set(context.feature.tags).intersection(tags_to_skip): return @@ -158,7 +158,7 @@ def after_scenario(context, scenario): # NOTE: gpconfig after_scenario cleanup is in the step `the gpconfig context is setup` tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpinitstandby', 'gpconfig', 'gpstop', 'gpinitsystem', 'cross_subnet', - 'gplogfilter'] + 'gplogfilter', 'ggrebalance'] if set(context.feature.tags).intersection(tags_to_skip): return diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature index c7a07c4f9d3c..8a4a0b9d6d6b 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature @@ -9,9 +9,9 @@ Feature: ggrebalance behave tests And segment information for content 2 is saved in context And segment information for content 3 is saved in context And all files in gpAdminLogs directory are deleted - When the user runs "ggrebalance -x 4" - Then ggrebalance should return a return code of 1 - And ggrebalance should print "Target segment count (4) >= current segment count (4)." to logfile with latest timestamp + When the user runs "ggrebalance -x 4 --skip-rebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Skipping rebalance" to logfile with latest timestamp When the user runs "ggrebalance -c" Then ggrebalance should return a return code of 0 And ggrebalance should print "Rebalance schema doesn't exist. Cleanup is not required." to logfile with latest timestamp @@ -24,11 +24,11 @@ Feature: ggrebalance behave tests When the user runs "ggrebalance -r" Then ggrebalance should return a return code of 0 And ggrebalance should print "Rebalance schema doesn't exist. Can't perform rollback." to logfile with latest timestamp - When the user runs "ggrebalance -x 2" + When the user runs "ggrebalance -x 2 --skip-rebalance" Then ggrebalance should return a return code of 0 And ggrebalance should print "Shrink is complete" to logfile with latest timestamp And verify no segment running for saved segment information - When the user runs "ggrebalance -x 1" + When the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 0 And ggrebalance should print "Previous run was completed successfully. Please execute cleanup before a new run." to logfile with latest timestamp When the user runs "ggrebalance -r" @@ -44,7 +44,7 @@ Feature: ggrebalance behave tests And a cluster is created with mirrors on "cdw" and "sdw1" And all files in gpAdminLogs directory are deleted And set fault inject "on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end" - When the user runs "ggrebalance -x 1" + When the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject @@ -68,14 +68,14 @@ Feature: ggrebalance behave tests And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When the user runs "ggrebalance -x 1" + When the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject When the user runs "ggrebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "Rebalance schema doesn't exists and no shrink plan is supplied. Please specify shrink plan." to logfile with latest timestamp - When the user runs "ggrebalance -x 1" + When the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 0 And ggrebalance should print "Shrink is complete" to logfile with latest timestamp And verify no segment running for saved segment information @@ -101,7 +101,7 @@ Feature: ggrebalance behave tests And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When the user runs "ggrebalance -x 1" + When the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject @@ -110,7 +110,7 @@ Feature: ggrebalance behave tests And ggrebalance should print "Can't determine next state. Try to execute cleanup." to logfile with latest timestamp When the user runs "ggrebalance -c -y" Then ggrebalance should return a return code of 0 - When the user runs "ggrebalance -x 1" + When the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 0 And ggrebalance should print "Shrink is complete" to logfile with latest timestamp And verify no segment running for saved segment information @@ -136,14 +136,14 @@ Feature: ggrebalance behave tests And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1" + When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject - When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1" + When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "Can't start a new operation, because the previous one was interrupted" to logfile with latest timestamp - When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1" + When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "Can't start a new operation, because the previous one was interrupted" to logfile with latest timestamp When the user runs "ggrebalance --parallel 1 --batch-size 1" @@ -197,7 +197,7 @@ Feature: ggrebalance behave tests And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows When set fault inject "" - And the user runs "ggrebalance -x 1" + And the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject @@ -232,7 +232,7 @@ Feature: ggrebalance behave tests And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When the user runs "ggrebalance -x 1" + When the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject @@ -262,7 +262,7 @@ Feature: ggrebalance behave tests And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1" + When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject @@ -311,7 +311,7 @@ Feature: ggrebalance behave tests And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When the user runs "ggrebalance -x 1" + When the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject @@ -352,7 +352,7 @@ Feature: ggrebalance behave tests And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows When set fault inject "" - And the user runs "ggrebalance -x 1" + And the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject @@ -404,7 +404,7 @@ Feature: ggrebalance behave tests And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When the user runs "ggrebalance -x 1" + When the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject @@ -443,7 +443,7 @@ Feature: ggrebalance behave tests And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When the user runs "ggrebalance -x 1" + When the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject @@ -504,7 +504,7 @@ Feature: ggrebalance behave tests And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When the user runs "ggrebalance -x 1" + When the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject @@ -546,7 +546,7 @@ Feature: ggrebalance behave tests And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When the user runs "ggrebalance -x 1" + When the user runs "ggrebalance -x 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject From 4b13164c7484e681f4e448444d8b1cda2b3fe539 Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Mon, 29 Dec 2025 12:09:11 +0600 Subject: [PATCH 14/22] Add basic support of resource planning (#2167) This patch implements the following changes: 1. The support of IP addresses in 'target-hosts, add-hosts, remove-hosts' is added. Their validation requires hostname resolution, thus, the HostResolver() class is added in rebalance_commons.py Without validation we may face the case when passed through options IP address corresponds to existing host but is interpreted by ggrebalance as a new one. 2. The support hosts files is added. 3. The target directories handling is reworked. TemplateParser() class is added to support several placeholders. Now if 'target-datadirs' options is not passed all moves will choose default template directories as target ones. 4. The port planning is added in simple form (since doing network communication is overhead here) via PortAllocator() class. It forms per host per segment type port patterns and assigns them incrementally to moves. 5. The storage estimation is implemented. DiskUsage, DiskFree commands are used. The source datadirs and tablespaces are taken into account and validation of available space is provided. Main datadirs and tablespaces are validated on available disk space on corresponding filesystems. Corresponding unit tests are added for basic scenarios. --- gpMgmt/bin/ggrebalance | 122 +- gpMgmt/bin/gppylib/commands/unix.py | 23 + gpMgmt/bin/gprebalance_modules/planner.py | 980 +++++++++- .../gprebalance_modules/rebalance_commons.py | 706 ++++++- gpMgmt/bin/gprebalance_modules/test/config.py | 23 +- .../test/data/unbalanced_9_ip.array | 23 + .../test/test_unit_commons.py | 831 ++++++++ .../test/test_unit_ggrebalance.py | 153 +- .../test/test_unit_resources.py | 1675 +++++++++++++++++ .../test/test_unit_solver.py | 4 +- .../behave/mgmt_utils/ggrebalance.feature | 27 + .../behave/mgmt_utils/steps/mgmt_utils.py | 4 + 12 files changed, 4385 insertions(+), 186 deletions(-) create mode 100644 gpMgmt/bin/gprebalance_modules/test/data/unbalanced_9_ip.array create mode 100644 gpMgmt/bin/gprebalance_modules/test/test_unit_commons.py create mode 100644 gpMgmt/bin/gprebalance_modules/test/test_unit_resources.py diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index 9df9b07e644d..5ba9b5e6181b 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -34,7 +34,7 @@ _usage = """ EXECNAME = os.path.split(__file__)[-1] MAX_BATCH_SIZE = 128 MAX_PARALLEL_WORKERS = 96 - +MAX_SEGMENTS = 4096 def parseargs(): parser = OptParser(option_class=OptChecker, @@ -72,8 +72,18 @@ def parseargs(): help='Show rebalance plan') parser.add_option('-d', '--target-datadirs', dest='target_datadirs', metavar='', help='segment datadirs on new hosts') - - parser.set_defaults(verbose=False) + parser.add_option('--target-hosts-file', dest='target_hosts_file', metavar='', + help='Target hosts list file. One host per line') + parser.add_option('--target-datadirs-file', dest='target_datadirs_file', metavar='', + help='Target segment data directories passed as file. One datadir per line') + parser.add_option('--add-hosts-file', dest='add_hosts_file', metavar='', + help='Hosts that user wants to add to a cluster. One host per line') + parser.add_option('--remove-hosts-file', dest='remove_hosts_file', metavar='', + help='Hosts that user wants to remove from a cluster. One host per line') + parser.add_option('--skip-resource-estimation', dest='skip_resource_estimation', metavar='', + action='store_true', default=False, help='Skip resource estimation (storage)') + parser.add_option('-v', '--verbose', action='store_true', default=False, + help='debug output.') # Parse the command line arguments options, args = parser.parse_args() @@ -106,65 +116,65 @@ def validate_options(options, args, parser): logger.error( 'Invalid argument. parallel value must be >= 1 and <= %d' % MAX_PARALLEL_WORKERS) parser.exit(1) - - if options.clean_required and options.rollback_required: - logger.error("Can't use together options '--clean' and '--rollback'") + + if options.target_segment_count and ( + options.target_segment_count < 1 + or options.target_segment_count > MAX_SEGMENTS): + logger.error( + 'Invalid value of --target-segment-count. It must be >= 1 and <= %d' % MAX_SEGMENTS) parser.exit(1) - if ((options.clean_required or options.rollback_required) and - options.target_segment_count != None): - logger.error("Can't use '--target-segment-count' together option '--clean' or '--rollback'") - parser.exit(1) + incompatibility_rules = [ + # (condition_option, incompatible_options) + ('target_hosts', ['add_hosts', 'add_hosts_file', 'remove_hosts', 'remove_hosts_file', 'target_hosts_file']), + ('target_hosts_file', ['add_hosts', 'add_hosts_file', 'remove_hosts', 'remove_hosts_file']), + ('add_hosts', ['add_hosts_file', 'remove_hosts', 'remove_hosts_file']), + ('add_hosts_file', ['remove_hosts', 'remove_hosts_file']), + ('remove_hosts', ['remove_hosts_file']), + + ('target_datadirs', ['target_datadirs_file']), + + ('skip_rebalance', ['mirror_mode']), + + ('clean_required', [ + 'target_hosts', 'target_hosts_file', 'add_hosts', 'add_hosts_file', + 'remove_hosts', 'remove_hosts_file', 'target_datadirs', 'target_datadirs_file', + 'target_segment_count', 'mirror_mode', 'skip_rebalance', 'show_plan', + 'skip_resource_estimation' + ]), + + ('rollback_required', [ + 'target_hosts', 'target_hosts_file', 'add_hosts', 'add_hosts_file', + 'remove_hosts', 'remove_hosts_file', 'target_datadirs', 'target_datadirs_file', + 'target_segment_count', 'mirror_mode', 'skip_rebalance', 'show_plan', + 'skip_resource_estimation', 'duration', 'end' + ]), + ] + + def option_is_set(opt_name): + """Check if option is set""" + if not hasattr(options, opt_name): + return False + value = getattr(options, opt_name) + return value is not None and value != False - if options.clean_required or options.rollback_required: - if options.target_hosts: - logger.error("Can't use '--target-hosts' with '--clean' or '--rollback'") - parser.exit(1) - - if options.add_hosts: - logger.error("Can't use '--add-hosts' with '--clean' or '--rollback'") - parser.exit(1) - - if options.remove_hosts: - logger.error("Can't use '--remove-hosts' with '--clean' or '--rollback'") - parser.exit(1) - - if options.mirror_mode: - logger.error("Can't use '--mirror-mode' with '--clean' or '--rollback'") - parser.exit(1) - - if options.skip_rebalance: - logger.error("Can't use '--skip-rebalance' with '--clean' or '--rollback'") - parser.exit(1) - - if options.target_datadirs: - logger.error("Can't use '--target-datadirs' with '--clean' or '--rollback'") - parser.exit(1) - - if options.target_hosts: - if options.add_hosts: - logger.error("Can't use together options '--target-hosts' and '--add-hosts'") - parser.exit(1) - elif options.remove_hosts: - logger.error("Can't use together options '--target-hosts' and '--remove-hosts'") - parser.exit(1) + for rule in incompatibility_rules: + condition_opt, incompatible_opts = rule + if option_is_set(condition_opt): + # Check if any incompatible option is set + for incomp_opt in incompatible_opts: + if option_is_set(incomp_opt): + error_msg = \ + f"Can't use together options '--{condition_opt.replace('_', '-')}' and '--{incomp_opt.replace('_', '-')}'" + logger.error(error_msg) + parser.exit(1) - if (options.target_hosts or options.add_hosts)\ - and not options.target_datadirs: - logger.error("-target-datadirs options is not provided") + if (option_is_set('target_hosts') or option_is_set('add_hosts')) and not option_is_set('target_datadirs'): + logger.error("--target-datadirs option is required when using --target-hosts or --add-hosts") parser.exit(1) - if not (options.target_hosts or options.add_hosts)\ - and options.target_datadirs: - logger.error("--target-hosts option is not provided") - parser.exit(1) - - if options.mirror_mode and options.mirror_mode not in ('grouped', 'spread'): - logger.error('Mirroring strategy %s is not supported' % - options.mirroring) - parser.exit(1) - elif options.mirror_mode and options.skip_rebalance: - logger.error("Can't use together options '--skip-rebalance' and '--mirror-mode'") + if option_is_set('mirror_mode') and options.mirror_mode not in ('grouped', 'spread'): + logger.error('Mirroring strategy %s is not supported' % options.mirror_mode) parser.exit(1) return options, args diff --git a/gpMgmt/bin/gppylib/commands/unix.py b/gpMgmt/bin/gppylib/commands/unix.py index d78e4b63db67..9ce764a3c0bf 100644 --- a/gpMgmt/bin/gppylib/commands/unix.py +++ b/gpMgmt/bin/gppylib/commands/unix.py @@ -691,6 +691,29 @@ def get_hostname(self): raise Exception('Command not yet executed') return self.results.stdout.strip() +# --------------port is not busy-------------------- +class PortIsAvailable(Command): + def __init__(self, name, port, ctxt=REMOTE, remoteHost=None): + self.port = port + # Check if port is listening: return code 0 if is in use, + # 1 if is available + cmdStr = ( + f"(command -v ss >/dev/null 2>&1 && " + f"ss -tuln | grep -q ':{port} ') && " + f"echo 'IN_USE' || echo 'AVAILABLE'" + ) + Command.__init__(self, name, cmdStr, ctxt, remoteHost) + + def is_port_available(self) -> bool: + """ + Check if port is available based on command results + """ + if not self.results: + return False + + output = self.results.stdout.strip() + + return output == 'AVAILABLE' # --------------tcp port is active ----------------------- class PgPortIsActive(Command): diff --git a/gpMgmt/bin/gprebalance_modules/planner.py b/gpMgmt/bin/gprebalance_modules/planner.py index 929907996862..f4209c063b5f 100755 --- a/gpMgmt/bin/gprebalance_modules/planner.py +++ b/gpMgmt/bin/gprebalance_modules/planner.py @@ -1,26 +1,22 @@ #!/usr/bin/env python3 from collections import defaultdict +from contextlib import closing import os import pickle -import re -from typing import Any, Set, Tuple, List, Dict +from typing import Any, Tuple, List, Dict from dataclasses import dataclass from copy import deepcopy from gppylib.db import dbconn import gppylib.gparray as gparray -from gprebalance_modules.rebalance_commons import Host, HostStatus, CandidateSegment +from gprebalance_modules.rebalance_commons import * from gprebalance_modules.solver import GreedySolver, HostId, SolverConfig +from gppylib.commands.unix import PortIsAvailable -class ValidationError(Exception): - pass class PlanningError(Exception): pass -PRIMARY_PATH = '/data/primary/gpseg' -MIRROR_PATH = '/data/mirror/gpseg' - GROUPED = 'grouped' SPREAD = 'spread' @@ -37,19 +33,20 @@ class LogicalMove: target_datadir: target data directory target_port: target port """ - seg: CandidateSegment + seg: Segment + srcHost: Host dstHost: Host target_datadir: str target_port: int + segment_size: Optional[SegmentSize] = None def __str__(self): """Pretty print logical move""" - seg_info = self.seg.info # Source information - src_host = self.seg.host.hostname - src_datadir = seg_info.datadir - src_port = seg_info.port + src_host = self.seg.getSegmentHostName() + src_datadir = self.seg.getSegmentDataDirectory() + src_port = self.seg.getSegmentPort() # Destination information dst_host = self.dstHost.hostname @@ -58,12 +55,12 @@ def __str__(self): # Size information (if available) size_str = "" - if self.seg.size: - size_str = str(self.seg.size) + if self.segment_size: + size_str = f" [{self.segment_size}]" return ( - f"Move Segment(content={seg_info.content}, dbid={seg_info.dbid}, " - f"role={seg_info.role}){size_str}\n" + f"Move Segment(content={self.seg.getSegmentContentId()}, dbid={self.seg.getSegmentDbId()}, " + f"role={self.seg.role}){size_str}\n" f" From: {src_host}:{src_port} → {src_datadir}\n" f" To: {dst_host}:{dst_port} → {dst_datadir}" ) @@ -173,8 +170,9 @@ class ConfigurationEncoder: @staticmethod def encode_configuration(gparray: gparray.GpArray, - target_hosts: List[Host], - strategy: str) -> Tuple[SolverConfig, HostMapping]: + target_hosts: List[Host], + strategy: str, + resolver: HostResolver = HostResolver()) -> Tuple[SolverConfig, HostMapping]: """ The rebalance solvers work with abstract input segment configuration. Each host must have an id in [0, n-1] interval, where n is a size @@ -199,9 +197,9 @@ def encode_configuration(gparray: gparray.GpArray, mirror_plcmnt = [0] * gparray.get_primary_count() for pair in gparray.segmentPairs: primary_plcmnt[pair.primaryDB.content] = host_mapping[Host(hostname=pair.primaryDB.hostname, - address=pair.primaryDB.address)] + address=resolver.get_address(pair.primaryDB.hostname))] mirror_plcmnt[pair.mirrorDB.content] = host_mapping[Host(hostname=pair.mirrorDB.hostname, - address=pair.mirrorDB.address)] + address=resolver.get_address(pair.mirrorDB.hostname))] n_initial = len(target_hosts) n_target = sum([1 for h in target_hosts if h.status != HostStatus.DECOMMISSIONED]) conf = SolverConfig(gparray.get_primary_count(), @@ -212,6 +210,264 @@ def encode_configuration(gparray: gparray.GpArray, strategy) return (conf, host_mapping) +class PortAllocator: + """ + Manages port allocation for segment moves + + Tracks existing ports and allocates new ones following the existing pattern + """ + + def __init__(self, gparray: gparray.GpArray, logger: Any, + verify_ports: bool = False): + """ + Initialize port allocator with existing segment information + + Args: + gparray: Current gp_segment_configuration + """ + # Map: hostname -> set of ports currently in use + self.existing_ports_by_host: Dict[str, Set[int]] = defaultdict(set) + + # Map: hostname -> (primary_base_port, mirror_base_port) + self.base_ports_by_host: Dict[str, Tuple[int, int]] = {} + + # Map: hostname -> set of ports planned to be used (for moves) + self.planned_ports_by_host: Dict[str, Set[int]] = defaultdict(set) + + # Map: hostname -> (set of primary ports, set of mirror ports) + self.existing_ports_by_role: Dict[str, Tuple[Set[int], Set[int]]] = defaultdict( + lambda: (set(), set()) + ) + + self.logger = logger + self.verify_ports = verify_ports + + # Initialize from existing segments + self._initialize_from_array(gparray) + + def _initialize_from_array(self, gparray: gparray.GpArray): + """ + Build initial port usage maps from gparray + Tracks separate port patterns for primaries and mirrors + """ + # First pass: collect all ports by role + for seg in gparray.getSegDbList(): + hostname = seg.getSegmentHostName() + port = seg.getSegmentPort() + + self.existing_ports_by_host[hostname].add(port) + + # Track ports by role (primary vs mirror) + primary_ports, mirror_ports = self.existing_ports_by_role[hostname] + if seg.isSegmentPrimary(): + primary_ports.add(port) + else: + mirror_ports.add(port) + + # Second pass: determine base ports for each role + for hostname in self.existing_ports_by_role.keys(): + primary_ports, mirror_ports = self.existing_ports_by_role[hostname] + + primary_base = min(primary_ports) if primary_ports else None + mirror_base = min(mirror_ports) if mirror_ports else None + + if primary_base is not None or mirror_base is not None: + self.base_ports_by_host[hostname] = (primary_base, mirror_base) + + def allocate_port(self, + host: Host, + current_port: int, + is_mirror: bool) -> int: + """ + Allocate a port for a segment being moved to target host + + Args: + host: Target host + current_port: Current port of segment on source host + is_mirror: True if segment is a mirror, False if primary + host_status: Status of target host (NEW or ACTIVE) + + Returns: + Port number to use on target host + """ + if host.status == HostStatus.NEW: + return self._allocate_port_new_host(host, current_port, is_mirror) + else: + return self._allocate_port_existing_host(host, current_port, is_mirror) + + def _allocate_port_new_host(self, host: Host, current_port: int, is_mirror: bool) -> int: + """ + Allocate port on a new host + + For new hosts, first primary/mirror establishes base port for that role. + Subsequent segments of same role follow the established pattern. + """ + primary_base, mirror_base = self.base_ports_by_host.get(host.hostname, (None, None)) + + # If this is the first segment of this role on new host + if is_mirror and mirror_base is None: + # Establish mirror base port + port = self._verify_and_allocate_port(host, current_port) + self.base_ports_by_host[host.hostname] = (primary_base, port) + self.planned_ports_by_host[host.hostname].add(port) + # Track by role + _, mirror_ports = self.existing_ports_by_role[host.hostname] + mirror_ports.add(port) + return port + elif not is_mirror and primary_base is None: + # Establish primary base port + port = self._verify_and_allocate_port(host, current_port) + self.base_ports_by_host[host.hostname] = (port, mirror_base) + self.planned_ports_by_host[host.hostname].add(port) + # Track by role + primary_ports, _ = self.existing_ports_by_role[host.hostname] + primary_ports.add(port) + return port + + # Base port for this role already established, find next available + return self._find_next_available_port(host, current_port, is_mirror) + + def _allocate_port_existing_host(self, host: Host, current_port: int, is_mirror: bool) -> int: + """ + Allocate port on an existing host + + Try to use segment's current port if available, otherwise find next free port + following the pattern for this role (primary/mirror) + """ + # Try to use the segment's current port if it's free + if self._is_port_available(host.hostname, current_port): + if self.verify_ports: + if not self._check_port_on_host(host, current_port): + self.logger.info(f"Port {current_port} on {host.hostname} " + "appears in use, finding alternative") + return self._find_next_available_port(host, current_port, is_mirror) + self.planned_ports_by_host[host.hostname].add(current_port) + primary_ports, mirror_ports = self.existing_ports_by_role[host.hostname] + if is_mirror: + mirror_ports.add(current_port) + else: + primary_ports.add(current_port) + return current_port + + # Port conflict - find next available port for this role + return self._find_next_available_port(host, current_port, is_mirror) + + def _is_port_available(self, hostname: str, port: int) -> bool: + """ + Check if port is available (not used by existing or planned segments) + """ + return (port not in self.existing_ports_by_host[hostname] and + port not in self.planned_ports_by_host[hostname]) + + def _check_port_on_host(self, host: Host, port: int) -> bool: + """ + Verify port is actually available on the target host + """ + if not self.verify_ports: + return True + + try: + cmd = PortIsAvailable( + name=f'check port {port} on {host.hostname}', + port=port, + ctxt=REMOTE, + remoteHost=host.address + ) + cmd.run() + + is_available = cmd.is_port_available() + + if self.logger: + status = "available" if is_available else "in use" + self.logger.debug(f"Port {port} on {host.hostname}: {status}") + + return is_available + + except Exception as e: + self.logger.warning(f"Failed to verify port {port} on {host.hostname}: {e}. Assuming available.") + # On error, assume available + return True + + def _verify_and_allocate_port(self, host: Host, preferred_port: int) -> int: + """ + Verify port is available on host and allocate it + + If verification fails, find next available port. + """ + if self._is_port_available(host.hostname, preferred_port): + if self.verify_ports and not self._check_port_on_host(host, preferred_port): + # Preferred port is actually in use, find alternative + self.logger.info(f"Preferred port {preferred_port} on {host.hostname} " + "is in use, searching for alternative") + self.existing_ports_by_host[host.hostname].add(preferred_port) + return self._find_verified_port(host, preferred_port + 1) + return preferred_port + + # Port not available in tracking, search from next + return self._find_verified_port(host, preferred_port + 1) + + def _find_verified_port(self, host: Host, start_port: int) -> int: + """ + Find an available port starting from start_port, with optional verification + """ + candidate_port = start_port + max_attempts = 1000 + + for _ in range(max_attempts): + if self._is_port_available(host.hostname, candidate_port): + # Port available in tracking + if self.verify_ports: + # Verify on actual host + if self._check_port_on_host(host, candidate_port): + return candidate_port + else: + # Port in use on host, mark and continue + self.existing_ports_by_host[host.hostname].add(candidate_port) + candidate_port += 1 + continue + else: + # No verification, trust tracking + return candidate_port + + candidate_port += 1 + + raise PlanningError( + f"Cannot find available port on host {host.hostname} " + f"after {max_attempts} attempts starting from {start_port}" + ) + + def _find_next_available_port(self, host: Host, preferred_port: int, is_mirror: bool) -> int: + """ + Find next available port following the pattern for this role + """ + # Get base port for this role on this host + primary_base, mirror_base = self.base_ports_by_host.get(host.hostname, (None, None)) + + if is_mirror: + base_port = mirror_base if mirror_base is not None else preferred_port + else: + base_port = primary_base if primary_base is not None else preferred_port + + # If no base port established yet, use preferred and establish it + if is_mirror and mirror_base is None: + self.base_ports_by_host[host.hostname] = (primary_base, base_port) + elif not is_mirror and primary_base is None: + self.base_ports_by_host[host.hostname] = (base_port, mirror_base) + + # Search for available port starting from base + candidate_port = self._find_verified_port(host, base_port) + + self.planned_ports_by_host[host.hostname].add(candidate_port) + + # Track by role + primary_ports, mirror_ports = self.existing_ports_by_role[host.hostname] + if is_mirror: + mirror_ports.add(candidate_port) + else: + primary_ports.add(candidate_port) + + return candidate_port + class Planner: """ Performs the main planning procedure. Decides the operations to be performed @@ -223,15 +479,110 @@ def __init__(self, logger: Any, dburl: dbconn.DbURL, gpArray: gparray.GpArray, o self.options = options self.logger = logger self.virtual_gparray = deepcopy(self.gparray) - if options.target_datadirs: - self.dir_template_p, self.dir_template_m = self.get_datadirs() - else: - self.dir_template_p, self.dir_template_m = PRIMARY_PATH, MIRROR_PATH + + if self.options.target_hosts_file: + self.options.target_hosts = get_hosts_from_file(self.options.target_hosts_file, "target-hosts-file") + if self.options.add_hosts_file: + self.options.add_hosts = get_hosts_from_file(self.options.add_hosts_file, "add-hosts-file") + if self.options.remove_hosts_file: + self.options.remove_hosts = get_hosts_from_file(self.options.remove_hosts_file, "remove-hosts-file") + + self.validate_options() + + self.dir_template_p, self.dir_template_m = self.get_datadirs() + + self.resolver = HostResolver() + target, add, remove = self.resolve_hosts() + self.target_hosts, self.host_set_changed = Planner.get_target_hosts(self.virtual_gparray, - options, self.dir_template_p, - self.dir_template_m) + self.dir_template_m, + target, add, remove, + self.resolver + ) + def validate_options(self): + # Provide basic validation of hostnames and addresses. + validate_hosts_basic(self.options.target_hosts, "target-hosts") + validate_hosts_basic(self.options.add_hosts, "add-hosts") + validate_hosts_basic(self.options.remove_hosts, "remove-hosts") + + def resolve_hosts(self) -> Tuple[List[str], List[str], List[str]]: + """ + Returns: + List of hostnames from target_hosts, add_hosts, + remove_hosts options + """ + existing_hosts = set() + for seg in self.gparray.getSegDbList(): + existing_hosts.add(seg.getSegmentHostName()) + + existing_hostname_list = list(existing_hosts) + target_hostname_list = [] + add_hostname_list = [] + remove_hostname_list = [] + + def _parse_and_resolve_hosts( + hosts_input: str, + option_name: str, + check_exists: bool = False, + check_not_exists: bool = False) -> List[str]: + """ + Parse comma-separated host list, resolve IPs to hostnames, and validate + + Returns: + List of resolved hostnames + """ + hosts_list = [h.strip() for h in hosts_input.split(',')] + resolved_hosts = [] + + for host in hosts_list: + # Resolve IP to hostname if needed + if is_ip_address(host): + hostname = self.resolver.resolve_ip(host) + if not hostname: + raise ValidationError(f"{option_name}: Cannot resolve IP {host}") + else: + hostname = host + self.resolver.resolve_hostname(hostname) + + # Check existence constraints + if check_exists or check_not_exists: + matching_host = self.resolver.find_matching_hostname(hostname, existing_hostname_list) + + if check_not_exists and matching_host: + raise ValidationError( + f"{option_name}: Host '{host}' already exists in cluster " + f"as '{matching_host}'" + ) + + if check_exists and not matching_host: + raise ValidationError( + f"{option_name}: Host '{host}' does not exist in cluster" + ) + + resolved_hosts.append(hostname) + + return resolved_hosts + + if self.options.target_hosts: + target_hostname_list = _parse_and_resolve_hosts( + self.options.target_hosts, '--target-hosts') + + if self.options.add_hosts: + add_hostname_list = _parse_and_resolve_hosts( + self.options.add_hosts, '--add-hosts', check_not_exists=True) + + if self.options.remove_hosts: + remove_hostname_list = _parse_and_resolve_hosts( + self.options.remove_hosts, '--remove-hosts', check_exists=True) + + # Resolve the rest of hostnames + for hostname in existing_hostname_list: + self.resolver.resolve_hostname(hostname) + + return target_hostname_list, add_hostname_list, remove_hostname_list + def plan(self) -> Plan: # TODO Remove with future development. Temp code before state machine is implemented. from gprebalance_modules.rebalance_schema import RebalanceSchema @@ -262,13 +613,28 @@ def plan(self) -> Plan: raise PlanningError("Expand is not supported yet") if self.options.skip_rebalance: - self.logger.info("Skipping rebalance") + self.logger.warning("Skipping rebalance") return plan rebalance_moves = self.form_moves() if rebalance_moves is None: self.logger.info("Cluster is already balanced, no segment moves will be held.") + else: + if not self.options.skip_resource_estimation: + try: + with closing(dbconn.connect(self.dburl, encoding='UTF8')) as conn: + estimator = ResourceEstimator( + self.logger, + conn, + self.virtual_gparray, + batch_size=getattr(self.options, 'batch_size', 16) + ) + estimator.estimate_and_validate_moves(rebalance_moves) + except ResourceError as e: + raise PlanningError(f"Resource validation failed: {e}") + else: + self.logger.warning("Skipping resource estimation") plan.setMoves(rebalance_moves) plan.setTargetSegmentCount(self.options.target_segment_count) @@ -307,91 +673,88 @@ def remove_segpair_from_array(self, segPair: gparray.SegmentPair): segs_list = list(filter(lambda x: x.dbid != segPair.mirrorDB.dbid, segs_list)) self.virtual_gparray.setSegmentsAsLoadedFromDb(segs_list) - + @staticmethod def get_target_hosts(array: gparray.GpArray, - options: Any, - dir_template_p: str = PRIMARY_PATH, - dir_template_m: str = MIRROR_PATH) -> Tuple[List[Host], bool]: + primary_template: str = DEFAULT_PRIMARY_TEMPLATE, + mirror_template: str = DEFAULT_MIRROR_TEMPLATE, + target_hostname_list: List[str] = [], + add_hostname_list: List[str] = [], + remove_hostname_list: List[str] = [], + resolver: HostResolver = HostResolver()) -> Tuple[List[Host], bool]: """ Form set of hosts where we need to rebalance segments on - """ + Returns: + - List of Host objects with: + * For existing hosts: templates + existing datadirs filled + * For new hosts: only templates, existing datadirs empty + - Boolean indicating if host set changed + """ hosts = {} - host_set_changed = False for seg in array.segmentPairs: if seg.primaryDB.content >= 0: - if seg.primaryDB.hostname not in hosts: - hosts[seg.primaryDB.hostname] = Host(hostname=seg.primaryDB.hostname,\ - address=seg.primaryDB.address,\ - primary_datadirs={dir_template_p},\ - mirror_datadirs={dir_template_m},\ + primary_host = seg.primaryDB.hostname + mirror_host = seg.mirrorDB.hostname + if primary_host not in hosts: + hosts[primary_host] = Host(hostname=primary_host, + address=resolver.get_address(seg.primaryDB.hostname), + datadir_info=DatadirInfo(primary_template, mirror_template), status = HostStatus.ACTIVE) - if seg.mirrorDB.hostname not in hosts: - hosts[seg.mirrorDB.hostname] = Host(hostname=seg.mirrorDB.hostname,\ - address=seg.mirrorDB.address,\ - primary_datadirs={dir_template_p},\ - mirror_datadirs={dir_template_m},\ + if mirror_host not in hosts: + hosts[mirror_host] = Host(hostname=mirror_host, + address=resolver.get_address(seg.mirrorDB.hostname), + datadir_info=DatadirInfo(primary_template, mirror_template), status = HostStatus.ACTIVE) for pair in array.segmentPairs: primary = pair.primaryDB mirror = pair.mirrorDB - hosts[primary.hostname].primary_datadirs.add( - re.sub(r'\d+$', '',primary.datadir)) + primary_base = TemplateParser.extract_parent_directory(primary.datadir) + hosts[primary.hostname].datadir_info.existing_primary_datadirs.add(primary_base) if mirror: - hosts[mirror.hostname].mirror_datadirs.add( - re.sub(r'\d+$', '',mirror.datadir)) - if options.target_hosts: - hl = list(map(str.strip, options.target_hosts.split(','))) + mirror_base = TemplateParser.extract_parent_directory(mirror.datadir) + hosts[mirror.hostname].datadir_info.existing_mirror_datadirs.add(mirror_base) + + host_set_changed = False + if target_hostname_list: for host in hosts.keys(): - if host not in hl: + if host not in target_hostname_list: hosts[host].status = HostStatus.DECOMMISSIONED host_set_changed = True - for host in hl: + for host in target_hostname_list: if host not in hosts: - hosts[host] = Host(hostname=host,\ - address=host,\ - primary_datadirs={dir_template_p},\ - mirror_datadirs={dir_template_m},\ + hosts[host] = Host(hostname=host, + address=resolver.get_address(host), + datadir_info=DatadirInfo(primary_template, mirror_template), status = HostStatus.NEW) host_set_changed = True - if options.add_hosts: - hl = list(map(str.strip, options.add_hosts.split(','))) - for host in hl: + + if add_hostname_list: + for host in add_hostname_list: if host not in hosts: - hosts[host] = Host(hostname=host,\ - address=host,\ - primary_datadirs={dir_template_p},\ - mirror_datadirs={dir_template_m},\ + hosts[host] = Host(hostname=host, + address=resolver.get_address(host), + datadir_info=DatadirInfo(primary_template, mirror_template), status = HostStatus.NEW) host_set_changed = True - if options.remove_hosts: - hl = list(map(str.strip, options.remove_hosts.split(','))) + + if remove_hostname_list: for host in hosts.keys(): - if host in hl: + if host in remove_hostname_list: hosts[host].status = HostStatus.DECOMMISSIONED host_set_changed = True - return hosts.values(), host_set_changed + return list(hosts.values()), host_set_changed def get_datadirs(self) -> Tuple[str, str]: - parts = self.options.target_datadirs.split(',') - if len(parts) != 2: - raise ValidationError('--target_datadirs options should have format like ' - '"/data/primary/gpseg{content} , /data/mirror/gpseg{content}"') - # Remove {content} placeholder from each part - cleaned_parts = [] - for part in parts: - # Strip whitespace - part = part.strip() - match = re.search(r'\{.*\}$', part) - if not match: - raise ValidationError('--target_datadirs options should have format like ' - '"/data/primary/gpseg{content} , /data/mirror/gpseg{content}"') - # Remove trailing {...} - part = re.sub(r'\{.*\}$', '', part) - cleaned_parts.append(part) - - return tuple(cleaned_parts) + """ + Get datadir templates from options or use defaults + """ + if self.options.target_datadirs: + return TemplateParser.parse_datadirs_input(self.options.target_datadirs) + elif self.options.target_datadirs_file: + return TemplateParser.parse_datadirs_file(self.options.target_datadirs_file) + else: + return DEFAULT_PRIMARY_TEMPLATE, DEFAULT_MIRROR_TEMPLATE def form_moves(self) -> List[LogicalMove]: self.logger.info("Validation of rebalance possibility") @@ -429,24 +792,46 @@ def form_moves(self) -> List[LogicalMove]: return None # dry movements are planned here - config, host_mapping = ConfigurationEncoder.encode_configuration(self.virtual_gparray, self.target_hosts, strat) + config, host_mapping = ConfigurationEncoder.encode_configuration(self.virtual_gparray, + self.target_hosts, + strat, + self.resolver) id_to_host = {v: k for k, v in host_mapping.items()} self.logger.info("Planning rebalance moves. Can take up to 60s.") planning_seed_value = int.from_bytes(os.urandom(16) , 'big') self.logger.info(f"Running randomized plan improvement with seed:{planning_seed_value}") - solution, cost = GreedySolver(config, seed=planning_seed_value).solve() + solution, _ = GreedySolver(config, seed=planning_seed_value).solve() + + self.logger.debug(f"Solution: {solution}") + self.logger.debug(f"Hosts: {id_to_host}") + + port_allocator = PortAllocator(self.virtual_gparray, + self.logger, + not self.options.skip_resource_estimation) moves = [] - for pair in self.virtual_gparray.segmentPairs: - prim = pair.primaryDB - mir = pair.mirrorDB - plcmnt = solution[prim.content] - # TODO - resource estimation, ports, directories, size planning - if host_mapping[Host(prim.hostname, prim.address)] != plcmnt[0]: - cseg = CandidateSegment(prim, [h for h in self.target_hosts if h == Host(prim.hostname, prim.address)][0], None) - moves.append(LogicalMove(cseg, id_to_host[plcmnt[0]], list(id_to_host[plcmnt[0]].primary_datadirs)[-1] + str(prim.content), 7002)) - if host_mapping[Host(mir.hostname, mir.address)] != plcmnt[1]: - cseg = CandidateSegment(mir, [h for h in self.target_hosts if h == Host(mir.hostname, mir.address)][0], None) - moves.append(LogicalMove(cseg, id_to_host[plcmnt[1]], list(id_to_host[plcmnt[1]].mirror_datadirs)[-1] + str(mir.content), 7003)) + for seg in self.virtual_gparray.getSegDbList(): + is_mirror = seg.isSegmentMirror() + if is_mirror: + hostId = config.initial_mirror_mapping[seg.content] + final_hostId = solution[seg.content][1] + else: + hostId = config.initial_primary_mapping[seg.content] + final_hostId = solution[seg.content][0] + + if hostId != final_hostId: + #segment is moved + source_host = id_to_host[hostId] + target_host = id_to_host[final_hostId] + target_datadir = self._get_datadir_for_segment(target_host, seg.content, is_mirror) + + # Allocate port for this segment on target host + target_port = port_allocator.allocate_port( + host=target_host, + current_port=seg.getSegmentPort(), + is_mirror=is_mirror) + + moves.append(LogicalMove(seg, source_host, target_host, target_datadir, target_port)) + if len(moves) == 0: return None @@ -470,3 +855,402 @@ def already_balanced(self, load: int) -> bool: if n != load: return False return True + + def _get_datadir_for_segment(self, host: Host, content_id: int, is_mirror: bool) -> str: + """ + Determine datadir for a segment on target host + + Logic: + - Always use template for either new/exising hosts + """ + datadir_info = host.datadir_info + + assert(host.status == HostStatus.NEW or host.status == HostStatus.ACTIVE) + # use template + template = datadir_info.mirror_template if is_mirror else datadir_info.primary_template + return TemplateParser.instantiate_template(template, host.hostname, content_id) + +class ResourceEstimator: + """ + Estimates and validates resource requirements for segment moves + + This class handles move-specific resource estimation logic, + using DiskSpaceChecker for invoking remote disk operations. + """ + + def __init__(self, logger: Any, conn: Any, gparray: gparray.GpArray, batch_size: int = 16): + """ + Initialize resource estimator + + Args: + logger: Logger instance + conn: Database connection + gparray: Current GpArray configuration + batch_size: Number of parallel operations + """ + self.logger = logger + self.conn = conn + self.gparray = gparray + self.disk_checker = DiskSpaceChecker(logger, batch_size) + + def estimate_and_validate_moves(self, moves: List['LogicalMove']) -> None: + """ + Estimate resource requirements and validate moves can be performed + + This method modifies the moves in-place, setting the segment_size attribute. + + Args: + moves: List of LogicalMove objects to validate + """ + if not moves: + return + + self.logger.info(f"Estimating resource requirements for {len(moves)} segment moves...") + + try: + # Step 1: Estimate segment sizes + self._estimate_segment_sizes(moves) + + # Step 2: Validate available space on target hosts + self._validate_target_space(moves) + + # Log summary + total_size_kb = sum(move.segment_size.total_size_kb for move in moves if move.segment_size) + total_size_gb = total_size_kb / 1024 / 1024 + self.logger.info(f"Total data to move: {total_size_gb:.2f} GB") + self.logger.info("Resource validation completed successfully") + + except Exception as e: + self.logger.error(f"Resource validation failed") + raise ResourceError(str(e)) + + def _estimate_segment_sizes(self, moves: List[LogicalMove]) -> None: + """ + Estimate sizes for all segments being moved + + Populates the segment_size attribute on each LogicalMove + """ + # Get unique segments to estimate + segments_to_estimate = {move.seg.dbid: move.seg for move in moves} + + # Query for tablespace locations + tablespace_map = self._get_tablespace_locations(list(segments_to_estimate.keys())) + + # Group segments by source host for batch disk usage + segments_by_host = defaultdict(list) + for move in moves: + segments_by_host[move.srcHost.address].append(move.seg) + + # Estimate datadir sizes + for host_addr, host_segments in segments_by_host.items(): + dirs = [seg.datadir for seg in host_segments] + + try: + disk_usage = self.disk_checker.get_disk_usage(host_addr, dirs) + + for seg in host_segments: + size_kb = disk_usage.get(seg.datadir, 0) + # Find all moves for this segment and set size + for move in moves: + if move.seg.dbid == seg.dbid: + move.segment_size = SegmentSize(datadir_size_kb=size_kb) + + except Exception as e: + raise ResourceError(f"Cannot estimate segment sizes on host {host_addr}: {e}") + + # Estimate tablespace sizes + self._estimate_tablespace_sizes(moves, tablespace_map) + + def _get_tablespace_locations(self, dbids: List[int]) -> Dict[int, List[str]]: + """ + Query database for tablespace locations for given segments + + Returns: + Dict mapping dbid to list of tablespace paths + """ + if not dbids: + return {} + + oid_subq = """ + (SELECT * + FROM ( + SELECT oid FROM pg_tablespace + WHERE spcname NOT IN ('pg_default', 'pg_global') + ) AS _q1, + LATERAL gp_tablespace_location(_q1.oid) + ) AS t + """ + + segment_dbids = ','.join(f'({dbid})' for dbid in dbids) + + tablespace_location_sql = f""" + SELECT c.dbid, t.tblspc_loc||'/'||c.dbid AS tblspc_loc + FROM {oid_subq} + JOIN gp_segment_configuration AS c + ON t.gp_segment_id = c.content + WHERE c.dbid IN (VALUES {segment_dbids}) + """ + + try: + cursor = dbconn.query(self.conn, tablespace_location_sql) + tablespaces = defaultdict(list) + + for dbid, loc in cursor: + tablespaces[dbid].append(loc) + + return tablespaces + except Exception as e: + raise ResourceError(f"Failed to query tablespace locations: {e}") + + def _estimate_tablespace_sizes(self, + moves: List['LogicalMove'], + tablespace_map: Dict[int, List[str]]) -> None: + """ + Estimate tablespace sizes and add to segment_size + + Modifies move.segment_size in-place + """ + if not tablespace_map: + return + + # Group tablespace directories by host + tblspace_by_host = defaultdict(list) + for move in moves: + dbid = move.seg.dbid + if dbid in tablespace_map: + for tblspace_dir in tablespace_map[dbid]: + tblspace_by_host[move.srcHost.address].append((dbid, tblspace_dir)) + + # Process each host + for host_addr, host_tablespaces in tblspace_by_host.items(): + # Get unique directories + dirs = list(set(tblspace_dir for _, tblspace_dir in host_tablespaces)) + + try: + disk_usage = self.disk_checker.get_disk_usage(host_addr, dirs) + + # Aggregate by segment + for dbid, tblspace_dir in host_tablespaces: + size_kb = disk_usage.get(tblspace_dir, 0) + + # Find all moves for this segment and add tablespace size + for move in moves: + if move.seg.dbid == dbid and move.segment_size: + if move.segment_size.tablespace_usage is None: + move.segment_size.tablespace_usage = {} + move.segment_size.tablespace_usage[tblspace_dir] = size_kb + + except Exception as e: + self.logger.warning(f"Failed to get tablespace disk usage for host {host_addr}: {e}") + # Continue without tablespace sizes + + def _validate_target_space(self, moves: List[LogicalMove]) -> None: + """ + Validate that target hosts have sufficient disk space + + Groups all space requirements by filesystem to handle cases where + datadirs and tablespaces share the same underlying filesystem. + """ + self.logger.info("Validating available disk space on target hosts...") + + # Collect all directories we need to check + dirs_by_host = defaultdict(set) + + # Track what each directory is used for (for better error reporting) + # (hostname, dir) -> moves in 'datadirs' if dir is datadir + # moves moves in 'tablespaces' if dir is tablespace + dir_usage = defaultdict(lambda: {'datadirs': [], 'tablespaces': []}) + + for move in moves: + if not move.segment_size: + continue + + target_host = move.dstHost.hostname + target_addr = move.dstHost.address + target_datadir = move.target_datadir + + # Track datadir + dirs_by_host[target_addr].add(target_datadir) + dir_usage[(target_host, target_datadir)]['datadirs'].append(move) + + # Track tablespace directories + if move.segment_size.tablespace_usage: + for tblspace_path, _ in move.segment_size.tablespace_usage.items(): + # Extract base path (remove /dbid suffix) + base_tblspace_path = os.path.dirname(tblspace_path) + + dirs_by_host[target_addr].add(base_tblspace_path) + dir_usage[(target_host, base_tblspace_path)]['tablespaces'].append(move) + + # Convert sets to lists for DiskFree + dirs_by_host = {host: list(dirs) for host, dirs in dirs_by_host.items()} + + # Check available space in batch + try: + space_info_by_host = self.disk_checker.check_batch_available_space(dirs_by_host) + except Exception as e: + raise ResourceError(f"Failed to check available disk space: {e}") + + # Group directories by filesystem and aggregate space requirements + self.logger.info("Aggregating space requirements by filesystem...") + issues = self._validate_filesystem_requirements( + dir_usage=dir_usage, + space_info_by_host=space_info_by_host + ) + + # Report all issues + if issues: + error_lines = ["Insufficient disk space for rebalance operation:\n"] + + for issue in issues: + error_lines.append(self._format_space_issue(issue)) + + error_lines.append( + f"\nNote: Estimates include {int(DISK_SPACE_SAFETY_MARGIN * 100)}% safety margin" + ) + raise ResourceError(''.join(error_lines)) + + self.logger.info("Disk space validation completed successfully") + + def _validate_filesystem_requirements(self, + dir_usage: Dict[Tuple[str, str], Dict], + space_info_by_host: Dict[str, Dict[str, DiskSpaceInfo]]) -> List[Dict]: + """ + Validate space requirements aggregated by filesystem + + This correctly handles cases where datadirs and tablespaces share the same filesystem. + + Args: + dir_usage: Dict mapping (hostname, directory) -> {'datadirs': [moves], 'tablespaces': [moves]} + space_info_by_host: Space information from DiskSpaceChecker + + Returns: + List of issue dicts for filesystems with insufficient space + """ + # Group by filesystem + filesystem_requirements = defaultdict(lambda: { + 'required_kb': 0, + 'datadir_moves': set(), + 'tablespace_moves': set(), + 'datadirs': set(), + 'tablespaces': set(), + 'space_info': None + }) + + for (hostname, directory), usage in dir_usage.items(): + # Find the host address for this hostname + host_address = None + for moves_list in [usage['datadirs'], usage['tablespaces']]: + if moves_list: + host_address = moves_list[0].dstHost.address + break + + if not host_address: + continue + + if host_address not in space_info_by_host: + raise ResourceError(f"No disk space information for host {hostname}") + + space_info = space_info_by_host[host_address].get(directory) + if not space_info: + raise ResourceError( + f"No disk space information for {hostname}:{directory}" + ) + + # Group by filesystem + fs_key = (hostname, space_info.filesystem) + fs_data = filesystem_requirements[fs_key] + fs_data['space_info'] = space_info + + # Process datadir moves + for move in usage['datadirs']: + if not move.segment_size: + continue + + # Only count each move once per filesystem + if move.seg.dbid not in fs_data['datadir_moves']: + fs_data['datadir_moves'].add(move.seg.dbid) + fs_data['datadirs'].add(directory) + size_kb = move.segment_size.datadir_size_kb + fs_data['required_kb'] += int(size_kb * (1 + DISK_SPACE_SAFETY_MARGIN)) + + # Process tablespace moves + for move in usage['tablespaces']: + if not move.segment_size or not move.segment_size.tablespace_usage: + continue + + # Extract tablespace sizes that belong to this directory + for tbl_path, size_kb in move.segment_size.tablespace_usage.items(): + tbl_base_path = os.path.dirname(tbl_path) + + if tbl_base_path == directory: + # Use a unique key to avoid double-counting the same tablespace + tbl_key = (move.seg.dbid, tbl_path) + + if tbl_key not in fs_data['tablespace_moves']: + fs_data['tablespace_moves'].add(tbl_key) + fs_data['tablespaces'].add(directory) + fs_data['required_kb'] += int(size_kb * (1 + DISK_SPACE_SAFETY_MARGIN)) + + # Validate each filesystem + issues = [] + + for (hostname, filesystem), fs_data in filesystem_requirements.items(): + required_kb = fs_data['required_kb'] + available_kb = fs_data['space_info'].available_kb + + required_gb = required_kb / 1024 / 1024 + available_gb = available_kb / 1024 / 1024 + + # Count unique segments + unique_segments = len(fs_data['datadir_moves']) + len({dbid for dbid, _ in fs_data['tablespace_moves']}) + + self.logger.debug( + f"Filesystem {filesystem} on {hostname}: " + f"Required {required_gb:.2f} GB, Available {available_gb:.2f} GB " + f"({unique_segments} segments, " + f"{len(fs_data['datadirs'])} datadirs, " + f"{len(fs_data['tablespaces'])} tablespace dirs)" + ) + + if available_kb < required_kb: + all_dirs = sorted(fs_data['datadirs'].union(fs_data['tablespaces'])) + + issues.append({ + 'type': 'filesystem', + 'hostname': hostname, + 'filesystem': filesystem, + 'target_dirs': all_dirs, + 'num_datadirs': len(fs_data['datadirs']), + 'num_tablespaces': len(fs_data['tablespaces']), + 'num_segments': unique_segments, + 'required_gb': required_gb, + 'available_gb': available_gb, + }) + + return issues + + def _format_space_issue(self, issue: Dict) -> str: + """ + Format a space issue for error reporting + """ + issue_type = issue.get('type', 'unknown').upper() + + # Build directory breakdown + dir_info = [] + if issue.get('num_datadirs', 0) > 0: + dir_info.append(f"{issue['num_datadirs']} datadir(s)") + if issue.get('num_tablespaces', 0) > 0: + dir_info.append(f"{issue['num_tablespaces']} tablespace(s)") + + dir_breakdown = ', '.join(dir_info) if dir_info else 'unknown' + + return ( + f"\n [{issue_type}] Host: {issue['hostname']}\n" + f" Filesystem: {issue['filesystem']}\n" + f" Directories: {dir_breakdown}\n" + f" Paths: {', '.join(issue['target_dirs'])}\n" + f" Segments affected: {issue['num_segments']}\n" + f" Required: {issue['required_gb']:.2f} GB\n" + f" Available: {issue['available_gb']:.2f} GB\n" + ) diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_commons.py b/gpMgmt/bin/gprebalance_modules/rebalance_commons.py index 912ff434edcb..8eea94d24344 100755 --- a/gpMgmt/bin/gprebalance_modules/rebalance_commons.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_commons.py @@ -1,15 +1,49 @@ #!/usr/bin/env python3 +import base64 from dataclasses import dataclass -from typing import Set, Optional +import ipaddress +import os +import pickle +import re +import socket +from typing import Any, Dict, List, Set, Optional, Tuple from enum import IntEnum -from gppylib.gparray import Segment +from gppylib.gparray import Segment, GpArray +from gppylib.commands.base import REMOTE, WorkerPool +from gppylib.commands.unix import Hostname, DiskFree, DiskUsage +from gppylib.operations.validate_disk_space import FileSystem + +class ValidationError(Exception): + pass + +class ResourceError(Exception): + pass + +DEFAULT_PRIMARY_TEMPLATE = '/data1/primary/gpseg{content}' +DEFAULT_MIRROR_TEMPLATE = '/data1/mirror/gpseg{content}' class HostStatus(IntEnum): ACTIVE = 1 NEW = 2 DECOMMISSIONED = 3 +@dataclass +class DatadirInfo: + """ + Stores both template and actual datadirs for a host + """ + primary_template: str + mirror_template: str + # Actual paths from existing segment + existing_primary_datadirs: Set[str] + existing_mirror_datadirs: Set[str] + + def __init__(self, primary_template: str, mirror_template: str): + self.primary_template = primary_template + self.mirror_template = mirror_template + self.existing_primary_datadirs = set() + self.existing_mirror_datadirs = set() @dataclass class Host: @@ -25,8 +59,7 @@ class Host: """ hostname: str address: str - primary_datadirs: Set[str] = None - mirror_datadirs: Set[str] = None + datadir_info: DatadirInfo = None status: HostStatus = None def __hash__(self): @@ -40,23 +73,666 @@ def __eq__(self, other): def __str__(self): pass +DISK_SPACE_SAFETY_MARGIN = 0.10 + +@dataclass class SegmentSize: """ - TODO. Should contain storage size of - segment instance with all tablespace info. + Storage size of segment instance with all tablespace info + + Attributes: + datadir_size_kb: Size of main datadir in KB + tablespace_usage: Dict mapping tablespace paths to their sizes in KB + total_size_kb: Total size including tablespaces """ - pass + datadir_size_kb: int + tablespace_usage: Optional[Dict[str, int]] = None + + @property + def total_size_kb(self) -> int: + """Calculate total size including tablespaces""" + total = self.datadir_size_kb + if self.tablespace_usage: + total += sum(self.tablespace_usage.values()) + return total + + def __str__(self): + """Human-readable size string""" + size_mb = self.total_size_kb / 1024 + if size_mb < 1024: + return f"{size_mb:.2f} MB" + else: + size_gb = size_mb / 1024 + return f"{size_gb:.2f} GB" + + def __repr__(self): + return f"SegmentSize(datadir={self.datadir_size_kb}KB, tablespaces={self.tablespace_usage})" + +class TemplateParser: + """ + Handles parsing and validation of directory templates + """ + + VALID_PLACEHOLDERS = {'hostname', 'content'} + PLACEHOLDER_PATTERN = r'\{(\w+)\}' + + @classmethod + def parse_datadirs_input(cls, input_str: str) -> Tuple[str, str]: + """ + Parse --target-datadirs input and return (primary_template, mirror_template) + + Handles: + - "/data/primary/gpseg{content}, /data/mirror/gpseg{content}" -> as is + - "/data/primary, /data/mirror" -> adds gpseg{content} + - "/data/primary/{hostname}, /data/mirror/{hostname}" -> adds gpseg{content} + - '"/data/primary", "/data/mirror"' -> removes quotes + - --target-datadirs="/dir1, /dir2" -> handles shell quoting + """ + # Split by comma, respecting quotes + parts = cls._split_respecting_quotes(input_str) + + if len(parts) != 2: + raise ValidationError( + '--target-datadirs should have format: ' + '"/data/primary/gpseg{content}, /data/mirror/gpseg{content}". ' + 'Available templated parameters: {hostname}, {content}' + ) + + # Clean and normalize each part + primary_template = cls._normalize_template(parts[0]) + mirror_template = cls._normalize_template(parts[1]) + + # Validate templates + cls._validate_templates(primary_template, mirror_template) + + return primary_template, mirror_template + + @classmethod + def _split_respecting_quotes(cls, input_str: str) -> list: + """ + Split input string by comma, respect quoted sections. + + Handles cases like: + - "/dir1, /dir2" -> ["/dir1", "/dir2"] + - '"/di,r1/", "/dir2"' -> ["/di,r1/", "/dir2"] + """ + # Manual parsing: split by comma while respecting quotes + parts = [] + current_part = [] + in_double_quotes = False + in_single_quotes = False + i = 0 + + while i < len(input_str): + char = input_str[i] + + # Track quote state + if char == '"' and not in_single_quotes: + in_double_quotes = not in_double_quotes + current_part.append(char) + elif char == "'" and not in_double_quotes: + in_single_quotes = not in_single_quotes + current_part.append(char) + elif char == ',' and not in_double_quotes and not in_single_quotes: + # Found unquoted comma - this is our delimiter + parts.append(''.join(current_part).strip()) + current_part = [] + else: + current_part.append(char) + + i += 1 + + # Add the last part + if current_part: + parts.append(''.join(current_part).strip()) + + # Clean up: remove empty parts + parts = [p for p in parts if p] + + return parts + + @classmethod + def _normalize_template(cls, path: str) -> str: + """ + Clean path from quotes and normalize template + - If it contains placeholders, validate and return as-is + - If it doesn't contain {content} placeholders, append gpseg{content} + """ + + path = path.strip() + if len(path) >= 2: + if (path[0] == '"' and path[-1] == '"') or (path[0] == "'" and path[-1] == "'"): + path = path[1:-1].strip() + + if not path: + raise ValidationError('Directory path cannot be empty') + + if not path.startswith('/'): + raise ValidationError( + f'Directory path must be absolute: {path}' + ) + + placeholders = re.findall(cls.PLACEHOLDER_PATTERN, path) + + # Validate placeholders + for placeholder in placeholders: + if placeholder not in cls.VALID_PLACEHOLDERS: + raise ValidationError( + f'Invalid placeholder {{{placeholder}}}. ' + f'Valid placeholders are: {", ".join("{" + p + "}" for p in cls.VALID_PLACEHOLDERS)}' + ) + + # If no placeholders, add default gpseg{content} + if not placeholders or ('content' not in placeholders): + # Remove trailing slash if present + path = path.rstrip('/') + return f'{path}/gpseg{{content}}' + + return path + + @classmethod + def _validate_templates(cls, primary_template: str, mirror_template: str) -> None: + """ + Validate primary and mirror templates for common issues + """ + # Check if templates are identical + if primary_template == mirror_template: + raise ValidationError( + 'Primary and mirror templates cannot be identical. ' + f'Both are: {primary_template}' + ) + + # Validate both contain {content} placeholder + # (should always be true after normalization, but double-check) + if '{content}' not in primary_template: + raise ValidationError( + f'Primary template must contain {{content}} placeholder: {primary_template}' + ) + + if '{content}' not in mirror_template: + raise ValidationError( + f'Mirror template must contain {{content}} placeholder: {mirror_template}' + ) + + @classmethod + def parse_datadirs_file(cls, filepath: str) -> Tuple[str, str]: + """ + Parse --target-datadirs-file + Expected format (2 lines): + /data/primary/gpseg{content} + /data/mirror/gpseg{content} + Available templated parameters: {hostname}, {content} + """ + + with open(filepath, 'r') as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + + if len(lines) != 2: + raise ValidationError( + f'File {filepath} should contain exactly 2 lines: ' + 'primary template and mirror template' + ) + + primary_template = cls._normalize_template(lines[0]) + mirror_template = cls._normalize_template(lines[1]) + + cls._validate_templates(primary_template, mirror_template) + + return primary_template, mirror_template + + @staticmethod + def extract_parent_directory(datadir: str) -> str: + """ + Extract parent directory from an actual segment datadir + + This handles arbitrary naming conventions by just getting the parent. + + Examples: + /data/primary/gpseg0 -> /data/primary + + Returns: + Parent directory path (without trailing slash) + """ + return os.path.dirname(datadir.rstrip('/')) + + @staticmethod + def instantiate_template(template: str, hostname: str = None, content: int = None) -> str: + """ + Instantiate a template with actual values + """ + result = template + if hostname is not None: + result = result.replace('{hostname}', hostname) + if content is not None: + result = result.replace('{content}', str(content)) + return result + +def is_ip_address(ip_str: str): + try: + ipaddress.ip_address(ip_str) + return True + except ValueError: + return False + +class HostResolver: + """ + Utility class to resolve and match hostnames with IP addresses + Supports multiple IPs per hostname and maintains bidirectional mappings + """ + def __init__(self): + # hostname -> set of IP addresses + self._hostname_to_ips: Dict[str, Set[str]] = {} + # IP address -> hostname + self._ip_to_hostname: Dict[str, str] = {} + + def get_address(self, hostname: str) -> str: + """ + Get primary IP address for hostname + Returns first IP address or hostname if not resolved + """ + ips = self._hostname_to_ips.get(hostname, set()) + if ips: + # Return first IP (sorted for consistency) + return sorted(ips)[0] + return hostname + + def get_all_addresses(self, hostname: str) -> Set[str]: + """ + Get all IP addresses associated with hostname + """ + return self._hostname_to_ips.get(hostname, set()) + + def get_hostname(self, ip: str) -> str: + """ + Get hostname for IP address + Returns hostname or IP if not resolved + """ + return self._ip_to_hostname.get(ip, ip) + + def resolve_hostname(self, hostname: str) -> Optional[str]: + """ + Resolve hostname to IP addresses + Caches all IP addresses associated with hostname + + Returns: + Primary IP address (first one found), or None if resolution fails + """ + # Already resolved + if hostname in self._hostname_to_ips: + return self.get_address(hostname) + + try: + # Get all addresses for this hostname (IPv4 and IPv6) + addr_info = socket.getaddrinfo( + hostname, + None, + socket.AF_UNSPEC, + socket.SOCK_STREAM + ) + + ips = set() + for info in addr_info: + ip = info[4][0] + # For IPv6 + if ':' in ip: + ip = ip.split('%')[0] + ips.add(ip) + + if ips: + # Store hostname -> IPs mapping + self._hostname_to_ips[hostname] = ips + + # Store reverse mappings (IP -> hostname) + for ip in ips: + self._ip_to_hostname[ip] = hostname + + # Return primary IP + return sorted(ips)[0] + else: + return None + + except (socket.gaierror, socket.error) as e: + return None + + def resolve_ip(self, ip_str: str) -> Optional[str]: + """ + Reverse resolve IP to hostname using remote command + + Returns: + Hostname or None if resolution fails + """ + # Already resolved + if ip_str in self._ip_to_hostname: + return self._ip_to_hostname[ip_str] + + try: + # Validate it's a valid IP first + ipaddress.ip_address(ip_str) + + # Get hostname from remote host + cmd = Hostname('hostname', ctxt=REMOTE, remoteHost=ip_str) + cmd.run() + + if not cmd.was_successful(): + return None + + hostname = cmd.get_hostname() + + if hostname: + # Store reverse mapping + self._ip_to_hostname[ip_str] = hostname + + # Also store forward mapping + if hostname not in self._hostname_to_ips: + self._hostname_to_ips[hostname] = set() + self._hostname_to_ips[hostname].add(ip_str) + + return hostname + else: + return None + + except (ValueError, Exception) as e: + return None + + def hosts_match(self, host1: str, host2: str) -> bool: + """ + Check if two hosts match (considering hostname/IP resolution) + Handles cases where hosts are specified as hostname or IP + + Returns: + True if hosts represent the same machine + """ + # Direct match + if host1 == host2: + return True + + host1_normalized = host1.split('%')[0] if ':' in host1 else host1 + host2_normalized = host2.split('%')[0] if ':' in host2 else host2 + + if host1_normalized == host2_normalized: + return True + + # Check if both are IPs + is_ip1 = is_ip_address(host1) + is_ip2 = is_ip_address(host2) + + if is_ip1 and is_ip2: + # Both are IPs - they don't match if not equal + return False + + # One or both are hostnames - resolve and compare + if is_ip1 and not is_ip2: + # host1 is IP, host2 is hostname + # Resolve host2 to get its IPs + ips_of_host2 = self._hostname_to_ips.get(host2) + if not ips_of_host2: + # Try to resolve + self.resolve_hostname(host2) + ips_of_host2 = self._hostname_to_ips.get(host2) + + if ips_of_host2: + return host1 in ips_of_host2 + + if not is_ip1 and is_ip2: + # host1 is hostname, host2 is IP + # Resolve host1 to get its IPs + ips_of_host1 = self._hostname_to_ips.get(host1) + if not ips_of_host1: + # Try to resolve + self.resolve_hostname(host1) + ips_of_host1 = self._hostname_to_ips.get(host1) + + if ips_of_host1: + return host2 in ips_of_host1 + + if not is_ip1 and not is_ip2: + # Both are hostnames + # Resolve both and check if they share any IPs + ips1 = self._hostname_to_ips.get(host1) + if not ips1: + self.resolve_hostname(host1) + ips1 = self._hostname_to_ips.get(host1, set()) + + ips2 = self._hostname_to_ips.get(host2) + if not ips2: + self.resolve_hostname(host2) + ips2 = self._hostname_to_ips.get(host2, set()) + + # Check if they share any IP addresses + if ips1 and ips2: + return bool(ips1 & ips2) + + return False + + def find_matching_hostname(self, target_host: str, existing_hosts: List[str]) -> Optional[str]: + """ + Find if target_host matches any existing host + + Returns: + The matching existing host name, or None if no match + """ + for existing_host in existing_hosts: + if self.hosts_match(target_host, existing_host): + return existing_host + return None + +def validate_hostname(hostname:str): + if len(hostname) > 255: + raise ValidationError(f"Hostname '{hostname}' exceeds maximum length of 255 characters") + + if not re.match(r'^[a-zA-Z0-9._-]+$', hostname): + raise ValidationError(f"Hostname '{hostname}' contains invalid characters. " + "Only ASCII letters, digits, hyphen, underscore, and dot are allowed") + +def validate_hosts_basic(hosts: str, option_name: str): + + if not hosts: + return + + target_hosts = list(map(str.strip, hosts.split(','))) + + # Remove empty strings + target_hosts = [h for h in target_hosts if h] + if not target_hosts: + raise ValidationError(f" --{option_name}: No valid hosts provided") + + seen_hosts = set() + has_ip = False + has_hostname = False + for host in target_hosts: + # Check for duplicates + if host in seen_hosts: + raise ValidationError(f" --{option_name}: Duplicate host '{host}' found") + seen_hosts.add(host) + + if is_ip_address(host): + has_ip = True + continue + has_hostname = True + validate_hostname(host) + if has_ip and has_hostname: + raise ValidationError(f" --{option_name} must not contain IP adress and hostname simultaniously") + +def get_hosts_from_file(file, option_name) -> str: + result = "" + with open(file, 'r') as fp: + i = 0 + for line in fp: + i += 1 + if i >= 1000: + raise ValidationError(f" --{option_name} contains more than 1000 hosts") + result = ", ".join(line.strip()) + return result + +@dataclass +class SegmentId: + """Identifier for a segment""" + dbid: int + content: int + + def __hash__(self): + return hash((self.dbid, self.content)) + + def __eq__(self, other): + if not isinstance(other, SegmentId): + return NotImplemented + return self.dbid == other.dbid and self.content == other.content @dataclass -class CandidateSegment: +class DiskSpaceInfo: """ - Accumulates the info of rebalance segments + Disk space information for a filesystem Attributes: - info: segment info from gparray - size: size estimation before execution - host: placement host + filesystem: Filesystem name/device + available_kb: Available disk space in KB + directory: Directory that was checked """ - info: Segment - host: Host - size: Optional[SegmentSize] \ No newline at end of file + filesystem: str + available_kb: int + directory: str + + @property + def available_mb(self) -> float: + return self.available_kb / 1024 + + @property + def available_gb(self) -> float: + return self.available_mb / 1024 + + def __str__(self): + return f"Available: {self.available_gb:.2f} GB on {self.filesystem}" + +class DiskSpaceChecker: + """ + Utility for checking disk space on local and remote hosts + """ + + def __init__(self, logger: Any, batch_size: int = 16): + """ + Initialize disk space checker + + Args: + logger: Logger instance + batch_size: Number of parallel operations + """ + self.logger = logger + self.batch_size = batch_size + + def get_disk_usage(self, hostaddr: str, directories: List[str]) -> Dict[str, int]: + """ + Get the disk usage for the given set of directories on the targeted host + + Args: + hostaddr: Host address (sometimes can be hostname) to check + directories: List of directories to check + + Returns: + Dictionary mapping directories to disk usage in KB + """ + dirs_disk_usage = {} + + if not directories: + return dirs_disk_usage + + pool = WorkerPool(numWorkers=min(len(directories), self.batch_size)) + try: + for directory in directories: + cmd = DiskUsage('check segment disk space used', + directory, ctxt=REMOTE, remoteHostAddr=hostaddr) + pool.addCommand(cmd) + pool.join() + finally: + pool.haltWork() + pool.joinWorkers() + + for cmd in pool.getCompletedItems(): + if not cmd.was_successful(): + raise Exception(f"Unable to check disk usage on segment: {cmd.get_results().stderr}") + + dirs_disk_usage[cmd.directory] = cmd.kbytes_used() + + return dirs_disk_usage + + def get_available_space(self, hostaddr: str, directories: List[str]) -> Dict[str, DiskSpaceInfo]: + """ + Get available disk space information for directories on remote host + + Uses DiskFree command which runs calculate_disk_free.py script. + This handles the case where directories don't exist yet by walking + up the path until it finds an existing directory. + + Args: + hostaddr: Host address to check + directories: List of directories/paths to check + + Returns: + Dictionary mapping directory to DiskSpaceInfo + """ + if not directories: + return {} + + filesystems = self._get_filesystems(hostaddr, directories) + + # Build result mapping + result = {} + for fs in filesystems: + # Each FileSystem has a list of directories it applies to + for directory in fs.directories: + result[directory] = DiskSpaceInfo( + filesystem=fs.name, + available_kb=fs.disk_free, + directory=directory + ) + + return result + + def _get_filesystems(self, hostaddr: str, directories: List[str]) -> List[FileSystem]: + """ + Get filesystem information for directories on target host + + Args: + hostaddr: Host address + directories: List of directories + + Returns: + List of FileSystem objects + + """ + filesystems = [] + + cmd = DiskFree(hostaddr, directories) + + cmd.run() + + if not cmd.was_successful(): + raise Exception(f"Failed to check disk free on target segment: {cmd.get_results().stderr}") + + # Decode the pickled result + filesystems = pickle.loads( + base64.urlsafe_b64decode(cmd.get_results().stdout)) + + return filesystems + + def check_batch_available_space(self, + directories_by_host: Dict[str, List[str]]) -> Dict[str, Dict[str, DiskSpaceInfo]]: + """ + Check available space for multiple directories across multiple hosts + + Args: + directories_by_host: Dict mapping host address to list of directories + + Returns: + Dict mapping host address to dict of (directory -> DiskSpaceInfo) + """ + results = {} + + for hostaddr, directories in directories_by_host.items(): + try: + space_info = self.get_available_space(hostaddr, directories) + results[hostaddr] = space_info + except Exception as e: + self.logger.error(f"Failed to check available space on {hostaddr}: {e}") + raise + + return results diff --git a/gpMgmt/bin/gprebalance_modules/test/config.py b/gpMgmt/bin/gprebalance_modules/test/config.py index aa658cadb2a9..be6b0bac957e 100644 --- a/gpMgmt/bin/gprebalance_modules/test/config.py +++ b/gpMgmt/bin/gprebalance_modules/test/config.py @@ -1,10 +1,8 @@ import os -import types -from typing import List, Tuple, Dict -from collections import defaultdict from gppylib.gparray import Segment, GpArray -from ..planner import ConfigurationEncoder, Planner +from gprebalance_modules.planner import ConfigurationEncoder, Planner, HostResolver + def initGparrayFromFile(basename): filename = os.path.dirname(__file__) + \ @@ -20,12 +18,21 @@ def getEncoding(file, strat, target_hosts, add_hosts, remove_hosts): def inner(func): def wrapper(self, *args, **kwargs): gparray = initGparrayFromFile(file) + target_hostnames = None + add_hostnames = None + remove_hostnames = None + if target_hosts: + target_hostnames = [h.strip() for h in target_hosts.split(',')] + if add_hosts: + add_hostnames = [h.strip() for h in add_hosts.split(',')] + if remove_hosts: + remove_hostnames = [h.strip() for h in remove_hosts.split(',')] self.encoding = ConfigurationEncoder.encode_configuration(gparray, Planner.get_target_hosts( - gparray, - types.SimpleNamespace(target_hosts=target_hosts, - add_hosts=add_hosts, - remove_hosts=remove_hosts) )[0], + array=gparray, + target_hostname_list=target_hostnames, + add_hostname_list=add_hostnames, + remove_hostname_list=remove_hostnames)[0], strat) res = func(self, *args, **kwargs) self.encoding = None diff --git a/gpMgmt/bin/gprebalance_modules/test/data/unbalanced_9_ip.array b/gpMgmt/bin/gprebalance_modules/test/data/unbalanced_9_ip.array new file mode 100644 index 000000000000..94d8bf5b2a31 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/unbalanced_9_ip.array @@ -0,0 +1,23 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|172.20.0.6|7000|/data/primary0 +3|1|p|p|s|u|sdw1|172.20.0.6|7001|/data/primary1 +4|2|p|p|s|u|sdw1|172.20.0.6|7002|/data/primary2 +14|3|m|m|s|u|sdw1|172.20.0.6|8003|/data/mirror3 +15|4|m|m|s|u|sdw1|172.20.0.6|8004|/data/mirror4 +16|5|m|m|s|u|sdw1|172.20.0.6|8005|/data/mirror5 +17|6|m|m|s|u|sdw1|172.20.0.6|8006|/data/mirror6 +18|7|m|m|s|u|sdw1|172.20.0.6|8007|/data/mirror7 +19|8|m|m|s|u|sdw1|172.20.0.6|8008|/data/mirror8 +# SDW2 +5|3|p|p|s|u|sdw2|172.20.0.7|7003|/data/primary3 +6|4|p|p|s|u|sdw2|172.20.0.7|7004|/data/primary4 +# SDW3 +7|5|p|p|s|u|sdw3|172.20.0.8|7005|/data/primary5 +8|6|p|p|s|u|sdw3|172.20.0.8|7006|/data/primary6 +9|7|p|p|s|u|sdw3|172.20.0.8|7007|/data/primary7 +10|8|p|p|s|u|sdw3|172.20.0.8|7008|/data/primary8 +11|0|m|m|s|u|sdw3|172.20.0.8|8000|/data/mirror0 +12|1|m|m|s|u|sdw3|172.20.0.8|8001|/data/mirror1 +13|2|m|m|s|u|sdw3|172.20.0.8|8002|/data/mirror2 diff --git a/gpMgmt/bin/gprebalance_modules/test/test_unit_commons.py b/gpMgmt/bin/gprebalance_modules/test/test_unit_commons.py new file mode 100644 index 000000000000..5b75f0232200 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/test_unit_commons.py @@ -0,0 +1,831 @@ +import os +import socket +import tempfile + +from gppylib.test.unit.gp_unittest import * +from mock import * +from gprebalance_modules.rebalance_commons import HostResolver, TemplateParser, ValidationError, is_ip_address + +class TestHostResolver(GpTestCase): + """ + Test suite for HostResolver class + """ + + def setUp(self): + self.resolver = HostResolver() + + def tearDown(self): + self.resolver = None + + def test_is_ip_address_valid_ipv4(self): + self.assertTrue(is_ip_address('192.168.1.1')) + self.assertTrue(is_ip_address('10.0.0.1')) + self.assertTrue(is_ip_address('255.255.255.255')) + self.assertTrue(is_ip_address('0.0.0.0')) + + def test_is_ip_address_valid_ipv6(self): + self.assertTrue(is_ip_address('2001:0db8:85a3::8a2e:0370:7334')) + self.assertTrue(is_ip_address('::1')) + self.assertTrue(is_ip_address('fe80::1')) + self.assertTrue(is_ip_address('::')) + + def test_is_ip_address_invalid(self): + self.assertFalse(is_ip_address('hostname')) + self.assertFalse(is_ip_address('not-an-ip')) + self.assertFalse(is_ip_address('999.999.999.999')) + self.assertFalse(is_ip_address('192.168.1')) + self.assertFalse(is_ip_address('')) + self.assertFalse(is_ip_address('192.168.1.1.1')) + + @patch('socket.getaddrinfo') + def test_resolve_hostname_success_ipv4(self, mock_getaddrinfo): + # Mock socket.getaddrinfo to return IPv4 address + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0)) + ] + + result = self.resolver.resolve_hostname('testhost') + + self.assertEqual(result, '192.168.1.10') + mock_getaddrinfo.assert_called_once_with( + 'testhost', None, socket.AF_UNSPEC, socket.SOCK_STREAM + ) + # Check caching - now returns a set + self.assertEqual(self.resolver._hostname_to_ips['testhost'], {'192.168.1.10'}) + # Check reverse mapping + self.assertEqual(self.resolver._ip_to_hostname['192.168.1.10'], 'testhost') + + @patch('socket.getaddrinfo') + def test_resolve_hostname_success_ipv6(self, mock_getaddrinfo): + mock_getaddrinfo.return_value = [ + (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('2001:db8::1', 0, 0, 0)) + ] + + result = self.resolver.resolve_hostname('testhost6') + + self.assertEqual(result, '2001:db8::1') + self.assertEqual(self.resolver._hostname_to_ips['testhost6'], {'2001:db8::1'}) + self.assertEqual(self.resolver._ip_to_hostname['2001:db8::1'], 'testhost6') + + @patch('socket.getaddrinfo') + def test_resolve_hostname_success_ipv6_with_scope(self, mock_getaddrinfo): + # Test IPv6 with scope ID normalization + mock_getaddrinfo.return_value = [ + (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('fe80::1%eth0', 0, 0, 0)) + ] + + result = self.resolver.resolve_hostname('testhost6') + + # Should normalize by removing scope ID + self.assertEqual(result, 'fe80::1') + self.assertEqual(self.resolver._hostname_to_ips['testhost6'], {'fe80::1'}) + + @patch('socket.getaddrinfo') + def test_resolve_hostname_multiple_ips(self, mock_getaddrinfo): + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0)), + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.10', 0)), + (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('fe80::1', 0, 0, 0)) + ] + + result = self.resolver.resolve_hostname('multihost') + + # Should return first IP when sorted + self.assertIn(result, {'10.0.0.10', '192.168.1.10', 'fe80::1'}) + # Should cache all IPs + self.assertEqual(self.resolver._hostname_to_ips['multihost'], + {'192.168.1.10', '10.0.0.10', 'fe80::1'}) + # All IPs should have reverse mappings + self.assertEqual(self.resolver._ip_to_hostname['192.168.1.10'], 'multihost') + self.assertEqual(self.resolver._ip_to_hostname['10.0.0.10'], 'multihost') + self.assertEqual(self.resolver._ip_to_hostname['fe80::1'], 'multihost') + + @patch('socket.getaddrinfo') + def test_resolve_hostname_cached(self, mock_getaddrinfo): + self.resolver._hostname_to_ips['cached'] = {'192.168.1.100'} + + result = self.resolver.resolve_hostname('cached') + + self.assertEqual(result, '192.168.1.100') + mock_getaddrinfo.assert_not_called() + + @patch('gprebalance_modules.rebalance_commons.Hostname') + def test_resolve_ip_success(self, mock_hostname_class): + # Mock the Hostname command + mock_cmd = MagicMock() + mock_cmd.was_successful.return_value = True + mock_cmd.get_hostname.return_value = 'testhost' + mock_hostname_class.return_value = mock_cmd + + result = self.resolver.resolve_ip('192.168.1.10') + + self.assertEqual(result, 'testhost') + mock_hostname_class.assert_called_once() + mock_cmd.run.assert_called_once() + # Check caching - reverse mapping + self.assertEqual(self.resolver._ip_to_hostname['192.168.1.10'], 'testhost') + # Check forward mapping was also created + self.assertIn('192.168.1.10', self.resolver._hostname_to_ips['testhost']) + + @patch('gprebalance_modules.rebalance_commons.Hostname') + def test_resolve_ip_cached(self, mock_hostname_class): + self.resolver._ip_to_hostname['192.168.1.100'] = 'cached' + + result = self.resolver.resolve_ip('192.168.1.100') + + self.assertEqual(result, 'cached') + mock_hostname_class.assert_not_called() + + @patch('gprebalance_modules.rebalance_commons.Hostname') + def test_resolve_ip_invalid_ip(self, mock_hostname_class): + result = self.resolver.resolve_ip('not-an-ip') + + self.assertIsNone(result) + mock_hostname_class.assert_not_called() + + @patch('gprebalance_modules.rebalance_commons.Hostname') + def test_resolve_ip_command_failure(self, mock_hostname_class): + mock_cmd = MagicMock() + mock_cmd.was_successful.return_value = False + mock_hostname_class.return_value = mock_cmd + + result = self.resolver.resolve_ip('192.168.1.99') + + self.assertIsNone(result) + + @patch('gprebalance_modules.rebalance_commons.Hostname') + def test_resolve_ip_exception(self, mock_hostname_class): + mock_cmd = MagicMock() + mock_cmd.run.side_effect = Exception('Resolution failed') + mock_hostname_class.return_value = mock_cmd + + result = self.resolver.resolve_ip('192.168.1.99') + + self.assertIsNone(result) + + @patch('gprebalance_modules.rebalance_commons.Hostname') + def test_resolve_ip_ipv6(self, mock_hostname_class): + mock_cmd = MagicMock() + mock_cmd.was_successful.return_value = True + mock_cmd.get_hostname.return_value = 'testhost6' + mock_hostname_class.return_value = mock_cmd + + result = self.resolver.resolve_ip('2001:db8::1') + + self.assertEqual(result, 'testhost6') + self.assertEqual(self.resolver._ip_to_hostname['2001:db8::1'], 'testhost6') + + def test_hosts_match_identical_hostnames(self): + self.assertTrue(self.resolver.hosts_match('testhost', 'testhost')) + + def test_hosts_match_identical_ips(self): + self.assertTrue(self.resolver.hosts_match('192.168.1.10', '192.168.1.10')) + + def test_hosts_match_ipv6_normalization(self): + # IPv6 addresses with scope IDs should match + self.assertTrue(self.resolver.hosts_match('fe80::1%eth0', 'fe80::1%eth1')) + self.assertTrue(self.resolver.hosts_match('fe80::1%eth0', 'fe80::1')) + + def test_hosts_match_different_ips(self): + self.assertFalse(self.resolver.hosts_match('192.168.1.10', '192.168.1.11')) + + @patch('socket.getaddrinfo') + def test_hosts_match_ip_to_hostname(self, mock_getaddrinfo): + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0)) + ] + + # IP matches hostname's resolved IP + self.assertTrue(self.resolver.hosts_match('192.168.1.10', 'testhost')) + # Should work in reverse too + self.assertTrue(self.resolver.hosts_match('testhost', '192.168.1.10')) + + @patch('socket.getaddrinfo') + def test_hosts_match_ip_to_hostname_multiple_ips(self, mock_getaddrinfo): + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0)), + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.10', 0)) + ] + + # Should match any of the IPs + self.assertTrue(self.resolver.hosts_match('192.168.1.10', 'testhost')) + self.assertTrue(self.resolver.hosts_match('10.0.0.10', 'testhost')) + self.assertFalse(self.resolver.hosts_match('192.168.1.99', 'testhost')) + + @patch('socket.getaddrinfo') + def test_hosts_match_ip_to_hostname_no_match(self, mock_getaddrinfo): + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0)) + ] + + # IP doesn't match hostname's resolved IP + self.assertFalse(self.resolver.hosts_match('192.168.1.99', 'testhost')) + + @patch('socket.getaddrinfo') + def test_hosts_match_hostname_to_hostname_shared_ip(self, mock_getaddrinfo): + # Two hostnames sharing an IP address + def getaddrinfo_side_effect(hostname, *args): + if hostname == 'host1': + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0))] + elif hostname == 'host2': + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0))] + return [] + + mock_getaddrinfo.side_effect = getaddrinfo_side_effect + + # Should match because they share an IP + self.assertTrue(self.resolver.hosts_match('host1', 'host2')) + + @patch('socket.getaddrinfo') + def test_hosts_match_hostname_to_hostname_no_shared_ip(self, mock_getaddrinfo): + def getaddrinfo_side_effect(hostname, *args): + if hostname == 'host1': + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0))] + elif hostname == 'host2': + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.11', 0))] + return [] + + mock_getaddrinfo.side_effect = getaddrinfo_side_effect + + # Should not match - no shared IPs + self.assertFalse(self.resolver.hosts_match('host1', 'host2')) + + def test_hosts_match_different_hostnames_no_resolution(self): + # Without resolution data, different hostnames don't match + self.assertFalse(self.resolver.hosts_match('host1', 'host2')) + + @patch('socket.getaddrinfo') + def test_hosts_match_resolution_failure(self, mock_getaddrinfo): + mock_getaddrinfo.side_effect = socket.gaierror('Resolution failed') + + # Should return False when resolution fails + self.assertFalse(self.resolver.hosts_match('192.168.1.10', 'unknown')) + + def test_hosts_match_ipv6(self): + self.assertTrue(self.resolver.hosts_match('2001:db8::1', '2001:db8::1')) + self.assertFalse(self.resolver.hosts_match('2001:db8::1', '2001:db8::2')) + + def test_find_matching_hostname_exact_match(self): + existing_hosts = ['host1', 'host2', 'host3'] + + result = self.resolver.find_matching_hostname('host2', existing_hosts) + + self.assertEqual(result, 'host2') + + @patch('socket.getaddrinfo') + def test_find_matching_hostname_ip_match(self, mock_getaddrinfo): + def getaddrinfo_side_effect(hostname, *args): + if hostname == 'host1': + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0))] + elif hostname == 'host2': + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.11', 0))] + return [] + + mock_getaddrinfo.side_effect = getaddrinfo_side_effect + + existing_hosts = ['host1', 'host2'] + + result = self.resolver.find_matching_hostname('192.168.1.10', existing_hosts) + + # Should match host1 + self.assertEqual(result, 'host1') + + def test_find_matching_hostname_no_match(self): + existing_hosts = ['host1', 'host2'] + + result = self.resolver.find_matching_hostname('nonexistent', existing_hosts) + + self.assertIsNone(result) + + def test_find_matching_hostname_empty_list(self): + result = self.resolver.find_matching_hostname('testhost', []) + + self.assertIsNone(result) + + def test_get_all_addresses(self): + self.resolver._hostname_to_ips['multihost'] = {'192.168.1.10', '10.0.0.10'} + + result = self.resolver.get_all_addresses('multihost') + + self.assertEqual(result, {'192.168.1.10', '10.0.0.10'}) + + def test_get_all_addresses_empty(self): + result = self.resolver.get_all_addresses('unknown') + + self.assertEqual(result, set()) + + def test_get_address_returns_first_sorted(self): + self.resolver._hostname_to_ips['multihost'] = {'192.168.1.10', '10.0.0.10', 'fe80::1'} + + result = self.resolver.get_address('multihost') + + # Should return first when sorted + sorted_ips = sorted(['192.168.1.10', '10.0.0.10', 'fe80::1']) + self.assertEqual(result, sorted_ips[0]) + + def test_get_hostname(self): + self.resolver._ip_to_hostname['192.168.1.10'] = 'testhost' + + result = self.resolver.get_hostname('192.168.1.10') + + self.assertEqual(result, 'testhost') + + def test_get_hostname_not_found(self): + result = self.resolver.get_hostname('192.168.1.99') + + # Should return IP itself if not found + self.assertEqual(result, '192.168.1.99') + + @patch('socket.getaddrinfo') + def test_caching_integration(self, mock_getaddrinfo): + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0)) + ] + + # First call should hit the network + result1 = self.resolver.resolve_hostname('testhost') + self.assertEqual(result1, '192.168.1.10') + self.assertEqual(mock_getaddrinfo.call_count, 1) + + # Second call should use cache + result2 = self.resolver.resolve_hostname('testhost') + self.assertEqual(result2, '192.168.1.10') + self.assertEqual(mock_getaddrinfo.call_count, 1) # Still 1, not called again + + # Third call to get from cache directly + result3 = self.resolver.get_address('testhost') + self.assertEqual(result3, '192.168.1.10') + + # get_all_addresses should also work + all_ips = self.resolver.get_all_addresses('testhost') + self.assertEqual(all_ips, {'192.168.1.10'}) + + @patch('socket.getaddrinfo') + def test_multiple_hosts(self, mock_getaddrinfo): + def getaddrinfo_side_effect(hostname, *args): + mapping = { + 'host1': [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0))], + 'host2': [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.11', 0))], + 'host3': [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.12', 0))] + } + return mapping.get(hostname, []) + + mock_getaddrinfo.side_effect = getaddrinfo_side_effect + + existing_hosts = ['host1', 'host2', 'host3'] + + # Test various scenarios + self.assertEqual(self.resolver.find_matching_hostname('host1', existing_hosts), 'host1') + self.assertEqual(self.resolver.find_matching_hostname('192.168.1.11', existing_hosts), 'host2') + self.assertIsNone(self.resolver.find_matching_hostname('192.168.1.99', existing_hosts)) + + # Test hosts_match + self.assertTrue(self.resolver.hosts_match('host1', '192.168.1.10')) + self.assertFalse(self.resolver.hosts_match('host1', '192.168.1.11')) + + # Test get_all_addresses + self.assertEqual(self.resolver.get_all_addresses('host2'), {'192.168.1.11'}) + + +class TestTemplateParser(GpTestCase): + """Test suite for TemplateParser""" + + def test_parse_basic_input(self): + """Test basic comma-separated input""" + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/primary, /data/mirror" + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_quoted_input_double_quotes(self): + """Test input with double quotes""" + primary, mirror = TemplateParser.parse_datadirs_input( + '"/data/primary", "/data/mirror"' + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_quoted_input_single_quotes(self): + """Test input with single quotes""" + primary, mirror = TemplateParser.parse_datadirs_input( + "'/data/primary', '/data/mirror'" + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_with_content_placeholder(self): + """Test input that already has {content}""" + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/primary/gpseg{content}, /data/mirror/gpseg{content}" + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_with_hostname_placeholder(self): + """Test input with {hostname} placeholder""" + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/{hostname}/primary, /data/{hostname}/mirror" + ) + self.assertEqual(primary, "/data/{hostname}/primary/gpseg{content}") + self.assertEqual(mirror, "/data/{hostname}/mirror/gpseg{content}") + + def test_parse_with_both_placeholders(self): + """Test input with both {hostname} and {content} placeholders""" + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/{hostname}/primary/gpseg{content}, /data/{hostname}/mirror/gpseg{content}" + ) + self.assertEqual(primary, "/data/{hostname}/primary/gpseg{content}") + self.assertEqual(mirror, "/data/{hostname}/mirror/gpseg{content}") + + def test_parse_with_trailing_slashes(self): + """Test input with trailing slashes""" + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/primary/, /data/mirror/" + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_with_extra_whitespace(self): + """Test input with extra whitespace""" + primary, mirror = TemplateParser.parse_datadirs_input( + " /data/primary , /data/mirror " + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_identical_templates_error(self): + """Test that identical templates raise error""" + with self.assertRaisesRegex(ValidationError, "cannot be identical"): + TemplateParser.parse_datadirs_input( + "/data/primary, /data/primary" + ) + + def test_identical_after_normalization_error(self): + """Test that templates identical after normalization raise error""" + with self.assertRaisesRegex(ValidationError, "cannot be identical"): + TemplateParser.parse_datadirs_input( + "/data/segments, /data/segments/" + ) + + def test_non_absolute_path_error(self): + """Test that relative paths raise error""" + with self.assertRaisesRegex(ValidationError, "must be absolute"): + TemplateParser.parse_datadirs_input( + "data/primary, data/mirror" + ) + + def test_non_absolute_second_path_error(self): + """Test that relative path in second position raises error""" + with self.assertRaisesRegex(ValidationError, "must be absolute"): + TemplateParser.parse_datadirs_input( + "/data/primary, data/mirror" + ) + + def test_empty_path_error(self): + """Test that empty paths raise error""" + with self.assertRaisesRegex(ValidationError, "cannot be empty"): + TemplateParser.parse_datadirs_input( + '"", /data/mirror' + ) + + def test_empty_after_quotes_error(self): + """Test that empty path after quote removal raises error""" + with self.assertRaisesRegex(ValidationError, "cannot be empty"): + TemplateParser.parse_datadirs_input( + ' "" , /data/mirror' + ) + + def test_invalid_placeholder_error(self): + """Test that invalid placeholders raise error""" + with self.assertRaisesRegex(ValidationError, "Invalid placeholder"): + TemplateParser.parse_datadirs_input( + "/data/{invalid}/primary, /data/mirror" + ) + + def test_multiple_invalid_placeholders_error(self): + """Test that multiple invalid placeholders are caught""" + with self.assertRaisesRegex(ValidationError, "Invalid placeholder"): + TemplateParser.parse_datadirs_input( + "/data/{foo}/{bar}/primary, /data/mirror" + ) + + def test_wrong_format_missing_second_dir_error(self): + """Test that missing second directory raises error""" + with self.assertRaisesRegex(ValidationError, "should have format"): + TemplateParser.parse_datadirs_input( + "/data/primary" + ) + + def test_wrong_format_too_many_dirs_error(self): + """Test that too many directories raise error""" + with self.assertRaisesRegex(ValidationError, "should have format"): + TemplateParser.parse_datadirs_input( + "/data/primary, /data/mirror, /data/extra" + ) + + def test_instantiate_template_both_placeholders(self): + """Test template instantiation with both placeholders""" + template = "/data/{hostname}/gpseg{content}" + result = TemplateParser.instantiate_template( + template, hostname="sdw1", content=0 + ) + self.assertEqual(result, "/data/sdw1/gpseg0") + + def test_instantiate_template_content_only(self): + """Test template instantiation with content only""" + template = "/data/primary/gpseg{content}" + result = TemplateParser.instantiate_template( + template, content=5 + ) + self.assertEqual(result, "/data/primary/gpseg5") + + def test_instantiate_template_hostname_only(self): + """Test template instantiation with hostname only""" + template = "/data/{hostname}/segments/gpseg{content}" + result = TemplateParser.instantiate_template( + template, hostname="sdw2" + ) + self.assertEqual(result, "/data/sdw2/segments/gpseg{content}") + + def test_instantiate_template_no_substitution(self): + """Test template instantiation with no substitution""" + template = "/data/{hostname}/gpseg{content}" + result = TemplateParser.instantiate_template(template) + self.assertEqual(result, "/data/{hostname}/gpseg{content}") + + def test_instantiate_template_large_content_id(self): + """Test template instantiation with large content ID""" + template = "/data/primary/gpseg{content}" + result = TemplateParser.instantiate_template( + template, content=12345 + ) + self.assertEqual(result, "/data/primary/gpseg12345") + + def test_extract_parent_directory_basic(self): + """Test parent directory extraction""" + result = TemplateParser.extract_parent_directory( + "/data/primary/gpseg0" + ) + self.assertEqual(result, "/data/primary") + + def test_extract_parent_directory_with_trailing_slash(self): + """Test parent directory extraction with trailing slash""" + result = TemplateParser.extract_parent_directory( + "/data/primary/gpseg0/" + ) + self.assertEqual(result, "/data/primary") + + def test_extract_parent_directory_deep_path(self): + """Test parent directory extraction with deep path""" + result = TemplateParser.extract_parent_directory( + "/mnt/disk1/data/segments/primary/gpseg0" + ) + self.assertEqual(result, "/mnt/disk1/data/segments/primary") + + def test_extract_parent_directory_custom_naming(self): + """Test parent directory extraction with custom segment naming""" + result = TemplateParser.extract_parent_directory( + "/data/primary/seg_data_0" + ) + self.assertEqual(result, "/data/primary") + + def test_parse_comma_in_double_quotes(self): + """Test paths with commas inside double quotes""" + primary, mirror = TemplateParser.parse_datadirs_input( + '"/di,r1/primary/", "/dir2/primary"' + ) + self.assertEqual(primary, "/di,r1/primary/gpseg{content}") + self.assertEqual(mirror, "/dir2/primary/gpseg{content}") + + def test_parse_comma_in_single_quotes(self): + """Test paths with commas inside single quotes""" + primary, mirror = TemplateParser.parse_datadirs_input( + "'/di,r1/primary/', '/dir2/primary'" + ) + self.assertEqual(primary, "/di,r1/primary/gpseg{content}") + self.assertEqual(mirror, "/dir2/primary/gpseg{content}") + + def test_parse_multiple_commas_in_quotes(self): + """Test paths with multiple commas inside quotes""" + primary, mirror = TemplateParser.parse_datadirs_input( + '"/path,with,many,commas/primary", "/simple/mirror"' + ) + self.assertEqual(primary, "/path,with,many,commas/primary/gpseg{content}") + self.assertEqual(mirror, "/simple/mirror/gpseg{content}") + +class TestTemplateParserFile(GpTestCase): + """Test suite for TemplateParser file parsing""" + + def setUp(self): + """Create temporary directory for test files""" + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + """Clean up temporary directory""" + import shutil + shutil.rmtree(self.temp_dir) + + def _create_test_file(self, filename, content): + """Helper to create test file""" + filepath = os.path.join(self.temp_dir, filename) + with open(filepath, 'w') as f: + f.write(content) + return filepath + + def test_parse_file_basic(self): + """Test parsing from file with basic paths""" + filepath = self._create_test_file( + "datadirs.txt", + "/data/primary\n/data/mirror\n" + ) + + primary, mirror = TemplateParser.parse_datadirs_file(filepath) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_file_with_quotes(self): + """Test parsing file with quoted paths""" + filepath = self._create_test_file( + "datadirs.txt", + '"/data/primary"\n"/data/mirror"\n' + ) + + primary, mirror = TemplateParser.parse_datadirs_file(filepath) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_file_with_blank_lines(self): + """Test parsing file with blank lines""" + filepath = self._create_test_file( + "datadirs.txt", + "\n/data/primary\n\n/data/mirror\n\n" + ) + + primary, mirror = TemplateParser.parse_datadirs_file(filepath) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_file_with_placeholders(self): + """Test parsing file with template placeholders""" + filepath = self._create_test_file( + "datadirs.txt", + "/data/{hostname}/primary/gpseg{content}\n" + "/data/{hostname}/mirror/gpseg{content}\n" + ) + + primary, mirror = TemplateParser.parse_datadirs_file(filepath) + self.assertEqual(primary, "/data/{hostname}/primary/gpseg{content}") + self.assertEqual(mirror, "/data/{hostname}/mirror/gpseg{content}") + + def test_parse_file_with_whitespace(self): + """Test parsing file with extra whitespace""" + filepath = self._create_test_file( + "datadirs.txt", + " /data/primary \n /data/mirror \n" + ) + + primary, mirror = TemplateParser.parse_datadirs_file(filepath) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_file_wrong_line_count_one_line(self): + """Test file with only one line""" + filepath = self._create_test_file( + "datadirs.txt", + "/data/primary\n" + ) + + with self.assertRaisesRegex(ValidationError, "exactly 2 lines"): + TemplateParser.parse_datadirs_file(filepath) + + def test_parse_file_wrong_line_count_three_lines(self): + """Test file with three lines""" + filepath = self._create_test_file( + "datadirs.txt", + "/data/primary\n/data/mirror\n/data/extra\n" + ) + + with self.assertRaisesRegex(ValidationError, "exactly 2 lines"): + TemplateParser.parse_datadirs_file(filepath) + + def test_parse_file_empty_file(self): + """Test empty file""" + filepath = self._create_test_file("datadirs.txt", "") + + with self.assertRaisesRegex(ValidationError, "exactly 2 lines"): + TemplateParser.parse_datadirs_file(filepath) + + def test_parse_file_only_blank_lines(self): + """Test file with only blank lines""" + filepath = self._create_test_file( + "datadirs.txt", + "\n\n\n" + ) + + with self.assertRaisesRegex(ValidationError, "exactly 2 lines"): + TemplateParser.parse_datadirs_file(filepath) + + def test_parse_file_not_exists(self): + """Test non-existent file""" + with self.assertRaises(FileNotFoundError): + TemplateParser.parse_datadirs_file("/nonexistent/file.txt") + + def test_parse_file_identical_paths(self): + """Test file with identical paths""" + filepath = self._create_test_file( + "datadirs.txt", + "/data/segments\n/data/segments\n" + ) + + with self.assertRaisesRegex(ValidationError, "cannot be identical"): + TemplateParser.parse_datadirs_file(filepath) + + def test_parse_file_invalid_placeholder(self): + """Test file with invalid placeholder""" + filepath = self._create_test_file( + "datadirs.txt", + "/data/{invalid}/primary\n/data/mirror\n" + ) + + with self.assertRaisesRegex(ValidationError, "Invalid placeholder"): + TemplateParser.parse_datadirs_file(filepath) + + def test_parse_file_non_absolute_path(self): + """Test file with relative path""" + filepath = self._create_test_file( + "datadirs.txt", + "data/primary\n/data/mirror\n" + ) + + with self.assertRaisesRegex(ValidationError, "must be absolute"): + TemplateParser.parse_datadirs_file(filepath) + +class TestTemplateParserEdgeCases(unittest.TestCase): + """Test edge cases and corner scenarios""" + + def test_parse_windows_style_path_error(self): + """Test that Windows-style paths are rejected""" + with self.assertRaisesRegex(ValidationError, "must be absolute"): + TemplateParser.parse_datadirs_input( + "C:\\data\\primary, C:\\data\\mirror" + ) + + def test_parse_mixed_quote_types(self): + """Test mixed quote types (one double, one single)""" + primary, mirror = TemplateParser.parse_datadirs_input( + '"/data/primary", \'/data/mirror\'' + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_only_first_quoted(self): + """Test only first path quoted""" + primary, mirror = TemplateParser.parse_datadirs_input( + '"/data/primary", /data/mirror' + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_special_characters_in_path(self): + """Test paths with special characters""" + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/primary-data_01, /data/mirror-data_01" + ) + self.assertEqual(primary, "/data/primary-data_01/gpseg{content}") + self.assertEqual(mirror, "/data/mirror-data_01/gpseg{content}") + + def test_parse_very_long_path(self): + """Test very long paths""" + long_path = "/data/" + "a" * 200 + primary, mirror = TemplateParser.parse_datadirs_input( + f"{long_path}/primary, {long_path}/mirror" + ) + self.assertEqual(primary, f"{long_path}/primary/gpseg{{content}}") + self.assertEqual(mirror, f"{long_path}/mirror/gpseg{{content}}") + + def test_instantiate_template_zero_content(self): + """Test instantiation with content=0""" + template = "/data/primary/gpseg{content}" + result = TemplateParser.instantiate_template(template, content=0) + self.assertEqual(result, "/data/primary/gpseg0") + + def test_instantiate_multiple_hostname_placeholders(self): + """Test instantiation with multiple hostname placeholders""" + template = "/data/{hostname}/primary/{hostname}/gpseg{content}" + result = TemplateParser.instantiate_template( + template, hostname="sdw1", content=0 + ) + self.assertEqual(result, "/data/sdw1/primary/sdw1/gpseg0") + + def test_collision_same_parent_different_suffix(self): + """Test that templates with same parent but different suffixes are valid""" + # This should NOT raise an error - they resolve to different paths + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/gpseg{content}, /data/gpmirror{content}" + ) + self.assertEqual(primary, "/data/gpseg{content}") + self.assertEqual(mirror, "/data/gpmirror{content}") + +if __name__ == '__main__': + run_tests() diff --git a/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py b/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py index 275b2ca0de4f..878a7fcdd358 100644 --- a/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py +++ b/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py @@ -1,6 +1,7 @@ import os import imp import sys +import socket from gppylib.test.unit.gp_unittest import * from mock import * @@ -8,7 +9,10 @@ from gppylib.gplog import * from gppylib.system.configurationInterface import GpConfigurationProvider from gppylib.system.environment import GpCoordinatorEnvironment -from .config import initGparrayFromFile + +from gprebalance_modules.test.config import initGparrayFromFile +from gprebalance_modules.planner import Planner +from gprebalance_modules.rebalance_commons import Host, HostStatus, ValidationError def rebalance_only(numsegs): def inner(func): @@ -25,8 +29,7 @@ def check_query(conn, query): return [0] return None -#Test options -class GpTestRebalanceValidation(GpTestCase): +class TestRebalanceUtilityCLI(GpTestCase): def setUp(self): gprebalance_file = os.path.abspath( os.path.dirname(__file__) + "/../../ggrebalance") @@ -69,7 +72,7 @@ def setUp(self): def tearDown(self): os.environ['COORDINATOR_DATA_DIRECTORY'] = self.previous_coordinator_data_directory sys.argv = self.old_sys_argv - super(GpTestRebalanceValidation, self).tearDown() + super(TestRebalanceUtilityCLI, self).tearDown() @patch('ggrebalance.GpArray.initFromCatalog', return_value=initGparrayFromFile("balanced_grouped_6")) @@ -100,12 +103,13 @@ def test_segment_down(self, mockCatalog, mockOsPath, mockCursor): @rebalance_only(numsegs = 24) def test_invalid_target_datadir(self, mockCatalog, mockOsPath, mockCursor): self.options.target_hosts = "sdw1, sdw2, sdw3" - self.options.target_datadirs = '/data/primary/gpseg{content}, /data/mirror/gpseg' + self.options.target_datadirs = '/data/primary/gpseg{content}' with self.assertRaises(SystemExit): self.subject.main(self.options, self.args, self.parser) self.subject.logger.error.assert_any_call( - 'ggrebalance failed: --target_datadirs options should have format like ' - '"/data/primary/gpseg{content} , /data/mirror/gpseg{content}" \n\nExiting...') + 'ggrebalance failed: --target-datadirs should have format: ' + '"/data/primary/gpseg{content}, /data/mirror/gpseg{content}". ' + 'Available templated parameters: {hostname}, {content} \n\nExiting...') @patch('ggrebalance.GpArray.initFromCatalog', return_value=initGparrayFromFile("role_mismatch")) @@ -117,7 +121,142 @@ def test_role_mistmatch(self, mockCatalog, mockOsPath, mockCursor): self.subject.main(self.options, self.args, self.parser) self.subject.logger.error.assert_any_call( "ggrebalance failed: Current role does not match preferred role for several segments. \n\nExiting...") + + +class TestHostsOptions(GpTestCase): + + def setUp(self): + self.options = Mock() + self.options.target_hosts = None + self.options.add_hosts = None + self.options.remove_hosts = None + self.options.target_datadirs = None + self.options.target_hosts_file = None + self.options.add_hosts_file = None + self.options.remove_hosts_file = None + self.options.target_datadirs_file = None + + def test_target_hosts_invalid_chars(self): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.target_hosts = "sdw1, sdw2, asfsa@" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("contains invalid characters", str(ctx.exception)) + + def test_target_hosts_duplicates(self): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.target_hosts = "sdw1, sdw2, sdw2" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("Duplicate host", str(ctx.exception)) + + def test_target_hosts_ip_and_name(self): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.target_hosts = "sdw1, sdw2, 192.168.0.15" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("must not contain IP adress and hostname simultaniously", str(ctx.exception)) + + @patch('gprebalance_modules.rebalance_commons.HostResolver.find_matching_hostname', return_value='sdw1') + def test_add_hosts_existing(self, MockResolver): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.add_hosts = "sdw1" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("--add-hosts: Host 'sdw1' already exists in cluster as 'sdw1'", str(ctx.exception)) + + @patch('gprebalance_modules.planner.HostResolver.resolve_ip', return_value='sdw1') + @patch('gprebalance_modules.rebalance_commons.HostResolver.find_matching_hostname', return_value='sdw1') + def test_add_hosts_existing_ip(self, MockResolver, mock_ip): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.add_hosts = "172.20.0.6" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("--add-hosts: Host '172.20.0.6' already exists in cluster as 'sdw1'", str(ctx.exception)) + + @patch('gprebalance_modules.rebalance_commons.HostResolver.find_matching_hostname', return_value=None) + def test_remove_hosts_unexisting(self, MockResolver): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.remove_hosts = "sdw3" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("--remove-hosts: Host 'sdw3' does not exist in cluster", str(ctx.exception)) + + @patch('gprebalance_modules.planner.HostResolver.resolve_ip', return_value='sdw3') + @patch('gprebalance_modules.rebalance_commons.HostResolver.find_matching_hostname', return_value=None) + def test_remove_hosts_unexisting_ip(self, MockResolver, mock_ip): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.remove_hosts = "172.20.0.9" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("--remove-hosts: Host '172.20.0.9' does not exist in cluster", str(ctx.exception)) + + @patch('socket.getaddrinfo') + def test_target_hosts_positive(self, mock_getaddrinfo): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.target_hosts = "sdw1, sdw2" + + mock_getaddrinfo.side_effect = [ + [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0))], + [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.11', 0))] + ] + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertTrue(planner.target_hosts == [Host('sdw1', '192.168.1.10'), Host('sdw2', '192.168.1.11')]) + + @patch('socket.getaddrinfo') + def test_target_hosts_positive_ip(self, mock_getaddrinfo): + gparray = initGparrayFromFile('unbalanced_9_ip') + self.options.target_hosts = "sdw1, sdw2" + mock_getaddrinfo.side_effect = [ + [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('172.20.0.6', 0))], + [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('172.20.0.7', 0))], + [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('172.20.0.8', 0))] + ] + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertTrue(planner.target_hosts == [Host('sdw1', '172.20.0.6'), Host('sdw3', '172.20.0.8'), Host('sdw2', '172.20.0.7')]) + self.assertTrue(planner.target_hosts[1].status == HostStatus.DECOMMISSIONED) + @patch('gprebalance_modules.planner.HostResolver') + def test_target_hosts_ip(self, mockHostResolver): + gparray = initGparrayFromFile('unbalanced_9_ip') + self.options.target_hosts = "172.20.0.7, 172.20.0.8, 172.20.0.9" + mockHostResolver.return_value.resolve_hostname.side_effect = ['172.20.0.6', '172.20.0.7', '172.20.0.8', '172.20.0.9'] + mockHostResolver.return_value.resolve_ip.side_effect = ['sdw2', 'sdw3', 'sdw4'] + mockHostResolver.return_value.get_address.side_effect = ['172.20.0.6', '172.20.0.7', '172.20.0.8', '172.20.0.9'] + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertEqual(len(planner.target_hosts), 4) + self.assertTrue(planner.target_hosts[3].status == HostStatus.NEW) + self.assertTrue(planner.target_hosts[0].status == HostStatus.DECOMMISSIONED) if __name__ == '__main__': run_tests() diff --git a/gpMgmt/bin/gprebalance_modules/test/test_unit_resources.py b/gpMgmt/bin/gprebalance_modules/test/test_unit_resources.py new file mode 100644 index 000000000000..f4a340b1d51e --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/test_unit_resources.py @@ -0,0 +1,1675 @@ +from gppylib.test.unit.gp_unittest import * +from mock import * + +from gprebalance_modules.planner import ResourceEstimator, ResourceError, LogicalMove, Planner, PortAllocator, PlanningError +from gppylib.db.dbconn import DbURL +from gppylib import gparray +from gprebalance_modules.test.config import initGparrayFromFile +from gprebalance_modules.rebalance_commons import ( + SegmentSize, + DiskSpaceChecker, + DiskSpaceInfo, + Host, + HostStatus +) + +def check_query(conn, query): + if "SELECT COUNT(1) FROM pg_namespace WHERE nspname =" in query: + return [0] + return None + +class TestDiskSpaceChecker(GpTestCase): + """Test cases for DiskSpaceChecker""" + + def setUp(self): + self.logger = Mock() + self.checker = DiskSpaceChecker(self.logger, batch_size=4) + + @patch('gprebalance_modules.rebalance_commons.WorkerPool') + @patch('gprebalance_modules.rebalance_commons.DiskUsage') + def test_get_disk_usage_success(self, mock_disk_usage_class, mock_pool_class): + """Test successful disk usage retrieval""" + # Setup mock pool + mock_pool = Mock() + mock_pool_class.return_value = mock_pool + + # Create mock commands + cmd1 = Mock() + cmd1.was_successful.return_value = True + cmd1.directory = '/data1/primary/gpseg0' + cmd1.kbytes_used.return_value = 1048576 + + cmd2 = Mock() + cmd2.was_successful.return_value = True + cmd2.directory = '/data1/primary/gpseg1' + cmd2.kbytes_used.return_value = 2097152 + + mock_pool.getCompletedItems.return_value = [cmd1, cmd2] + + # Execute + directories = ['/data1/primary/gpseg0', '/data1/primary/gpseg1'] + result = self.checker.get_disk_usage('sdw1', directories) + + # Verify results + self.assertEqual(len(result), 2) + self.assertEqual(result['/data1/primary/gpseg0'], 1048576) + self.assertEqual(result['/data1/primary/gpseg1'], 2097152) + + # Verify pool usage + self.assertEqual(mock_pool.addCommand.call_count, 2) + mock_pool.join.assert_called_once() + mock_pool.haltWork.assert_called_once() + mock_pool.joinWorkers.assert_called_once() + + @patch('gprebalance_modules.rebalance_commons.WorkerPool') + def test_get_disk_usage_empty_list(self, mock_pool_class): + """Test disk usage with empty directory list""" + result = self.checker.get_disk_usage('sdw1', []) + + self.assertEqual(result, {}) + mock_pool_class.assert_not_called() + + @patch('gprebalance_modules.rebalance_commons.WorkerPool') + @patch('gprebalance_modules.rebalance_commons.DiskUsage') + def test_get_disk_usage_command_failure(self, mock_disk_usage_class, mock_pool_class): + """Test disk usage command failure""" + mock_pool = Mock() + mock_pool_class.return_value = mock_pool + + cmd = Mock() + cmd.was_successful.return_value = False + cmd.get_results.return_value.stderr = "Permission denied" + + mock_pool.getCompletedItems.return_value = [cmd] + + with self.assertRaises(Exception) as context: + self.checker.get_disk_usage('sdw1', ['/data1/primary/gpseg0']) + + self.assertIn("Unable to check disk usage", str(context.exception)) + + @patch('gprebalance_modules.rebalance_commons.WorkerPool') + @patch('gprebalance_modules.rebalance_commons.DiskFree') + @patch('gprebalance_modules.rebalance_commons.pickle') + @patch('gprebalance_modules.rebalance_commons.base64') + def test_get_available_space_success(self, mock_base64, mock_pickle, + mock_disk_free_class, mock_pool_class): + """Test successful available space retrieval""" + from gppylib.operations.validate_disk_space import FileSystem + + # Setup mock pool + mock_pool = Mock() + mock_pool_class.return_value = mock_pool + + # Mock FileSystem objects + fs1 = Mock(spec=FileSystem) + fs1.name = '/dev/sdb1' + fs1.disk_free = 10485760 + fs1.directories = ['/data1/primary/gpseg0'] + + fs2 = Mock(spec=FileSystem) + fs2.name = '/dev/sdb1' + fs2.disk_free = 10485760 + fs2.directories = ['/data1/primary/gpseg1'] + + mock_pickle.loads.return_value = [fs1, fs2] + mock_base64.urlsafe_b64decode.return_value = b'pickled_data' + + cmd = Mock() + cmd.was_successful.return_value = True + cmd.get_results.return_value.stdout = 'encoded_data' + + mock_pool.getCompletedItems.return_value = [cmd] + + # Execute + directories = ['/data1/primary/gpseg0', '/data1/primary/gpseg1'] + result = self.checker.get_available_space('sdw1', directories) + + # Verify + self.assertEqual(len(result), 2) + self.assertIn('/data1/primary/gpseg0', result) + self.assertIn('/data1/primary/gpseg1', result) + + self.assertEqual(result['/data1/primary/gpseg0'].filesystem, '/dev/sdb1') + self.assertEqual(result['/data1/primary/gpseg0'].available_kb, 10485760) + self.assertEqual(result['/data1/primary/gpseg0'].available_gb, 10.0) + + @patch('gprebalance_modules.rebalance_commons.DiskFree') + def test_get_available_space_command_failure(self, mock_disk_free_class): + """Test available space command failure""" + + cmd = Mock() + mock_disk_free_class.return_value = cmd + cmd.was_successful.return_value = False + cmd.get_results.return_value.stderr = "No such file or directory" + + with self.assertRaises(Exception) as context: + self.checker.get_available_space('sdw1', ['/data1/primary/gpseg0']) + + self.assertIn("Failed to check disk free", str(context.exception)) + +class TestResourceEstimator(GpTestCase): + """Test cases for ResourceEstimator using real GpArray configuration""" + + def setUp(self): + """Set up test fixtures""" + self.logger = Mock() + self.logger.info = Mock() + self.logger.debug = Mock() + self.logger.warning = Mock() + self.logger.error = Mock() + + self.conn = Mock() + self.dburl = Mock(spec=DbURL) + + # Load real GpArray configuration + self.gparray = initGparrayFromFile('unbalanced_9_ip') + + # Options for planner + self.options = Mock() + self.options.target_segment_count = 9 + self.options.target_hosts = None + self.options.add_hosts = None + self.options.remove_hosts = None + self.options.target_datadirs = None + self.options.target_hosts_file = None + self.options.add_hosts_file = None + self.options.remove_hosts_file = None + self.options.target_datadirs_file = None + self.options.mirror_mode = 'grouped' + self.options.skip_rebalance = False + self.options.skip_resource_estimation = False + self.options.batch_size = 16 + + @patch('gprebalance_modules.planner.dbconn') + def test_estimate_segment_sizes_from_unbalanced_cluster(self, mock_dbconn): + """Test segment size estimation from unbalanced cluster""" + # Create a realistic move: move primary seg0 from sdw1 to sdw2 + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + self.assertIsNotNone(seg0) + self.assertEqual(seg0.hostname, 'sdw1') + self.assertEqual(seg0.address, '172.20.0.6') + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=None + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock disk checker to return segment size + estimator.disk_checker.get_disk_usage = Mock(return_value={ + '/data/primary0': 2097152 # 2GB + }) + + # Mock tablespace query (no tablespaces) + mock_dbconn.query.return_value = [] + + estimator._estimate_segment_sizes(moves) + + # Verify segment size was set + self.assertIsNotNone(moves[0].segment_size) + self.assertEqual(moves[0].segment_size.datadir_size_kb, 2097152) + self.assertEqual(moves[0].segment_size.total_size_kb, 2097152) + + # Verify disk usage was called with correct parameters + estimator.disk_checker.get_disk_usage.assert_called_once_with( + '172.20.0.6', + ['/data/primary0'] + ) + + @patch('gprebalance_modules.planner.dbconn') + def test_estimate_multiple_segments_same_host(self, mock_dbconn): + """ + Test estimating multiple segments from same source host + """ + # Get primaries from sdw1: seg0, seg1, seg2 + segs_from_sdw1 = [] + for seg in self.gparray.getSegDbList(): + if seg.getSegmentHostName() == 'sdw1' and seg.isSegmentPrimary(): + segs_from_sdw1.append(seg) + + self.assertEqual(len(segs_from_sdw1), 3) + + # Create moves for all three segments + moves = [ + LogicalMove( + seg=segs_from_sdw1[0], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=None + ), + LogicalMove( + seg=segs_from_sdw1[1], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary1', + target_port=7001, + segment_size=None + ), + LogicalMove( + seg=segs_from_sdw1[2], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary2', + target_port=7002, + segment_size=None + ) + ] + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock disk checker to return sizes for all segments + estimator.disk_checker.get_disk_usage = Mock(return_value={ + '/data/primary0': 1048576, # 1GB + '/data/primary1': 2097152, # 2GB + '/data/primary2': 1572864 # 1.5GB + }) + + # Mock tablespace query + mock_dbconn.query.return_value = [] + + estimator._estimate_segment_sizes(moves) + + # Verify all segment sizes were set + self.assertEqual(moves[0].segment_size.datadir_size_kb, 1048576) + self.assertEqual(moves[1].segment_size.datadir_size_kb, 2097152) + self.assertEqual(moves[2].segment_size.datadir_size_kb, 1572864) + + # Verify single call to get_disk_usage with all directories + estimator.disk_checker.get_disk_usage.assert_called_once() + call_args = estimator.disk_checker.get_disk_usage.call_args[0] + self.assertEqual(call_args[0], '172.20.0.6') + self.assertEqual(set(call_args[1]), {'/data/primary0', '/data/primary1', '/data/primary2'}) + + @patch('gprebalance_modules.planner.dbconn') + def test_estimate_with_tablespaces(self, mock_dbconn): + # Get primary seg0 from sdw1 + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=None + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock disk checker for datadir and tablespaces + call_count = [0] + def mock_disk_usage_side_effect(host, dirs): + call_count[0] += 1 + if call_count[0] == 1: + # First call: datadir + return {'/data/primary0': 2097152} # 2GB + else: + # Second call: tablespaces + return { + '/tablespace1/2': 524288, # 512MB + '/tablespace2/2': 1048576 # 1GB + } + + estimator.disk_checker.get_disk_usage = Mock(side_effect=mock_disk_usage_side_effect) + + # Mock tablespace query to return tablespace locations + mock_dbconn.query.return_value = [ + (2, '/tablespace1/2'), + (2, '/tablespace2/2') + ] + + estimator._estimate_segment_sizes(moves) + + # Verify segment size includes tablespaces + self.assertIsNotNone(moves[0].segment_size) + self.assertEqual(moves[0].segment_size.datadir_size_kb, 2097152) + self.assertIsNotNone(moves[0].segment_size.tablespace_usage) + self.assertEqual(moves[0].segment_size.tablespace_usage['/tablespace1/2'], 524288) + self.assertEqual(moves[0].segment_size.tablespace_usage['/tablespace2/2'], 1048576) + + # Total should be datadir + tablespaces + expected_total = 2097152 + 524288 + 1048576 + self.assertEqual(moves[0].segment_size.total_size_kb, expected_total) + + def test_validate_target_space_sufficient(self): + """Test validation passes when sufficient space is available""" + # Get primary seg0 + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize(datadir_size_kb=2097152) # 2GB + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock disk checker - 20GB available + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=20971520, # 20GB + directory='/data/primary0' + ) + } + }) + + # Should not raise exception + try: + estimator._validate_target_space(moves) + except ResourceError: + self.fail("ResourceError raised when space is sufficient") + + # Verify disk space check was called + estimator.disk_checker.check_batch_available_space.assert_called_once() + call_args = estimator.disk_checker.check_batch_available_space.call_args[0][0] + self.assertIn('172.20.0.7', call_args) + self.assertIn('/data/primary0', call_args['172.20.0.7']) + + def test_validate_target_space_insufficient(self): + """Test validation fails when insufficient space for datadir""" + # Get primary seg0 + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize(datadir_size_kb=10485760) # 10GB + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock disk checker - only 2GB available + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=2097152, # 2GB available + directory='/data/primary0' + ) + } + }) + + # Should raise ResourceError + with self.assertRaises(ResourceError) as context: + estimator._validate_target_space(moves) + + error_msg = str(context.exception) + self.assertIn("Insufficient disk space", error_msg) + self.assertIn("sdw2", error_msg) + self.assertIn("/data/primary0", error_msg) + + def test_validate_multiple_moves_same_filesystem_insufficient(self): + """Test validation aggregates space requirements on same filesystem""" + # Get primaries from sdw1 + segs_from_sdw1 = [] + for seg in self.gparray.getDbList(): + if seg.hostname == 'sdw1' and seg.isSegmentPrimary() and seg.content < 2: + segs_from_sdw1.append(seg) + + moves = [ + LogicalMove( + seg=segs_from_sdw1[0], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize(datadir_size_kb=5242880) # 5GB + ), + LogicalMove( + seg=segs_from_sdw1[1], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary1', + target_port=7001, + segment_size=SegmentSize(datadir_size_kb=5242880) # 5GB + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Total needed: (5GB + 5GB) * 1.1 = 11GB + # Available: only 8GB (insufficient) + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=8388608, # 8GB + directory='/data/primary0' + ), + '/data/primary1': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem + available_kb=8388608, # 8GB (same) + directory='/data/primary1' + ) + } + }) + + with self.assertRaises(ResourceError) as context: + estimator._validate_target_space(moves) + + error_msg = str(context.exception) + self.assertIn("Insufficient disk space", error_msg) + # Both directories should be mentioned + self.assertIn("/data/primary0", error_msg) + self.assertIn("/data/primary1", error_msg) + # Should show aggregated requirement (~11GB) vs available (8GB) + self.assertIn("11", error_msg) + self.assertIn("8", error_msg) + + def test_validate_multiple_moves_same_filesystem_sufficient(self): + """Test validation passes when aggregated space is sufficient""" + # Get primaries from sdw1 + segs_from_sdw1 = [] + for seg in self.gparray.getDbList(): + if seg.hostname == 'sdw1' and seg.isSegmentPrimary() and seg.content < 2: + segs_from_sdw1.append(seg) + + moves = [ + LogicalMove( + seg=segs_from_sdw1[0], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize(datadir_size_kb=5242880) # 5GB + ), + LogicalMove( + seg=segs_from_sdw1[1], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary1', + target_port=7001, + segment_size=SegmentSize(datadir_size_kb=5242880) # 5GB + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Total needed: (5GB + 5GB) * 1.1 = 11GB + # Available: 15GB (sufficient) + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=15728640, # 15GB + directory='/data/primary0' + ), + '/data/primary1': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem + available_kb=15728640, # 15GB (same) + directory='/data/primary1' + ) + } + }) + + # Should not raise + try: + estimator._validate_target_space(moves) + except ResourceError: + self.fail("ResourceError raised when space is sufficient") + + def test_validate_target_space_no_space_info(self): + """Test validation fails when no space info is available""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize(datadir_size_kb=2097152) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Return empty space info + estimator.disk_checker.check_batch_available_space = Mock(return_value={}) + + with self.assertRaises(ResourceError) as context: + estimator._validate_target_space(moves) + + self.assertIn("No disk space information for host sdw2", str(context.exception)) + + def test_validate_tablespace_space_sufficient(self): + """Test validation succeeds when tablespace has sufficient space on separate filesystem""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=2097152, # 2GB + tablespace_usage={'/tablespace1/2': 1048576} # 1GB + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock sufficient space for both datadir and tablespace on different filesystems + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=20971520, # 20GB + directory='/data/primary0' + ), + '/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdc1', # Different filesystem + available_kb=10485760, # 10GB + directory='/tablespace1' + ) + } + }) + + # Should not raise exception + try: + estimator._validate_target_space(moves) + except ResourceError: + self.fail("ResourceError raised when space is sufficient") + + def test_validate_tablespace_space_insufficient(self): + """Test validation fails when tablespace has insufficient space""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=2097152, # 2GB + tablespace_usage={'/tablespace1/2': 10485760} # 10GB + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock sufficient space for datadir but insufficient for tablespace + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=20971520, # 20GB + directory='/data/primary0' + ), + '/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdc1', + available_kb=1048576, # Only 1GB available + directory='/tablespace1' + ) + } + }) + + with self.assertRaises(ResourceError) as context: + estimator._validate_target_space(moves) + + error_msg = str(context.exception) + self.assertIn("Insufficient disk space", error_msg) + self.assertIn("/tablespace1", error_msg) + + def test_validate_multiple_tablespaces_different_filesystems(self): + """Test validation with multiple tablespaces on different filesystems""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=2097152, # 2GB + tablespace_usage={ + '/tablespace1/2': 3145728, # 3GB + '/tablespace2/2': 4194304 # 4GB + } + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock space - datadir OK, tablespace1 OK, tablespace2 insufficient + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=20971520, # 20GB + directory='/data/primary0' + ), + '/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdc1', + available_kb=10485760, # 10GB - sufficient + directory='/tablespace1' + ), + '/tablespace2': DiskSpaceInfo( + filesystem='/dev/sdd1', + available_kb=2097152, # Only 2GB - insufficient + directory='/tablespace2' + ) + } + }) + + with self.assertRaises(ResourceError) as context: + estimator._validate_target_space(moves) + + error_msg = str(context.exception) + self.assertIn("Insufficient disk space", error_msg) + # Only tablespace2 should be in error + self.assertIn("/tablespace2", error_msg) + + def test_validate_tablespace_same_filesystem_as_datadir_insufficient(self): + """Test when tablespace and datadir share the same filesystem - insufficient space""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=5242880, # 5GB + tablespace_usage={'/data/tablespace1/2': 5242880} # 5GB + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Both datadir and tablespace on same filesystem /dev/sdb1 + # Total needed: (5GB + 5GB) * 1.1 = 11GB + # Available: only 8GB (insufficient) + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=8388608, # 8GB + directory='/data/primary0' + ), + '/data/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem! + available_kb=8388608, # 8GB (same as above - shared filesystem!) + directory='/data/tablespace1' + ) + } + }) + + with self.assertRaises(ResourceError) as context: + estimator._validate_target_space(moves) + + error_msg = str(context.exception) + self.assertIn("Insufficient disk space", error_msg) + # Should show BOTH directories since they're on the same filesystem + self.assertIn("/data/primary0", error_msg) + self.assertIn("/data/tablespace1", error_msg) + # Should show it needs ~11GB but only has 8GB + self.assertIn("11", error_msg) + self.assertIn("8", error_msg) + + def test_validate_tablespace_same_filesystem_as_datadir_sufficient(self): + """Test when tablespace and datadir share filesystem with sufficient space""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=5242880, # 5GB + tablespace_usage={'/data/tablespace1/2': 5242880} # 5GB + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Both on same filesystem, need 11GB, have 20GB - should pass + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=20971520, # 20GB + directory='/data/primary0' + ), + '/data/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem + available_kb=20971520, # 20GB (same) + directory='/data/tablespace1' + ) + } + }) + + # Should not raise exception + try: + estimator._validate_target_space(moves) + except ResourceError: + self.fail("ResourceError raised when space is sufficient") + + def test_validate_complex_overlapping_filesystems(self): + """Test multiple segments with overlapping filesystem usage""" + segs = [seg for seg in self.gparray.getDbList() + if seg.isSegmentPrimary() and seg.content < 2] + + moves = [ + # Seg 0: datadir on /dev/sdb1, tablespace on /dev/sdc1 + LogicalMove( + seg=segs[0], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=3145728, # 3GB + tablespace_usage={'/tablespace1/2': 2097152} # 2GB + ) + ), + # Seg 1: datadir on /dev/sdb1 (SAME as seg0 datadir), + # tablespace on /dev/sdb1 (SAME filesystem!) + LogicalMove( + seg=segs[1], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary1', + target_port=7001, + segment_size=SegmentSize( + datadir_size_kb=4194304, # 4GB + tablespace_usage={'/data/tblspace2/3': 3145728} # 3GB + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # /dev/sdb1: needs (3GB + 4GB + 3GB) * 1.1 = 11GB, has 12GB - PASS + # /dev/sdc1: needs 2GB * 1.1 = 2.2GB, has 5GB - PASS + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=12582912, # 12GB + directory='/data/primary0' + ), + '/data/primary1': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same as primary0 + available_kb=12582912, + directory='/data/primary1' + ), + '/data/tblspace2': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem! + available_kb=12582912, + directory='/data/tblspace2' + ), + '/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdc1', # Different filesystem + available_kb=5242880, # 5GB + directory='/tablespace1' + ) + } + }) + + # Should not raise + try: + estimator._validate_target_space(moves) + except ResourceError: + self.fail("ResourceError raised when space is sufficient") + + def test_validate_complex_overlapping_filesystems_insufficient(self): + """Test multiple segments with overlapping filesystems - insufficient space""" + segs = [seg for seg in self.gparray.getDbList() + if seg.isSegmentPrimary() and seg.content < 2] + + moves = [ + # Seg 0: datadir on /dev/sdb1, tablespace on /dev/sdc1 + LogicalMove( + seg=segs[0], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=3145728, # 3GB + tablespace_usage={'/tablespace1/2': 2097152} # 2GB + ) + ), + # Seg 1: datadir on /dev/sdb1, tablespace on /dev/sdb1 (SAME filesystem!) + LogicalMove( + seg=segs[1], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary1', + target_port=7001, + segment_size=SegmentSize( + datadir_size_kb=4194304, # 4GB + tablespace_usage={'/data/tblspace2/3': 3145728} # 3GB + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # /dev/sdb1: needs (3GB + 4GB + 3GB) * 1.1 = 11GB, has only 9GB - FAIL + # /dev/sdc1: needs 2GB * 1.1 = 2.2GB, has 5GB - PASS + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=9437184, # 9GB - insufficient! + directory='/data/primary0' + ), + '/data/primary1': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same as primary0 + available_kb=9437184, + directory='/data/primary1' + ), + '/data/tblspace2': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem! + available_kb=9437184, + directory='/data/tblspace2' + ), + '/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdc1', # Different filesystem + available_kb=5242880, # 5GB - sufficient + directory='/tablespace1' + ) + } + }) + + with self.assertRaises(ResourceError) as context: + estimator._validate_target_space(moves) + + error_msg = str(context.exception) + self.assertIn("Insufficient disk space", error_msg) + # Should show all three directories on /dev/sdb1 + self.assertIn("/data/primary0", error_msg) + self.assertIn("/data/primary1", error_msg) + self.assertIn("/data/tblspace2", error_msg) + # Should show aggregated requirement (~11GB) vs available (9GB) + self.assertIn("11", error_msg) + self.assertIn("9", error_msg) + + def test_validate_no_double_counting_same_segment(self): + """Test that the same segment is not double-counted on same filesystem""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + # Create two moves for the same segment (shouldn't happen in reality, + # but tests deduplication logic) + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize(datadir_size_kb=5242880) # 5GB + ), + LogicalMove( + seg=seg0, # Same segment! + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0_copy', + target_port=7001, + segment_size=SegmentSize(datadir_size_kb=5242880) # 5GB + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Both on same filesystem + # Should only count once: 5GB * 1.1 = 5.5GB, not 11GB + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=6291456, # 6GB + directory='/data/primary0' + ), + '/data/primary0_copy': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem + available_kb=6291456, + directory='/data/primary0_copy' + ) + } + }) + + # Should NOT raise - only needs 5.5GB, has 6GB + try: + estimator._validate_target_space(moves) + except ResourceError: + self.fail("ResourceError raised - segment was double-counted!") + + @patch('gprebalance_modules.planner.PortIsAvailable') + @patch('gprebalance_modules.planner.DiskSpaceChecker') + @patch('gprebalance_modules.planner.HostResolver.resolve_hostname') + @patch('gprebalance_modules.planner.HostResolver.get_address') + @patch('gprebalance_modules.planner.GreedySolver') + @patch('gprebalance_modules.rebalance_schema.dbconn.queryRow', side_effect=check_query) + @patch('gprebalance_modules.planner.dbconn') + def test_planner_with_resource_estimation(self, mock_dbconn, mock_schema, mock_solver, + mock_get_address, mock_resolve, mock_disk_check, mock_port): + """Test Planner integration with resource estimation""" + # Setup resolver mocks + mock_resolve.return_value = None + def address_side_effect(hostname): + addr_map = { + 'sdw1': '172.20.0.6', + 'sdw2': '172.20.0.7', + 'sdw3': '172.20.0.8' + } + return addr_map.get(hostname, hostname) + mock_get_address.side_effect = address_side_effect + + # Setup solver mock + mock_solver_instance = Mock() + mock_solver.return_value = mock_solver_instance + + # Mock solution: move segments to balance cluster + solution = {0: (0, 1), + 1: (0, 1), + 2: (0, 1), + 3: (2, 0), + 4: (2, 0), + 5: (1, 2), + 6: (1, 2), + 7: (1, 2), + 8: (2, 0)} + + mock_solver_instance.solve.return_value = (solution, {}) + + self.options.target_datadirs="/data/primary{content}, /data/mirror{content}" + + mock_port.return_value._is_port_available.return_value = True + # Create planner + planner = Planner( + logger=self.logger, + dburl=self.dburl, + gpArray=self.gparray, + options=self.options + ) + + # Mock disk usage - all segments are 2GB + mock_disk_check.return_value.get_disk_usage.return_value = { + '/data/primary0': 2097152, + '/data/primary1': 2097152, + '/data/primary2': 2097152, + '/data/mirror3': 2097152, + '/data/mirror4': 2097152, + '/data/mirror5': 2097152, + '/data/mirror6': 2097152, + '/data/mirror7': 2097152, + '/data/mirror8': 2097152, + } + + # Mock available space - on all targets where we move segments to + mock_disk_check.return_value.check_batch_available_space.return_value = { + '172.20.0.7': { + '/data/primary8': DiskSpaceInfo('/dev/sdb1', 52428800, '/data/primary8'), + '/data/mirror5': DiskSpaceInfo('/dev/sdb1', 52428800, '/data/mirror5'), + '/data/mirror6': DiskSpaceInfo('/dev/sdb1', 52428800, '/data/mirror6'), + '/data/mirror7': DiskSpaceInfo('/dev/sdb1', 52428800, '/data/mirror7'), + }, + '172.20.0.8': { + '/data/mirror0': DiskSpaceInfo('/dev/sdc1', 52428800, '/data/mirror0'), + '/data/mirror1': DiskSpaceInfo('/dev/sdc1', 52428800, '/data/mirror1'), + '/data/mirror2': DiskSpaceInfo('/dev/sdc1', 52428800, '/data/mirror2'), + }, + } + + mock_dbconn.connect.return_value = self.conn + mock_dbconn.query.return_value = [] + + # Execute planning + plan = planner.plan() + + # Verify moves were created + self.assertIsNotNone(plan.getMoves()) + + # Verify resource estimation was performed + for move in plan.getMoves(): + self.assertIsNotNone(move.segment_size, + f"Segment size not set for move: {move}") + + @patch('gprebalance_modules.planner.dbconn') + @patch('gprebalance_modules.rebalance_schema.dbconn.queryRow', side_effect=check_query) + @patch('gprebalance_modules.planner.HostResolver.resolve_hostname') + @patch('gprebalance_modules.planner.HostResolver.get_address') + def test_planner_skips_resource_estimation_when_requested(self, mock_get_address, + mock_resolve, mock_schema, mock_conn): + """Test Planner skips resource estimation when skip_resource_estimation=True""" + mock_resolve.return_value = None + def address_side_effect(hostname): + addr_map = { + 'sdw1': '172.20.0.6', + 'sdw2': '172.20.0.7', + 'sdw3': '172.20.0.8' + } + return addr_map.get(hostname, hostname) + mock_get_address.side_effect = address_side_effect + + # Enable skip flag + self.options.skip_resource_estimation = True + + planner = Planner( + logger=self.logger, + dburl=self.dburl, + gpArray=self.gparray, + options=self.options + ).plan() + + # Verify warning was logged + self.logger.warning.assert_any_call("Skipping resource estimation") + +class TestPortAllocator(GpTestCase): + """ + Unit tests for PortAllocator class + """ + + def setUp(self): + self.logger = Mock() + + def _create_mock_segment(self, dbid, content, hostname, port, role='p', preferred_role='p'): + """ + Helper to create mock segment + """ + seg = Mock(spec=gparray.Segment) + seg.dbid = dbid + seg.content = content + seg.hostname = hostname + seg.datadir = f'/data{content}/seg{content}' + seg.port = port + seg.role = role + seg.preferred_role = preferred_role + + seg.getSegmentDbId.return_value = dbid + seg.getSegmentContentId.return_value = content + seg.getSegmentHostName.return_value = hostname + seg.getSegmentDataDirectory.return_value = seg.datadir + seg.getSegmentPort.return_value = port + seg.isSegmentPrimary.return_value = (role == 'p') + seg.isSegmentMirror.return_value = (role == 'm') + + return seg + + def _create_mock_gparray(self, segments): + """ + Helper to create mock GpArray + """ + array = Mock(spec=gparray.GpArray) + array.getSegDbList.return_value = segments + return array + + def test_initialization_simple(self): + """ + Test basic initialization with simple segment configuration + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + self._create_mock_segment(2, 0, 'host2', 6000, 'm'), + self._create_mock_segment(3, 1, 'host1', 6001, 'p'), + self._create_mock_segment(4, 1, 'host2', 6001, 'm'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger) + + # Verify existing ports tracked + self.assertIn(6000, allocator.existing_ports_by_host['host1']) + self.assertIn(6001, allocator.existing_ports_by_host['host1']) + self.assertIn(6000, allocator.existing_ports_by_host['host2']) + self.assertIn(6001, allocator.existing_ports_by_host['host2']) + + # Verify base ports detected + self.assertEqual(allocator.base_ports_by_host['host1'], (6000, None)) + self.assertEqual(allocator.base_ports_by_host['host2'], (None, 6000)) + + def test_initialization_with_primaries_and_mirrors(self): + """ + Test initialization with both primaries and mirrors on same host + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + self._create_mock_segment(2, 1, 'host1', 6001, 'p'), + self._create_mock_segment(3, 0, 'host2', 7000, 'm'), + self._create_mock_segment(4, 1, 'host2', 7001, 'm'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger) + + # Host1 has primaries + primary_ports, mirror_ports = allocator.existing_ports_by_role['host1'] + self.assertEqual(primary_ports, {6000, 6001}) + self.assertEqual(mirror_ports, set()) + + # Host2 has mirrors + primary_ports, mirror_ports = allocator.existing_ports_by_role['host2'] + self.assertEqual(primary_ports, set()) + self.assertEqual(mirror_ports, {7000, 7001}) + + # Base ports + self.assertEqual(allocator.base_ports_by_host['host1'], (6000, None)) + self.assertEqual(allocator.base_ports_by_host['host2'], (None, 7000)) + + def test_allocate_port_existing_host_port_available(self): + """ + Test allocating port on existing host when current port is available + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + self._create_mock_segment(2, 1, 'host1', 6001, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + host = Host(hostname='host2', address='10.0.0.2', status=HostStatus.ACTIVE) + + # Allocate port 7000 on host2 (doesn't exist in cluster yet) + allocated_port = allocator.allocate_port(host, current_port=7000, is_mirror=False) + + self.assertEqual(allocated_port, 7000) + self.assertIn(7000, allocator.planned_ports_by_host['host2']) + + def test_allocate_port_existing_host_port_conflict(self): + """ + Test allocating port on existing host when current port conflicts + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + self._create_mock_segment(2, 1, 'host1', 6001, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.ACTIVE) + + # Try to allocate port 6000 which is already used + allocated_port = allocator.allocate_port(host, current_port=6000, is_mirror=False) + + # Should get next available port + self.assertNotEqual(allocated_port, 6000) + self.assertGreaterEqual(allocated_port, 6000) + self.assertIn(allocated_port, allocator.planned_ports_by_host['host1']) + + def test_allocate_port_new_host_first_primary(self): + """ + Test allocating first primary port on new host + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + new_host = Host(hostname='host2', address='10.0.0.2', status=HostStatus.NEW) + + # Allocate first primary on new host + allocated_port = allocator.allocate_port(new_host, current_port=6000, is_mirror=False) + + self.assertEqual(allocated_port, 6000) + self.assertEqual(allocator.base_ports_by_host['host2'], (6000, None)) + self.assertIn(6000, allocator.planned_ports_by_host['host2']) + + def test_allocate_port_new_host_first_mirror(self): + """ + Test allocating first mirror port on new host + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + new_host = Host(hostname='host2', address='10.0.0.2', status=HostStatus.NEW) + + # Allocate first mirror on new host + allocated_port = allocator.allocate_port(new_host, current_port=7000, is_mirror=True) + + self.assertEqual(allocated_port, 7000) + self.assertEqual(allocator.base_ports_by_host['host2'], (None, 7000)) + self.assertIn(7000, allocator.planned_ports_by_host['host2']) + + def test_allocate_port_new_host_subsequent_primaries(self): + """ + Test allocating subsequent primary ports on new host follow pattern + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + new_host = Host(hostname='host2', address='10.0.0.2', status=HostStatus.NEW) + + # Allocate first primary + port1 = allocator.allocate_port(new_host, current_port=6000, is_mirror=False) + self.assertEqual(port1, 6000) + + # Allocate second primary - should follow pattern + port2 = allocator.allocate_port(new_host, current_port=6001, is_mirror=False) + self.assertEqual(port2, 6001) + + # Allocate third primary + port3 = allocator.allocate_port(new_host, current_port=6002, is_mirror=False) + self.assertEqual(port3, 6002) + + def test_allocate_port_new_host_mirrors_separate_from_primaries(self): + """ + Test that mirrors and primaries maintain separate port ranges on new host + """ + segments = [] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + new_host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + + # Allocate primaries starting at 6000 + port_p1 = allocator.allocate_port(new_host, current_port=6000, is_mirror=False) + port_p2 = allocator.allocate_port(new_host, current_port=6001, is_mirror=False) + + # Allocate mirrors starting at 7000 + port_m1 = allocator.allocate_port(new_host, current_port=7000, is_mirror=True) + port_m2 = allocator.allocate_port(new_host, current_port=7001, is_mirror=True) + + self.assertEqual(port_p1, 6000) + self.assertEqual(port_p2, 6001) + self.assertEqual(port_m1, 7000) + self.assertEqual(port_m2, 7001) + + # Verify base ports + self.assertEqual(allocator.base_ports_by_host['host1'], (6000, 7000)) + + def test_is_port_available(self): + """ + Test port availability checking + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger) + + # Port 6000 is used + self.assertFalse(allocator._is_port_available('host1', 6000)) + + # Port 6001 is free + self.assertTrue(allocator._is_port_available('host1', 6001)) + + # Mark 6001 as planned + allocator.planned_ports_by_host['host1'].add(6001) + self.assertFalse(allocator._is_port_available('host1', 6001)) + + @patch('gprebalance_modules.planner.PortIsAvailable') + def test_check_port_on_host_available(self, mock_port_is_available): + """Test actual port verification on host when port is available""" + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=True) + + # Mock PortIsAvailable command + mock_cmd = Mock() + mock_cmd.is_port_available.return_value = True + mock_port_is_available.return_value = mock_cmd + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + result = allocator._check_port_on_host(host, 6000) + + self.assertTrue(result) + mock_cmd.run.assert_called_once() + + @patch('gprebalance_modules.planner.PortIsAvailable') + def test_check_port_on_host_in_use(self, mock_port_is_available): + """Test actual port verification when port is in use""" + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=True) + + # Mock PortIsAvailable command + mock_cmd = Mock() + mock_cmd.is_port_available.return_value = False + mock_port_is_available.return_value = mock_cmd + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + result = allocator._check_port_on_host(host, 6000) + + self.assertFalse(result) + + @patch('gprebalance_modules.planner.PortIsAvailable') + def test_check_port_on_host_verification_disabled(self, mock_port_is_available): + """Test that verification is skipped when disabled""" + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + result = allocator._check_port_on_host(host, 6000) + + # Should return True without checking + self.assertTrue(result) + mock_port_is_available.assert_not_called() + + @patch('gprebalance_modules.planner.PortIsAvailable') + def test_verify_and_allocate_port_preferred_available(self, mock_port_is_available): + """Test verify_and_allocate when preferred port is available""" + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=True) + + # Mock port as available + mock_cmd = Mock() + mock_cmd.is_port_available.return_value = True + mock_port_is_available.return_value = mock_cmd + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + allocated = allocator._verify_and_allocate_port(host, 6000) + + self.assertEqual(allocated, 6000) + + @patch('gprebalance_modules.planner.PortIsAvailable') + def test_verify_and_allocate_port_preferred_in_use(self, mock_port_is_available): + """ + Test verify_and_allocate when preferred port is in use + """ + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=True) + + # Mock first port as in use, second as available + def mock_is_available_side_effect(): + call_count = [0] + def side_effect(): + call_count[0] += 1 + return call_count[0] > 1 + return side_effect + + mock_cmd = Mock() + mock_cmd.is_port_available.side_effect = mock_is_available_side_effect() + mock_port_is_available.return_value = mock_cmd + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + allocated = allocator._verify_and_allocate_port(host, 6000) + + # Should get next port + self.assertEqual(allocated, 6001) + self.assertIn(6000, allocator.existing_ports_by_host['host1']) # Marked as used + + def test_find_next_available_port_with_base(self): + """Test finding next available port with established base""" + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + self._create_mock_segment(2, 1, 'host1', 6001, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.ACTIVE) + + # Find next primary port (base is 6000) + next_port = allocator._find_next_available_port(host, 6000, is_mirror=False) + + # Should skip 6000, 6001 and return 6002 + self.assertEqual(next_port, 6002) + + def test_find_verified_port_max_attempts(self): + """Test that finding port fails after max attempts""" + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + + # Fill up many ports + for i in range(1000): + allocator.existing_ports_by_host['host1'].add(6000 + i) + + # Should raise error after max attempts + with self.assertRaises(PlanningError) as ctx: + allocator._find_verified_port(host, 6000) + + self.assertIn("Cannot find available port", str(ctx.exception)) + self.assertIn("after 1000 attempts", str(ctx.exception)) + + def test_complex_scenario_mixed_roles(self): + """ + Test complex scenario with primaries and mirrors on same host + """ + segments = [ + # Host1: primaries on 6000-6002 + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + self._create_mock_segment(2, 1, 'host1', 6001, 'p'), + self._create_mock_segment(3, 2, 'host1', 6002, 'p'), + # Host1: mirrors on 7000-7001 + self._create_mock_segment(4, 3, 'host1', 7000, 'm'), + self._create_mock_segment(5, 4, 'host1', 7001, 'm'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + host1 = Host(hostname='host1', address='10.0.0.1', status=HostStatus.ACTIVE) + + # Allocate new primary (should get 6003) + port_p = allocator.allocate_port(host1, current_port=6003, is_mirror=False) + self.assertEqual(port_p, 6003) + + # Allocate new mirror (should get 7002) + port_m = allocator.allocate_port(host1, current_port=7002, is_mirror=True) + self.assertEqual(port_m, 7002) + + # Verify role separation maintained + primary_ports, mirror_ports = allocator.existing_ports_by_role['host1'] + self.assertIn(6003, primary_ports) + self.assertIn(7002, mirror_ports) + + def test_empty_gparray(self): + """Test initialization with empty gparray""" + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger) + + self.assertEqual(len(allocator.existing_ports_by_host), 0) + self.assertEqual(len(allocator.base_ports_by_host), 0) + + # Should still be able to allocate on new host + new_host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + port = allocator.allocate_port(new_host, current_port=6000, is_mirror=False) + self.assertEqual(port, 6000) + +class TestPortAllocatorIntegration(GpTestCase): + """Integration tests simulating realistic rebalance scenarios""" + + def setUp(self): + self.logger = Mock() + + def _create_segment(self, dbid, content, hostname, port, role='p'): + """Helper to create mock segment""" + seg = Mock(spec=gparray.Segment) + seg.dbid = dbid + seg.content = content + seg.hostname = hostname + seg.datadir = f'/data{content}/seg{content}' + seg.port = port + seg.role = role + seg.preferred_role = role + + seg.getSegmentDbId.return_value = dbid + seg.getSegmentContentId.return_value = content + seg.getSegmentHostName.return_value = hostname + seg.getSegmentDataDirectory.return_value = seg.datadir + seg.getSegmentPort.return_value = port + seg.isSegmentPrimary.return_value = (role == 'p') + seg.isSegmentMirror.return_value = (role == 'm') + + return seg + + def test_expansion_scenario(self): + """Test port allocation during cluster expansion (add new host)""" + # Initial: 2 hosts with 2 primaries each + segments = [ + self._create_segment(1, 0, 'host1', 6000, 'p'), + self._create_segment(2, 1, 'host1', 6001, 'p'), + self._create_segment(3, 0, 'host2', 7000, 'm'), + self._create_segment(4, 1, 'host2', 7001, 'm'), + ] + + gparray_mock = Mock(spec=gparray.GpArray) + gparray_mock.getSegDbList.return_value = segments + + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + # Add new host3 - move some mirrors there + new_host = Host(hostname='host3', address='10.0.0.3', status=HostStatus.NEW) + + # Move mirror of content 0 to new host + port1 = allocator.allocate_port(new_host, current_port=7000, is_mirror=True) + self.assertEqual(port1, 7000) # Establishes base + + # Move mirror of content 1 to new host + port2 = allocator.allocate_port(new_host, current_port=7001, is_mirror=True) + self.assertEqual(port2, 7001) # Follows pattern + + def test_shrink_scenario(self): + """Test port allocation during cluster shrink (remove host)""" + # Initial: 3 hosts + segments = [ + self._create_segment(1, 0, 'host1', 6000, 'p'), + self._create_segment(2, 1, 'host2', 6000, 'p'), + self._create_segment(3, 2, 'host3', 6000, 'p'), + self._create_segment(4, 0, 'host2', 7000, 'm'), + self._create_segment(5, 1, 'host3', 7000, 'm'), + self._create_segment(6, 2, 'host1', 7000, 'm'), + ] + + gparray_mock = Mock(spec=gparray.GpArray) + gparray_mock.getSegDbList.return_value = segments + + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + # Remove host3 - redistribute its segments + host1 = Host(hostname='host1', address='10.0.0.1', status=HostStatus.ACTIVE) + host2 = Host(hostname='host2', address='10.0.0.2', status=HostStatus.ACTIVE) + + # Move content 2 primary from host3 to host1 + port_p = allocator.allocate_port(host1, current_port=6000, is_mirror=False) + self.assertEqual(port_p, 6001) # 6000 in use, get next + + # Move content 2 mirror from host1 to host2 + port_m = allocator.allocate_port(host2, current_port=7000, is_mirror=True) + self.assertEqual(port_m, 7001) # 7000 in use, get next + + def test_rebalance_after_failure(self): + """Test rebalance scenario after segment failure and recovery""" + # Scenario: segment recovered with different port, need to rebalance + segments = [ + self._create_segment(1, 0, 'host1', 6000, 'p'), + self._create_segment(2, 1, 'host1', 6001, 'p'), + self._create_segment(3, 2, 'host1', 6002, 'p'), + self._create_segment(4, 0, 'host2', 7000, 'm'), + self._create_segment(5, 1, 'host2', 7001, 'm'), + self._create_segment(6, 2, 'host2', 7999, 'm'), # Recovered with odd port + ] + + gparray_mock = Mock(spec=gparray.GpArray) + gparray_mock.getSegDbList.return_value = segments + + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + # Rebalance: move content 2 mirror back to proper port + host2 = Host(hostname='host2', address='10.0.0.2', status=HostStatus.ACTIVE) + + # Try to use 7002 (following pattern) + port = allocator.allocate_port(host2, current_port=7002, is_mirror=True) + self.assertEqual(port, 7002) # Should be available + +if __name__ == '__main__': + run_tests() diff --git a/gpMgmt/bin/gprebalance_modules/test/test_unit_solver.py b/gpMgmt/bin/gprebalance_modules/test/test_unit_solver.py index ce1e4eb18ae3..2cc1cd3cbe83 100644 --- a/gpMgmt/bin/gprebalance_modules/test/test_unit_solver.py +++ b/gpMgmt/bin/gprebalance_modules/test/test_unit_solver.py @@ -1,5 +1,4 @@ import random -from typing import List, Dict, Tuple from collections import defaultdict from gppylib.test.unit.gp_unittest import * @@ -17,7 +16,8 @@ def _validate_solution_allassign(self, solution: Solution, if len(solution) != solver.n_segments: return False - return True + return True + def _validate_solution_host_ids(self, solution: Solution, solver: GreedySolver) -> bool: for seg_id, (primary, mirror) in solution.items(): diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature index 8a4a0b9d6d6b..79d89cc040fa 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature @@ -1,6 +1,33 @@ @ggrebalance Feature: ggrebalance behave tests + Scenario Outline: Validate incompatible option combinations + Given a standard local demo cluster is running + When the user runs "ggrebalance " + Then ggrebalance should return a return code of 1 + And ggrebalance should print "" to stdout + Given + + Examples: Mutually exclusive options + | options | error_message | run_status | + | --target-hosts sdw1,sdw2 --target-hosts-file /tmp/hosts.txt | Can't use together options '--target-hosts' and '--target-hosts-file' | stub | + | --target-hosts sdw1,sdw2 --add-hosts sdw3 | Can't use together options '--target-hosts' and '--add-hosts' | stub | + | --target-hosts sdw1,sdw2 --remove-hosts sdw3 | Can't use together options '--target-hosts' and '--remove-hosts' | stub | + | --add-hosts sdw3 --add-hosts-file /tmp/add.txt | Can't use together options '--add-hosts' and '--add-hosts-file' | stub | + | --remove-hosts sdw3 --remove-hosts-file /tmp/rm.txt | Can't use together options '--remove-hosts' and '--remove-hosts-file' | stub | + | --add-hosts sdw3 --remove-hosts sdw3 | Can't use together options '--add-hosts' and '--remove-hosts' | stub | + | --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' --target-datadirs-file /tmp/datadirs.txt | Can't use together options '--target-datadirs' and '--target-datadirs-file' | stub | + | --mirror-mode grouped --skip-rebalance | Can't use together options '--skip-rebalance' and '--mirror-mode' | stub | + | -m spread --skip-rebalance | Can't use together options '--skip-rebalance' and '--mirror-mode' | stub | + | -c --target-hosts sdw1,sdw2 | Can't use together options '--clean-required' and '--target-hosts' | stub | + | -c --add-hosts sdw3 | Can't use together options '--clean-required' and '--add-hosts' | stub | + | -c --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' | Can't use together options '--clean-required' and '--target-datadirs' | stub | + | -c -x 2 | Can't use together options '--clean-required' and '--target-segment-count' | stub | + | -c --mirror-mode grouped | Can't use together options '--clean-required' and '--mirror-mode' | stub | + | -c --skip-rebalance | Can't use together options '--clean-required' and '--skip-rebalance' | stub | + | -c --show-plan | Can't use together options '--clean-required' and '--show-plan' | stub | + | -c --skip-resource-estimation | Can't use together options '--clean-required' and '--skip-resource-estimation' | the database is not running | + Scenario: test 1.1 ggrebalance simple scenarios Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 21b9f0784597..620f4d6ec0f4 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -4460,3 +4460,7 @@ def impl(context, fault): @when('unset fault inject') def impl(context): os.environ['GPMGMT_FAULT_POINT'] = "" + +@given('stub') +def impl(context): + pass From 1a89824415dfeea2a96f4ef9352be4bd5ee4584b Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Sat, 24 Jan 2026 09:09:51 +1000 Subject: [PATCH 15/22] Implement cluster rebalance for ggrebalance utility (#2240) List of changes: 1. This patch adds rebalance functionality. Main part of the related logic is located in the 'RebalanceSM' class. Rebalance is done according to the list of moves from the supplied plan, and includes following steps: - move (via gpmovemirrors) all mirrors from the list of moves; - for all primaries from the list of moves switch them with their mirrors; - move (via gpmovemirrors) all these segments which were primaries; - switch all these segments back to primaries roles. 2. As the rebalance functionality should be correctly coordinated with the existing shrink logic, this patch adds the high level state-machine implementation in 'GGRebalanceMainSM' class. It is responsible for proper flow of high level states like planning, rebalance schema creation and deletion, invocation of shrink and rebalance execution, invocation of cleanup and shrink. Therefore: - some states and logic are moved from the existing shrink state-machine to 'GGRebalanceMainSM'; - temp code is removed from the planner; - code in 'ggrebalance' is updated to call only 'GGRebalanceMainSM', that will do the rest. 3. As now we need to handle states from shrink, rebalance and main state-machines, 'RebalanceSchema' code is updated to store and access these state categories. 4. New behave tests for rebalance functionality are added. As the ggrebalance test suit became too large and long too execute, it is split into 3 files: - 'ggrebalance_basics.feature' - contains the existing basic checks from the old file; - 'ggrebalance_shrink.feature' - contains the existing checks for shrink from the old file; - 'ggrebalance_rebalance.feature' - contains the new tests for the rebalance. Also, some notes about changes related to tests: - Old test named 'test 2.2. shrink' is merged into the test with a new name 'test 1.3. shrink', as the usage of the new top-level state-machine allows now to continue shrink execution in this test case; - New step definition is added into 'mgmt_util.py', that allows to get the number of segments which satisfy a certain condition. It is used in the new tests. - New step definition is added into 'mgmt_util.py', that allows to set a delay for a fault to happen. The respective changes are added into the fault injector code. It is used in the new tests, when we test interruption during the work of gpmovemirrors or gprecoverseg. --- gpMgmt/bin/ggrebalance | 58 ++- gpMgmt/bin/gppylib/commands/gp.py | 14 + gpMgmt/bin/gppylib/fault_injection.py | 18 +- gpMgmt/bin/gprebalance_modules/Makefile | 3 +- .../ggrebalance_main_sm.py | 301 +++++++++++++++ .../bin/gprebalance_modules/ggrebalance_sm.py | 363 ++++++++++++++++++ gpMgmt/bin/gprebalance_modules/planner.py | 19 - .../gprebalance_modules/rebalance_schema.py | 41 +- gpMgmt/bin/gprebalance_modules/shrink.py | 287 +++++--------- gpMgmt/test/behave/mgmt_utils/environment.py | 6 +- .../mgmt_utils/ggrebalance_basics.feature | 80 ++++ .../mgmt_utils/ggrebalance_rebalance.feature | 256 ++++++++++++ ...nce.feature => ggrebalance_shrink.feature} | 159 ++------ .../behave/mgmt_utils/steps/mgmt_utils.py | 21 + 14 files changed, 1252 insertions(+), 374 deletions(-) create mode 100755 gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py create mode 100755 gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py create mode 100755 gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature create mode 100755 gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature rename gpMgmt/test/behave/mgmt_utils/{ggrebalance.feature => ggrebalance_shrink.feature} (79%) diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index 5ba9b5e6181b..d3836a3a3c65 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -14,6 +14,7 @@ try: from gppylib.system.environment import GpCoordinatorEnvironment from gppylib.system import configurationInterface, configurationImplGpdb from gprebalance_modules.shrink import GGShrink, DBNAME + from gprebalance_modules.ggrebalance_main_sm import GGRebalanceMainSM from gprebalance_modules.planner import * except ImportError as e: sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) @@ -228,14 +229,43 @@ def remove_pid_file(coordinator_data_directory: str): except IOError: pass -gg_shrink = None +gg_main_rebalance_SM = None +cmd_recoverseg = None def sig_handler(sig, arg): - if gg_shrink is not None: - gg_shrink.shutdown() + if gg_main_rebalance_SM is not None: + gg_main_rebalance_SM.shutdown() else: + if cmd_recoverseg != None: + cmd_recoverseg.cancel() sys.exit(1) +def check_down_segments(logger: Any, dburl: dbconn.DbURL): + # check if some of the segments are down - it can happen if we're recovering from an + # interrupted state, and such segments are left by the interrupted gprecoverseg + conn = dbconn.connect(dburl, encoding='UTF8', allowSystemTableMods=True) + dbconn.execSQL(conn, "SELECT gp_request_fts_probe_scan()") + cnt_primaries_down = int(dbconn.queryRow(conn, f"SELECT COUNT(1) FROM gp_segment_configuration WHERE role = 'p' AND status = 'd'")[0]) + cnt_mirrors_down = int(dbconn.queryRow(conn, f"SELECT COUNT(1) FROM gp_segment_configuration WHERE role = 'm' AND status = 'd'")[0]) + conn.close() + + if cnt_primaries_down != 0: + raise Exception('Detected some primary segments are down, please recover manually') + + if cnt_mirrors_down != 0: + logger.info("Some mirrors are down, trying to recover them, it may take some time...") + recoverseg_options = "-a -F" + global cmd_recoverseg + try: + cmd_recoverseg = GpRecoverSeg("Running gprecoverseg", options=recoverseg_options) + cmd_recoverseg.run(validateAfter=True) + except Exception as e: + logger.error(str(e)) + error_msg = f"Failed to execute 'gprecoverseg {recoverseg_options}'" + raise Exception(error_msg) + finally: + cmd_recoverseg = None + def main(options, args, parser): try: signal.signal(signal.SIGTERM, sig_handler) @@ -268,6 +298,8 @@ def main(options, args, parser): gparray_dump_filename = options.coordinator_data_directory + '/gparraydump' + check_down_segments(logger, dburl) + logger.info('Init gparray from catalog') try: gparray = GpArray.initFromCatalog(dburl, utility=True) @@ -281,23 +313,9 @@ def main(options, args, parser): logger.error(f"{str(ex)}") sys.exit(1) - global gg_shrink - gg_shrink = GGShrink(logger, dburl, options, gpenv, gparray, gparray_dump_filename) - - # Note: the plan for a shrink later will be provided by the planner component. - # But for now we simply create a Plan object from the options directly. - # We'll keep this stub till planner is ready. - # If shrink_plan is None, we assume that we need to continue previous operation - plan = None - if options.target_segment_count != None: - plan = Planner(logger, dburl, gparray, options).plan() - - if options.target_segment_count != None and options.show_plan: - logger.info(f"Final plan:\n{plan}") - sys.exit(0) - - if plan is None or isinstance(plan, ShrinkPlan): - gg_shrink.run(plan) + global gg_main_rebalance_SM + gg_main_rebalance_SM = GGRebalanceMainSM(logger, dburl, options, gpenv, gparray, gparray_dump_filename) + gg_main_rebalance_SM.run() sys.exit(0) except Exception as e: diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index 214206adaf33..f6b7ed70b42e 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -1688,6 +1688,20 @@ def __init__(self, name, options = "", ctxt = LOCAL, remoteHost = None): cmdStr = "$GPHOME/bin/gprecoverseg %s" % (options) Command.__init__(self,name,cmdStr,ctxt,remoteHost) +class GpMoveMirrors(Command): + """ + This command will execute the gpmovemirror utility + """ + + def __init__(self, name, options = "", ctxt = LOCAL, remoteHost = None): + self.name = name + self.options = options + self.ctxt = ctxt + self.remoteHost = remoteHost + + cmdStr = "$GPHOME/bin/gpmovemirrors %s" % (options) + Command.__init__(self,name,cmdStr,ctxt,remoteHost) + class IfAddrs: @staticmethod def list_addrs(hostname=None, include_loopback=False): diff --git a/gpMgmt/bin/gppylib/fault_injection.py b/gpMgmt/bin/gppylib/fault_injection.py index 43f437cc7671..142ab17dfcfd 100755 --- a/gpMgmt/bin/gppylib/fault_injection.py +++ b/gpMgmt/bin/gppylib/fault_injection.py @@ -1,12 +1,28 @@ #!/usr/bin/env python3 import os +import threading +import time +import signal GPMGMT_FAULT_POINT = 'GPMGMT_FAULT_POINT' +GPMGMT_FAULT_DELAY_MS = 'GPMGMT_FAULT_DELAY_MS' def inject_fault(fault_point): if GPMGMT_FAULT_POINT in os.environ and fault_point == os.environ[GPMGMT_FAULT_POINT]: - raise Exception('Fault Injection %s' % os.environ[GPMGMT_FAULT_POINT]) + if GPMGMT_FAULT_DELAY_MS in os.environ and int(os.environ[GPMGMT_FAULT_DELAY_MS]) > 0: + delay_ms = int(os.environ[GPMGMT_FAULT_DELAY_MS]) + + def raise_exception(delay: int): + time.sleep(delay / 1000) + print('Fault Injection %s' % os.environ[GPMGMT_FAULT_POINT]) + os.kill(os.getpid(), signal.SIGINT) + + thread = threading.Thread(target=raise_exception, kwargs={"delay": delay_ms}) + thread.daemon = True + thread.start() + else: + raise Exception('Fault Injection %s' % os.environ[GPMGMT_FAULT_POINT]) # decorator for test purposes def wrap_state_func_with_faults(func): diff --git a/gpMgmt/bin/gprebalance_modules/Makefile b/gpMgmt/bin/gprebalance_modules/Makefile index ef363a6ab844..b1e8b2c959e0 100644 --- a/gpMgmt/bin/gprebalance_modules/Makefile +++ b/gpMgmt/bin/gprebalance_modules/Makefile @@ -11,7 +11,8 @@ rebalance_status.py \ shrink.py planner.py \ rebalance_commons.py \ rebalance_schema.py \ -solver.py +solver.py ggrebalance_sm.py \ +ggrebalance_main_sm.py installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)/gprebalance_modules' diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py new file mode 100755 index 000000000000..c900bb9d6bec --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 + +from transitions import Machine + +try: + from gppylib.commands.unix import * + from gppylib.commands.gp import * + from gppylib.gplog import * + from gppylib.system.environment import * + from gprebalance_modules.planner import * + from gprebalance_modules.rebalance_schema import RebalanceSchema + from gppylib.fault_injection import * + from gprebalance_modules.shrink import GGShrink + from gprebalance_modules.ggrebalance_sm import RebalanceSM +except ImportError as e: + sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) + +class GGRebalanceMainSM: + + states_not_logged = [ + 'STATE_START', + 'STATE_OPTIONS_VALIDATION', + 'STATE_CLEANUP', + 'STATE_ROLLBACK', + 'STATE_PLANNING_STARTED', + 'STATE_PLANNING_DONE', + 'STATE_CHECK_PREVIOUS_RUN', + 'STATE_END', + 'STATE_ERROR' + ] + + states_logged = [ + 'STATE_SETUP_SCHEMA_STARTED', + 'STATE_SETUP_SCHEMA_DONE', + 'STATE_EXECUTOR_STARTED', + 'STATE_EXECUTOR_DONE', + 'STATE_SHRINK_STARTED', + 'STATE_SHRINK_DONE', + 'STATE_REBALANCE_STARTED', + 'STATE_REBALANCE_DONE', + ] + + transitions = [ + { + 'trigger': 'start', + 'source': 'STATE_START', + 'dest': 'STATE_OPTIONS_VALIDATION' + }, + { + 'trigger': 'move_to_STATE_CLEANUP', + 'source': 'STATE_OPTIONS_VALIDATION', + 'dest': 'STATE_CLEANUP' + }, + { + 'trigger': 'move_to_STATE_ROLLBACK', + 'source': 'STATE_OPTIONS_VALIDATION', + 'dest': 'STATE_ROLLBACK' + }, + { + 'trigger': 'move_to_STATE_PLANNING_STARTED', + 'source': 'STATE_OPTIONS_VALIDATION', + 'dest': 'STATE_PLANNING_STARTED' + }, + { + 'trigger': 'move_to_STATE_PLANNING_DONE', + 'source': 'STATE_PLANNING_STARTED', + 'dest': 'STATE_PLANNING_DONE' + }, + { + 'trigger': 'move_to_STATE_CHECK_PREVIOUS_RUN', + 'source': 'STATE_PLANNING_DONE', + 'dest': 'STATE_CHECK_PREVIOUS_RUN' + }, + { + 'trigger': 'move_to_STATE_SETUP_SCHEMA_STARTED', + 'source': 'STATE_CHECK_PREVIOUS_RUN', + 'dest': 'STATE_SETUP_SCHEMA_STARTED' + }, + { + 'trigger': 'move_to_STATE_SETUP_SCHEMA_DONE', + 'source': 'STATE_SETUP_SCHEMA_STARTED', + 'dest': 'STATE_SETUP_SCHEMA_DONE' + }, + { + 'trigger': 'move_to_STATE_EXECUTOR_STARTED', + 'source': ['STATE_SETUP_SCHEMA_DONE', 'STATE_CHECK_PREVIOUS_RUN'], + 'dest': 'STATE_EXECUTOR_STARTED' + }, + { + 'trigger': 'move_to_STATE_SHRINK_STARTED', + 'source': 'STATE_EXECUTOR_STARTED', + 'dest': 'STATE_SHRINK_STARTED' + }, + { + 'trigger': 'move_to_STATE_SHRINK_DONE', + 'source': 'STATE_SHRINK_STARTED', + 'dest': 'STATE_SHRINK_DONE' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_STARTED', + 'source': ['STATE_EXECUTOR_STARTED', 'STATE_SHRINK_DONE'], + 'dest': 'STATE_REBALANCE_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_DONE', + 'source': 'STATE_REBALANCE_STARTED', + 'dest': 'STATE_REBALANCE_DONE' + }, + { + 'trigger': 'move_to_STATE_EXECUTOR_DONE', + 'source': ['STATE_EXECUTOR_STARTED', 'STATE_SHRINK_DONE', 'STATE_REBALANCE_DONE'], + 'dest': 'STATE_EXECUTOR_DONE' + }, + { + 'trigger': 'move_to_STATE_END', + 'source': ['STATE_EXECUTOR_DONE', 'STATE_CHECK_PREVIOUS_RUN', 'STATE_CLEANUP', 'STATE_ROLLBACK'], + 'dest': 'STATE_END' + }, + { + 'trigger': 'move_to_STATE_ERROR', + 'source': '*', + 'dest': 'STATE_ERROR' + } + ] + + def __init__(self, logger: Any, dburl: dbconn.DbURL, options: Any, gpEnv: GpCoordinatorEnvironment, gpArray: gparray.GpArray, gpArrayDumpFilename: str): + self.logger = logger + self.dburl = dburl + self.options = options + self.gparray = gpArray + self.conn = dbconn.connect( + self.dburl, encoding='UTF8', allowSystemTableMods=True) + + self.rebalance_schema = RebalanceSchema(self.conn) + + self.machine = Machine(model = self, + queued=True, + states = self.states_not_logged + self.states_logged, + transitions = self.transitions, + initial = 'STATE_START', + before_state_change = 'on_every_state') + + self.gg_shrink = GGShrink(self.conn, self.rebalance_schema, self.logger, self.options, gpEnv, self.gparray, gpArrayDumpFilename) + self.gg_rebalance = RebalanceSM(self.conn, self.rebalance_schema, self.logger, self.options, self.gparray) + + self.plan = None + self.main_state_from_prev_run = self.rebalance_schema.getMainStateFromPreviousRun() + + def on_every_state(self) -> None: + if self.state in self.states_logged: + self.rebalance_schema.storeMainState(self.state) + + def run(self) -> None: + self.trigger('start') + + def shutdown(self) -> None: + need_exit = True + + if self.gg_shrink is not None: + self.gg_shrink.shutdown() + need_exit = False + + if self.gg_rebalance is not None: + self.gg_rebalance.shutdown() + need_exit = False + + if need_exit: + sys.exit(1) + + # state callbacks start here + + @wrap_state_func_with_faults + def on_enter_STATE_OPTIONS_VALIDATION(self) -> None: + if self.options.clean_required: + self.trigger('move_to_STATE_CLEANUP') + elif self.options.rollback_required: + self.trigger('move_to_STATE_ROLLBACK') + else: + self.trigger('move_to_STATE_PLANNING_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_CLEANUP(self) -> None: + if not self.rebalance_schema.schemaExists(): + self.logger.info(f"Rebalance schema doesn't exist. Cleanup is not required.") + else: + prev_run_was_complete = (self.main_state_from_prev_run == 'STATE_EXECUTOR_DONE' or + self.main_state_from_prev_run == 'STATE_ROLLBACK') + self.gg_shrink.cleanup(prev_run_was_complete) + self.rebalance_schema.dropSchema() + self.logger.info('Cleanup is complete') + self.trigger('move_to_STATE_END') + + @wrap_state_func_with_faults + def on_enter_STATE_ROLLBACK(self) -> None: + self.plan = self.rebalance_schema.retrieveSavedPlan() + self.gg_shrink.rollback(self.plan) + self.trigger('move_to_STATE_END') + + @wrap_state_func_with_faults + def on_enter_STATE_PLANNING_STARTED(self) -> None: + if self.options.target_segment_count != None: + self.plan = Planner(self.logger, self.dburl, self.gparray, self.options).plan() + + if self.options.target_segment_count != None and self.options.show_plan: + self.logger.info(f"Final plan:\n{self.plan}") + + self.trigger('move_to_STATE_PLANNING_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_PLANNING_DONE(self) -> None: + self.trigger('move_to_STATE_CHECK_PREVIOUS_RUN') + + @wrap_state_func_with_faults + def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: + if not self.rebalance_schema.schemaExists(): + if self.plan == None: + self.logger.error("Rebalance schema doesn't exists and no shrink plan is supplied. Please specify shrink plan.") + self.trigger('move_to_STATE_ERROR') + return + if self.gparray.get_segment_count() < self.plan.target_segment_count: + logger.error('Target segment count (%s) > current segment count (%s).\n' + 'Currently only shrink is supported (target segment count < current segment count).' + % (self.plan.target_segment_count, self.gparray.get_segment_count())) + self.trigger('move_to_STATE_ERROR') + return + self.trigger('move_to_STATE_SETUP_SCHEMA_STARTED') + else: + # Schema already exists from the previous run. + # In this case we already have a plan saved in the schema, + # and we'll continue (or rollback) according to it. + # Or, if everything is complete, just exit. + if self.main_state_from_prev_run == 'STATE_EXECUTOR_DONE': + self.logger.info('Previous run was completed successfully. Please execute cleanup before a new run.') + return + + if self.plan != None: + self.logger.error("Can't start a new operation, because the previous one was interrupted. " + "Please try to launch again without a plan to continue from the interrupted state, " + "or use '--rollback' or '--cleanup' options.") + self.trigger('move_to_STATE_ERROR') + return + + self.plan = self.rebalance_schema.retrieveSavedPlan() + + self.trigger('move_to_STATE_EXECUTOR_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_SETUP_SCHEMA_STARTED(self) -> None: + # Create schema and status tables. + # It will also save plan in order to use it for recovering after interruption + self.rebalance_schema.createSchema(self.plan) + self.trigger('move_to_STATE_SETUP_SCHEMA_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SETUP_SCHEMA_DONE(self) -> None: + self.logger.info(f'Created "{self.rebalance_schema.getSchemaName()}" schema') + self.trigger('move_to_STATE_EXECUTOR_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_EXECUTOR_STARTED(self) -> None: + if isinstance(self.plan, ShrinkPlan): + shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() + if not self.gg_shrink.state_is_final(shrink_state_from_prev_run): + self.trigger('move_to_STATE_SHRINK_STARTED') + return + self.trigger('move_to_STATE_REBALANCE_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_EXECUTOR_DONE(self) -> None: + self.trigger('move_to_STATE_END') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_STARTED(self) -> None: + self.gg_shrink.run(self.plan) + self.trigger('move_to_STATE_SHRINK_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_STARTED(self) -> None: + if self.plan is not None and self.plan.getMoves() is not None: + self.gg_rebalance.run(self.plan) + self.logger.info('Rebalance is complete') + + self.trigger('move_to_STATE_REBALANCE_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_DONE(self) -> None: + self.trigger('move_to_STATE_EXECUTOR_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_END(self) -> None: + self.conn.close() + + @wrap_state_func_with_faults + def on_enter_STATE_ERROR(self) -> None: + raise Exception('Main SM entered STATE_ERROR') + + # state callbacks end here diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py new file mode 100755 index 000000000000..b774c88fdc4b --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 + +from transitions import Machine +from enum import Enum + +try: + from gppylib.commands.unix import * + from gppylib.commands.gp import * + from gppylib.gplog import * + from gppylib.commands.gp import GpMoveMirrors + from gppylib.system.environment import * + from gprebalance_modules.planner import * + from gprebalance_modules.rebalance_schema import RebalanceSchema, STATE_NOT_DEFINED + from gppylib.fault_injection import * +except ImportError as e: + sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) + +class RebalanceSM: + + states_not_logged = [ + 'STATE_REBALANCE_INIT', + 'STATE_CHECK_PREVIOUS_RUN' + ] + + states_main_rebalance_flow = [ + 'STATE_REBALANCE_STARTED', + 'STATE_REBALANCE_MOVE_MIRRORS_STARTED', + 'STATE_REBALANCE_MOVE_MIRRORS_DONE', + 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED', + 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE', + 'STATE_REBALANCE_MOVE_PRIMARIES_STARTED', + 'STATE_REBALANCE_MOVE_PRIMARIES_DONE', + 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED', + 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE', + 'STATE_REBALANCE_DONE' + ] + + transitions = [ + { + 'trigger': 'start', + 'source': 'STATE_REBALANCE_INIT', + 'dest': 'STATE_CHECK_PREVIOUS_RUN' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_STARTED', + 'source': 'STATE_CHECK_PREVIOUS_RUN', + 'dest': 'STATE_REBALANCE_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_MOVE_MIRRORS_STARTED', + 'source': 'STATE_REBALANCE_STARTED', + 'dest': 'STATE_REBALANCE_MOVE_MIRRORS_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_MOVE_MIRRORS_DONE', + 'source': 'STATE_REBALANCE_MOVE_MIRRORS_STARTED', + 'dest': 'STATE_REBALANCE_MOVE_MIRRORS_DONE' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED', + 'source': 'STATE_REBALANCE_MOVE_MIRRORS_DONE', + 'dest': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE', + 'source': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED', + 'dest': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_MOVE_PRIMARIES_STARTED', + 'source': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE', + 'dest': 'STATE_REBALANCE_MOVE_PRIMARIES_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_MOVE_PRIMARIES_DONE', + 'source': 'STATE_REBALANCE_MOVE_PRIMARIES_STARTED', + 'dest': 'STATE_REBALANCE_MOVE_PRIMARIES_DONE' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED', + 'source': 'STATE_REBALANCE_MOVE_PRIMARIES_DONE', + 'dest': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE', + 'source': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED', + 'dest': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_DONE', + 'source': ['STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE', 'STATE_REBALANCE_MOVE_MIRRORS_DONE'], + 'dest': 'STATE_REBALANCE_DONE' + } + ] + + class RoleSwapDirection(Enum): + PRIMARY_TO_MIRROR = 1 + MIRROR_TO_PRIMARY = 2 + + def __init__(self, conn: dbconn.Connection, schema: RebalanceSchema, logger: Any, options: Any, gpArray: gparray.GpArray): + self.logger = logger + self.options = options + self.shutdown_requested = False + self.gparray = gpArray + self.conn = conn + self.rebalance_schema = schema + self.cmd = None + + self.machine = Machine(model = self, + queued=True, + states = self.states_main_rebalance_flow + self.states_not_logged, + transitions = self.transitions, + initial = 'STATE_REBALANCE_INIT', + before_state_change = 'on_every_state') + + def on_every_state(self) -> None: + if self.shutdown_requested: + self.logger.info('Rebalance was interrupted') + raise Exception('Rebalance was interrupted') + + if self.state in self.states_main_rebalance_flow: + self.rebalance_schema.storeRebalanceState(self.state) + + def run(self, plan: Plan) -> None: + self.rebalance_plan = plan + if not self.rebalance_plan.getMoves(): + return + + self.moves_primaries = [] + self.moves_mirrors = [] + for move in self.rebalance_plan.getMoves(): + if move.seg.isSegmentPrimary() : + self.moves_primaries.append(move) + else: + self.moves_mirrors.append(move) + + self.primary_segments_to_move = [move.seg for move in self.moves_primaries] + + self.trigger('start') + + def process_moves(self, moves: List[LogicalMove]): + filename = self.create_config_file(moves) + gpmovemirrors_options = f'-a -i {filename}' + + if self.options.batch_size is not None: + batch_size = self.options.batch_size + # gpmovemirrors has its own limitation for batch size, + # need to consider it here. + if batch_size > MAX_COORDINATOR_NUM_WORKERS: + batch_size = MAX_COORDINATOR_NUM_WORKERS + gpmovemirrors_options += f' -B {batch_size}' + + try: + self.cmd = GpMoveMirrors("Running gpmovemirrors", options=gpmovemirrors_options) + self.cmd.run(validateAfter=True) + except Exception as e: + logger.error(str(e)) + error_msg = f"Failed to execute 'gpmovemirrors {gpmovemirrors_options}'" + raise Exception(error_msg) + finally: + self.cmd = None + + if os.path.exists(filename): + os.remove(filename) + + def execute_role_swaps(self, segments_to_move: List[Segment], direction: RoleSwapDirection): + """Execute multiple role swaps in single gprecoverseg -r call""" + + assert (len(segments_to_move) > 0) + + segids = [segment.getSegmentContentId() for segment in segments_to_move] + dbids = [segment.getSegmentDbId() for segment in segments_to_move] + + seg_list = ', '.join(str(seg) for seg in segids) + dbid_list = ', '.join(str(dbid) for dbid in dbids) + + dbconn.execSQL(self.conn, "BEGIN") + + # check the current status of 'preferred_role' and 'role' for all requested dbids + # in order to recover properly from the previous interrupted run (if any) + + cnt_preferred_role_p = \ + int(dbconn.queryRow(self.conn, + f"SELECT COUNT(1) FROM gp_segment_configuration WHERE preferred_role = 'p' AND dbid IN ({dbid_list})")[0]) + cnt_role_p = \ + int(dbconn.queryRow(self.conn, + f"SELECT COUNT(1) FROM gp_segment_configuration WHERE role = 'p' AND dbid IN ({dbid_list})")[0]) + cnt_preferred_role_m = \ + int(dbconn.queryRow(self.conn, + f"SELECT COUNT(1) FROM gp_segment_configuration WHERE preferred_role = 'm' AND dbid IN ({dbid_list})")[0]) + cnt_role_m = \ + int(dbconn.queryRow(self.conn, + f"SELECT COUNT(1) FROM gp_segment_configuration WHERE role = 'm' AND dbid IN ({dbid_list})")[0]) + + # if some have 'preferred_role'='p' and some have 'preferred_role'='m' - shouldn't happen, error out, needs to be resolved manually. + # also some sanity check that there are no other values in catalog except 'm' and 'p' for 'preferred_role'. + if ((cnt_preferred_role_p > 0 and cnt_preferred_role_m > 0) or + (cnt_preferred_role_p + cnt_preferred_role_m != len(segids))): + raise Exception("Error in catalog configuration: " + f"for dbid list ({dbid_list}) " + f"{cnt_preferred_role_p} have 'p' preferred role, and " + f"{cnt_preferred_role_m} have 'm' preferred role") + + is_catalog_update_required = False + is_gprecoverseg_required = False + + if direction == self.RoleSwapDirection.PRIMARY_TO_MIRROR: + # if all have 'preferred_role'='p' - it is our first run, need to update catalog and launch gprecoverseg + if cnt_preferred_role_p == len(segids): + is_catalog_update_required = True + is_gprecoverseg_required = True + else: + # if all have 'preferred_role'='m' and not all have 'role'='m' - previous gprecoverseg was interrupted, need to launch it again + if cnt_role_m != cnt_preferred_role_m: + is_gprecoverseg_required = True + # if all have 'preferred_role'='m' and 'role'='m' - we've done everything on previous interrupted run, nothing to do + else: + # moving back in MIRROR_TO_PRIMARY direction + # if all have 'preferred_role'='m' - it is our first run, need to update catalog and launch gprecoverseg + if cnt_preferred_role_m == len(segids): + is_catalog_update_required = True + is_gprecoverseg_required = True + else: + # if all have 'preferred_role'='p' and not all have 'role'='p' - previous gprecoverseg was interrupted, need to launch it again + if cnt_role_p != cnt_preferred_role_p: + is_gprecoverseg_required = True + # if all have 'preferred_role'='p' and 'role'='p' - we've done everything on previous interrupted run, nothing to do + + if is_catalog_update_required: + dbconn.execSQL(self.conn, "UPDATE gp_segment_configuration SET preferred_role = 't' WHERE " + f"content IN ({seg_list}) AND preferred_role = 'm'") + dbconn.execSQL(self.conn, "UPDATE gp_segment_configuration SET preferred_role = 'm' WHERE " + f"content IN ({seg_list}) AND preferred_role = 'p'") + dbconn.execSQL(self.conn, "UPDATE gp_segment_configuration SET preferred_role = 'p' WHERE " + f"content IN ({seg_list}) AND preferred_role = 't'") + + dbconn.execSQL(self.conn, "COMMIT") + + if direction == self.RoleSwapDirection.PRIMARY_TO_MIRROR: + inject_fault('FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR') + else: + inject_fault('FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY') + + if is_gprecoverseg_required: + recoverseg_options = "-r -a" + try: + self.cmd = GpRecoverSeg("Running gprecoverseg", options=recoverseg_options) + self.cmd.run(validateAfter=True) + except Exception as e: + logger.error(str(e)) + error_msg = f"Failed to execute 'gprecoverseg {recoverseg_options}'" + raise Exception(error_msg) + finally: + self.cmd = None + + def lookup_seg(self, seg: Segment) -> bool: + """ Look up the segment gpdb by address, port, and dataDirectory """ + for db in self.gparray.getDbList(): + if (seg.getSegmentHostName() == db.getSegmentHostName() and + seg.getSegmentPort() == db.getSegmentPort() and + seg.getSegmentDataDirectory() == db.getSegmentDataDirectory()): + return True + return False + + def create_config_file(self, moves: List[LogicalMove]) -> str: + filename = f'/tmp/ggrebalance_move_config_pid{os.getpid()}' + with open(filename, 'w') as fp: + for move in moves: + segment_current_info = move.seg + if not self.lookup_seg(segment_current_info): + self.logger.info(f'Skip segment for gpmovemirrors: {str(segment_current_info)}') + continue + cfg_line = f'{segment_current_info.getSegmentHostName()}|{segment_current_info.getSegmentPort()}|{segment_current_info.getSegmentDataDirectory()} ' + cfg_line += f'{move.dstHost.hostname}|{move.target_port}|{move.target_datadir}\n' + fp.write(cfg_line) + return filename + + def shutdown(self) -> None: + self.shutdown_requested = True + if self.cmd != None: + self.cmd.cancel() + + def state_is_final(self, state: str) -> bool: + return state == self.states_main_rebalance_flow[-1] + + def get_state_after_interrupt(self, prev_state) -> str: + prev_idx = self.states_main_rebalance_flow.index(prev_state) + return self.states_main_rebalance_flow[prev_idx + 1] + + # state callbacks start here + + @wrap_state_func_with_faults + def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: + state_from_prev_run = self.rebalance_schema.getRebalanceStateFromPreviousRun() + + if state_from_prev_run == STATE_NOT_DEFINED: + self.trigger('move_to_STATE_REBALANCE_STARTED') + elif self.state_is_final(state_from_prev_run): + self.logger.info('Cluster is already rebalanced...') + else: + self.logger.info('Continue interrupted rebalance operation...') + self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") + try: + next_state = self.get_state_after_interrupt(state_from_prev_run) + except: + self.logger.error("Can't determine next state. Try to execute cleanup.") + self.trigger('move_to_STATE_ERROR') + return + # use auto to_«state» method to recover + self.trigger(f'to_{next_state}') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_STARTED(self) -> None: + self.trigger('move_to_STATE_REBALANCE_MOVE_MIRRORS_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_MOVE_MIRRORS_STARTED(self) -> None: + self.logger.info('Rebalance - start moving mirrors') + self.process_moves(self.moves_mirrors) + self.logger.info('Rebalance - end moving mirrors') + self.trigger('move_to_STATE_REBALANCE_MOVE_MIRRORS_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_MOVE_MIRRORS_DONE(self) -> None: + if self.primary_segments_to_move: + self.trigger('move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED') + else: + self.trigger('move_to_STATE_REBALANCE_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED(self) -> None: + self.execute_role_swaps(self.primary_segments_to_move, self.RoleSwapDirection.PRIMARY_TO_MIRROR) + self.trigger('move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_MOVE_PRIMARIES_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_MOVE_PRIMARIES_STARTED(self) -> None: + self.logger.info('Rebalance - start moving primaries') + self.process_moves(self.moves_primaries) + self.logger.info('Rebalance - end moving primaries') + self.trigger('move_to_STATE_REBALANCE_MOVE_PRIMARIES_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_MOVE_PRIMARIES_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED(self) -> None: + self.execute_role_swaps(self.primary_segments_to_move, self.RoleSwapDirection.MIRROR_TO_PRIMARY) + self.trigger('move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_DONE(self) -> None: + pass + + # state callbacks end here diff --git a/gpMgmt/bin/gprebalance_modules/planner.py b/gpMgmt/bin/gprebalance_modules/planner.py index f4209c063b5f..eb60d95553a6 100755 --- a/gpMgmt/bin/gprebalance_modules/planner.py +++ b/gpMgmt/bin/gprebalance_modules/planner.py @@ -584,25 +584,6 @@ def _parse_and_resolve_hosts( return target_hostname_list, add_hostname_list, remove_hostname_list def plan(self) -> Plan: - # TODO Remove with future development. Temp code before state machine is implemented. - from gprebalance_modules.rebalance_schema import RebalanceSchema - conn = dbconn.connect(self.dburl, encoding='UTF8') - rebalance_schema = RebalanceSchema(conn) - if rebalance_schema.schemaExists(): - state_from_prev_run = rebalance_schema.getStateFromPreviousRun() - if state_from_prev_run in ['STATE_SHRINK_TABLES_DONE', - 'STATE_SHRINK_CATALOG_STARTED', - 'STATE_SHRINK_CATALOG_DONE', - 'STATE_SHRINK_SEGMENTS_STOP_STARTED', - 'STATE_SHRINK_SEGMENTS_STOP_DONE', - 'STATE_SHRINK_DONE']: - plan = ShrinkPlan([]) - plan.setTargetSegmentCount(self.options.target_segment_count) - conn.close() - return plan - conn.close() - - # Planning starts here plan = Plan() self.validate_segment_status() diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py index 000ce689dc75..34ef1c317e83 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py @@ -4,6 +4,8 @@ from gppylib.db import dbconn from gprebalance_modules.planner import Plan, deserializePlan +STATE_NOT_DEFINED = 'not defined' + def get_table_distr_segment_count(conn: dbconn.Connection, schema_name: str, table_name: str) -> int: row = dbconn.queryRow(conn, f'''SELECT p.numsegments @@ -13,6 +15,10 @@ def get_table_distr_segment_count(conn: dbconn.Connection, schema_name: str, tab return int(row[0]) class RebalanceSchema: + STATE_CATEGORY_SHRINK = 'SHRINK' + STATE_CATEGORY_REBALANCE = 'REBALANCE' + STATE_CATEGORY_MAIN = 'MAIN' + def __init__(self, conn: dbconn.Connection): self.schema_name = 'ggrebalance' self.rebalance_status = 'rebalance_status' @@ -25,7 +31,7 @@ def createSchema(self, plan: Plan) -> None: dbconn.execSQL(self.conn, f'CREATE SCHEMA {self.schema_name}') dbconn.execSQL(self.conn, f'''CREATE TABLE {self.schema_name}.{self.rebalance_status} - (status TEXT, updated TIMESTAMP WITH TIME ZONE) + (state TEXT, state_category TEXT, updated TIMESTAMP WITH TIME ZONE) DISTRIBUTED REPLICATED''') dbconn.execSQL(self.conn, f'''CREATE TABLE {self.schema_name}.{self.table_rebalance_status_detail} @@ -70,11 +76,21 @@ def schemaExists(self) -> bool: row = dbconn.queryRow(self.conn, f"SELECT COUNT(1) FROM pg_namespace WHERE nspname = '{self.schema_name}'") return int(row[0]) == 1 - def getStateFromPreviousRun(self) -> str: - cursor = dbconn.query(self.conn, f'SELECT status FROM {self.schema_name}.{self.rebalance_status} ORDER BY updated DESC LIMIT 1') - if cursor.rowcount > 0: - return str(cursor.fetchone()[0]) - return 'not defined' + def getStateFromPreviousRun(self, state_category: str) -> str: + if self.schemaExists(): + cursor = dbconn.query(self.conn, f"SELECT state FROM {self.schema_name}.{self.rebalance_status} WHERE state_category = '{state_category}' ORDER BY updated DESC LIMIT 1") + if cursor.rowcount > 0: + return str(cursor.fetchone()[0]) + return STATE_NOT_DEFINED + + def getShrinkStateFromPreviousRun(self) -> str: + return self.getStateFromPreviousRun(self.STATE_CATEGORY_SHRINK) + + def getRebalanceStateFromPreviousRun(self) -> str: + return self.getStateFromPreviousRun(self.STATE_CATEGORY_REBALANCE) + + def getMainStateFromPreviousRun(self) -> str: + return self.getStateFromPreviousRun(self.STATE_CATEGORY_MAIN) def rebalanceSchema(self, target_segment_count: int) -> None: # Before rebalancing check if the tables are already rebalanced @@ -94,11 +110,20 @@ def rebalanceSchema(self, target_segment_count: int) -> None: f'''ALTER TABLE "{self.schema_name}"."{self.saved_plan}" REBALANCE {target_segment_count}''') - def storeState(self, state: str) -> None: + def storeState(self, state: str, state_category: str) -> None: if self.schemaExists(): dbconn.execSQL(self.conn, f'''INSERT INTO {self.schema_name}.{self.rebalance_status} - VALUES ('{state}', NOW())''') + VALUES ('{state}', '{state_category}', NOW())''') + + def storeShrinkState(self, state: str) -> None: + self.storeState(state, self.STATE_CATEGORY_SHRINK) + + def storeRebalanceState(self, state: str) -> None: + self.storeState(state, self.STATE_CATEGORY_REBALANCE) + + def storeMainState(self, state: str) -> None: + self.storeState(state, self.STATE_CATEGORY_MAIN) def clearTablesToRebalanceWithStatus(self, status: str) -> None: dbconn.execSQL(self.conn, diff --git a/gpMgmt/bin/gprebalance_modules/shrink.py b/gpMgmt/bin/gprebalance_modules/shrink.py index da73f5849929..9c4616f87c3e 100644 --- a/gpMgmt/bin/gprebalance_modules/shrink.py +++ b/gpMgmt/bin/gprebalance_modules/shrink.py @@ -17,7 +17,7 @@ from gppylib.commands.gp import SEGMENT_STOP_TIMEOUT_DEFAULT, SegmentStop from gppylib.system.environment import * from gprebalance_modules.planner import * - from gprebalance_modules.rebalance_schema import RebalanceSchema + from gprebalance_modules.rebalance_schema import RebalanceSchema, STATE_NOT_DEFINED except ImportError as e: sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) @@ -52,12 +52,9 @@ class GGShrink: states = [ 'STATE_START', - 'STATE_OPTIONS_VALIDATION', 'STATE_CHECK_PREVIOUS_RUN', 'STATE_END', - 'STATE_CLEANUP', 'STATE_ERROR', - 'STATE_ROLLBACK', 'STATE_END_FROM_ROLLBACK' ] @@ -68,8 +65,6 @@ class GGShrink: # so they are reflected in the status table inside the schema, # and we can re-enter the interrupted state. states_main_shrink_flow = [ - 'STATE_SETUP_SHRINK_SCHEMA_STARTED', - 'STATE_SETUP_SHRINK_SCHEMA_DONE', 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED', 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE', 'STATE_PREPARE_SHRINK_SCHEMA_STARTED', @@ -102,36 +97,11 @@ class GGShrink: { 'trigger': 'start', 'source': 'STATE_START', - 'dest': 'STATE_OPTIONS_VALIDATION' - }, - { - 'trigger': 'move_to_STATE_CLEANUP', - 'source': 'STATE_OPTIONS_VALIDATION', - 'dest': 'STATE_CLEANUP' - }, - { - 'trigger': 'move_to_STATE_ROLLBACK', - 'source': 'STATE_OPTIONS_VALIDATION', - 'dest': 'STATE_ROLLBACK' - }, - { - 'trigger': 'move_to_STATE_CHECK_PREVIOUS_RUN', - 'source': ['STATE_OPTIONS_VALIDATION', 'STATE_ROLLBACK'], 'dest': 'STATE_CHECK_PREVIOUS_RUN' }, - { - 'trigger': 'move_to_STATE_SETUP_SHRINK_SCHEMA_STARTED', - 'source': 'STATE_CHECK_PREVIOUS_RUN', - 'dest': 'STATE_SETUP_SHRINK_SCHEMA_STARTED' - }, - { - 'trigger': 'move_to_STATE_SETUP_SHRINK_SCHEMA_DONE', - 'source': 'STATE_SETUP_SHRINK_SCHEMA_STARTED', - 'dest': 'STATE_SETUP_SHRINK_SCHEMA_DONE' - }, { 'trigger': 'move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED', - 'source': 'STATE_SETUP_SHRINK_SCHEMA_DONE', + 'source': 'STATE_CHECK_PREVIOUS_RUN', 'dest': 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED' }, { @@ -186,7 +156,7 @@ class GGShrink: }, { 'trigger': 'move_to_STATE_END', - 'source': ['STATE_SHRINK_DONE', 'STATE_CHECK_PREVIOUS_RUN', 'STATE_CLEANUP', 'STATE_END_FROM_ROLLBACK'], + 'source': ['STATE_SHRINK_DONE', 'STATE_CHECK_PREVIOUS_RUN', 'STATE_END_FROM_ROLLBACK'], 'dest': 'STATE_END' }, { @@ -231,7 +201,7 @@ class GGShrink: }, { 'trigger': 'move_to_STATE_END_FROM_ROLLBACK', - 'source': ['STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE', 'STATE_ROLLBACK', 'STATE_CHECK_PREVIOUS_RUN'], + 'source': ['STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE', 'STATE_CHECK_PREVIOUS_RUN', 'STATE_START'], 'dest': 'STATE_END_FROM_ROLLBACK' }, { @@ -241,19 +211,18 @@ class GGShrink: } ] - def __init__(self, logger: Any, dburl: dbconn.DbURL, options: Any, gpEnv: GpCoordinatorEnvironment, gpArray: gparray.GpArray, gpArrayDumpFilename: str) -> None: + def __init__(self, conn: dbconn.Connection, + schema: RebalanceSchema, logger: Any, options: Any, gpEnv: GpCoordinatorEnvironment, gpArray: gparray.GpArray, gpArrayDumpFilename: str) -> None: self.logger = logger - self.dburl = dburl self.options = options self.gpEnv = gpEnv - self.conn = dbconn.connect( - self.dburl, encoding='UTF8', allowSystemTableMods=True) + self.conn = conn self.shutdown_requested = False self.workers_for_tables_rebalance = None self.workers_for_segment_stop = None self.gparray = gpArray self.gparray_dump_file = gpArrayDumpFilename - self.rebalance_schema = RebalanceSchema(self.conn) + self.rebalance_schema = schema self.shrink_plan = None self.needs_repopulate = False self.dumped_gparray = gparray.GpArray.initFromFile(self.gparray_dump_file) if os.path.exists(self.gparray_dump_file) else None @@ -269,6 +238,28 @@ def run(self, shrinkPlan: ShrinkPlan) -> None: self.shrink_plan = shrinkPlan self.trigger('start') + def rollback(self, shrinkPlan: ShrinkPlan) -> None: + self.shrink_plan = shrinkPlan + if not self.rebalance_schema.schemaExists(): + self.logger.info("Rebalance schema doesn't exist. Can't perform rollback.") + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + return + else: + state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() + if state_from_prev_run != STATE_NOT_DEFINED: + # check maybe the state is the final one + if self.state_is_final(state_from_prev_run): + self.logger.info("Previous run was completed successfully. Can't perform rollback.") + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + return + + if not self.state_can_rollback(state_from_prev_run) or self.is_gp_segment_configuration_shrinked(): + self.logger.info("Can't perform rollback as the catalog is already updated") + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + return + + self.trigger('start') + def get_state_after_interrupt(self, prev_state) -> str: prev_idx = self.states_main_shrink_flow.index(prev_state) lower = self.states_main_shrink_flow.index('STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED') @@ -298,100 +289,87 @@ def on_every_state(self) -> None: assert self.state in self.states + self.states_main_shrink_flow + self.states_rollback_flow if self.state in self.states_main_shrink_flow + self.states_rollback_flow: - self.rebalance_schema.storeState(self.state) + self.rebalance_schema.storeShrinkState(self.state) + + def cleanup(self, prev_run_was_complete: bool) -> None: + if not prev_run_was_complete: + self.logger.warning("ggrebalance hasn't finished shrink process properly. Previous run was interrupted. " + "Some unbalanced tables can still exist.") + + # get default num segments + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + row = dbconn.queryRow(self.conn, 'SELECT gp_toolkit.gp_rebalance_numsegments_is_set()') + numsegments_is_set = bool(row[0]) + dbconn.execSQL(self.conn, 'END') + + if numsegments_is_set: + self.logger.warning('Current numsegments is not equal to default value.') + self.logger.info('Suggestion: explicitly reset the value before cleanup. Note: cluster restart will implicitly reset the value.') + + if (self.options.interactive and + not userinput.ask_yesno(None, "\nContinue with cleanup?", 'Y')): + self.logger.info('Cleanup was interrupted...') + return + + if (numsegments_is_set and + (not self.options.interactive or userinput.ask_yesno(None, "\nReset numsegments to default?", 'Y'))): + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + dbconn.execSQL(self.conn, 'SELECT gp_toolkit.gp_reset_rebalance_numsegments()') + dbconn.execSQL(self.conn, 'COMMIT') + self.logger.info('Reset numsegments to default is done.') + + if os.path.exists(self.gparray_dump_file): + os.remove(self.gparray_dump_file) + + def state_is_final(self, state: str) -> bool: + return state == self.states_main_shrink_flow[-1] # state callbacks start here - @wrap_state_func_with_faults - def on_enter_STATE_OPTIONS_VALIDATION(self) -> None: - if self.options.clean_required: - self.trigger('move_to_STATE_CLEANUP') - elif self.options.rollback_required: - self.trigger('move_to_STATE_ROLLBACK') - else: - self.trigger('move_to_STATE_CHECK_PREVIOUS_RUN') @wrap_state_func_with_faults def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: - # check if rebalance schema exists - # and whether we can get the state where we stopped in previous run + assert self.rebalance_schema.schemaExists() + # check whether we can get the state where we stopped in previous run # in order to proceed from the same point - if self.rebalance_schema.schemaExists(): - self.logger.info('Rebalance schema exists') - state_from_prev_run = self.rebalance_schema.getStateFromPreviousRun() - # check maybe the state is the final one - if state_from_prev_run == self.states_main_shrink_flow[-1]: + state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() + # check maybe the state is the final one + if self.state_is_final(state_from_prev_run): + if self.options.rollback_required: + self.logger.info(f"Previous run was completed successfully. Can't perform rollback.") + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + else: + if state_from_prev_run in self.states_rollback_flow: + self.logger.info('Continue interrupted shrink rollback operation...') + self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") + try: + next_state = self.states_rollback_flow[ self.states_rollback_flow.index(state_from_prev_run) + 1 ] + except: + self.logger.error("Can't determine next rollback state") + self.trigger('move_to_STATE_ERROR') + return + else: if self.options.rollback_required: - self.logger.info(f"Previous run was completed successfully. Can't perform rollback.") - self.trigger('move_to_STATE_END_FROM_ROLLBACK') - else: - self.logger.error('Previous run was completed successfully. Please execute cleanup before a new run.') - self.trigger('move_to_STATE_END') - return + self.trigger('move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START') + return - elif self.shrink_plan != None: - self.logger.error("Can't start a new operation, because the previous one was interrupted. " - "Please try to launch again without a plan to continue from the interrupted state, " - "or use '--rollback' or '--cleanup' options.") - self.trigger('move_to_STATE_ERROR') - return + # no state so far, so start from the beginning + if state_from_prev_run == STATE_NOT_DEFINED: + self.trigger('move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED') + return - else: - self.shrink_plan = self.rebalance_schema.retrieveSavedPlan() - if self.shrink_plan == None: - self.logger.error('No saved plan found. Try to execute cleanup.') + self.logger.info('Continue interrupted shrink operation...') + self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") + try: + next_state = self.get_state_after_interrupt(state_from_prev_run) + except: + self.logger.error("Can't determine next state. Try to execute cleanup.") self.trigger('move_to_STATE_ERROR') return - if state_from_prev_run in self.states_rollback_flow: - self.logger.info('Continue interrupted rollback operation...') - self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") - try: - next_state = self.states_rollback_flow[ self.states_rollback_flow.index(state_from_prev_run) + 1 ] - except: - self.logger.error("Can't determine next rollback state") - self.trigger('move_to_STATE_ERROR') - return - else: - if self.options.rollback_required: - self.trigger('move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START') - return - - self.logger.info('Continue interrupted operation...') - self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") - try: - next_state = self.get_state_after_interrupt(state_from_prev_run) - except: - self.logger.error("Can't determine next state. Try to execute cleanup.") - self.trigger('move_to_STATE_ERROR') - return - - # use auto to_«state» method to recover - self.trigger(f'to_{next_state}') - else: - if self.shrink_plan == None: - self.logger.error("Rebalance schema doesn't exists and no shrink plan is supplied. Please specify shrink plan.") - self.trigger('move_to_STATE_ERROR') - return - if self.gparray.get_segment_count() < self.shrink_plan.target_segment_count: - logger.error('Target segment count (%s) > current segment count (%s).\n' - 'Currently only shrink is supported (target segment count < current segment count).' - % (self.shrink_plan.target_segment_count, self.gparray.get_segment_count())) - self.trigger('move_to_STATE_ERROR') - return - - self.trigger('move_to_STATE_SETUP_SHRINK_SCHEMA_STARTED') - - @wrap_state_func_with_faults - def on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED(self) -> None: - # Create schema and status tables. - # It will also save plan in order to use it for recovering after interruption - self.rebalance_schema.createSchema(self.shrink_plan) - self.trigger('move_to_STATE_SETUP_SHRINK_SCHEMA_DONE') - - @wrap_state_func_with_faults - def on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE(self) -> None: - self.logger.info(f'Created "{self.rebalance_schema.getSchemaName()}" schema') - self.trigger('move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED') + # use auto to_«state» method to recover + self.trigger(f'to_{next_state}') @wrap_state_func_with_faults def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED(self) -> None: @@ -505,69 +483,6 @@ def on_enter_STATE_SHRINK_DONE(self) -> None: self.logger.info('Shrink is complete') self.trigger('move_to_STATE_END') - @wrap_state_func_with_faults - def on_enter_STATE_CLEANUP(self) -> None: - if not self.rebalance_schema.schemaExists(): - self.logger.info(f"Rebalance schema doesn't exist. Cleanup is not required.") - else: - state_from_prev_run = self.rebalance_schema.getStateFromPreviousRun() - if state_from_prev_run != self.states_main_shrink_flow[-1]: - self.logger.warning("ggrebalance hasn't finished shrink process properly. Previous run was interrupted. " - "Some unbalanced tables can still exist.") - - # get default num segments - dbconn.execSQL(self.conn, 'BEGIN') - dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') - row = dbconn.queryRow(self.conn, 'SELECT gp_toolkit.gp_rebalance_numsegments_is_set()') - numsegments_is_set = bool(row[0]) - dbconn.execSQL(self.conn, 'END') - - if numsegments_is_set: - self.logger.warning('Current numsegments is not equal to default value.') - self.logger.info('Suggestion: explicitly reset the value before cleanup. Note: cluster restart will implicitly reset the value.') - - if (self.options.interactive and - not userinput.ask_yesno(None, "\nContinue with cleanup?", 'Y')): - self.logger.info('Cleanup was interrupted...') - self.trigger('move_to_STATE_END') - return - - if (numsegments_is_set and - (not self.options.interactive or userinput.ask_yesno(None, "\nReset numsegments to default?", 'Y'))): - dbconn.execSQL(self.conn, 'BEGIN') - dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') - dbconn.execSQL(self.conn, 'SELECT gp_toolkit.gp_reset_rebalance_numsegments()') - dbconn.execSQL(self.conn, 'COMMIT') - self.logger.info('Reset numsegments to default is done.') - - if os.path.exists(self.gparray_dump_file): - os.remove(self.gparray_dump_file) - self.rebalance_schema.dropSchema() - self.logger.info('Cleanup is complete') - self.trigger('move_to_STATE_END') - - @wrap_state_func_with_faults - def on_enter_STATE_ROLLBACK(self) -> None: - if not self.rebalance_schema.schemaExists(): - self.logger.info("Rebalance schema doesn't exist. Can't perform rollback.") - self.trigger('move_to_STATE_END_FROM_ROLLBACK') - return - else: - state_from_prev_run = self.rebalance_schema.getStateFromPreviousRun() - if state_from_prev_run != 'not defined': - # check maybe the state is the final one - if state_from_prev_run == self.states_main_shrink_flow[-1]: - self.logger.info("Previous run was completed successfully. Can't perform rollback.") - self.trigger('move_to_STATE_END_FROM_ROLLBACK') - return - - if not self.state_can_rollback(state_from_prev_run) or self.is_gp_segment_configuration_shrinked(): - self.logger.info("Can't perform rollback as the catalog is already updated") - self.trigger('move_to_STATE_END_FROM_ROLLBACK') - return - - self.trigger('move_to_STATE_CHECK_PREVIOUS_RUN') - @wrap_state_func_with_faults def on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START(self) -> None: dbconn.execSQL(self.conn, 'BEGIN') @@ -575,7 +490,7 @@ def on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START(self) -> N dbconn.execSQL(self.conn, 'SELECT gp_toolkit.gp_reset_rebalance_numsegments()') # Store state here in case we fail before we enter 'on_every_state()' # because after COMMIT we are on a one-way road of rollback. - self.rebalance_schema.storeState(self.state) + self.rebalance_schema.storeShrinkState(self.state) dbconn.execSQL(self.conn, 'COMMIT') self.trigger('move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE') @@ -622,7 +537,7 @@ def on_enter_STATE_END_FROM_ROLLBACK(self) -> None: @wrap_state_func_with_faults def on_enter_STATE_END(self) -> None: - self.conn.close() + pass @wrap_state_func_with_faults def on_enter_STATE_ERROR(self) -> None: diff --git a/gpMgmt/test/behave/mgmt_utils/environment.py b/gpMgmt/test/behave/mgmt_utils/environment.py index 00fca9002b1b..7e3c68456f43 100644 --- a/gpMgmt/test/behave/mgmt_utils/environment.py +++ b/gpMgmt/test/behave/mgmt_utils/environment.py @@ -22,7 +22,7 @@ def before_feature(context, feature): # we should be able to run gpexpand without having a cluster initialized tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors', 'gpconfig', 'gpssh-exkeys', 'gpstop', 'gpinitsystem', 'cross_subnet', - 'gplogfilter', 'ggrebalance'] + 'gplogfilter', 'ggrebalance_basics', 'ggrebalance_shrink', 'ggrebalance_rebalance'] if set(context.feature.tags).intersection(tags_to_skip): return @@ -125,7 +125,7 @@ def before_scenario(context, scenario): tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors', 'gpconfig', 'gpssh-exkeys', 'gpstop', 'gpinitsystem', 'cross_subnet', - 'gplogfilter', 'ggrebalance'] + 'gplogfilter', 'ggrebalance_basics', 'ggrebalance_shrink', 'ggrebalance_rebalance'] if set(context.feature.tags).intersection(tags_to_skip): return @@ -158,7 +158,7 @@ def after_scenario(context, scenario): # NOTE: gpconfig after_scenario cleanup is in the step `the gpconfig context is setup` tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpinitstandby', 'gpconfig', 'gpstop', 'gpinitsystem', 'cross_subnet', - 'gplogfilter', 'ggrebalance'] + 'gplogfilter', 'ggrebalance_basics', 'ggrebalance_shrink', 'ggrebalance_rebalance'] if set(context.feature.tags).intersection(tags_to_skip): return diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature new file mode 100755 index 000000000000..942d225105af --- /dev/null +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature @@ -0,0 +1,80 @@ +@ggrebalance_basics +Feature: ggrebalance behave tests + + Scenario Outline: test 1. validate incompatible option combinations + Given a standard local demo cluster is running + When the user runs "ggrebalance " + Then ggrebalance should return a return code of 1 + And ggrebalance should print "" to stdout + Given + + Examples: Mutually exclusive options + | options | error_message | run_status | + | --target-hosts sdw1,sdw2 --target-hosts-file /tmp/hosts.txt | Can't use together options '--target-hosts' and '--target-hosts-file' | stub | + | --target-hosts sdw1,sdw2 --add-hosts sdw3 | Can't use together options '--target-hosts' and '--add-hosts' | stub | + | --target-hosts sdw1,sdw2 --remove-hosts sdw3 | Can't use together options '--target-hosts' and '--remove-hosts' | stub | + | --add-hosts sdw3 --add-hosts-file /tmp/add.txt | Can't use together options '--add-hosts' and '--add-hosts-file' | stub | + | --remove-hosts sdw3 --remove-hosts-file /tmp/rm.txt | Can't use together options '--remove-hosts' and '--remove-hosts-file' | stub | + | --add-hosts sdw3 --remove-hosts sdw3 | Can't use together options '--add-hosts' and '--remove-hosts' | stub | + | --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' --target-datadirs-file /tmp/datadirs.txt | Can't use together options '--target-datadirs' and '--target-datadirs-file' | stub | + | --mirror-mode grouped --skip-rebalance | Can't use together options '--skip-rebalance' and '--mirror-mode' | stub | + | -m spread --skip-rebalance | Can't use together options '--skip-rebalance' and '--mirror-mode' | stub | + | -c --target-hosts sdw1,sdw2 | Can't use together options '--clean-required' and '--target-hosts' | stub | + | -c --add-hosts sdw3 | Can't use together options '--clean-required' and '--add-hosts' | stub | + | -c --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' | Can't use together options '--clean-required' and '--target-datadirs' | stub | + | -c -x 2 | Can't use together options '--clean-required' and '--target-segment-count' | stub | + | -c --mirror-mode grouped | Can't use together options '--clean-required' and '--mirror-mode' | stub | + | -c --skip-rebalance | Can't use together options '--clean-required' and '--skip-rebalance' | stub | + | -c --show-plan | Can't use together options '--clean-required' and '--show-plan' | stub | + | -c --skip-resource-estimation | Can't use together options '--clean-required' and '--skip-resource-estimation' | the database is not running | + + Scenario: test 2. ggrebalance simple scenarios + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2" + And segment information for content 1 is saved in context + And segment information for content 2 is saved in context + And segment information for content 3 is saved in context + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 4 --skip-rebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Skipping rebalance" to logfile with latest timestamp + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance schema doesn't exist. Cleanup is not required." to logfile with latest timestamp + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance schema doesn't exist. Can't perform rollback." to logfile with latest timestamp + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance schema doesn't exist. Can't perform rollback." to logfile with latest timestamp + When the user runs "ggrebalance -x 2 --skip-rebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + When the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Previous run was completed successfully. Please execute cleanup before a new run." to logfile with latest timestamp + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Previous run was completed successfully. Can't perform rollback." to logfile with latest timestamp + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Cleanup is complete" to logfile with latest timestamp + + Scenario: test 3. check cleanup after the target segment count was updated + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end" + When the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -c -y" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Reset numsegments to default is done." to logfile with latest timestamp + And ggrebalance should print "Cleanup is complete" to logfile with latest timestamp diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature new file mode 100755 index 000000000000..b2a1ceefa703 --- /dev/null +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -0,0 +1,256 @@ +@ggrebalance_rebalance +Feature: ggrebalance behave tests (rebalance scenarios) + + Scenario Outline: test 1. rebalance - check scenario, when we remove/add a host and rebalance the cluster (with different batch size). + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -B -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Previous run was completed successfully. Please execute cleanup before a new run" to logfile with latest timestamp + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Cluster is already balanced, no segment moves will be held." to logfile with latest timestamp + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -B -x 6 --add-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Previous run was completed successfully. Please execute cleanup before a new run" to logfile with latest timestamp + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Cluster is already balanced, no segment moves will be held." to logfile with latest timestamp + + Examples: + | batch_size | + | 1 | + | 16 | + | 64 | + | 128 | + + Scenario: test 2. rebalance - check rebalance after shrink. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 3, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 3, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 3, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 3, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 3, row count = 100 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 3" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Cluster is already balanced, no segment moves will be held." to logfile with latest timestamp + + Scenario Outline: 3. rebalance - interrupt and continue. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And set fault inject delay ms + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And unset fault inject delay + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | fault_delay_ms | + | on_enter_STATE_REBALANCE_STARTED_begin | 0 | + | on_enter_STATE_REBALANCE_STARTED_end | 0 | + | on_enter_STATE_REBALANCE_MOVE_MIRRORS_STARTED_begin | 0 | + | on_enter_STATE_REBALANCE_MOVE_MIRRORS_STARTED_end | 0 | + | on_enter_STATE_REBALANCE_MOVE_MIRRORS_DONE_begin | 0 | + | on_enter_STATE_REBALANCE_MOVE_MIRRORS_DONE_end | 0 | + | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED_begin | 0 | + | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED_end | 0 | + | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE_begin | 0 | + | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE_end | 0 | + | on_enter_STATE_REBALANCE_MOVE_PRIMARIES_STARTED_begin | 0 | + | on_enter_STATE_REBALANCE_MOVE_PRIMARIES_STARTED_end | 0 | + | on_enter_STATE_REBALANCE_MOVE_PRIMARIES_DONE_begin | 0 | + | on_enter_STATE_REBALANCE_MOVE_PRIMARIES_DONE_end | 0 | + | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED_begin | 0 | + | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED_end | 0 | + | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE_begin | 0 | + | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE_end | 0 | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 0 | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 0 | + | on_enter_STATE_REBALANCE_DONE_begin | 0 | + | on_enter_STATE_REBALANCE_DONE_end | 0 | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 1500 | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 3000 | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 1500 | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 3000 | + | on_enter_STATE_REBALANCE_MOVE_MIRRORS_STARTED_begin | 3000 | + | on_enter_STATE_REBALANCE_MOVE_PRIMARIES_STARTED_begin | 3000 | + + Scenario: 4. rebalance - check rebalance after interrupted shrink. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "fault_rebalance_table_test_db_2.test_schema_2.test_table_1" + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Continue interrupted shrink operation..." to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 + + Scenario: 5. rebalance - check interrupted rebalance after shrink. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_REBALANCE_MOVE_MIRRORS_STARTED_end" + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Continue interrupted rebalance operation..." to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_shrink.feature similarity index 79% rename from gpMgmt/test/behave/mgmt_utils/ggrebalance.feature rename to gpMgmt/test/behave/mgmt_utils/ggrebalance_shrink.feature index 79d89cc040fa..5f8f705d7846 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_shrink.feature @@ -1,92 +1,13 @@ -@ggrebalance +@ggrebalance_shrink Feature: ggrebalance behave tests - Scenario Outline: Validate incompatible option combinations - Given a standard local demo cluster is running - When the user runs "ggrebalance " - Then ggrebalance should return a return code of 1 - And ggrebalance should print "" to stdout - Given - - Examples: Mutually exclusive options - | options | error_message | run_status | - | --target-hosts sdw1,sdw2 --target-hosts-file /tmp/hosts.txt | Can't use together options '--target-hosts' and '--target-hosts-file' | stub | - | --target-hosts sdw1,sdw2 --add-hosts sdw3 | Can't use together options '--target-hosts' and '--add-hosts' | stub | - | --target-hosts sdw1,sdw2 --remove-hosts sdw3 | Can't use together options '--target-hosts' and '--remove-hosts' | stub | - | --add-hosts sdw3 --add-hosts-file /tmp/add.txt | Can't use together options '--add-hosts' and '--add-hosts-file' | stub | - | --remove-hosts sdw3 --remove-hosts-file /tmp/rm.txt | Can't use together options '--remove-hosts' and '--remove-hosts-file' | stub | - | --add-hosts sdw3 --remove-hosts sdw3 | Can't use together options '--add-hosts' and '--remove-hosts' | stub | - | --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' --target-datadirs-file /tmp/datadirs.txt | Can't use together options '--target-datadirs' and '--target-datadirs-file' | stub | - | --mirror-mode grouped --skip-rebalance | Can't use together options '--skip-rebalance' and '--mirror-mode' | stub | - | -m spread --skip-rebalance | Can't use together options '--skip-rebalance' and '--mirror-mode' | stub | - | -c --target-hosts sdw1,sdw2 | Can't use together options '--clean-required' and '--target-hosts' | stub | - | -c --add-hosts sdw3 | Can't use together options '--clean-required' and '--add-hosts' | stub | - | -c --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' | Can't use together options '--clean-required' and '--target-datadirs' | stub | - | -c -x 2 | Can't use together options '--clean-required' and '--target-segment-count' | stub | - | -c --mirror-mode grouped | Can't use together options '--clean-required' and '--mirror-mode' | stub | - | -c --skip-rebalance | Can't use together options '--clean-required' and '--skip-rebalance' | stub | - | -c --show-plan | Can't use together options '--clean-required' and '--show-plan' | stub | - | -c --skip-resource-estimation | Can't use together options '--clean-required' and '--skip-resource-estimation' | the database is not running | - - Scenario: test 1.1 ggrebalance simple scenarios - Given the database is not running - And a working directory of the test as '/data/gpdata/ggrebalance' - And a cluster is created with mirrors on "cdw" and "sdw1, sdw2" - And segment information for content 1 is saved in context - And segment information for content 2 is saved in context - And segment information for content 3 is saved in context - And all files in gpAdminLogs directory are deleted - When the user runs "ggrebalance -x 4 --skip-rebalance" - Then ggrebalance should return a return code of 0 - And ggrebalance should print "Skipping rebalance" to logfile with latest timestamp - When the user runs "ggrebalance -c" - Then ggrebalance should return a return code of 0 - And ggrebalance should print "Rebalance schema doesn't exist. Cleanup is not required." to logfile with latest timestamp - When the user runs "ggrebalance -c" - Then ggrebalance should return a return code of 0 - And ggrebalance should print "Rebalance schema doesn't exist. Cleanup is not required." to logfile with latest timestamp - When the user runs "ggrebalance -r" - Then ggrebalance should return a return code of 0 - And ggrebalance should print "Rebalance schema doesn't exist. Can't perform rollback." to logfile with latest timestamp - When the user runs "ggrebalance -r" - Then ggrebalance should return a return code of 0 - And ggrebalance should print "Rebalance schema doesn't exist. Can't perform rollback." to logfile with latest timestamp - When the user runs "ggrebalance -x 2 --skip-rebalance" - Then ggrebalance should return a return code of 0 - And ggrebalance should print "Shrink is complete" to logfile with latest timestamp - And verify no segment running for saved segment information - When the user runs "ggrebalance -x 1 --skip-rebalance" - Then ggrebalance should return a return code of 0 - And ggrebalance should print "Previous run was completed successfully. Please execute cleanup before a new run." to logfile with latest timestamp - When the user runs "ggrebalance -r" - Then ggrebalance should return a return code of 0 - And ggrebalance should print "Previous run was completed successfully. Can't perform rollback." to logfile with latest timestamp - When the user runs "ggrebalance -c" - Then ggrebalance should return a return code of 0 - And ggrebalance should print "Cleanup is complete" to logfile with latest timestamp - - Scenario: test 1.2. check cleanup after the target segment count was updated - Given the database is not running - And a working directory of the test as '/data/gpdata/ggrebalance' - And a cluster is created with mirrors on "cdw" and "sdw1" - And all files in gpAdminLogs directory are deleted - And set fault inject "on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end" - When the user runs "ggrebalance -x 1 --skip-rebalance" - Then ggrebalance should return a return code of 1 - And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp - And unset fault inject - When the user runs "ggrebalance -c -y" - Then ggrebalance should return a return code of 0 - And ggrebalance should print "Reset numsegments to default is done." to logfile with latest timestamp - And ggrebalance should print "Cleanup is complete" to logfile with latest timestamp - - Scenario: test 2.1. shrink - check continue after interrupted state, if interruption is done before the rebalance schema creation + Scenario: test 1.1. shrink - check continue after interrupted state, if interruption is done before the rebalance schema creation Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1" And segment information for content 1 is saved in context And all files in gpAdminLogs directory are deleted - And set fault inject "on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED_begin" + And set fault inject "on_enter_STATE_SETUP_SCHEMA_STARTED_begin" And database "test_db_1" exists And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows @@ -113,42 +34,7 @@ Feature: ggrebalance behave tests When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 1, row count = 100 - Scenario: test 2.2. shrink - check continue after interrupted state, if interruption is done after the rebalance schema creation, but before any state is saved there - Given the database is not running - And a working directory of the test as '/data/gpdata/ggrebalance' - And a cluster is created with mirrors on "cdw" and "sdw1" - And segment information for content 1 is saved in context - And all files in gpAdminLogs directory are deleted - And set fault inject "on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED_end" - And database "test_db_1" exists - And schema "test_schema_1" exists in "test_db_1" - And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows - And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows - And database "test_db_2" exists - And schema "test_schema_2" exists in "test_db_2" - And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows - And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows - When the user runs "ggrebalance -x 1 --skip-rebalance" - Then ggrebalance should return a return code of 1 - And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp - And unset fault inject - When the user runs "ggrebalance" - Then ggrebalance should return a return code of 1 - And ggrebalance should print "Can't determine next state. Try to execute cleanup." to logfile with latest timestamp - When the user runs "ggrebalance -c -y" - Then ggrebalance should return a return code of 0 - When the user runs "ggrebalance -x 1 --skip-rebalance" - Then ggrebalance should return a return code of 0 - And ggrebalance should print "Shrink is complete" to logfile with latest timestamp - And verify no segment running for saved segment information - And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 - And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 - And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 - And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 - When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows - Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 1, row count = 100 - - Scenario Outline: test 2.3. shrink - check continue after interrupted state + Scenario Outline: test 1.2. shrink - check continue after interrupted state Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1" @@ -186,8 +72,9 @@ Feature: ggrebalance behave tests Examples: | fault_name | - | on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE_begin | - | on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE_end | + | on_enter_STATE_SETUP_SCHEMA_STARTED_end | + | on_enter_STATE_SETUP_SCHEMA_DONE_begin | + | on_enter_STATE_SETUP_SCHEMA_DONE_end | | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_begin | | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end | | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | @@ -209,7 +96,7 @@ Feature: ggrebalance behave tests | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | | fault_segment_stop_dbid_3 | - Scenario Outline: test 2.4. test shrink continue after cluster restart + Scenario Outline: test 1.3. test shrink continue after cluster restart Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1" @@ -245,12 +132,12 @@ Feature: ggrebalance behave tests | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_begin | | on_enter_STATE_SHRINK_TABLES_DONE_begin | - Scenario: test 3.1. shrink - check rollback after interrupted state, if interruption is done before the rebalance schema creation + Scenario: test 2.1. shrink - check rollback after interrupted state, if interruption is done before the rebalance schema creation Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1" And all files in gpAdminLogs directory are deleted - And set fault inject "on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED_begin" + And set fault inject "on_enter_STATE_SETUP_SCHEMA_STARTED_begin" And database "test_db_1" exists And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows @@ -275,7 +162,7 @@ Feature: ggrebalance behave tests When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 2, row count = 100 - Scenario Outline: test 3.2. shrink - check rollback after interrupted state + Scenario Outline: test 2.2. shrink - check rollback after interrupted state Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1" @@ -305,9 +192,9 @@ Feature: ggrebalance behave tests Examples: | fault_name | - | on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED_end | - | on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE_begin | - | on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE_end | + | on_enter_STATE_SETUP_SCHEMA_STARTED_end | + | on_enter_STATE_SETUP_SCHEMA_DONE_begin | + | on_enter_STATE_SETUP_SCHEMA_DONE_end | | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_begin | | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end | | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | @@ -323,7 +210,7 @@ Feature: ggrebalance behave tests | on_enter_STATE_SHRINK_CATALOG_STARTED_begin | | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | - Scenario Outline: test 3.3. shrink - check rollback after interrupted state (interruption is done after the point of no return). Rollback fails. So just continue shrink. + Scenario Outline: test 2.3. shrink - check rollback after interrupted state (interruption is done after the point of no return). Rollback fails. So just continue shrink. Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1" @@ -365,7 +252,7 @@ Feature: ggrebalance behave tests | on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED_end | | fault_segment_stop_dbid_3 | - Scenario Outline: test 3.4. shrink - check rollback after interrupted state and cluster restart + Scenario Outline: test 2.4. shrink - check rollback after interrupted state and cluster restart Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1" @@ -398,9 +285,9 @@ Feature: ggrebalance behave tests And distribution information from table "test_schema_2.test_table_4" with data in "test_db_2" is equal to segment count = 2, row count = 300 Examples: | fault_name | - | on_enter_STATE_SETUP_SHRINK_SCHEMA_STARTED_end | - | on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE_begin | - | on_enter_STATE_SETUP_SHRINK_SCHEMA_DONE_end | + | on_enter_STATE_SETUP_SCHEMA_STARTED_end | + | on_enter_STATE_SETUP_SCHEMA_DONE_begin | + | on_enter_STATE_SETUP_SCHEMA_DONE_end | | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_begin | | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end | | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | @@ -416,7 +303,7 @@ Feature: ggrebalance behave tests | on_enter_STATE_SHRINK_CATALOG_STARTED_begin | | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | - Scenario Outline: test 4.1. shrink - check continue after interrupted rollback state. In this case we fail in rollback too early, and normal shrink will be complete. + Scenario Outline: test 3.1. shrink - check continue after interrupted rollback state. In this case we fail in rollback too early, and normal shrink will be complete. Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1" @@ -456,7 +343,7 @@ Feature: ggrebalance behave tests | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_begin | | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_begin | - Scenario Outline: test 4.2. shrink - check continue after interrupted rollback state + Scenario Outline: test 3.2. shrink - check continue after interrupted rollback state Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1" @@ -517,7 +404,7 @@ Feature: ggrebalance behave tests | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_begin | | on_enter_STATE_SHRINK_TABLES_DONE_begin | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | - Scenario Outline: test 4.3. shrink - check continue after interrupted rollback state (interruption is done after rebalance schema is dropped) + Scenario Outline: test 3.3. shrink - check continue after interrupted rollback state (interruption is done after rebalance schema is dropped) Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1" @@ -559,7 +446,7 @@ Feature: ggrebalance behave tests | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE_begin | | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE_end | - Scenario Outline: test 4.4. shrink - check continue after interrupted rollback state and cluster restart + Scenario Outline: test 3.4. shrink - check continue after interrupted rollback state and cluster restart Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1" diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 620f4d6ec0f4..e8767249a819 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -1603,6 +1603,15 @@ def impl(context, filter): Given the user runs psql with "-c 'BEGIN; CREATE TEMP TABLE tempt(a int); COMMIT'" against database "postgres" ''') +@given('the cluster configuration has {segment_count} segments where "{filter}"') +@when('the cluster configuration has {segment_count} segments where "{filter}"') +@then('the cluster configuration has {segment_count} segments where "{filter}"') +def impl(context, segment_count, filter): + sql = "SELECT count(*) FROM gp_segment_configuration WHERE %s" % filter + with closing(dbconn.connect(dbconn.DbURL(), unsetSearchPath=False)) as conn: + row = dbconn.queryRow(conn, sql) + if int(row[0]) != int(segment_count): + raise Exception(f"Expected {segment_count} segments, but got {row[0]}") @given('the cluster configuration is saved for "{when}"') @then('the cluster configuration is saved for "{when}"') @@ -4461,6 +4470,18 @@ def impl(context, fault): def impl(context): os.environ['GPMGMT_FAULT_POINT'] = "" +@given('set fault inject delay {delay} ms') +@then('set fault inject delay {delay} ms') +@when('set fault inject delay {delay} ms') +def impl(context, delay): + os.environ['GPMGMT_FAULT_DELAY_MS'] = delay + +@given('unset fault inject delay') +@then('unset fault inject delay') +@when('unset fault inject delay') +def impl(context): + os.environ['GPMGMT_FAULT_DELAY_MS'] = "" + @given('stub') def impl(context): pass From 28ebc247ae64376a7800f147a5a018689ab667ee Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 5 Feb 2026 10:45:13 +1000 Subject: [PATCH 16/22] Implement proper parallel flow of the rebalance execution (#2243) Problem description: Need to update rebalance execution flow in a way that it can support parallel segment movement, and at the same time the flow must consider following limitations: - ggrebalance should save every move step and it's status in persistance storage so that failed steps may be retried, rollbacked or cancelled (rollback, retry or cancel of particular movement will be implemented later in a separate patch); - switchover actions (primary to mirror, mirror to primary) will require user approval once we implement interactive mode (later in a separate patch); - ggrebalance should consider the order of the planned movements in the primary-mirror swap scenario using 3rd intermediate, transitional host. It means that the executor can't swap the order of mirror and primary movements. Therefore, this patch: 1. Adds an entity of RebalanceStep, that contain the state of execution together with the movement definition. List of such steps is now saved to the rebalance schema. 2. Updates the state machine of the rebalance execution. Now new states, where approval will be later requested from the user, are added. And the state machine can switch between segment processing and approval request as many times as required, till all steps are processed. Execution of the rebalance steps is performed in batches. Each batch is comprised from the same type of rebalance steps, without duplication of dbids. 3. Updates the code to use '--parallel' option to config 'gpmovemirrors'/'gprecoverseg'. 4. Updates behave tests according to changes described above. --- gpMgmt/bin/ggrebalance | 7 +- gpMgmt/bin/gprebalance_modules/Makefile | 2 +- .../ggrebalance_main_sm.py | 7 +- .../bin/gprebalance_modules/ggrebalance_sm.py | 271 +++++++++++++----- .../gprebalance_modules/rebalance_schema.py | 47 +++ .../bin/gprebalance_modules/rebalance_step.py | 77 +++++ .../mgmt_utils/ggrebalance_rebalance.feature | 51 ++-- 7 files changed, 364 insertions(+), 98 deletions(-) create mode 100755 gpMgmt/bin/gprebalance_modules/rebalance_step.py diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index d3836a3a3c65..94029b590e18 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -267,6 +267,7 @@ def check_down_segments(logger: Any, dburl: dbconn.DbURL): cmd_recoverseg = None def main(options, args, parser): + conn = None try: signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGHUP, sig_handler) @@ -313,8 +314,10 @@ def main(options, args, parser): logger.error(f"{str(ex)}") sys.exit(1) + conn = dbconn.connect(dburl, encoding='UTF8', allowSystemTableMods=True) + global gg_main_rebalance_SM - gg_main_rebalance_SM = GGRebalanceMainSM(logger, dburl, options, gpenv, gparray, gparray_dump_filename) + gg_main_rebalance_SM = GGRebalanceMainSM(conn, logger, dburl, options, gpenv, gparray, gparray_dump_filename) gg_main_rebalance_SM.run() sys.exit(0) @@ -322,6 +325,8 @@ def main(options, args, parser): logger.error(f'ggrebalance failed: {e} \n\nExiting...') sys.exit(1) finally: + if conn != None: + conn.close() remove_pid_file(get_coordinatordatadir()) if __name__ == '__main__': diff --git a/gpMgmt/bin/gprebalance_modules/Makefile b/gpMgmt/bin/gprebalance_modules/Makefile index b1e8b2c959e0..b9b823bb02f3 100644 --- a/gpMgmt/bin/gprebalance_modules/Makefile +++ b/gpMgmt/bin/gprebalance_modules/Makefile @@ -12,7 +12,7 @@ shrink.py planner.py \ rebalance_commons.py \ rebalance_schema.py \ solver.py ggrebalance_sm.py \ -ggrebalance_main_sm.py +ggrebalance_main_sm.py rebalance_step.py installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)/gprebalance_modules' diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py index c900bb9d6bec..a1d347d27899 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py @@ -123,13 +123,12 @@ class GGRebalanceMainSM: } ] - def __init__(self, logger: Any, dburl: dbconn.DbURL, options: Any, gpEnv: GpCoordinatorEnvironment, gpArray: gparray.GpArray, gpArrayDumpFilename: str): + def __init__(self, conn: dbconn.Connection, logger: Any, dburl: dbconn.DbURL, options: Any, gpEnv: GpCoordinatorEnvironment, gpArray: gparray.GpArray, gpArrayDumpFilename: str): self.logger = logger self.dburl = dburl self.options = options self.gparray = gpArray - self.conn = dbconn.connect( - self.dburl, encoding='UTF8', allowSystemTableMods=True) + self.conn = conn self.rebalance_schema = RebalanceSchema(self.conn) @@ -292,7 +291,7 @@ def on_enter_STATE_REBALANCE_DONE(self) -> None: @wrap_state_func_with_faults def on_enter_STATE_END(self) -> None: - self.conn.close() + pass @wrap_state_func_with_faults def on_enter_STATE_ERROR(self) -> None: diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index b774c88fdc4b..7578ba0d2168 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -11,6 +11,7 @@ from gppylib.system.environment import * from gprebalance_modules.planner import * from gprebalance_modules.rebalance_schema import RebalanceSchema, STATE_NOT_DEFINED + from gprebalance_modules.rebalance_step import * from gppylib.fault_injection import * except ImportError as e: sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) @@ -24,14 +25,13 @@ class RebalanceSM: states_main_rebalance_flow = [ 'STATE_REBALANCE_STARTED', - 'STATE_REBALANCE_MOVE_MIRRORS_STARTED', - 'STATE_REBALANCE_MOVE_MIRRORS_DONE', - 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED', - 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE', - 'STATE_REBALANCE_MOVE_PRIMARIES_STARTED', - 'STATE_REBALANCE_MOVE_PRIMARIES_DONE', - 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED', - 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE', + 'STATE_REBALANCE_PREPARE_MOVES_STARTED', + 'STATE_REBALANCE_PREPARE_MOVES_DONE', + 'STATE_REBALANCE_EXECUTION_STARTED', + 'STATE_REBALANCE_MOVES_SUCCEEDED', + 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED', + 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE', + 'STATE_REBALANCE_EXECUTION_DONE', 'STATE_REBALANCE_DONE' ] @@ -47,48 +47,43 @@ class RebalanceSM: 'dest': 'STATE_REBALANCE_STARTED' }, { - 'trigger': 'move_to_STATE_REBALANCE_MOVE_MIRRORS_STARTED', + 'trigger': 'move_to_STATE_REBALANCE_PREPARE_MOVES_STARTED', 'source': 'STATE_REBALANCE_STARTED', - 'dest': 'STATE_REBALANCE_MOVE_MIRRORS_STARTED' + 'dest': 'STATE_REBALANCE_PREPARE_MOVES_STARTED' }, { - 'trigger': 'move_to_STATE_REBALANCE_MOVE_MIRRORS_DONE', - 'source': 'STATE_REBALANCE_MOVE_MIRRORS_STARTED', - 'dest': 'STATE_REBALANCE_MOVE_MIRRORS_DONE' + 'trigger': 'move_to_STATE_REBALANCE_PREPARE_MOVES_DONE', + 'source': 'STATE_REBALANCE_PREPARE_MOVES_STARTED', + 'dest': 'STATE_REBALANCE_PREPARE_MOVES_DONE' }, { - 'trigger': 'move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED', - 'source': 'STATE_REBALANCE_MOVE_MIRRORS_DONE', - 'dest': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED' + 'trigger': 'move_to_STATE_REBALANCE_EXECUTION_STARTED', + 'source': ['STATE_REBALANCE_PREPARE_MOVES_DONE', 'STATE_REBALANCE_MOVES_SUCCEEDED', 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE'], + 'dest': 'STATE_REBALANCE_EXECUTION_STARTED' }, { - 'trigger': 'move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE', - 'source': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED', - 'dest': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE' + 'trigger': 'move_to_STATE_REBALANCE_MOVES_SUCCEEDED', + 'source': 'STATE_REBALANCE_EXECUTION_STARTED', + 'dest': 'STATE_REBALANCE_MOVES_SUCCEEDED' }, { - 'trigger': 'move_to_STATE_REBALANCE_MOVE_PRIMARIES_STARTED', - 'source': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE', - 'dest': 'STATE_REBALANCE_MOVE_PRIMARIES_STARTED' + 'trigger': 'move_to_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED', + 'source': 'STATE_REBALANCE_EXECUTION_STARTED', + 'dest': 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED' }, { - 'trigger': 'move_to_STATE_REBALANCE_MOVE_PRIMARIES_DONE', - 'source': 'STATE_REBALANCE_MOVE_PRIMARIES_STARTED', - 'dest': 'STATE_REBALANCE_MOVE_PRIMARIES_DONE' + 'trigger': 'move_to_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE', + 'source': 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED', + 'dest': 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE' }, { - 'trigger': 'move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED', - 'source': 'STATE_REBALANCE_MOVE_PRIMARIES_DONE', - 'dest': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED' - }, - { - 'trigger': 'move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE', - 'source': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED', - 'dest': 'STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE' + 'trigger': 'move_to_STATE_REBALANCE_EXECUTION_DONE', + 'source': 'STATE_REBALANCE_EXECUTION_STARTED', + 'dest': 'STATE_REBALANCE_EXECUTION_DONE' }, { 'trigger': 'move_to_STATE_REBALANCE_DONE', - 'source': ['STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE', 'STATE_REBALANCE_MOVE_MIRRORS_DONE'], + 'source': 'STATE_REBALANCE_EXECUTION_DONE', 'dest': 'STATE_REBALANCE_DONE' } ] @@ -139,11 +134,14 @@ def run(self, plan: Plan) -> None: self.trigger('start') def process_moves(self, moves: List[LogicalMove]): + if len(moves) == 0: + return + filename = self.create_config_file(moves) gpmovemirrors_options = f'-a -i {filename}' - if self.options.batch_size is not None: - batch_size = self.options.batch_size + if self.options.parallel is not None: + batch_size = self.options.parallel # gpmovemirrors has its own limitation for batch size, # need to consider it here. if batch_size > MAX_COORDINATOR_NUM_WORKERS: @@ -243,6 +241,15 @@ def execute_role_swaps(self, segments_to_move: List[Segment], direction: RoleSwa if is_gprecoverseg_required: recoverseg_options = "-r -a" + + if self.options.parallel is not None: + batch_size = self.options.parallel + # gprecoverseg has its own limitation for batch size, + # need to consider it here. + if batch_size > MAX_COORDINATOR_NUM_WORKERS: + batch_size = MAX_COORDINATOR_NUM_WORKERS + recoverseg_options += f' -B {batch_size}' + try: self.cmd = GpRecoverSeg("Running gprecoverseg", options=recoverseg_options) self.cmd.run(validateAfter=True) @@ -284,9 +291,69 @@ def state_is_final(self, state: str) -> bool: return state == self.states_main_rebalance_flow[-1] def get_state_after_interrupt(self, prev_state) -> str: + if (prev_state == 'STATE_REBALANCE_EXECUTION_STARTED' or + prev_state == 'STATE_REBALANCE_MOVES_SUCCEEDED' or + prev_state == 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE'): + return 'STATE_REBALANCE_EXECUTION_STARTED' + prev_idx = self.states_main_rebalance_flow.index(prev_state) + return self.states_main_rebalance_flow[prev_idx + 1] + def reset_in_progress_execution_steps(self) -> None: + in_progress_steps = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.IN_PROGRESS]) + for step in in_progress_steps: + step.setStatus(RebalanceStep.Status.PLANNED) + self.rebalance_schema.updateExecutionStep(step) + + @staticmethod + def convert_moves_to_rebalance_steps(moves: List[LogicalMove]) -> List[RebalanceStep]: + # In the loop below we create a list of rebalance execution steps from the plan's list of moves. + # Mirror's movemenet is simply translated into single step. + # But primary's movement is translated into 3 steps: + # 1st is switchover with the mirror; + # 2nd is movement itself; + # 3rd is the back switchover. + # If there are several consequent primary movements, we assume that they can be done in parallel, + # and, to allow batch processing for them, we re-order their steps to combine together + # the respective switchover and movement steps. + rebalance_steps = [] + + batch_mirror_steps = [] + batch_primary_steps = [[],[],[]] + + def fill_rebalance_steps(): + nonlocal rebalance_steps + nonlocal batch_mirror_steps + nonlocal batch_primary_steps + assert not (len(batch_mirror_steps) > 0 and len(batch_primary_steps[0]) > 0) + if len(batch_mirror_steps) > 0: + rebalance_steps = rebalance_steps + batch_mirror_steps + if len(batch_primary_steps[0]) > 0: + flattened_batch_primary_steps = [step for sublist in batch_primary_steps for step in sublist] + rebalance_steps = rebalance_steps + flattened_batch_primary_steps + # clear the batches + batch_mirror_steps = [] + batch_primary_steps = [[],[],[]] + + prev_move = None + for move in moves: + # if the move type switched, fill rebalance_steps with the content of + # previously gathered batches + if prev_move != None and prev_move.seg.getSegmentRole() != move.seg.getSegmentRole(): + fill_rebalance_steps() + prev_move = move + # add steps to the current batch + if move.seg.isSegmentMirror(): + batch_mirror_steps.append(RebalanceStepMoveMirror(move)) + else: + batch_primary_steps[0].append(RebalanceStepSwitchoverToMirror(move)) + batch_primary_steps[1].append(RebalanceStepMoveMirror(move)) + batch_primary_steps[2].append(RebalanceStepSwitchoverToPrimary(move)) + fill_rebalance_steps() # fill what's left + + return rebalance_steps + # state callbacks start here @wrap_state_func_with_faults @@ -311,50 +378,124 @@ def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: @wrap_state_func_with_faults def on_enter_STATE_REBALANCE_STARTED(self) -> None: - self.trigger('move_to_STATE_REBALANCE_MOVE_MIRRORS_STARTED') + self.trigger('move_to_STATE_REBALANCE_PREPARE_MOVES_STARTED') @wrap_state_func_with_faults - def on_enter_STATE_REBALANCE_MOVE_MIRRORS_STARTED(self) -> None: - self.logger.info('Rebalance - start moving mirrors') - self.process_moves(self.moves_mirrors) - self.logger.info('Rebalance - end moving mirrors') - self.trigger('move_to_STATE_REBALANCE_MOVE_MIRRORS_DONE') + def on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED(self) -> None: + if not self.rebalance_plan.getMoves(): + raise Exception('Rebalance executor was launched with a plan without segment movements') - @wrap_state_func_with_faults - def on_enter_STATE_REBALANCE_MOVE_MIRRORS_DONE(self) -> None: - if self.primary_segments_to_move: - self.trigger('move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED') - else: - self.trigger('move_to_STATE_REBALANCE_DONE') + rebalance_steps = self.convert_moves_to_rebalance_steps(self.rebalance_plan.getMoves()) + + self.logger.info('Saving following rebalance execution steps:') + + move_order = 0 + for step in rebalance_steps: + step.setMoveOrder(move_order) + move_order += 1 + self.logger.info(str(step)) + + self.rebalance_schema.saveExecutionSteps(rebalance_steps) + + self.logger.info('Saved rebalance execution steps') + + self.trigger('move_to_STATE_REBALANCE_PREPARE_MOVES_DONE') @wrap_state_func_with_faults - def on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED(self) -> None: - self.execute_role_swaps(self.primary_segments_to_move, self.RoleSwapDirection.PRIMARY_TO_MIRROR) - self.trigger('move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE') + def on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') @wrap_state_func_with_faults - def on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE(self) -> None: - self.trigger('move_to_STATE_REBALANCE_MOVE_PRIMARIES_STARTED') + def on_enter_STATE_REBALANCE_EXECUTION_STARTED(self) -> None: + + if self.rebalance_schema.allExecutionStepsAreDone(): + self.trigger('move_to_STATE_REBALANCE_EXECUTION_DONE') + return + + # In normal execution we shouldn't have IN_PROGRESS steps at this moment. + # If they are presented, it means they are left from previous interrupted run. + # Bring them back to PLANNED state, so we can try to process them again. + self.reset_in_progress_execution_steps() + + steps_to_execute = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) + + if len(steps_to_execute) > 0: + + if steps_to_execute[0].getStatus() == RebalanceStep.Status.APPROVE_REQUIRED: + self.trigger('move_to_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED') + return + + current_batch = [] + for step in steps_to_execute: + # fill current batch until we found a step that requires approve + # (it will be handled the next time we enter this state), + if (step.getStatus() == RebalanceStep.Status.APPROVE_REQUIRED or + # or till the type of RebalanceStep changes. + (len(current_batch) > 0 and (type(current_batch[0]) is not type(step)))): + break + + step.setStatus(RebalanceStep.Status.IN_PROGRESS) + self.rebalance_schema.updateExecutionStep(step) + current_batch.append(step) + + if isinstance(current_batch[0], RebalanceStepMoveMirror): + self.logger.info('Rebalance - start moving segments:') + moves = [step.getMove() for step in current_batch] + for move in moves: + self.logger.info(str(move)) + self.process_moves(moves) + self.logger.info('Rebalance - end moving segments') + else: + direction = self.RoleSwapDirection.PRIMARY_TO_MIRROR + if not isinstance(current_batch[0], RebalanceStepSwitchoverToMirror): + direction = self.RoleSwapDirection.MIRROR_TO_PRIMARY + segments = [step.getMove().seg for step in current_batch] + self.logger.info(f'Rebalance - start role swap {str(direction)} for segments:') + for segment in segments: + self.logger.info(str(segment)) + self.execute_role_swaps(segments, direction) + self.logger.info('Rebalance - end role swap') + + # TODO: check the errored segments here, once we implement rollback for the rebalance. + # For now if some error happened, the entire tool will halt its work, so if we reached this point + # just mark all steps as done. + for step in steps_to_execute: + if step.getStatus() == RebalanceStep.Status.IN_PROGRESS: + step.setStatus(RebalanceStep.Status.DONE) + self.rebalance_schema.updateExecutionStep(step) + + self.trigger('move_to_STATE_REBALANCE_MOVES_SUCCEEDED') @wrap_state_func_with_faults - def on_enter_STATE_REBALANCE_MOVE_PRIMARIES_STARTED(self) -> None: - self.logger.info('Rebalance - start moving primaries') - self.process_moves(self.moves_primaries) - self.logger.info('Rebalance - end moving primaries') - self.trigger('move_to_STATE_REBALANCE_MOVE_PRIMARIES_DONE') + def on_enter_STATE_REBALANCE_MOVES_SUCCEEDED(self) -> None: + self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') @wrap_state_func_with_faults - def on_enter_STATE_REBALANCE_MOVE_PRIMARIES_DONE(self) -> None: - self.trigger('move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED') + def on_enter_STATE_REBALANCE_EXECUTION_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_DONE') @wrap_state_func_with_faults - def on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED(self) -> None: - self.execute_role_swaps(self.primary_segments_to_move, self.RoleSwapDirection.MIRROR_TO_PRIMARY) - self.trigger('move_to_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE') + def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED(self) -> None: + + # Approve all consequent steps that require approval + steps = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) + + assert len(steps) > 0 + + for step in steps: + if step.getStatus() != RebalanceStep.Status.APPROVE_REQUIRED: + break + # TODO: we'll need to add logic here to get approval from the user in the interactive mode, + # once we start implementing the interactive mode. + # In non-interactive mode we assume that the switchover is always approved. + step.setStatus(RebalanceStep.Status.PLANNED) + self.rebalance_schema.updateExecutionStep(step) + + self.trigger('move_to_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE') @wrap_state_func_with_faults - def on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE(self) -> None: - self.trigger('move_to_STATE_REBALANCE_DONE') + def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') @wrap_state_func_with_faults def on_enter_STATE_REBALANCE_DONE(self) -> None: diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py index 34ef1c317e83..a7bdbdf35814 100644 --- a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py @@ -2,7 +2,9 @@ from psycopg2.extensions import cursor from gppylib.db import dbconn +from typing import List from gprebalance_modules.planner import Plan, deserializePlan +from gprebalance_modules.rebalance_step import * STATE_NOT_DEFINED = 'not defined' @@ -24,6 +26,7 @@ def __init__(self, conn: dbconn.Connection): self.rebalance_status = 'rebalance_status' self.table_rebalance_status_detail = 'table_rebalance_status_detail' self.saved_plan = 'saved_plan' + self.segment_move_steps = 'segment_move_steps' self.conn = conn def createSchema(self, plan: Plan) -> None: @@ -143,3 +146,47 @@ def setStatusForTableToRebalance(self, db: str, schema_name: str, rel_name: str, def getTablesToRebalanceWithStatus(self, status: str) -> cursor: return dbconn.query(self.conn, f"""SELECT db_name, schema_name, rel_name FROM {self.schema_name}.{self.table_rebalance_status_detail} WHERE status = '{status}'""") + + def saveExecutionSteps(self, steps: List[RebalanceStep]) -> None: + dbconn.execSQL(self.conn, 'BEGIN') + + dbconn.execSQL(self.conn, f'DROP TABLE IF EXISTS {self.schema_name}.{self.segment_move_steps}') + + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.segment_move_steps} + (move_order INT NOT NULL UNIQUE, status TEXT, step BYTEA) + DISTRIBUTED REPLICATED''') + + for step in steps: + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.schema_name}.{self.segment_move_steps} + VALUES ({step.getMoveOrder()}, '{step.getStatus().name}', '\\x{step.serializeStep().hex()}')''') + + dbconn.execSQL(self.conn, 'COMMIT') + + def updateExecutionStep(self, step: RebalanceStep) -> None: + dbconn.execSQL(self.conn, + f'''UPDATE {self.schema_name}.{self.segment_move_steps} + SET status='{step.getStatus().name}', step='\\x{step.serializeStep().hex()}' WHERE move_order = {step.getMoveOrder()}''') + + def allExecutionStepsAreDone(self) -> bool: + row = dbconn.queryRow(self.conn, + f"SELECT count(1) FROM {self.schema_name}.{self.segment_move_steps} WHERE status <> '{RebalanceStep.Status.DONE.name}'") + not_done_count = int(row[0]) + return not_done_count == 0 + + def getExecutionSteps(self, status_filter: List[RebalanceStep.Status]) -> List[RebalanceStep]: + result = [] + + filter = "" + if len(status_filter) > 0: + status_list = ', '.join("'" + status.name + "'" for status in status_filter) + filter = f" WHERE status IN ({status_list})" + + cursor = dbconn.query(self.conn, + f'SELECT step FROM {self.schema_name}.{self.segment_move_steps} {filter} ORDER BY move_order') + for row in cursor: + result.append(deserializeStep(row[0])) + + return result + diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_step.py b/gpMgmt/bin/gprebalance_modules/rebalance_step.py new file mode 100755 index 000000000000..39bc176d1616 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_step.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +from enum import Enum +import pickle +from gprebalance_modules.planner import * + +class RebalanceStep: + class Status(Enum): + APPROVE_REQUIRED = 1 + PLANNED = 2 + IN_PROGRESS = 3 + ERROR = 4 + ROLLBACK_PLANNED = 5 + ROLLED_BACK = 6 + CANCELLED = 7 + DONE = 8 + + def __init__(self, move: LogicalMove): + self.move_order = -1 + self.move = move + self.status = self.Status.PLANNED + + def __str__(self): + return ( + f"Rebalance step with move_order: {self.getMoveOrder()}, status: {self.getStatus()}" + ) + + def getMoveOrder(self): + return self.move_order + + def setMoveOrder(self, move_order: int): + self.move_order = move_order + + def getStatus(self): + return self.status + + def getMove(self): + return self.move + + def setStatus(self, status: Status): + self.status = status + + def serializeStep(self) -> bytes: + return pickle.dumps(self) + +class RebalanceStepMoveMirror(RebalanceStep): + def __init__(self, move: LogicalMove): + super().__init__(move) + + def __str__(self): + return ( + f"{super().__str__()}, type: RebalanceStepMoveMirror:\n" + f"{str(self.move)}" + ) + +class RebalanceStepSwitchoverToMirror(RebalanceStep): + def __init__(self, move: LogicalMove): + super().__init__(move) + self.status = self.Status.APPROVE_REQUIRED + + def __str__(self): + return ( + f"{super().__str__()}, type: RebalanceStepSwitchoverToMirror, DBID {str(self.move.seg.getSegmentDbId())}" + ) + +class RebalanceStepSwitchoverToPrimary(RebalanceStep): + def __init__(self, move: LogicalMove): + super().__init__(move) + self.status = self.Status.APPROVE_REQUIRED + + def __str__(self): + return ( + f"{super().__str__()}, type: RebalanceStepSwitchoverToPrimary, DBID {str(self.move.seg.getSegmentDbId())}" + ) + +def deserializeStep(input: bytes) -> RebalanceStep: + return pickle.loads(input) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index b2a1ceefa703..a66020dc26b0 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -1,7 +1,7 @@ @ggrebalance_rebalance Feature: ggrebalance behave tests (rebalance scenarios) - Scenario Outline: test 1. rebalance - check scenario, when we remove/add a host and rebalance the cluster (with different batch size). + Scenario Outline: test 1. rebalance - check scenario, when we remove/add a host and rebalance the cluster (with different parallel size). Given the database is not running And a working directory of the test as '/data/gpdata/ggrebalance' And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" @@ -14,7 +14,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows And all files in gpAdminLogs directory are deleted - When the user runs "ggrebalance -B -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + When the user runs "ggrebalance --parallel -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" Then ggrebalance should return a return code of 0 And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" @@ -44,7 +44,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) When the user runs "ggrebalance -c" Then ggrebalance should return a return code of 0 And all files in gpAdminLogs directory are deleted - When the user runs "ggrebalance -B -x 6 --add-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + When the user runs "ggrebalance --parallel -x 6 --add-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" Then ggrebalance should return a return code of 0 And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" @@ -72,11 +72,11 @@ Feature: ggrebalance behave tests (rebalance scenarios) And ggrebalance should print "Cluster is already balanced, no segment moves will be held." to logfile with latest timestamp Examples: - | batch_size | - | 1 | - | 16 | - | 64 | - | 128 | + | parallel_size | + | 1 | + | 16 | + | 64 | + | 96 | Scenario: test 2. rebalance - check rebalance after shrink. Given the database is not running @@ -156,22 +156,20 @@ Feature: ggrebalance behave tests (rebalance scenarios) | fault_name | fault_delay_ms | | on_enter_STATE_REBALANCE_STARTED_begin | 0 | | on_enter_STATE_REBALANCE_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_MOVE_MIRRORS_STARTED_begin | 0 | - | on_enter_STATE_REBALANCE_MOVE_MIRRORS_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_MOVE_MIRRORS_DONE_begin | 0 | - | on_enter_STATE_REBALANCE_MOVE_MIRRORS_DONE_end | 0 | - | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED_begin | 0 | - | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE_begin | 0 | - | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_PRIMARY_TO_MIRROR_DONE_end | 0 | - | on_enter_STATE_REBALANCE_MOVE_PRIMARIES_STARTED_begin | 0 | - | on_enter_STATE_REBALANCE_MOVE_PRIMARIES_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_MOVE_PRIMARIES_DONE_begin | 0 | - | on_enter_STATE_REBALANCE_MOVE_PRIMARIES_DONE_end | 0 | - | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED_begin | 0 | - | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_STARTED_end | 0 | - | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE_begin | 0 | - | on_enter_STATE_REBALANCE_SWAP_PREFERRED_ROLES_MIRROR_TO_PRIMARY_DONE_end | 0 | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_begin | 0 | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_end | 0 | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE_begin | 0 | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE_end | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_end | 0 | + | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_begin | 0 | + | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_end | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_begin | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_end | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_begin | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_end | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_DONE_begin | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_DONE_end | 0 | | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 0 | | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 0 | | on_enter_STATE_REBALANCE_DONE_begin | 0 | @@ -180,8 +178,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 3000 | | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 1500 | | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 3000 | - | on_enter_STATE_REBALANCE_MOVE_MIRRORS_STARTED_begin | 3000 | - | on_enter_STATE_REBALANCE_MOVE_PRIMARIES_STARTED_begin | 3000 | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | 3000 | Scenario: 4. rebalance - check rebalance after interrupted shrink. Given the database is not running @@ -232,7 +229,7 @@ Feature: ggrebalance behave tests (rebalance scenarios) And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows And all files in gpAdminLogs directory are deleted - And set fault inject "on_enter_STATE_REBALANCE_MOVE_MIRRORS_STARTED_end" + And set fault inject "on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin" When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp From 804cd8b673c5f111485ccddf4e2176e1e53c7aa1 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Sat, 7 Feb 2026 12:01:26 +1000 Subject: [PATCH 17/22] Add a batch of ggrebalance behave tests (#2244) This patch adds a new 'ggrebalance_misc_options' test suite, which currently has checks for: 1. '--target-hosts-file' option; 2. '--target-hosts' option; 3. '--target-datadirs-file' option; 4. '--target-datadirs' option; 5. '--mirror-mode' option; 6. '--add-hosts-file' option; 7. '--remove-hosts-file' option; 8. scenario with no mirrors in the cluster; 9. scenario when the cluster can't be rebalanced with the given parameters; 10. scenario when the cluster is in coordinator-only mode; 11. scenario when another instance of ggrebalance is running; 12. scenario when another conflicting tool is running; Also, this patch updates and adds some new step definitions, required by the new tests. Noticeable change: now we can bring up a test cluster with configurable number of segments (before it was hardcoded to 2 segments). And this patch adds a set of small fixes in the ggrebalance code to support the tested scenarios: - Move the validation that the cluster has mirrors to an earlier stage. Otherwise, without this check, ggrebalance crashed on accessing the non-existing mirror information, before it actually checked the mirror's presence. - Fix function 'get_hosts_from_file()'. Before this change, it tried to split hostname into letters (for ex., instead of 'sdw1', it returned 4 hosts: 's', 'd', 'w', '1'). Also, added a validation that the file is not empty. - Add checks for 'gpexpand' and 'pg_basebackup' tools running in parallel. --- gpMgmt/bin/ggrebalance | 18 + gpMgmt/bin/gprebalance_modules/planner.py | 6 +- .../gprebalance_modules/rebalance_commons.py | 12 +- gpMgmt/test/behave/mgmt_utils/environment.py | 6 +- .../ggrebalance_misc_options.feature | 342 ++++++++++++++++++ .../behave/mgmt_utils/steps/gpstate_utils.py | 1 + .../behave/mgmt_utils/steps/mgmt_utils.py | 68 +++- gpMgmt/test/behave_utils/cluster_setup.py | 4 +- 8 files changed, 435 insertions(+), 22 deletions(-) create mode 100755 gpMgmt/test/behave/mgmt_utils/ggrebalance_misc_options.feature diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index 94029b590e18..48c0d2daaef2 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -16,6 +16,7 @@ try: from gprebalance_modules.shrink import GGShrink, DBNAME from gprebalance_modules.ggrebalance_main_sm import GGRebalanceMainSM from gprebalance_modules.planner import * + from gppylib.programs.clsRecoverSegment_triples import get_segments_with_running_basebackup except ImportError as e: sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) @@ -196,7 +197,18 @@ def check_running_gputils(dburl: dbconn.DbURL, coordinator_data_directory: str): except IOError: pass + gpexpand_status_file = 'gpexpand.status' + if os.path.exists(os.path.join(coordinator_data_directory, gpexpand_status_file)): + logger.error(f'{gpexpand_status_file} file exists. Assuming gpexpand is already running.') + sys.exit(1) + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + cursor = dbconn.query(conn, "SELECT oid FROM pg_namespace WHERE nspname='gpexpand'") + if cursor.rowcount > 0: + logger.error( + '''gpexpand schema exists. Assuming gpexpand is already running.''') + sys.exit(1) + cursor = dbconn.query(conn, '''SELECT datid FROM pg_stat_activity WHERE application_name LIKE 'gpbackup%' OR @@ -211,6 +223,12 @@ def check_running_gputils(dburl: dbconn.DbURL, coordinator_data_directory: str): '''gprecoverseg is already running.''') sys.exit(1) + segments_with_running_basebackup = get_segments_with_running_basebackup() + if len(segments_with_running_basebackup) > 0: + logger.error( + f'''Segments {segments_with_running_basebackup} have running pg_basebackup.''') + sys.exit(1) + def create_pid_file(coordinator_data_directory: str): """ diff --git a/gpMgmt/bin/gprebalance_modules/planner.py b/gpMgmt/bin/gprebalance_modules/planner.py index eb60d95553a6..e5e14b64fa34 100755 --- a/gpMgmt/bin/gprebalance_modules/planner.py +++ b/gpMgmt/bin/gprebalance_modules/planner.py @@ -480,6 +480,9 @@ def __init__(self, logger: Any, dburl: dbconn.DbURL, gpArray: gparray.GpArray, o self.logger = logger self.virtual_gparray = deepcopy(self.gparray) + if not self.virtual_gparray.hasMirrors: + raise ValidationError("Cluster has mirroring disabled. Can't proceed with rebalance") + if self.options.target_hosts_file: self.options.target_hosts = get_hosts_from_file(self.options.target_hosts_file, "target-hosts-file") if self.options.add_hosts_file: @@ -740,9 +743,6 @@ def get_datadirs(self) -> Tuple[str, str]: def form_moves(self) -> List[LogicalMove]: self.logger.info("Validation of rebalance possibility") - if not self.virtual_gparray.hasMirrors: - raise ValidationError("Cluster has mirroring disabled. Can't proceed with rebalance") - for pair in self.virtual_gparray.segmentPairs: prim = pair.primaryDB mir = pair.mirrorDB diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_commons.py b/gpMgmt/bin/gprebalance_modules/rebalance_commons.py index 8eea94d24344..98429b0f7703 100755 --- a/gpMgmt/bin/gprebalance_modules/rebalance_commons.py +++ b/gpMgmt/bin/gprebalance_modules/rebalance_commons.py @@ -554,15 +554,19 @@ def validate_hosts_basic(hosts: str, option_name: str): raise ValidationError(f" --{option_name} must not contain IP adress and hostname simultaniously") def get_hosts_from_file(file, option_name) -> str: - result = "" + hosts = [] with open(file, 'r') as fp: i = 0 for line in fp: - i += 1 if i >= 1000: raise ValidationError(f" --{option_name} contains more than 1000 hosts") - result = ", ".join(line.strip()) - return result + hostname = line.strip() + if hostname != '': + hosts.append(line.strip()) + i += 1 + if len(hosts) == 0: + raise Exception(f"Empty '{file}' file") + return ", ".join(hosts) @dataclass class SegmentId: diff --git a/gpMgmt/test/behave/mgmt_utils/environment.py b/gpMgmt/test/behave/mgmt_utils/environment.py index 7e3c68456f43..90e1f06cd948 100644 --- a/gpMgmt/test/behave/mgmt_utils/environment.py +++ b/gpMgmt/test/behave/mgmt_utils/environment.py @@ -22,7 +22,7 @@ def before_feature(context, feature): # we should be able to run gpexpand without having a cluster initialized tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors', 'gpconfig', 'gpssh-exkeys', 'gpstop', 'gpinitsystem', 'cross_subnet', - 'gplogfilter', 'ggrebalance_basics', 'ggrebalance_shrink', 'ggrebalance_rebalance'] + 'gplogfilter', 'ggrebalance_basics', 'ggrebalance_shrink', 'ggrebalance_rebalance', 'ggrebalance_misc_options'] if set(context.feature.tags).intersection(tags_to_skip): return @@ -125,7 +125,7 @@ def before_scenario(context, scenario): tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors', 'gpconfig', 'gpssh-exkeys', 'gpstop', 'gpinitsystem', 'cross_subnet', - 'gplogfilter', 'ggrebalance_basics', 'ggrebalance_shrink', 'ggrebalance_rebalance'] + 'gplogfilter', 'ggrebalance_basics', 'ggrebalance_shrink', 'ggrebalance_rebalance', 'ggrebalance_misc_options'] if set(context.feature.tags).intersection(tags_to_skip): return @@ -158,7 +158,7 @@ def after_scenario(context, scenario): # NOTE: gpconfig after_scenario cleanup is in the step `the gpconfig context is setup` tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpinitstandby', 'gpconfig', 'gpstop', 'gpinitsystem', 'cross_subnet', - 'gplogfilter', 'ggrebalance_basics', 'ggrebalance_shrink', 'ggrebalance_rebalance'] + 'gplogfilter', 'ggrebalance_basics', 'ggrebalance_shrink', 'ggrebalance_rebalance', 'ggrebalance_misc_options'] if set(context.feature.tags).intersection(tags_to_skip): return diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_misc_options.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_misc_options.feature new file mode 100755 index 000000000000..06e7c7633722 --- /dev/null +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_misc_options.feature @@ -0,0 +1,342 @@ +@ggrebalance_misc_options +Feature: ggrebalance behave tests (misc options scenarios) + + Scenario: test 1. Check if cluster has no mirroring. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with no mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Cluster has mirroring disabled. Can't proceed with rebalance" to logfile with latest timestamp + + Scenario: test 2. Check if cluster can't be rebalanced to a balanced state with given parameters. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 5 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Cannot evenly distribute 5 segments across 2 hosts." to logfile with latest timestamp + When the user runs "ggrebalance -x 6 --remove-hosts 'sdw2, sdw3' -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Cannot perform rebalance at 1 host" to logfile with latest timestamp + + Scenario: test 3. Check rebalance with 'grouped' mirror configuration. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 3 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6 --mirror-mode grouped -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And verify that mirror segments are in "group" configuration + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + + Scenario: test 4. Check rebalance with 'spread' mirror configuration. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 3 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6 --mirror-mode spread -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And verify that mirror segments are in "spread" configuration + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + + Scenario: test 5. Check rebalance against coordinator-only mode. + Given the database is not running + And the user runs "gpstart -ma" + And "gpstart -ma" should return a return code of 0 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "System was started in single node mode - only utility mode connections are allowed" to logfile with latest timestamp + + Scenario: test 6. Check rebalance with '--target-hosts'. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1", with 4 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 4 --target-hosts 'sdw2, sdw3' -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 0 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + + Scenario: test 7. Check rebalance with '--target-hosts-file'. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1", with 4 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When there is a file "/tmp/ggrebalance_target_hosts" with hosts " " + And the user runs "ggrebalance -x 4 --target-hosts-file /tmp/ggrebalance_target_hosts -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Empty '/tmp/ggrebalance_target_hosts' file" to logfile with latest timestamp + And the temporary file "/tmp/ggrebalance_target_hosts" is removed + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 4 --target-hosts-file /tmp/ggrebalance_non_existing_file -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "No such file or directory: '/tmp/ggrebalance_non_existing_file'" to logfile with latest timestamp + And all files in gpAdminLogs directory are deleted + When there is a file "/tmp/ggrebalance_target_hosts" with hosts "sdw2|sdw3" + And the user runs "ggrebalance -x 4 --target-hosts-file /tmp/ggrebalance_target_hosts -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 0 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And the temporary file "/tmp/ggrebalance_target_hosts" is removed + + Scenario: test 8. Check rebalance with '--add-hosts-file'. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1", with 3 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And there is a file "/tmp/ggrebalance_add_hosts" with hosts "sdw2|sdw3" + When the user runs "ggrebalance -x 3 --add-hosts-file /tmp/ggrebalance_add_hosts -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 3, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 3, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 3, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 3, row count = 100 + And the temporary file "/tmp/ggrebalance_add_hosts" is removed + + Scenario: test 9. Check rebalance with '--remove-hosts-file'. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 2 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And there is a file "/tmp/ggrebalance_remove_hosts" with hosts "sdw3" + When the user runs "ggrebalance -x 4 --remove-hosts-file /tmp/ggrebalance_remove_hosts -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And the temporary file "/tmp/ggrebalance_remove_hosts" is removed + + Scenario: test 10. Check rebalance with '--target-datadirs' plain paths (without template substitution). + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 2 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 --target-datadirs '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs_new/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs_new/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has some segments where "datadir like '/home/gpadmin/gpdb\_src/gpAux/gpdemo/datadirs\_new/dbfast/gpseg_'" + And the cluster configuration has some segments where "datadir like '/home/gpadmin/gpdb\_src/gpAux/gpdemo/datadirs\_new/dbfast_mirror/gpseg_'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + + Scenario: test 11. Check rebalance with '--target-datadirs' {content} and {hostname} substitution. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 2 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 --target-datadirs '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs_new/dbfast/new_seg{content}_on_host_{hostname}, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs_new/dbfast_mirror/new_seg{content}_on_host_{hostname}'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has some segments where "datadir like '/home/gpadmin/gpdb\_src/gpAux/gpdemo/datadirs\_new/dbfast/new\_seg_\_on\_host\_sdw_'" + And the cluster configuration has some segments where "datadir like '/home/gpadmin/gpdb\_src/gpAux/gpdemo/datadirs\_new/dbfast\_mirror/new\_seg_\_on\_host\_sdw_'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + + Scenario: test 12. Check rebalance with '--target-datadirs-file' {content} and {hostname} substitution. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 2 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And there is a file "/tmp/ggrebalance_target_datadirs" with hosts "/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs_{hostname}/dbfast/new_seg{content}|/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs_{hostname}/dbfast_mirror/new_seg{content}" + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 --target-datadirs-file /tmp/ggrebalance_target_datadirs" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has some segments where "datadir like '/home/gpadmin/gpdb\_src/gpAux/gpdemo/datadirs\_sdw_/dbfast/new\_seg_'" + And the cluster configuration has some segments where "datadir like '/home/gpadmin/gpdb\_src/gpAux/gpdemo/datadirs\_sdw_/dbfast\_mirror/new\_seg_'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And the temporary file "/tmp/ggrebalance_target_datadirs" is removed + + Scenario: test 13. Check ggrebalance launch when pid file (or any other mark) for other tool exists. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + When we run a sample background script to generate a pid on "coordinator" segment + And a sample ggrebalance.pid file is created using the background pid in the coordinator_data_directory + And the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance is already running." to logfile with latest timestamp + And the background pid is killed on "coordinator" segment + And a sample ggrebalance.pid file is removed from the coordinator_data_directory + And all files in gpAdminLogs directory are deleted + When we run a sample background script to generate a pid on "coordinator" segment + And a sample gpexpand.pid file is created using the background pid in the coordinator_data_directory + And the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "gpexpand is already running." to logfile with latest timestamp + And the background pid is killed on "coordinator" segment + And a sample gpexpand.pid file is removed from the coordinator_data_directory + And all files in gpAdminLogs directory are deleted + When schema "gpexpand" exists in "postgres" + And the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "gpexpand schema exists. Assuming gpexpand is already running." to logfile with latest timestamp + And schema "gpexpand" is removed in "postgres" + And all files in gpAdminLogs directory are deleted + When we run a sample background script to generate a pid on "coordinator" segment + And a sample gpexpand.status file is created using the background pid in the coordinator_data_directory + And the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "gpexpand.status file exists. Assuming gpexpand is already running." to logfile with latest timestamp + And the background pid is killed on "coordinator" segment + And a sample gpexpand.status file is removed from the coordinator_data_directory + And all files in gpAdminLogs directory are deleted + When we run a sample background script to generate a pid on "coordinator" segment + And a sample gprecoverseg.lock directory is created using the background pid in coordinator_data_directory + And the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "gprecoverseg is already running." to logfile with latest timestamp + And the background pid is killed on "coordinator" segment + And the gprecoverseg lock directory is removed + And all files in gpAdminLogs directory are deleted + + Scenario: test 14. Check ggrebalance if pg_basebackup is already running + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 1 segments on each + And all the segments are running + And the segments are synchronized + And all files in gpAdminLogs directory are deleted + And the information of contents 0,1,2 is saved + And user immediately stops all mirror processes for content 0,1,2 + And user can start transactions + And the user suspend the walsender on the primary on content 0 + And the user asynchronously runs "gprecoverseg -aF" and the process is saved + And the user just waits until recovery_progress.file is created in gpAdminLogs + And user waits until gp_stat_replication table has no pg_basebackup entries for content 1,2 + And an FTS probe is triggered + And the user waits until mirror on content 1,2 is up + And verify that mirror on content 0 is down + And the gprecoverseg lock directory is removed + And user immediately stops all mirror processes for content 1,2 + And the user waits until mirror on content 1,2 is down + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Segments {0} have running pg_basebackup." to logfile with latest timestamp + And the user reset the walsender on the primary on content 0 + And the user waits until saved async process is completed + And verify that mirror on content 0 is up diff --git a/gpMgmt/test/behave/mgmt_utils/steps/gpstate_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/gpstate_utils.py index 271b1433d7de..e9e1db548523 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/gpstate_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/gpstate_utils.py @@ -19,6 +19,7 @@ def impl(context): fp.write("full:5: 1164848/1371715 kB (84%), 0/1 tablespace (...t1/demoDataDir0/base/16384/40962)\n") fp.write("incremental:6: 1/1371875 kB (1%)") +@when('a sample {lock_file} directory is created using the background pid in coordinator_data_directory') @then('a sample {lock_file} directory is created using the background pid in coordinator_data_directory') @given('a sample {lock_file} directory is created using the background pid in coordinator_data_directory') def impl(context, lock_file): diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index e8767249a819..d1dbbdd9cf79 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -963,6 +963,7 @@ def impl(context, tname, dbname, nrows): check_row_count(context, tname, dbname, int(nrows)) @given('schema "{schema_list}" exists in "{dbname}"') +@when('schema "{schema_list}" exists in "{dbname}"') @then('schema "{schema_list}" exists in "{dbname}"') def impl(context, schema_list, dbname): schemas = [s.strip() for s in schema_list.split(',')] @@ -970,6 +971,13 @@ def impl(context, schema_list, dbname): drop_schema_if_exists(context, s.strip(), dbname) create_schema(context, s.strip(), dbname) +@given('schema "{schema_list}" is removed in "{dbname}"') +@then('schema "{schema_list}" is removed in "{dbname}"') +def impl(context, schema_list, dbname): + schemas = [s.strip() for s in schema_list.split(',')] + for s in schemas: + drop_schema_if_exists(context, s.strip(), dbname) + @then('the temporary file "{filename}" is removed') def impl(context, filename): @@ -977,19 +985,25 @@ def impl(context, filename): os.remove(filename) -def create_table_file_locally(context, filename, table_list, location=os.getcwd()): - tables = table_list.split('|') +def create_value_list_file_locally(context, filename, value_list, location=os.getcwd()): + values = value_list.split('|') file_path = os.path.join(location, filename) with open(file_path, 'w') as fp: - for t in tables: + for t in values: fp.write(t + '\n') context.filename = file_path -@given('there is a file "{filename}" with tables "{table_list}"') -@then('there is a file "{filename}" with tables "{table_list}"') -def impl(context, filename, table_list): - create_table_file_locally(context, filename, table_list) +@given('there is a file "{filename}" with tables "{list}"') +@then('there is a file "{filename}" with tables "{list}"') +@given('there is a file "{filename}" with hosts "{list}"') +@when('there is a file "{filename}" with hosts "{list}"') +@then('there is a file "{filename}" with hosts "{list}"') +@given('there is a file "{filename}" with datadirs "{list}"') +@when('there is a file "{filename}" with datadirs "{list}"') +@then('there is a file "{filename}" with datadirs "{list}"') +def impl(context, filename, list): + create_value_list_file_locally(context, filename, list) @given('the row "{row_values}" is inserted into "{table}" in "{dbname}"') @@ -1610,7 +1624,10 @@ def impl(context, segment_count, filter): sql = "SELECT count(*) FROM gp_segment_configuration WHERE %s" % filter with closing(dbconn.connect(dbconn.DbURL(), unsetSearchPath=False)) as conn: row = dbconn.queryRow(conn, sql) - if int(row[0]) != int(segment_count): + if segment_count == 'some': + if int(row[0]) == 0: + raise Exception(f"Expected some segments, but got 0") + elif int(row[0]) != int(segment_count): raise Exception(f"Expected {segment_count} segments, but got {row[0]}") @given('the cluster configuration is saved for "{when}"') @@ -1681,6 +1698,30 @@ def impl(context, seg): cmd.run(validateAfter=True) +@given('a sample {lock_file} file is created using the background pid in the coordinator_data_directory') +@when('a sample {lock_file} file is created using the background pid in the coordinator_data_directory') +@then('a sample {lock_file} file is created using the background pid in the coordinator_data_directory') +def impl(context, lock_file): + if 'bg_pid' in context: + bg_pid = context.bg_pid + if not unix.check_pid(bg_pid): + raise Exception("The background process with PID {} is not running.".format(bg_pid)) + else: + bg_pid = "" + + utility_pidfile = os.path.join(get_coordinatordatadir(), lock_file) + + with open(utility_pidfile, 'w') as f: + f.write(bg_pid) + +@given('a sample {lock_file} file is removed from the coordinator_data_directory') +@when('a sample {lock_file} file is removed from the coordinator_data_directory') +@then('a sample {lock_file} file is removed from the coordinator_data_directory') +def impl(context, lock_file): + utility_pidfile = os.path.join(get_coordinatordatadir(), lock_file) + if os.path.exists(utility_pidfile): + os.remove(utility_pidfile) + @when('{process} is killed on mirror with content {contentids}') @then('{process} is killed on mirror with content {contentids}') def impl(context, process, contentids): @@ -2854,7 +2895,7 @@ def _create_working_directory(context, working_directory, mode=''): os.mkdir(context.working_directory) -def _create_cluster(context, coordinator_host, segment_host_list, hba_hostnames='0', with_mirrors=False, mirroring_configuration='group'): +def _create_cluster(context, coordinator_host, segment_host_list, hba_hostnames='0', with_mirrors=False, mirroring_configuration='group', number_of_segments=2): if segment_host_list == "": segment_host_list = [] else: @@ -2876,7 +2917,10 @@ def _create_cluster(context, coordinator_host, segment_host_list, hba_hostnames= except: pass - testcluster = TestCluster(hosts=[coordinator_host]+segment_host_list, base_dir=context.working_directory,hba_hostnames=hba_hostnames) + testcluster = TestCluster(hosts=[coordinator_host]+segment_host_list, + base_dir=context.working_directory, + hba_hostnames=hba_hostnames, + number_of_segments=number_of_segments) testcluster.reset_cluster() testcluster.create_cluster(with_mirrors=with_mirrors, mirroring_configuration=mirroring_configuration) context.gpexpand_mirrors_enabled = with_mirrors @@ -2901,6 +2945,10 @@ def impl(context, coordinator_host, segment_host_list): def impl(context, mirroring_configuration, coordinator_host, segment_host_list): _create_cluster(context, coordinator_host, segment_host_list, with_mirrors=True, mirroring_configuration=mirroring_configuration) +@given('a cluster is created with mirrors on "{coordinator_host}" and "{segment_host_list}", with {number_of_segments} segments on each') +def impl(context, coordinator_host, segment_host_list, number_of_segments): + _create_cluster(context, coordinator_host, segment_host_list, with_mirrors=True, mirroring_configuration='group', number_of_segments=int(number_of_segments)) + @given('the user runs gpexpand interview to add {num_of_segments} new segment and {num_of_hosts} new host "{hostnames}"') @when('the user runs gpexpand interview to add {num_of_segments} new segment and {num_of_hosts} new host "{hostnames}"') def impl(context, num_of_segments, num_of_hosts, hostnames): diff --git a/gpMgmt/test/behave_utils/cluster_setup.py b/gpMgmt/test/behave_utils/cluster_setup.py index bf88e04571f6..ec5e7a35c030 100755 --- a/gpMgmt/test/behave_utils/cluster_setup.py +++ b/gpMgmt/test/behave_utils/cluster_setup.py @@ -21,7 +21,7 @@ def run(self, validate=True): class TestCluster: - def __init__(self, hosts = None, base_dir = '/tmp/default_gpinitsystem', hba_hostnames='0'): + def __init__(self, hosts = None, base_dir = '/tmp/default_gpinitsystem', hba_hostnames='0', number_of_segments = 2): """ hosts: lists of cluster hosts. coordinator host will be assumed to be the first element. base_dir: cluster directory @@ -55,7 +55,7 @@ def __init__(self, hosts = None, base_dir = '/tmp/default_gpinitsystem', hba_hos # Test metadata # Whether to do gpinitsystem or not self.mirror_enabled = False - self.number_of_segments = 2 + self.number_of_segments = number_of_segments self.number_of_hosts = len(self.hosts)-1 self.number_of_expansion_hosts = 0 From 730af796baf23d483decb01b2596753c615e54d4 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 11 Feb 2026 15:53:06 +1000 Subject: [PATCH 18/22] Implement miscellaneous minor options for rebalance execution (#2245) This patch adds support for the following options: - '--hba-hostnames' It determines whether to use hostnames in pg_hba.conf. Passed directly to 'gpmovemirrors' tool. - '--replay-lag ' It determines replay lag (in GBs) allowed on the mirror when rebalancing the segments. Passed directly to the 'gprecoverseg' tool. - '--log-dir ' It determines the directory to store logs of the tool and all tools that are called by it. - '--analyze' It determines whether to run ANALYZE after rebalancing table redistribution. Also, this patch adds: - tests for the mentioned options; - definition of new steps required by the tests; - a small fix in the 'gpmovemirrors' tool to support log-dir with spaces in the name; - definition of STATE_ERROR into rebalance executor SM; --- gpMgmt/bin/ggrebalance | 19 ++- gpMgmt/bin/gpmovemirrors | 2 +- .../bin/gprebalance_modules/ggrebalance_sm.py | 22 ++- gpMgmt/bin/gprebalance_modules/shrink.py | 3 + .../mgmt_utils/ggrebalance_basics.feature | 3 + .../ggrebalance_misc_options.feature | 138 ++++++++++++++++++ .../behave/mgmt_utils/steps/mgmt_utils.py | 20 ++- 7 files changed, 197 insertions(+), 10 deletions(-) diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index 48c0d2daaef2..194d090170b4 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -84,6 +84,15 @@ def parseargs(): help='Hosts that user wants to remove from a cluster. One host per line') parser.add_option('--skip-resource-estimation', dest='skip_resource_estimation', metavar='', action='store_true', default=False, help='Skip resource estimation (storage)') + parser.add_option('-l', '--log-dir', dest='logfile_directory', metavar='', + help='The directory to write the log files. The default location is ~/gpAdminLogs.') + parser.add_option('-a', '--analyze', dest="analyze", action='store_true', default=False, + help='Run ANALYZE after rebalance tables redistribution.' + + 'Ignored if no shrink or expand had been performed.') + parser.add_option('--hba-hostnames', dest="hba_hostnames", action='store_true', default=False, + help='Use host names when updating the pg_hba.conf file.') + parser.add_option('--replay-lag', dest="replay_lag", type="float", + help='Replay lag(in GBs) allowed on mirror when rebalancing the segments.') parser.add_option('-v', '--verbose', action='store_true', default=False, help='debug output.') @@ -142,7 +151,7 @@ def validate_options(options, args, parser): 'target_hosts', 'target_hosts_file', 'add_hosts', 'add_hosts_file', 'remove_hosts', 'remove_hosts_file', 'target_datadirs', 'target_datadirs_file', 'target_segment_count', 'mirror_mode', 'skip_rebalance', 'show_plan', - 'skip_resource_estimation' + 'skip_resource_estimation', 'analyze', 'replay_lag', 'hba_hostnames' ]), ('rollback_required', [ @@ -258,7 +267,7 @@ def sig_handler(sig, arg): cmd_recoverseg.cancel() sys.exit(1) -def check_down_segments(logger: Any, dburl: dbconn.DbURL): +def check_down_segments(logger: Any, options: Any, dburl: dbconn.DbURL): # check if some of the segments are down - it can happen if we're recovering from an # interrupted state, and such segments are left by the interrupted gprecoverseg conn = dbconn.connect(dburl, encoding='UTF8', allowSystemTableMods=True) @@ -273,6 +282,8 @@ def check_down_segments(logger: Any, dburl: dbconn.DbURL): if cnt_mirrors_down != 0: logger.info("Some mirrors are down, trying to recover them, it may take some time...") recoverseg_options = "-a -F" + if options.logfile_directory is not None: + recoverseg_options = recoverseg_options + f' -l "{str(options.logfile_directory)}"' global cmd_recoverseg try: cmd_recoverseg = GpRecoverSeg("Running gprecoverseg", options=recoverseg_options) @@ -292,7 +303,7 @@ def main(options, args, parser): signal.signal(signal.SIGINT, sig_handler) logger = get_default_logger() - setup_tool_logging(EXECNAME, getLocalHostname(), getUserName()) + setup_tool_logging(EXECNAME, getLocalHostname(), getUserName(), options.logfile_directory) options, args = validate_options(options, args, parser) @@ -317,7 +328,7 @@ def main(options, args, parser): gparray_dump_filename = options.coordinator_data_directory + '/gparraydump' - check_down_segments(logger, dburl) + check_down_segments(logger, options, dburl) logger.info('Init gparray from catalog') try: diff --git a/gpMgmt/bin/gpmovemirrors b/gpMgmt/bin/gpmovemirrors index 11e459469206..463a7d692c99 100755 --- a/gpMgmt/bin/gpmovemirrors +++ b/gpMgmt/bin/gpmovemirrors @@ -400,7 +400,7 @@ try: recoversegOptions = "-i " + newConfig.inputFile + " -B " + str(options.batch_size) + \ " -b " + str(options.segment_batch_size) + " -v -a -d " + options.coordinator_data_directory if options.logfile_directory != None: - recoversegOptions = recoversegOptions + " -l " + str(options.logfile_directory) + recoversegOptions = recoversegOptions + f' -l "{str(options.logfile_directory)}"' logger.info('About to run gprecoverseg with options: ' + recoversegOptions) cmd = GpRecoverSeg("Running gprecoverseg", options=recoversegOptions) cmd.run(validateAfter=True) diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index 7578ba0d2168..3d371fea5d27 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -20,7 +20,8 @@ class RebalanceSM: states_not_logged = [ 'STATE_REBALANCE_INIT', - 'STATE_CHECK_PREVIOUS_RUN' + 'STATE_CHECK_PREVIOUS_RUN', + 'STATE_ERROR' ] states_main_rebalance_flow = [ @@ -85,6 +86,11 @@ class RebalanceSM: 'trigger': 'move_to_STATE_REBALANCE_DONE', 'source': 'STATE_REBALANCE_EXECUTION_DONE', 'dest': 'STATE_REBALANCE_DONE' + }, + { + 'trigger': 'move_to_STATE_ERROR', + 'source': '*', + 'dest': 'STATE_ERROR' } ] @@ -147,7 +153,10 @@ def process_moves(self, moves: List[LogicalMove]): if batch_size > MAX_COORDINATOR_NUM_WORKERS: batch_size = MAX_COORDINATOR_NUM_WORKERS gpmovemirrors_options += f' -B {batch_size}' - + if self.options.hba_hostnames: + gpmovemirrors_options = gpmovemirrors_options + ' --hba-hostnames' + if self.options.logfile_directory is not None: + gpmovemirrors_options = gpmovemirrors_options + f' -l "{str(self.options.logfile_directory)}"' try: self.cmd = GpMoveMirrors("Running gpmovemirrors", options=gpmovemirrors_options) self.cmd.run(validateAfter=True) @@ -249,7 +258,10 @@ def execute_role_swaps(self, segments_to_move: List[Segment], direction: RoleSwa if batch_size > MAX_COORDINATOR_NUM_WORKERS: batch_size = MAX_COORDINATOR_NUM_WORKERS recoverseg_options += f' -B {batch_size}' - + if self.options.replay_lag is not None: + recoverseg_options = recoverseg_options + f' --replay-lag {self.options.replay_lag}' + if self.options.logfile_directory is not None: + recoverseg_options = recoverseg_options + f' -l "{str(self.options.logfile_directory)}"' try: self.cmd = GpRecoverSeg("Running gprecoverseg", options=recoverseg_options) self.cmd.run(validateAfter=True) @@ -501,4 +513,8 @@ def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE(self) -> def on_enter_STATE_REBALANCE_DONE(self) -> None: pass + @wrap_state_func_with_faults + def on_enter_STATE_ERROR(self) -> None: + raise Exception('Rebalance execution entered STATE_ERROR') + # state callbacks end here diff --git a/gpMgmt/bin/gprebalance_modules/shrink.py b/gpMgmt/bin/gprebalance_modules/shrink.py index 9c4616f87c3e..a93c5d6e091b 100644 --- a/gpMgmt/bin/gprebalance_modules/shrink.py +++ b/gpMgmt/bin/gprebalance_modules/shrink.py @@ -614,6 +614,9 @@ def run(self) -> None: f'''ALTER TABLE "{self.schema_name}"."{self.rel_name}" REBALANCE {self.target_segment_count}''') self.shrink.rebalance_schema.setStatusForTableToRebalance(self.db_name, self.schema_name, self.rel_name, self.table_status_after_rebalance) + if self.shrink.options.analyze: + dbconn.execSQL(conn, + f'''ANALYZE "{self.schema_name}"."{self.rel_name}"''') dbconn.execSQL(conn, 'COMMIT') self.shrink.logger.info(f'Complete table rebalance for "{self.db_name}"."{self.schema_name}"."{self.rel_name}"') self.set_results(CommandResult(0, b'', b'', True, False)) diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature index 942d225105af..5300a340ca10 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature @@ -26,6 +26,9 @@ Feature: ggrebalance behave tests | -c --mirror-mode grouped | Can't use together options '--clean-required' and '--mirror-mode' | stub | | -c --skip-rebalance | Can't use together options '--clean-required' and '--skip-rebalance' | stub | | -c --show-plan | Can't use together options '--clean-required' and '--show-plan' | stub | + | -c --analyze | Can't use together options '--clean-required' and '--analyze' | stub | + | -c --replay-lag 1 | Can't use together options '--clean-required' and '--replay-lag' | stub | + | -c --hba-hostnames | Can't use together options '--clean-required' and '--hba-hostnames' | stub | | -c --skip-resource-estimation | Can't use together options '--clean-required' and '--skip-resource-estimation' | the database is not running | Scenario: test 2. ggrebalance simple scenarios diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_misc_options.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_misc_options.feature index 06e7c7633722..2ce9f0f7b1f8 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_misc_options.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_misc_options.feature @@ -340,3 +340,141 @@ Feature: ggrebalance behave tests (misc options scenarios) And the user reset the walsender on the primary on content 0 And the user waits until saved async process is completed And verify that mirror on content 0 is up + + Scenario: test 15. Check '--log-dir' option. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -l "/tmp/ggrebalance_logs" -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And gpAdminLogs directory has no "ggrebalance*" files + And gpAdminLogs directory has no "gpmovemirrors*" files + And gpAdminLogs directory has no "gprecoverseg*" files + And "/tmp/ggrebalance_logs" directory has "ggrebalance*" files + And "/tmp/ggrebalance_logs" directory has "gpmovemirrors*" files + And "/tmp/ggrebalance_logs" directory has "gprecoverseg*" files + And all files in "/tmp/ggrebalance_logs" directory are deleted + When the user runs "ggrebalance -c" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance --log-dir "/tmp/ggrebalance logs" -x 6 --add-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And gpAdminLogs directory has no "ggrebalance*" files + And gpAdminLogs directory has no "gpmovemirrors*" files + And gpAdminLogs directory has no "gprecoverseg*" files + And "/tmp/ggrebalance logs" directory has "ggrebalance*" files + And "/tmp/ggrebalance logs" directory has "gpmovemirrors*" files + And "/tmp/ggrebalance logs" directory has "gprecoverseg*" files + And all files in "/tmp/ggrebalance logs" directory are deleted + + Scenario: test 16.1. Check without '--analyze' option. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And all files in gpAdminLogs directory are deleted + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS not_analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | not_analyzed_tables_cnt | + | 2 | + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NOT NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | analyzed_tables_cnt | + | 0 | + When the user runs "ggrebalance -x 3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS not_analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | not_analyzed_tables_cnt | + | 2 | + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NOT NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | analyzed_tables_cnt | + | 0 | + + Scenario: test 16.2. Check with '--analyze' option. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And all files in gpAdminLogs directory are deleted + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS not_analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | not_analyzed_tables_cnt | + | 2 | + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NOT NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | analyzed_tables_cnt | + | 0 | + When the user runs "ggrebalance --analyze -x 3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS not_analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | not_analyzed_tables_cnt | + | 0 | + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NOT NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | analyzed_tables_cnt | + | 2 | + + Scenario: test 17. Check '--replay-lag' option. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance --replay-lag 0 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And gprecoverseg should print "0 bytes of wal is still to be replayed on mirror with dbid.*, let mirror catchup on replay then trigger rebalance" regex to logfile + And all files in gpAdminLogs directory are deleted + + Scenario: test 18-1. Check '--hba-hostnames' option. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance --hba-hostnames -x 2 --add-hosts sdw2 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And pg_hba file "/data/gpdata/ggrebalance/data/primary/gpseg0/pg_hba.conf" on host "sdw1" contains entries for "sdw2" + + Scenario: test 18-2. Check without '--hba-hostnames' option. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 2 --add-hosts sdw2 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And pg_hba file "/data/gpdata/ggrebalance/data/primary/gpseg0/pg_hba.conf" on host "sdw1" contains only cidr addresses diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index d1dbbdd9cf79..059f6e27c8f5 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -2666,7 +2666,15 @@ def impl(context, location): @when('all files in gpAdminLogs directory are deleted') @then('all files in gpAdminLogs directory are deleted') def impl(context): - log_dir = _get_gpAdminLogs_directory() + remove_all_logfiles(_get_gpAdminLogs_directory()) + +@given('all files in "{log_dir}" directory are deleted') +@when('all files in "{log_dir}" directory are deleted') +@then('all files in "{log_dir}" directory are deleted') +def impl(context, log_dir): + remove_all_logfiles(log_dir) + +def remove_all_logfiles(log_dir): files_found = glob.glob('%s/*' % (log_dir)) for file in files_found: os.remove(file) @@ -2707,7 +2715,15 @@ def impl(context): @then('gpAdminLogs directory {has} "{expected_file}" files') def impl(context, has, expected_file): - log_dir = _get_gpAdminLogs_directory() + check_logs_directory(_get_gpAdminLogs_directory(), has, expected_file) + + +@then('"{log_dir}" directory {has} "{expected_file}" files') +def impl(context, log_dir, has, expected_file): + check_logs_directory(log_dir, has, expected_file) + + +def check_logs_directory(log_dir, has, expected_file): files_found = glob.glob('%s/%s' % (log_dir, expected_file)) if files_found and (has == 'has no'): raise Exception("expected no %s files in %s, but found %s" % (expected_file, log_dir, files_found)) From a3eb478a3e8db40acf08a2e0bbab07b9fd20376a Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 13 Feb 2026 17:59:13 +1000 Subject: [PATCH 19/22] Fix ALTER MATERIALIZED VIEW ... REBALANCE (#2246) Problem description: Attempts to rebalance a materialized view via 'ALTER MATERIALIZED VIEW ... REBALANCE' command (or via equivalently working for materialized views 'ALTER TABLE ... REBALANCE') ended with an error: 'ERROR: cannot change materialized view ...' Root cause: The table rebalance logic tried to insert the data directly into the materialized view as if it were an ordinary table. It is prohibited for materialized views. Fix: Skip the call of 'ATExecShrinkTable()' for the materialized views. So during 'ALTER ... REBALANCE' only the distribution policy for the materialized view is updated. And the user needs to perform 'REFRESH MATERIALIZED VIEW ...' after the rebalance. --- src/backend/commands/tablecmds.c | 5 ++ src/test/regress/expected/alter_rebalance.out | 66 +++++++++++++++++++ src/test/regress/sql/alter_rebalance.sql | 42 ++++++++++++ 3 files changed, 113 insertions(+) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 7a1cf6d681b5..224b1c235c22 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -17651,6 +17651,11 @@ ATExecRebalanceTable(List **wqueue, Relation rel, AlterTableCmd *cmd) * child partitions. */ } + else if (rel->rd_rel->relkind == RELKIND_MATVIEW) + { + ereport((Gp_role == GP_ROLE_EXECUTE) ? DEBUG1 : NOTICE, + (errmsg("Materialized view requires REFRESH after rebalance"))); + } else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) { if (rel_is_external_table(relid)) diff --git a/src/test/regress/expected/alter_rebalance.out b/src/test/regress/expected/alter_rebalance.out index 1c2b65ae8703..0f1e88d1d6ee 100644 --- a/src/test/regress/expected/alter_rebalance.out +++ b/src/test/regress/expected/alter_rebalance.out @@ -3555,3 +3555,69 @@ alter table table_distr_hashed rebalance 3; ERROR: cannot rebalance table "table_distr_hashed" DETAIL: table has already been rebalanced drop table table_distr_hashed; +-- Check rebalance of materialized view +create table test_table(a int) distributed by (a); +insert into test_table select generate_series(1, 10); +create materialized view mv_test_table as select a from test_table distributed by (a); +alter table test_table rebalance 1; +alter materialized view mv_test_table rebalance 1; +NOTICE: Materialized view requires REFRESH after rebalance +refresh materialized view mv_test_table; +select count(1), gp_segment_id from test_table group by gp_segment_id; + count | gp_segment_id +-------+--------------- + 10 | 0 +(1 row) + +select count(1), gp_segment_id from mv_test_table group by gp_segment_id; + count | gp_segment_id +-------+--------------- + 10 | 0 +(1 row) + +drop materialized view mv_test_table; +drop table test_table; +-- Check rollback of the rebalance of materialized view +create table test_table(a int) distributed by (a); +insert into test_table select generate_series(1, 10); +create materialized view mv_test_table as select a from test_table distributed by (a); +select count(1), gp_segment_id from test_table group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 5 | 0 + 1 | 1 + 4 | 2 +(3 rows) + +select count(1), gp_segment_id from mv_test_table group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 5 | 0 + 1 | 1 + 4 | 2 +(3 rows) + +begin; +alter table test_table rebalance 1; +alter materialized view mv_test_table rebalance 1; +NOTICE: Materialized view requires REFRESH after rebalance +refresh materialized view mv_test_table; +rollback; +select count(1), gp_segment_id from test_table group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 5 | 0 + 1 | 1 + 4 | 2 +(3 rows) + +select count(1), gp_segment_id from mv_test_table group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 5 | 0 + 1 | 1 + 4 | 2 +(3 rows) + +drop materialized view mv_test_table; +drop table test_table; diff --git a/src/test/regress/sql/alter_rebalance.sql b/src/test/regress/sql/alter_rebalance.sql index accacc6db888..f11c3b3ddb19 100644 --- a/src/test/regress/sql/alter_rebalance.sql +++ b/src/test/regress/sql/alter_rebalance.sql @@ -449,3 +449,45 @@ select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id or alter table table_distr_hashed rebalance 3; drop table table_distr_hashed; + +-- Check rebalance of materialized view +-- start_ignore +drop materialized view if exists mv_test_table; +drop table if exists test_table; +-- end_ignore + +create table test_table(a int) distributed by (a); +insert into test_table select generate_series(1, 10); + +create materialized view mv_test_table as select a from test_table distributed by (a); + +alter table test_table rebalance 1; +alter materialized view mv_test_table rebalance 1; +refresh materialized view mv_test_table; + +select count(1), gp_segment_id from test_table group by gp_segment_id; +select count(1), gp_segment_id from mv_test_table group by gp_segment_id; + +drop materialized view mv_test_table; +drop table test_table; + +-- Check rollback of the rebalance of materialized view +create table test_table(a int) distributed by (a); +insert into test_table select generate_series(1, 10); + +create materialized view mv_test_table as select a from test_table distributed by (a); + +select count(1), gp_segment_id from test_table group by gp_segment_id order by gp_segment_id; +select count(1), gp_segment_id from mv_test_table group by gp_segment_id order by gp_segment_id; + +begin; +alter table test_table rebalance 1; +alter materialized view mv_test_table rebalance 1; +refresh materialized view mv_test_table; +rollback; + +select count(1), gp_segment_id from test_table group by gp_segment_id order by gp_segment_id; +select count(1), gp_segment_id from mv_test_table group by gp_segment_id order by gp_segment_id; + +drop materialized view mv_test_table; +drop table test_table; From 0385d979d304b5a3ee828b55dd4d7955c5cc26fc Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Tue, 24 Feb 2026 21:44:30 +1000 Subject: [PATCH 20/22] Use CTAS approach for rebalancing the materialized view (#2249) Problem description: Before this patch, in order to rebalance a materialized view, 2 steps were required: the actual rebalance where distribution policy was updated, and the refresh step to update the data in the materialized view. This approach had 2 problems with respect to usage in 'ggrebalance' tool for cluster shrink: 1. It could change the actual data in the materialized view before the cluster shrink, and after the shrink, if the view was not up-to-date. We intend to keep the logical data in the cluster not altered. 2. If a materialized view depends on another materialized view, there could be a race condition when doing the refresh, when we try to refresh based on the yet-not-refreshed one. Fix: Use the CTAS approach from the EXPAND TABLE specifically when we are rebalancing a materialized view. It creates a temp table with a correct distribution policy, where all data from the materialized view is copied, and then the relfilenode of the materialized view is swapped with the temp table. It keeps the data as it was before the rebalance, even if it was not up-to-date (therefore we will not surprise the user with the not expected view content), and it eliminates dependencies on other objects besides the materialized view itself. --- src/backend/commands/tablecmds.c | 28 +++++++++++++------ src/test/regress/expected/alter_rebalance.out | 4 --- src/test/regress/sql/alter_rebalance.sql | 2 -- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 224b1c235c22..f4c4412e640a 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -513,7 +513,9 @@ static void ATExecRebalanceTable(List **wqueue, Relation rel, AlterTableCmd *cmd static void ATRepackTable(Relation origTable, AlteredTableInfo *tab); static void ATExecExpandPartitionTablePrepare(Relation rel); -static void ATExecExpandTableCTAS(AlterTableCmd *rootCmd, Relation rel, AlterTableCmd *cmd); +static void ATExecRebalanceTableCTAS(AlterTableCmd *rootCmd, + Relation rel, AlterTableCmd *cmd, + int targetNumSegments); static void ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd); @@ -17513,7 +17515,7 @@ ATExecExpandTable(List **wqueue, Relation rel, AlterTableCmd *cmd) } else { - ATExecExpandTableCTAS(rootCmd, rel, cmd); + ATExecRebalanceTableCTAS(rootCmd, rel, cmd, getgpsegmentCount()); } /* Update numsegments to cluster size */ @@ -17651,11 +17653,6 @@ ATExecRebalanceTable(List **wqueue, Relation rel, AlterTableCmd *cmd) * child partitions. */ } - else if (rel->rd_rel->relkind == RELKIND_MATVIEW) - { - ereport((Gp_role == GP_ROLE_EXECUTE) ? DEBUG1 : NOTICE, - (errmsg("Materialized view requires REFRESH after rebalance"))); - } else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) { if (rel_is_external_table(relid)) @@ -17677,6 +17674,18 @@ ATExecRebalanceTable(List **wqueue, Relation rel, AlterTableCmd *cmd) return; } } + else if (rel->rd_rel->relkind == RELKIND_MATVIEW) + { + /* + * We can't insert data directly to an existing matview, + * therefore the approach from ATExecShrinkTable() is not suitable, + * and we use CTAS method for matviews. + */ + AlteredTableInfo *tab = linitial(*wqueue); + AlterTableCmd *rootCmd = + (AlterTableCmd *)linitial(tab->subcmds[AT_PASS_MISC]); + ATExecRebalanceTableCTAS(rootCmd, rel, cmd, targetNumSegments); + } else { ATExecShrinkTable(rel, newPolicy); @@ -17778,7 +17787,8 @@ ATExecExpandPartitionTablePrepare(Relation rel) } static void -ATExecExpandTableCTAS(AlterTableCmd *rootCmd, Relation rel, AlterTableCmd *cmd) +ATExecRebalanceTableCTAS(AlterTableCmd *rootCmd, Relation rel, + AlterTableCmd *cmd, int targetNumSegments) { RangeVar *tmprv; Oid tmprelid; @@ -17817,7 +17827,7 @@ ATExecExpandTableCTAS(AlterTableCmd *rootCmd, Relation rel, AlterTableCmd *cmd) /* Step (b) - build CTAS */ distby = make_distributedby_for_rel(rel); - distby->numsegments = getgpsegmentCount(); + distby->numsegments = targetNumSegments; queryDesc = build_ctas_with_dist(rel, distby, untransformRelOptions(get_rel_opts(rel)), diff --git a/src/test/regress/expected/alter_rebalance.out b/src/test/regress/expected/alter_rebalance.out index 0f1e88d1d6ee..24a6a1c892a2 100644 --- a/src/test/regress/expected/alter_rebalance.out +++ b/src/test/regress/expected/alter_rebalance.out @@ -3561,8 +3561,6 @@ insert into test_table select generate_series(1, 10); create materialized view mv_test_table as select a from test_table distributed by (a); alter table test_table rebalance 1; alter materialized view mv_test_table rebalance 1; -NOTICE: Materialized view requires REFRESH after rebalance -refresh materialized view mv_test_table; select count(1), gp_segment_id from test_table group by gp_segment_id; count | gp_segment_id -------+--------------- @@ -3600,8 +3598,6 @@ select count(1), gp_segment_id from mv_test_table group by gp_segment_id order b begin; alter table test_table rebalance 1; alter materialized view mv_test_table rebalance 1; -NOTICE: Materialized view requires REFRESH after rebalance -refresh materialized view mv_test_table; rollback; select count(1), gp_segment_id from test_table group by gp_segment_id order by gp_segment_id; count | gp_segment_id diff --git a/src/test/regress/sql/alter_rebalance.sql b/src/test/regress/sql/alter_rebalance.sql index f11c3b3ddb19..09c9555f5b06 100644 --- a/src/test/regress/sql/alter_rebalance.sql +++ b/src/test/regress/sql/alter_rebalance.sql @@ -463,7 +463,6 @@ create materialized view mv_test_table as select a from test_table distributed b alter table test_table rebalance 1; alter materialized view mv_test_table rebalance 1; -refresh materialized view mv_test_table; select count(1), gp_segment_id from test_table group by gp_segment_id; select count(1), gp_segment_id from mv_test_table group by gp_segment_id; @@ -483,7 +482,6 @@ select count(1), gp_segment_id from mv_test_table group by gp_segment_id order b begin; alter table test_table rebalance 1; alter materialized view mv_test_table rebalance 1; -refresh materialized view mv_test_table; rollback; select count(1), gp_segment_id from test_table group by gp_segment_id order by gp_segment_id; From a57039b0848283e1c417d9e8ea03eb08ffef7ce1 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 25 Feb 2026 21:21:05 +1000 Subject: [PATCH 21/22] Implement cluster shrink (2nd phase) (#2247) List of changes: - Add support for redistribution of materialized views, external writable tables, partitioned tables, unlogged tables. Skip processing of temp tables. It is done to comply with the requirements. - Add checks that the database and the table exists before we actually start to rebalance the table. It is needed as one could drop it in parallel after we have created the rebalance table list. - Add retry logic into table rebalance worker. It is needed, when for ex., other session opens a transaction after we have created the rebalance table list, drops the table before we started to rebalance it, and commits the transaction when we started to rebalance the table (and are hanging on the table's locks). - Change the order of shrunk segment processes stopping. Now mirrors are stopped strictly after primaries in order to avoid hanging replication processes. - Do not stop the tool execution in case we couldn't stop some of the shrinked segments. Now we only emit a warning. It is done to comply with the requirements. - Rework fault injection when stopping a segment due to the item above, as now we will not stop in case of an exception inside the 'SegmentStopAfterShrink' worker. So now, when a fault is injected, send SIGINT to the ggrebalance process to halt its work. - Improve logging inside 'SegmentStopAfterShrink'. - Remove not used flag 'needs_repopulate'. - Add new behave test cases and update old ones to cover the new functionality. - Add new behave step definitions to support the updates in the tests. - Fix behave test steps for view/matview creation - they opened a connection, but didn't use it. Instead, they tried to use the connection from the context, which was not properly configured. - Update code in the behave utils to support new test step definitions for materialized views and unlogged tables. - Add into the fault injector the ability to suspend execution instead of crashing it. --- gpMgmt/bin/gppylib/fault_injection.py | 8 + gpMgmt/bin/gprebalance_modules/shrink.py | 129 +++++--- .../mgmt_utils/ggrebalance_shrink.feature | 294 +++++++++++++++++- .../mgmt_utils/steps/analyzedb_mgmt_utils.py | 4 +- .../behave/mgmt_utils/steps/mgmt_utils.py | 69 +++- gpMgmt/test/behave_utils/utils.py | 15 +- 6 files changed, 468 insertions(+), 51 deletions(-) diff --git a/gpMgmt/bin/gppylib/fault_injection.py b/gpMgmt/bin/gppylib/fault_injection.py index 142ab17dfcfd..388097634575 100755 --- a/gpMgmt/bin/gppylib/fault_injection.py +++ b/gpMgmt/bin/gppylib/fault_injection.py @@ -7,9 +7,17 @@ GPMGMT_FAULT_POINT = 'GPMGMT_FAULT_POINT' GPMGMT_FAULT_DELAY_MS = 'GPMGMT_FAULT_DELAY_MS' +GPMGMT_FAULT_TYPE = 'GPMGMT_FAULT_TYPE' +GPMGMT_FAULT_FILE_FLAG = 'GPMGMT_FAULT_FILE_FLAG' + +GPMGMT_FAULT_TYPE_SYSPEND = 'suspend' def inject_fault(fault_point): if GPMGMT_FAULT_POINT in os.environ and fault_point == os.environ[GPMGMT_FAULT_POINT]: + if GPMGMT_FAULT_TYPE in os.environ and os.environ[GPMGMT_FAULT_TYPE] == GPMGMT_FAULT_TYPE_SYSPEND: + while GPMGMT_FAULT_FILE_FLAG in os.environ and os.path.exists(os.environ[GPMGMT_FAULT_FILE_FLAG]): + time.sleep(0.1) + return if GPMGMT_FAULT_DELAY_MS in os.environ and int(os.environ[GPMGMT_FAULT_DELAY_MS]) > 0: delay_ms = int(os.environ[GPMGMT_FAULT_DELAY_MS]) diff --git a/gpMgmt/bin/gprebalance_modules/shrink.py b/gpMgmt/bin/gprebalance_modules/shrink.py index a93c5d6e091b..48798267ed6c 100644 --- a/gpMgmt/bin/gprebalance_modules/shrink.py +++ b/gpMgmt/bin/gprebalance_modules/shrink.py @@ -224,7 +224,6 @@ def __init__(self, conn: dbconn.Connection, self.gparray_dump_file = gpArrayDumpFilename self.rebalance_schema = schema self.shrink_plan = None - self.needs_repopulate = False self.dumped_gparray = gparray.GpArray.initFromFile(self.gparray_dump_file) if os.path.exists(self.gparray_dump_file) else None self.machine = Machine(model = self, @@ -276,7 +275,6 @@ def get_state_after_interrupt(self, prev_state) -> str: # means that target rebalance numsegments is reset, and new tables are created at old segment count if bool(row[0]) is False: self.logger.info("Cluster restarted after previous run, trying to repopulate the relation queue") - self.needs_repopulate = True return 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED' return self.states_main_shrink_flow[prev_idx + 1] @@ -444,29 +442,28 @@ def on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED(self) -> None: gp_array = self.gparray segments_to_stop = gp_array.get_segment_count() - self.shrink_plan.getTargetSegmentCount() - segments_to_stop = segments_to_stop * 2 # consider mirrors self.workers_for_segment_stop = WorkerPool(numWorkers=min(segments_to_stop, self.options.batch_size)) - for seg_pair in gp_array.getSegmentList(): - primary_seg = seg_pair.primaryDB - mirror_seg = seg_pair.mirrorDB - if primary_seg.getSegmentContentId() >= self.shrink_plan.getTargetSegmentCount(): - if primary_seg.isSegmentUp(): - cmd = self.SegmentStopAfterShrink(self, primary_seg) + # Stop primaries first, and mirrors after primaries, + # to avoid hanging replication processes + seg_roles = [gparray.ROLE_PRIMARY, gparray.ROLE_MIRROR] + for seg_role in seg_roles: + self.logger.info(f"Prepare to stop segments with role '{seg_role}'") + for seg in gp_array.getSegDbList(): + if (seg.getSegmentContentId() >= self.shrink_plan.getTargetSegmentCount() and + seg.getSegmentRole() == seg_role and seg.isSegmentUp()): + cmd = self.SegmentStopAfterShrink(self, seg) self.workers_for_segment_stop.addCommand(cmd) - - if mirror_seg != None and mirror_seg.isSegmentUp(): - cmd = self.SegmentStopAfterShrink(self, mirror_seg) - self.workers_for_segment_stop.addCommand(cmd) - - print_progress(self.workers_for_segment_stop, interval=1) + if self.shutdown_requested: + break + print_progress(self.workers_for_segment_stop, interval=1) self.workers_for_segment_stop.haltWork() self.workers_for_segment_stop.joinWorkers() for task in self.workers_for_segment_stop.getCompletedItems(): if not task.was_successful(): - raise Exception('Failed to stop segments') + self.logger.warning('Failed to stop segments') self.workers_for_segment_stop = None @@ -566,20 +563,28 @@ def __init__(self, shrink: 'GGShrink', segment: Segment) -> None: # decorator to inject a fault before running SegmentStopAfterShrink for a specific dbid def wrap_segment_stop_with_faults(fun): def func_with_faults(self): - inject_fault(f'fault_segment_stop_dbid_{self.segment.getSegmentDbId()}') + try: + inject_fault(f'fault_segment_stop_dbid_{self.segment.getSegmentDbId()}') + except Exception as e: + os.kill(os.getpid(), signal.SIGINT) + return fun(self) return func_with_faults @wrap_segment_stop_with_faults def run(self) -> None: - self.shrink.logger.info(f'Stopping shrinked segment dbid {self.segment.getSegmentDbId()} @ host={self.remoteHost}, datadir={self.segment.getSegmentDataDirectory()}') + self.shrink.logger.info(f'Stopping shrinked segment {str(self.segment)}') self.checkRunningSegment.run() if self.checkRunningSegment.is_shutdown(): - self.shrink.logger.info(f'Segment dbid {self.segment.getSegmentDbId()} is already down @ host={self.remoteHost}, datadir={self.segment.getSegmentDataDirectory()} ') + self.shrink.logger.info(f'Segment {str(self.segment)} is already down') self.set_results(CommandResult(0, b'', b'', True, False)) else: - SegmentStop.run(self) - self.shrink.logger.info(f'Stopped shrinked segment dbid {self.segment.getSegmentDbId()} @ host={self.remoteHost}, datadir={self.segment.getSegmentDataDirectory()}') + try: + SegmentStop.run(self, validateAfter = True) + except ExecutionError: + self.shrink.logger.info(f'Failed to stop shrinked segment {str(self.segment)}') + return + self.shrink.logger.info(f'Stopped shrinked segment {str(self.segment)}') class TableRebalanceTask(SQLCommand): def __init__(self, @@ -599,28 +604,70 @@ def __init__(self, # decorator to inject a fault before running TableRebalanceTask for a specific {db_name, schema_name, rel_name} def wrap_table_rebalance_with_faults(fun): - def func_with_faults(self): + def func_with_faults(self, attempt: int): inject_fault(f'fault_rebalance_table_{self.db_name}.{self.schema_name}.{self.rel_name}') - fun(self) + fun(self, attempt) return func_with_faults + def table_exists(self, conn: dbconn.Connection, schema_name: str, rel_name: str) -> bool: + if dbconn.querySingleton(conn, f""" + SELECT count(1) + FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE c.relname = '{rel_name}' AND n.nspname = '{schema_name}' AND c.relnamespace = n.oid + """) == 0: + return False + return True + + def db_exists(self, conn: dbconn.Connection, db_name: str) -> bool: + if dbconn.querySingleton(conn, f"""SELECT count(*) FROM pg_database WHERE datname = '{db_name}'""") == 0: + return False + return True + @wrap_table_rebalance_with_faults - def run(self) -> None: - self.shrink.logger.info(f'Start table rebalance for "{self.db_name}"."{self.schema_name}"."{self.rel_name}" to {self.target_segment_count} segments') - dburl = dbconn.DbURL(dbname=self.db_name, port=self.shrink.gpEnv.getCoordinatorPort()) - with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: - dbconn.execSQL(conn, 'BEGIN') - dbconn.execSQL(conn, - f'''ALTER TABLE "{self.schema_name}"."{self.rel_name}" - REBALANCE {self.target_segment_count}''') - self.shrink.rebalance_schema.setStatusForTableToRebalance(self.db_name, self.schema_name, self.rel_name, self.table_status_after_rebalance) - if self.shrink.options.analyze: - dbconn.execSQL(conn, - f'''ANALYZE "{self.schema_name}"."{self.rel_name}"''') - dbconn.execSQL(conn, 'COMMIT') + def process_table(self, attempt: int) -> None: + self.shrink.logger.info(f'Start table rebalance for "{self.db_name}"."{self.schema_name}"."{self.rel_name}" to {self.target_segment_count} segments (attempt {attempt})') + if self.db_exists(self.shrink.rebalance_schema.conn, self.db_name): + dburl = dbconn.DbURL(dbname=self.db_name, port=self.shrink.gpEnv.getCoordinatorPort()) + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + dbconn.execSQL(conn, 'BEGIN') + + table_exists = self.table_exists(conn, self.schema_name, self.rel_name) + if table_exists: + dbconn.execSQL(conn, + f'''ALTER TABLE "{self.schema_name}"."{self.rel_name}" + REBALANCE {self.target_segment_count}''') + if self.shrink.options.analyze: + dbconn.execSQL(conn, + f'''ANALYZE "{self.schema_name}"."{self.rel_name}"''') + else: + self.shrink.logger.info(f'''Table "{self.db_name}"."{self.schema_name}"."{self.rel_name}" doesn't exist, skipping actual rebalance''') + + self.shrink.rebalance_schema.setStatusForTableToRebalance(self.db_name, self.schema_name, self.rel_name, self.table_status_after_rebalance) + dbconn.execSQL(conn, 'COMMIT') + else: + self.shrink.logger.info(f'''DB "{self.db_name}" doesn't exist, skipping actual rebalance for "{self.schema_name}"."{self.rel_name}"''') self.shrink.logger.info(f'Complete table rebalance for "{self.db_name}"."{self.schema_name}"."{self.rel_name}"') self.set_results(CommandResult(0, b'', b'', True, False)) + def run(self) -> None: + # Give 2 attempts to process a table. It is needed, when, for example, + # other session opens a transaction after we have created the rebalance table + # list, drops the table before we started to rebalance it, and commits the + # transaction when we've started to rebalance the table. + attempt_max_cnt = 2 + for i in range(attempt_max_cnt): + attempt = i + 1 + try: + self.process_table(attempt) + except Exception as e: + if attempt < attempt_max_cnt: + logger.warning(f"{str(e)}") + else: + logger.error(f"{str(e)}") + raise Exception(f'Failed to process the db object for {attempt_max_cnt} attempts') + continue + break + def prepare_shrink_schema(self, is_rollback: bool) -> None: status = 'done' if is_rollback else 'none' cmp = '<=' if is_rollback else '>' @@ -642,14 +689,18 @@ def prepare_shrink_schema(self, is_rollback: bool) -> None: dburl = dbconn.DbURL(dbname=db, port=self.gpEnv.getCoordinatorPort()) with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: cursor = dbconn.query(conn, - f'''SELECT n.nspname, c.relname + f'''SELECT n.nspname, c.relname, c.relkind, pe.writable is not null as external_writable FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid JOIN gp_distribution_policy p ON c.oid = p.localoid - WHERE c.relkind IN ('r', 'p') AND c.relispartition = FALSE AND + LEFT JOIN pg_exttable pe on (c.oid=pe.reloid and pe.writable) + WHERE c.relkind IN ('r', 'p', 'm', 'f') AND c.relispartition = FALSE AND + c.relpersistence != 't' AND p.numsegments {cmp} {self.shrink_plan.getTargetSegmentCount()} AND n.nspname NOT IN ('pg_catalog', 'information_schema', '{self.rebalance_schema.getSchemaName()}')''') - for schema_name, rel_name in cursor: + for schema_name, rel_name, rel_kind, external_writable in cursor: + if rel_kind == 'f' and not external_writable: + continue self.rebalance_schema.addTableToRebalance(db, schema_name, rel_name, status) dbconn.execSQL(self.conn, 'COMMIT') diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_shrink.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_shrink.feature index 5f8f705d7846..48241783d44f 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_shrink.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_shrink.feature @@ -12,6 +12,12 @@ Feature: ggrebalance behave tests And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" And database "test_db_2" exists And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows @@ -29,6 +35,11 @@ Feature: ggrebalance behave tests And verify no segment running for saved segment information And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows @@ -45,6 +56,14 @@ Feature: ggrebalance behave tests And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And the user connects to "gptest" with named connection "test_connection" + And the user executes "CREATE TEMP TABLE temp_table(a int);" with named connection "test_connection" + And database "gptest" exists + And the user create a writable external table with name "ext_test" And database "test_db_2" exists And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows @@ -53,6 +72,14 @@ Feature: ggrebalance behave tests Then ggrebalance should return a return code of 1 And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp And unset fault inject + And the user drops the named connection "test_connection" + When execute following sql in db "postgres" and store result in the context + """ + select count(1) as temp_tables_for_redistribute from ggrebalance.table_rebalance_status_detail where schema_name LIKE 'pg\_temp\_%'; + """ + Then validate that following rows are in the stored rows + | temp_tables_for_redistribute | + | 0 | When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1 --skip-rebalance" Then ggrebalance should return a return code of 1 And ggrebalance should print "Can't start a new operation, because the previous one was interrupted" to logfile with latest timestamp @@ -65,6 +92,11 @@ Feature: ggrebalance behave tests And verify no segment running for saved segment information And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows @@ -106,6 +138,12 @@ Feature: ggrebalance behave tests And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" And database "test_db_2" exists And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows @@ -119,18 +157,27 @@ Feature: ggrebalance behave tests Then gpstart should return a return code of 0 When there is a "heap" table "test_schema_2.test_table_3" in "test_db_2" with data And the user runs "ggrebalance" - Then ggrebalance should print "Cluster restarted after previous run, trying to repopulate the relation queue" to logfile + Then ggrebalance should return a return code of 0 And ggrebalance should print "Shrink is complete" to logfile with latest timestamp And verify no segment running for saved segment information And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 And distribution information from table "test_schema_2.test_table_3" with data in "test_db_2" is equal to segment count = 1, row count = 1094 - And ggrebalance should return a return code of 0 + Examples: | fault_name | | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_begin | | on_enter_STATE_SHRINK_TABLES_DONE_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_end | + | on_enter_STATE_SHRINK_CATALOG_STARTED_begin | + | on_enter_STATE_SHRINK_CATALOG_STARTED_end | Scenario: test 2.1. shrink - check rollback after interrupted state, if interruption is done before the rebalance schema creation Given the database is not running @@ -142,6 +189,12 @@ Feature: ggrebalance behave tests And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" And database "test_db_2" exists And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows @@ -157,6 +210,11 @@ Feature: ggrebalance behave tests Then ggrebalance should return a return code of 0 And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And the numsegments of table "ext_test" is 2 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows @@ -172,6 +230,12 @@ Feature: ggrebalance behave tests And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" And database "test_db_2" exists And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows @@ -185,6 +249,11 @@ Feature: ggrebalance behave tests And ggrebalance should print "Rollback is complete" to logfile with latest timestamp And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And the numsegments of table "ext_test" is 2 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows @@ -221,6 +290,12 @@ Feature: ggrebalance behave tests And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" And database "test_db_2" exists And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows @@ -238,6 +313,11 @@ Feature: ggrebalance behave tests And verify no segment running for saved segment information And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows @@ -261,6 +341,12 @@ Feature: ggrebalance behave tests And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" And database "test_db_2" exists And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows @@ -279,6 +365,11 @@ Feature: ggrebalance behave tests And ggrebalance should print "Rollback is complete" to logfile with latest timestamp And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And the numsegments of table "ext_test" is 2 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_2.test_table_3" with data in "test_db_2" is equal to segment count = 2, row count = 200 @@ -314,6 +405,12 @@ Feature: ggrebalance behave tests And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" And database "test_db_2" exists And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows @@ -333,6 +430,11 @@ Feature: ggrebalance behave tests And verify no segment running for saved segment information And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows @@ -353,6 +455,12 @@ Feature: ggrebalance behave tests And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" And database "test_db_2" exists And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows @@ -371,6 +479,11 @@ Feature: ggrebalance behave tests And ggrebalance should print "Rollback is complete" to logfile with latest timestamp And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And the numsegments of table "ext_test" is 2 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows @@ -414,6 +527,12 @@ Feature: ggrebalance behave tests And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" And database "test_db_2" exists And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows @@ -432,6 +551,11 @@ Feature: ggrebalance behave tests And ggrebalance should print "Rebalance schema doesn't exists and no shrink plan is supplied. Please specify shrink plan." to logfile with latest timestamp And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And the numsegments of table "ext_test" is 2 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows @@ -456,6 +580,12 @@ Feature: ggrebalance behave tests And schema "test_schema_1" exists in "test_db_1" And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" And database "test_db_2" exists And schema "test_schema_2" exists in "test_db_2" And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows @@ -478,6 +608,11 @@ Feature: ggrebalance behave tests And ggrebalance should print "Rollback is complete" to logfile with latest timestamp And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And the numsegments of table "ext_test" is 2 And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 And distribution information from table "test_schema_2.test_table_3" with data in "test_db_2" is equal to segment count = 2, row count = 200 @@ -512,3 +647,158 @@ Feature: ggrebalance behave tests | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_end | | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_begin | + Scenario: test 4. test shrink continue, when a table planned for rebalance was dropped + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When set fault inject "fault_rebalance_table_test_db_2.test_schema_2.test_table_1" + And the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And table "test_schema_2.test_table_1" is dropped in "test_db_2" + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + + Scenario: test 4.1. test shrink continue, when a mat view planned for rebalance was dropped + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When set fault inject "fault_rebalance_table_test_db_1.test_schema_1.mv_test_table_1" + And the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And materialized view "test_schema_1.mv_test_table_1" is dropped in "test_db_1" + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + + Scenario: test 5. test shrink continue, when a db with the table planned for rebalance was dropped + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When set fault inject "fault_rebalance_table_test_db_2.test_schema_2.test_table_1" + And the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And the database "test_db_2" does not exist + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 + + Scenario: test 6. test shrink, when a table, planned for rebalance, is dropped in a parallel transaction, committed after the start of table redistribution + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And there is a "heap" table "test_table_1" in "gptest" with "100" rows + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And set fault inject "on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_begin" + And set fault inject type to suspend + When the user asynchronously runs "ggrebalance -x 1 --skip-rebalance" and the process is saved + And the user waits till ggrebalance prints "Updated target segment count to 1" in the logs (with timeout of "60" sec) + And the user connects to "gptest" with named connection "test_connection" + And the user executes "BEGIN; DROP TABLE test_table_1;" with named connection "test_connection" + And unset fault inject + And the user waits till ggrebalance prints "Start table rebalance for \"gptest\".\"public\".\"test_table_1\" to 1 segments" in the logs (with timeout of "60" sec) + And waiting "5" seconds + And the user executes "COMMIT;" with named connection "test_connection" + And the user drops the named connection "test_connection" + Then the async process finished with a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 diff --git a/gpMgmt/test/behave/mgmt_utils/steps/analyzedb_mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/analyzedb_mgmt_utils.py index f7bf6f1bdfa7..43c2a172e982 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/analyzedb_mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/analyzedb_mgmt_utils.py @@ -94,13 +94,13 @@ def impl(context, view_name, table_name, schema_name): @given('a view "{view_name}" exists on table "{table_name}"') def impl(context, view_name, table_name): with closing(dbconn.connect(dbconn.DbURL(dbname=context.dbname))) as conn: - create_view_on_table(context.conn, view_name, table_name) + create_view_on_table(conn, view_name, table_name) @given('a materialized view "{view_name}" exists on table "{table_name}"') def impl(context, view_name, table_name): with closing(dbconn.connect(dbconn.DbURL(dbname=context.dbname))) as conn: - create_materialized_view_on_table_in_schema(context.conn, viewname=view_name, + create_materialized_view_on_table_in_schema(conn, viewname=view_name, tablename=table_name) diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 059f6e27c8f5..3d6f3936aa3f 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -39,6 +39,7 @@ from gppylib import pgconf from gppylib.commands.gp import get_coordinatordatadir from gppylib.parseutils import canonicalize_address +from gppylib import fault_injection coordinator_data_dir = gp.get_coordinatordatadir() if coordinator_data_dir is None: @@ -156,6 +157,8 @@ def impl(context, query, db, contentids): @given('the user connects to "{dbname}" with named connection "{cname}"') +@when('the user connects to "{dbname}" with named connection "{cname}"') +@then('the user connects to "{dbname}" with named connection "{cname}"') def impl(context, dbname, cname): if not hasattr(context, 'named_conns'): context.named_conns = {} @@ -197,11 +200,14 @@ def impl(conetxt, tabname): @given('the user executes "{sql}" with named connection "{cname}"') +@when('the user executes "{sql}" with named connection "{cname}"') +@then('the user executes "{sql}" with named connection "{cname}"') def impl(context, cname, sql): conn = context.named_conns[cname] dbconn.execSQL(conn, sql) +@when('the user drops the named connection "{cname}"') @then('the user drops the named connection "{cname}"') def impl(context, cname): if cname in context.named_conns: @@ -660,6 +666,31 @@ def impl(context, kill_process_name, log_msg, logfile_name): "fi; done" % (log_msg, logfile_name, kill_process_name) run_async_command(context, command) +@given('the user waits till {process_name} prints "{log_msg}" in the logs (with timeout of "{timeout}" sec)') +@when('the user waits till {process_name} prints "{log_msg}" in the logs (with timeout of "{timeout}" sec)') +@then('the user waits till {process_name} prints "{log_msg}" in the logs (with timeout of "{timeout}" sec)') +def impl(context, process_name, log_msg, timeout): + poll_period = 0.1 + max_iteration_cnt = int(int(timeout) / poll_period) + command = f""" + ITERATION=0 + MAX_ITERATION_CNT={max_iteration_cnt} + while sleep {poll_period}; do + if grep -E --quiet '{log_msg}' ~/gpAdminLogs/{process_name}*log ; + then break 2; + fi; + + ITERATION=$((ITERATION + 1)) + if [ $ITERATION -ge $MAX_ITERATION_CNT ]; then + echo "Timeout after {timeout} seconds waiting for '{log_msg}' in {process_name} logs" >&2 + exit 1 + fi + done + """ + rc, _, error = run_cmd(command) + if rc: + raise Exception(error) + @given('the user asynchronously sets up to end {process_name} process with {signal_name}') @when('the user asynchronously sets up to end {process_name} process with {signal_name}') @then('the user asynchronously sets up to end {process_name} process with {signal_name}') @@ -1561,6 +1592,11 @@ def get_opened_files(filename, pidfile): def impl(context, tablename, dbname): drop_table_if_exists(context, table_name=tablename, dbname=dbname) +@when('materialized view "{viewname}" is dropped in "{dbname}"') +@then('materialized view "{viewname}" is dropped in "{dbname}"') +@given('materialized view "{viewname}" is dropped in "{dbname}"') +def impl(context, viewname, dbname): + drop_materialized_view_if_exists(context, view_name=viewname, dbname=dbname) @given('all the segments are running') @when('all the segments are running') @@ -2338,6 +2374,11 @@ def impl(context): def impl(context, tabletype, tablename, dbname, numrows): populate_regular_table_data(context, tabletype, tablename, dbname, compression_type=None, with_data=True, rowcount=int(numrows)) +@given('there is an unlogged "{tabletype}" table "{tablename}" in "{dbname}" with "{numrows}" rows') +@then('there is an unlogged "{tabletype}" table "{tablename}" in "{dbname}" with "{numrows}" rows') +@when('there is an unlogged "{tabletype}" table "{tablename}" in "{dbname}" with "{numrows}" rows') +def impl(context, tabletype, tablename, dbname, numrows): + populate_regular_table_data(context, tabletype, tablename, dbname, compression_type=None, with_data=True, rowcount=int(numrows), unlogged=True) @given('there is a "{tabletype}" table "{tablename}" in "{dbname}" with data') @then('there is a "{tabletype}" table "{tablename}" in "{dbname}" with data') @@ -2358,6 +2399,12 @@ def impl(context, tabletype, tablename, dbname): def impl(context, tabletype, table_name, dbname): create_partition(context, tablename=table_name, storage_type=tabletype, dbname=dbname, with_data=True) +@given('there is a "{tabletype}" partition table "{table_name}" in "{dbname}" with "{numrows}" rows') +@then('there is a "{tabletype}" partition table "{table_name}" in "{dbname}" with "{numrows}" rows') +@when('there is a "{tabletype}" partition table "{table_name}" in "{dbname}" with "{numrows}" rows') +def impl(context, tabletype, table_name, dbname, numrows): + create_partition(context, tablename=table_name, storage_type=tabletype, dbname=dbname, with_data=True, rowcount=int(numrows)) + @given('there is a view without columns in "{dbname}"') @then('there is a view without columns in "{dbname}"') @when('there is a view without columns in "{dbname}"') @@ -4526,25 +4573,39 @@ def step_impl(context, address): @then('set fault inject "{fault}"') @when('set fault inject "{fault}"') def impl(context, fault): - os.environ['GPMGMT_FAULT_POINT'] = fault + os.environ[fault_injection.GPMGMT_FAULT_POINT] = fault @given('unset fault inject') @then('unset fault inject') @when('unset fault inject') def impl(context): - os.environ['GPMGMT_FAULT_POINT'] = "" + os.environ[fault_injection.GPMGMT_FAULT_POINT] = "" + os.environ[fault_injection.GPMGMT_FAULT_TYPE] = "" + os.environ[fault_injection.GPMGMT_FAULT_FILE_FLAG] = "" + if hasattr(context, 'fault_flag_filename') and os.path.exists(context.fault_flag_filename): + os.remove(context.fault_flag_filename) @given('set fault inject delay {delay} ms') @then('set fault inject delay {delay} ms') @when('set fault inject delay {delay} ms') def impl(context, delay): - os.environ['GPMGMT_FAULT_DELAY_MS'] = delay + os.environ[fault_injection.GPMGMT_FAULT_DELAY_MS] = delay + +@given('set fault inject type to suspend') +@then('set fault inject type to suspend') +@when('set fault inject type to suspend') +def impl(context): + os.environ[fault_injection.GPMGMT_FAULT_TYPE] = fault_injection.GPMGMT_FAULT_TYPE_SYSPEND + context.fault_flag_filename = "/tmp/ggrebalance_fault_suspend_flag" + with open(context.fault_flag_filename, "w"): + pass + os.environ[fault_injection.GPMGMT_FAULT_FILE_FLAG] = context.fault_flag_filename @given('unset fault inject delay') @then('unset fault inject delay') @when('unset fault inject delay') def impl(context): - os.environ['GPMGMT_FAULT_DELAY_MS'] = "" + os.environ[fault_injection.GPMGMT_FAULT_DELAY_MS] = "" @given('stub') def impl(context): diff --git a/gpMgmt/test/behave_utils/utils.py b/gpMgmt/test/behave_utils/utils.py index 976d1a87be72..53cac4169ffd 100644 --- a/gpMgmt/test/behave_utils/utils.py +++ b/gpMgmt/test/behave_utils/utils.py @@ -358,6 +358,10 @@ def drop_table(context, table_name, dbname, host=None, port=0, user=None): if check_table_exists(context, table_name=table_name, dbname=dbname, host=host, port=port, user=user): raise Exception('Unable to successfully drop the table %s' % table_name) +def drop_materialized_view_if_exists(context, view_name, dbname, host=None, port=0, user=None): + SQL = 'drop materialized view if exists %s' % view_name + with closing(dbconn.connect(dbconn.DbURL(hostname=host, port=port, username=user, dbname=dbname), unsetSearchPath=False)) as conn: + dbconn.execSQL(conn, SQL) def check_schema_exists(context, schema_name, dbname): schema_check_sql = "select * from pg_namespace where nspname='%s';" % schema_name @@ -430,11 +434,14 @@ def create_external_partition(context, tablename, dbname, port, filename): def create_partition(context, tablename, storage_type, dbname, compression_type=None, partition=True, rowcount=1094, - with_data=True, with_desc=False, host=None, port=0, user=None): + with_data=True, with_desc=False, host=None, port=0, user=None, unlogged=False): interval = '1 year' table_definition = 'Column1 int, Column2 varchar(20), Column3 date' - create_table_str = "Create table " + tablename + "(" + table_definition + ")" + create_table_str = "Create table " + if unlogged: + create_table_str = "Create unlogged table " + create_table_str = create_table_str + tablename + "(" + table_definition + ")" storage_type_dict = {'ao': 'row', 'co': 'column'} part_table = " Distributed Randomly Partition by list(Column2) \ @@ -732,11 +739,11 @@ def validate_local_path(path): def populate_regular_table_data(context, tabletype, table_name, dbname, compression_type=None, rowcount=1094, - with_data=False, with_desc=False, host=None, port=0, user=None): + with_data=False, with_desc=False, host=None, port=0, user=None, unlogged=False): create_database_if_not_exists(context, dbname, host=host, port=port, user=user) drop_table_if_exists(context, table_name=table_name, dbname=dbname, host=host, port=port, user=user) create_partition(context, table_name, tabletype, dbname, compression_type=compression_type, partition=False, - rowcount=rowcount, with_data=with_data, with_desc=with_desc, host=host, port=port, user=user) + rowcount=rowcount, with_data=with_data, with_desc=with_desc, host=host, port=port, user=user, unlogged=unlogged) def is_process_running(proc_name, host=None): From 0c8cb58dbe9bbea86c6b4696938e7c4c95889d2f Mon Sep 17 00:00:00 2001 From: Alexander Kondakov Date: Fri, 6 Mar 2026 08:56:21 +0300 Subject: [PATCH 22/22] [GG-181] Use intermediate host for swap moves (#2248) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, primary and mirror could coexist at the same host during execution of moves, where segments just swap their hosts. This violates the HA rule for the whole cluster. When the suboptimal rebalance plan requires swapping the locations of a primary segment and its mirror, the planner now decomposes this into three safe phases using an intermediate host to prevent primary-mirror coexistence violations. planner.py now detects swap moves in form_moves() and chooses the appropriate 3rd host for mirror movement. The search is performed based on available space, considering other moves, host status, and on other swap counts. Thus, plan, which previously looked like: ``` ---------------------------------BALANCE MOVES---------------------------------- Total moves planned: 2 [1] Move Segment(content=3, dbid=5, role=p) [254.73 MB] From: sdw1:7005 → /home/gpadmin/.data/primary/gpseg3 To: sdw2:7005 → /home/gpadmin/.data/primary/gpseg3 [2] Move Segment(content=3, dbid=11, role=m) [190.44 MB] From: sdw2:7053 → /home/gpadmin/.data/mirror/gpseg3 To: sdw1:7053 → /home/gpadmin/.data/mirror/gpseg3 ``` now expands into three moves ``` ---------------------------------BALANCE MOVES---------------------------------- Total moves planned: 3 [1] Move Segment(content=2, dbid=10, role=m) [190.45 MB] From: sdw2:7052 → /home/gpadmin/.data/mirror/gpseg2 To: sdw3:7052 → /home/gpadmin/.data/mirror/gpseg2 [2] Move Segment(content=2, dbid=4, role=p) [254.74 MB] From: sdw1:7004 → /home/gpadmin/.data/primary/gpseg2 To: sdw2:7004 → /home/gpadmin/.data/primary/gpseg2 [3] Move Segment(content=2, dbid=10, role=m) [190.45 MB] From: sdw3:7052 → /home/gpadmin/.data/mirror/gpseg2 To: sdw1:7054 → /home/gpadmin/.data/mirror/gpseg2 ``` Moreover, available space check for intermediate host now uses cached filesystem info. Thus, the ResourceEstimator class is refactored. It's unit tests are adjusted. Additionally some unit tests were fixed, because we've forgotten to check them in previous patches. --- gpMgmt/bin/ggrebalance | 8 +- .../bin/gprebalance_modules/ggrebalance_sm.py | 3 +- gpMgmt/bin/gprebalance_modules/planner.py | 927 +++++++++++++----- .../test/test_unit_ggrebalance.py | 12 +- .../test/test_unit_resources.py | 80 +- .../mgmt_utils/ggrebalance_basics.feature | 57 +- .../mgmt_utils/ggrebalance_rebalance.feature | 127 +++ .../behave/mgmt_utils/steps/mgmt_utils.py | 20 + 8 files changed, 908 insertions(+), 326 deletions(-) diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance index 194d090170b4..05837ee8ec5c 100755 --- a/gpMgmt/bin/ggrebalance +++ b/gpMgmt/bin/ggrebalance @@ -95,6 +95,8 @@ def parseargs(): help='Replay lag(in GBs) allowed on mirror when rebalancing the segments.') parser.add_option('-v', '--verbose', action='store_true', default=False, help='debug output.') + parser.add_option('--inplace-swap-roles', dest="inplace_swap_roles", action='store_true', default=False, + help='Allows to place primary and mirror at the same host during rebalance.') # Parse the command line arguments options, args = parser.parse_args() @@ -145,20 +147,20 @@ def validate_options(options, args, parser): ('target_datadirs', ['target_datadirs_file']), - ('skip_rebalance', ['mirror_mode']), + ('skip_rebalance', ['mirror_mode', 'inplace_swap_roles']), ('clean_required', [ 'target_hosts', 'target_hosts_file', 'add_hosts', 'add_hosts_file', 'remove_hosts', 'remove_hosts_file', 'target_datadirs', 'target_datadirs_file', 'target_segment_count', 'mirror_mode', 'skip_rebalance', 'show_plan', - 'skip_resource_estimation', 'analyze', 'replay_lag', 'hba_hostnames' + 'skip_resource_estimation', 'analyze', 'replay_lag', 'hba_hostnames', 'inplace_swap_roles' ]), ('rollback_required', [ 'target_hosts', 'target_hosts_file', 'add_hosts', 'add_hosts_file', 'remove_hosts', 'remove_hosts_file', 'target_datadirs', 'target_datadirs_file', 'target_segment_count', 'mirror_mode', 'skip_rebalance', 'show_plan', - 'skip_resource_estimation', 'duration', 'end' + 'skip_resource_estimation', 'duration', 'end', 'inplace_swap_roles' ]), ] diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py index 3d371fea5d27..cbb2bc2a7c4d 100755 --- a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -286,7 +286,8 @@ def create_config_file(self, moves: List[LogicalMove]) -> str: with open(filename, 'w') as fp: for move in moves: segment_current_info = move.seg - if not self.lookup_seg(segment_current_info): + is_swap_phase3 = (move.swap_metadata is not None and move.swap_metadata.get('phase') == 3) + if not is_swap_phase3 and not self.lookup_seg(segment_current_info): self.logger.info(f'Skip segment for gpmovemirrors: {str(segment_current_info)}') continue cfg_line = f'{segment_current_info.getSegmentHostName()}|{segment_current_info.getSegmentPort()}|{segment_current_info.getSegmentDataDirectory()} ' diff --git a/gpMgmt/bin/gprebalance_modules/planner.py b/gpMgmt/bin/gprebalance_modules/planner.py index e5e14b64fa34..095cc61e09b2 100755 --- a/gpMgmt/bin/gprebalance_modules/planner.py +++ b/gpMgmt/bin/gprebalance_modules/planner.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 from collections import defaultdict -from contextlib import closing +import copy import os import pickle from typing import Any, Tuple, List, Dict -from dataclasses import dataclass +from contextlib import closing +from dataclasses import dataclass, field from copy import deepcopy +import math from gppylib.db import dbconn import gppylib.gparray as gparray @@ -32,6 +34,8 @@ class LogicalMove: dst: destination host target_datadir: target data directory target_port: target port + segment_size: segment volume including tablespaces + swap_metadata: defines the order of swap moves """ seg: Segment srcHost: Host @@ -39,6 +43,7 @@ class LogicalMove: target_datadir: str target_port: int segment_size: Optional[SegmentSize] = None + swap_metadata: Optional[Dict[str, Any]] = None def __str__(self): """Pretty print logical move""" @@ -604,21 +609,6 @@ def plan(self) -> Plan: if rebalance_moves is None: self.logger.info("Cluster is already balanced, no segment moves will be held.") - else: - if not self.options.skip_resource_estimation: - try: - with closing(dbconn.connect(self.dburl, encoding='UTF8')) as conn: - estimator = ResourceEstimator( - self.logger, - conn, - self.virtual_gparray, - batch_size=getattr(self.options, 'batch_size', 16) - ) - estimator.estimate_and_validate_moves(rebalance_moves) - except ResourceError as e: - raise PlanningError(f"Resource validation failed: {e}") - else: - self.logger.warning("Skipping resource estimation") plan.setMoves(rebalance_moves) plan.setTargetSegmentCount(self.options.target_segment_count) @@ -789,6 +779,7 @@ def form_moves(self) -> List[LogicalMove]: port_allocator = PortAllocator(self.virtual_gparray, self.logger, not self.options.skip_resource_estimation) + final_moves = [] moves = [] for seg in self.virtual_gparray.getSegDbList(): is_mirror = seg.isSegmentMirror() @@ -816,8 +807,38 @@ def form_moves(self) -> List[LogicalMove]: if len(moves) == 0: return None + + resource_estimator = None + + if not self.options.skip_resource_estimation: + try: + resource_estimator = ResourceEstimator( + self.logger, + self.dburl, + self.virtual_gparray, + batch_size=getattr(self.options, 'batch_size', 16) + ) + resource_estimator.estimate_and_validate_moves(moves) + except ResourceError as e: + raise PlanningError(f"Resource validation failed: {e}") + else: + self.logger.warning("Skipping resource estimation") + # Detect swaps + swap_pairs = None + if not self.options.inplace_swap_roles: + swap_pairs = self.detect_swap_pairs(moves) + if swap_pairs: + self.logger.info(f"Detected {len(swap_pairs)} primary-mirror pairs which just swap hosts") + final_moves = self.handle_swaps(swap_pairs, moves, port_allocator, resource_estimator) + else: + final_moves = moves + if resource_estimator: + self.logger.info( + f"Estimated total data to move: " + f"{resource_estimator.total_gb(final_moves):.2f} GB" + ) - return moves + return final_moves def already_balanced(self, load: int) -> bool: primaries_by_host = defaultdict(int) @@ -850,6 +871,406 @@ def _get_datadir_for_segment(self, host: Host, content_id: int, is_mirror: bool) # use template template = datadir_info.mirror_template if is_mirror else datadir_info.primary_template return TemplateParser.instantiate_template(template, host.hostname, content_id) + + def detect_swap_pairs(self, moves: List[LogicalMove]) -> List[Tuple[LogicalMove, LogicalMove]]: + """ + Detect primary-mirror swap scenarios + + A swap occurs when: + - Primary of content N moves from host A to host B + - Mirror of content N moves from host B to host A + + Returns: + List of (primary_move, mirror_move) tuples for detected swaps + """ + # Index moves by content + moves_by_content = defaultdict(dict) + for move in moves: + content_id = move.seg.getSegmentContentId() + if move.seg.isSegmentPrimary(): + moves_by_content[content_id]['primary'] = move + else: + moves_by_content[content_id]['mirror'] = move + + swap_pairs = [] + for content_id, seg_moves in moves_by_content.items(): + # Check if both primary and mirror are moved + if 'primary' not in seg_moves or 'mirror' not in seg_moves: + continue + + prim_move = seg_moves['primary'] + mir_move = seg_moves['mirror'] + + # Check if they're swapping hosts + prim_src = prim_move.seg.getSegmentHostName() + prim_dst = prim_move.dstHost.hostname + mir_src = mir_move.seg.getSegmentHostName() + mir_dst = mir_move.dstHost.hostname + + if (prim_src == mir_dst and + prim_dst == mir_src): + swap_pairs.append((prim_move, mir_move)) + self.logger.debug( + f"Detected swap for content {content_id}: " + f"{prim_src} <-> {prim_dst}" + ) + + return swap_pairs + + def select_intermediate_host(self, + primary_move: LogicalMove, + mirror_move: LogicalMove, + used_intermediate_hosts: Dict[str, int], + resource_estimator: Optional['ResourceEstimator'] = None + ) -> Host: + """ + Select an intermediate host for swap operation with basic space validation + + Naive criteria (since the executor runs all mirror moves before role swap): + 1. Not involved in this swap (not source or dest for either segment) + 2. Has sufficient space for the mirror PLUS other planned moves to that host + 3. Prefer hosts with fewer intermediate segments already assigned + + Args: + primary_move: Primary segment move + mirror_move: Mirror segment move + used_intermediate_hosts: Dict tracking hostname -> count of intermediates assigned + resource_estimator: Optional ResourceEstimator for space validation + + Returns: + Selected intermediate host + """ + content_id = primary_move.seg.getSegmentContentId() + + # Collect excluded hostnames + excluded_hosts = { + primary_move.dstHost.hostname, + mirror_move.dstHost.hostname + } + + # Find candidate hosts + candidates = [] + for host in self.target_hosts: + if host.status not in [HostStatus.ACTIVE, HostStatus.NEW]: + continue + + # Check if excluded + if host.hostname in excluded_hosts: + continue + + candidates.append(host) + + if not candidates: + raise PlanningError( + f"No intermediate host available for swap of content {content_id}. " + f"Need at least 3 hosts to perform swaps safely. Or you may allow " + f"primary-mirror coexistence by --inplace-swap-roles." + ) + + # Get mirror size for space check + mirror_size_kb = 0 + if mirror_move.segment_size: + mirror_size_kb = int(mirror_move.segment_size.total_size_kb * (1 + DISK_SPACE_SAFETY_MARGIN)) + + # Score each candidate + scored_candidates = [] + + for host in candidates: + # Check space if resource estimation is enabled + has_space = True + available_space_kb = float('inf') + filesystems = 'unknown' + + if resource_estimator: + try: + intermediate_datadir = self._get_datadir_for_segment( + host, content_id, is_mirror=True) + + has_space, available_space_kb, filesystems = \ + resource_estimator.check_space( + hostname=host.hostname, + host_address=host.address, + data_directory=intermediate_datadir, + required_size=mirror_move.segment_size + ) + + if not has_space: + self.logger.debug( + f"Host {host.hostname}: insufficient space insufficient space on {filesystems}\n" + f" Directory: {intermediate_datadir}\n" + f" (available: {available_space_kb / 1024 / 1024:.2f} GB, " + f" required: {mirror_size_kb / 1024 / 1024:.2f} GB)" + ) + continue + else: + self.logger.debug( + f"Host {host.hostname}: viable candidate\n" + f" Directory: {intermediate_datadir}\n" + f" Filesystem: {filesystems}\n" + f" Available: {available_space_kb / 1024 / 1024:.2f} GB\n" + f" Required: {mirror_size_kb / 1024 / 1024:.2f} GB" + ) + except Exception as e: + self.logger.error(f"Could not check space on {host.hostname}: {e}") + has_space = False + + if not has_space: + continue + + # Calculate score + # Factors: available space, intermediate usage + if mirror_size_kb <= 0: + space_score = 1.0 + else: + ratio = available_space_kb / mirror_size_kb + # sigmoid. more sensitive around ratio=1 + space_score = 1.0 / (1.0 + math.exp(-3.0 * (ratio - 1.0))) + + # Penalize hosts already used as intermediate + intermediate_count = used_intermediate_hosts.get(host.hostname, 0) + intermediate_score = math.exp(-1.0 * intermediate_count) + + # make score lie in [0, 1] + score = space_score * 0.7 + intermediate_score * 0.3 + + scored_candidates.append((host, score, available_space_kb, filesystems)) + + if not scored_candidates: + raise PlanningError( + f"No suitable intermediate host found for swap of content {content_id}. " + f"Need at least 3 hosts with sufficient space to perform swaps safely." + ) + + # Sort by score (higher is better) + scored_candidates.sort(key=lambda x: x[1], reverse=True) + + selected_host, selected_score, selected_space, selected_fs = scored_candidates[0] + + # Track usage + used_intermediate_hosts[selected_host.hostname] = \ + used_intermediate_hosts.get(selected_host.hostname, 0) + 1 + + self.logger.info( + f"Selected intermediate host {selected_host.hostname} for content {content_id}\n" + f" Score: {selected_score:.1f}\n" + f" Filesystems: {selected_fs}\n" + f" Available: {selected_space / 1024 / 1024:.2f} GB\n" + f" Required: {mirror_size_kb / 1024 / 1024:.2f} GB") + + if resource_estimator: + resource_estimator.reserve_space( + hostname=host.hostname, + host_address=host.address, + data_directory=self._get_datadir_for_segment(host, content_id, is_mirror=True), + dbid=mirror_move.seg.dbid, + required_size=mirror_move.segment_size + ) + + return selected_host + + def decompose_swap(self, + primary_move: LogicalMove, + mirror_move: LogicalMove, + intermediate_host: Host, + port_allocator: PortAllocator) -> Tuple[LogicalMove, LogicalMove, LogicalMove]: + """ + Decompose a swap into 3 moves through intermediate host + + Phase 1: Mirror -> Intermediate host + Phase 2: Primary -> Mirror's original host (direct move) + Phase 3: Mirror (from intermediate) -> Primary's original host + + Returns: + (phase1_move, phase2_move, phase3_move) + """ + content_id = primary_move.seg.getSegmentContentId() + + # Phase 1: Move mirror to intermediate host + intermediate_datadir = self._get_datadir_for_segment( + intermediate_host, + content_id, + is_mirror=True + ) + intermediate_port = port_allocator.allocate_port( + host=intermediate_host, + current_port=mirror_move.seg.getSegmentPort(), + is_mirror=True + ) + + phase1_move = LogicalMove( + seg=mirror_move.seg, + srcHost=mirror_move.srcHost, + dstHost=intermediate_host, + target_datadir=intermediate_datadir, + target_port=intermediate_port, + segment_size=mirror_move.segment_size, + swap_metadata={ + 'phase': 1, + 'content_id': content_id, + 'intermediate_host': intermediate_host.hostname + } + ) + + # Phase 2: Primary move (direct to final location) + # This keeps the original primary move unchanged + phase2_move = primary_move + phase2_move.swap_metadata = {'phase': 2, + 'content_id': content_id} + + # Phase 3: Move mirror from intermediate to final location (primary's original host) + final_datadir = mirror_move.target_datadir + final_port = port_allocator.allocate_port( + host=mirror_move.dstHost, + current_port=mirror_move.seg.getSegmentPort(), + is_mirror=True + ) + + # Create a copy of mirror segment with updated location (now on intermediate) + intermediate_seg = copy.deepcopy(mirror_move.seg) + intermediate_seg.hostname = intermediate_host.hostname + intermediate_seg.address = intermediate_host.address + intermediate_seg.datadir = intermediate_datadir + intermediate_seg.port = intermediate_port + + phase3_move = LogicalMove( + seg=intermediate_seg, + srcHost=intermediate_host, + dstHost=mirror_move.dstHost, # Final location (primary's original host) + target_datadir=final_datadir, + target_port=final_port, + segment_size=mirror_move.segment_size, + swap_metadata={ + 'phase': 3, + 'content_id': content_id, + 'intermediate_host': intermediate_host.hostname + } + ) + + return phase1_move, phase2_move, phase3_move + + def handle_swaps(self, + swap_pairs: List[Tuple[LogicalMove, LogicalMove]], + moves: List[LogicalMove], + port_allocator: PortAllocator, + resource_estimator: 'ResourceEstimator' + ) -> List[LogicalMove]: + swap_move_ids = set() + # Track intermediate host usage for better distribution + used_intermediate_hosts = {} + # Decompose swaps with intermediate host selection + swap_phase1_moves = [] + swap_phase2_moves = [] + swap_phase3_moves = [] + + for prim_move, mir_move in swap_pairs: + swap_move_ids.add(prim_move.seg.getSegmentDbId()) + swap_move_ids.add(mir_move.seg.getSegmentDbId()) + try: + # Select intermediate host + intermediate_host = self.select_intermediate_host( + prim_move, + mir_move, + used_intermediate_hosts, + resource_estimator # Pass the estimator with cached filesystem data + ) + # Decompose into 3 phases + phase1, phase2, phase3 = self.decompose_swap( + prim_move, + mir_move, + intermediate_host, + port_allocator + ) + swap_phase1_moves.append(phase1) + swap_phase2_moves.append(phase2) + swap_phase3_moves.append(phase3) + except Exception as e: + raise PlanningError(f"Failed to plan swap for content {prim_move.seg.getSegmentContentId()}: {e}") + + # Collect non-swap moves + non_swap_mirror_moves = [] + non_swap_primary_moves = [] + + for move in moves: + if move.seg.getSegmentDbId() not in swap_move_ids: + if move.seg.isSegmentPrimary(): + non_swap_primary_moves.append(move) + else: + non_swap_mirror_moves.append(move) + + moves_with_swap = [] + # Batch 1: All initial mirror moves (regular + phase1) + moves_with_swap.extend(non_swap_mirror_moves) + moves_with_swap.extend(swap_phase1_moves) + + # Batch 2: All primary moves (regular + phase2) + moves_with_swap.extend(non_swap_primary_moves) + moves_with_swap.extend(swap_phase2_moves) + + # Batch 3: Phase 3 mirrors (must execute last) + moves_with_swap.extend(swap_phase3_moves) + + return moves_with_swap + +@dataclass +class FilesystemRequirement: + """ + Tracks space requirements and allocations for a single filesystem on a host. + """ + space_info: DiskSpaceInfo + required_kb: int = 0 + # dbids and datadirs, which contribute to this filesystem + dbid_datadirs: Dict[int, Tuple[str, int]] = field(default_factory=dict) + # dict(dbid, dict(tablespace_base, size_kb)) contributing to this filesystem + dbid_tablespaces: Dict[int, Dict[str, int]] = field(default_factory=dict) + # tablespace_base/dbid for error msgs + tablespace_paths: Set[str] = field(default_factory=set) + + def add_datadir(self, dbid: int, directory: str, size_kb: int) -> None: + """ + Add a datadir contribution. + """ + request = int(size_kb * (1 + DISK_SPACE_SAFETY_MARGIN)) + self.dbid_datadirs[dbid] = (directory, request) + self.required_kb += request + + def add_tablespace(self, dbid: int, tbl_path: str, tbl_base: str, size_kb: int) -> None: + """ + Add a tablespace contribution. + """ + request = int(size_kb * (1 + DISK_SPACE_SAFETY_MARGIN)) + self.tablespace_paths.add(tbl_path) + if dbid not in self.dbid_tablespaces: + self.dbid_tablespaces[dbid] = defaultdict(int) + self.dbid_tablespaces[dbid][tbl_base] += request + self.required_kb += request + + def remove_datadir(self, dbid: int) -> None: + if dbid in self.dbid_datadirs: + item = self.dbid_datadirs.pop(dbid) + self.required_kb -= item[1] + + def remove_tablespaces(self, dbid: int) -> None: + if dbid in self.dbid_tablespaces: + tblsps = self.dbid_tablespaces.pop(dbid) + for _, size in tblsps.items(): + self.required_kb -= size + + @property + def available_kb(self) -> int: + return self.space_info.available_kb + + @property + def datadir_paths(self) -> Set[str]: + """for error reporting""" + paths = set() + for _, item in self.dbid_datadirs.items(): + paths.add(item[0]) + return paths + + @property + def unique_segment_count(self) -> int: + return len(self.dbid_datadirs) + \ + len(set(self.dbid_tablespaces.keys()) - set(self.dbid_datadirs.keys())) class ResourceEstimator: """ @@ -859,20 +1280,25 @@ class ResourceEstimator: using DiskSpaceChecker for invoking remote disk operations. """ - def __init__(self, logger: Any, conn: Any, gparray: gparray.GpArray, batch_size: int = 16): + def __init__(self, logger: Any, dburl: dbconn.DbURL, gparray: gparray.GpArray, batch_size: int = 16): """ Initialize resource estimator Args: logger: Logger instance - conn: Database connection + dburl: Database url gparray: Current GpArray configuration batch_size: Number of parallel operations """ self.logger = logger - self.conn = conn + self.dburl = dburl self.gparray = gparray self.disk_checker = DiskSpaceChecker(logger, batch_size) + # Cache filesystem allocation data after validation + # Maps (hostname, filesystem) -> {required_kb, available_kb, datadir_moves, ...} + self.filesystem_allocations: Dict[Tuple[str, str], FilesystemRequirement] = {} + # Cache space_info_by_host for reuse + self.space_info_by_host: Dict[str, Dict[str, DiskSpaceInfo]] = {} def estimate_and_validate_moves(self, moves: List['LogicalMove']) -> None: """ @@ -887,23 +1313,15 @@ def estimate_and_validate_moves(self, moves: List['LogicalMove']) -> None: return self.logger.info(f"Estimating resource requirements for {len(moves)} segment moves...") - - try: - # Step 1: Estimate segment sizes - self._estimate_segment_sizes(moves) - - # Step 2: Validate available space on target hosts - self._validate_target_space(moves) - - # Log summary - total_size_kb = sum(move.segment_size.total_size_kb for move in moves if move.segment_size) - total_size_gb = total_size_kb / 1024 / 1024 - self.logger.info(f"Total data to move: {total_size_gb:.2f} GB") - self.logger.info("Resource validation completed successfully") - - except Exception as e: - self.logger.error(f"Resource validation failed") - raise ResourceError(str(e)) + # Step 1: Estimate segment sizes + self._estimate_segment_sizes(moves) + self.logger.info("Validating available disk space on target hosts...") + # Step 2: Validate available space on target hosts + self._validate_and_build_allocations(moves) + + def total_gb(self, moves: List[LogicalMove]) -> int: + total_size_kb = sum(move.segment_size.total_size_kb for move in moves if move.segment_size) + return total_size_kb / 1024 / 1024 def _estimate_segment_sizes(self, moves: List[LogicalMove]) -> None: """ @@ -911,36 +1329,25 @@ def _estimate_segment_sizes(self, moves: List[LogicalMove]) -> None: Populates the segment_size attribute on each LogicalMove """ - # Get unique segments to estimate - segments_to_estimate = {move.seg.dbid: move.seg for move in moves} - - # Query for tablespace locations - tablespace_map = self._get_tablespace_locations(list(segments_to_estimate.keys())) - - # Group segments by source host for batch disk usage - segments_by_host = defaultdict(list) + tablespace_map = self._get_tablespace_locations([m.seg.dbid for m in moves]) + self._estimate_datadir_sizes(moves) + if tablespace_map: + self._estimate_tablespace_sizes(moves, tablespace_map) + + def _estimate_datadir_sizes(self, moves: List[LogicalMove]) -> None: + moves_by_host: Dict[str, List[LogicalMove]] = defaultdict(list) for move in moves: - segments_by_host[move.srcHost.address].append(move.seg) + moves_by_host[move.srcHost.address].append(move) - # Estimate datadir sizes - for host_addr, host_segments in segments_by_host.items(): - dirs = [seg.datadir for seg in host_segments] - + for host_addr, host_moves in moves_by_host.items(): + dirs = [m.seg.datadir for m in host_moves] try: disk_usage = self.disk_checker.get_disk_usage(host_addr, dirs) - - for seg in host_segments: - size_kb = disk_usage.get(seg.datadir, 0) - # Find all moves for this segment and set size - for move in moves: - if move.seg.dbid == seg.dbid: - move.segment_size = SegmentSize(datadir_size_kb=size_kb) - + for move in host_moves: + size_kb = disk_usage.get(move.seg.datadir, 0) + move.segment_size = SegmentSize(datadir_size_kb=size_kb) except Exception as e: raise ResourceError(f"Cannot estimate segment sizes on host {host_addr}: {e}") - - # Estimate tablespace sizes - self._estimate_tablespace_sizes(moves, tablespace_map) def _get_tablespace_locations(self, dbids: List[int]) -> Dict[int, List[str]]: """ @@ -949,6 +1356,7 @@ def _get_tablespace_locations(self, dbids: List[int]) -> Dict[int, List[str]]: Returns: Dict mapping dbid to list of tablespace paths """ + if not dbids: return {} @@ -973,13 +1381,14 @@ def _get_tablespace_locations(self, dbids: List[int]) -> Dict[int, List[str]]: """ try: - cursor = dbconn.query(self.conn, tablespace_location_sql) - tablespaces = defaultdict(list) - - for dbid, loc in cursor: - tablespaces[dbid].append(loc) - - return tablespaces + with closing(dbconn.connect(self.dburl, encoding='UTF8')) as conn: + cursor = dbconn.query(conn, tablespace_location_sql) + tablespaces = defaultdict(list) + + for dbid, loc in cursor: + tablespaces[dbid].append(loc) + + return tablespaces except Exception as e: raise ResourceError(f"Failed to query tablespace locations: {e}") @@ -997,10 +1406,8 @@ def _estimate_tablespace_sizes(self, # Group tablespace directories by host tblspace_by_host = defaultdict(list) for move in moves: - dbid = move.seg.dbid - if dbid in tablespace_map: - for tblspace_dir in tablespace_map[dbid]: - tblspace_by_host[move.srcHost.address].append((dbid, tblspace_dir)) + for tbl_dir in tablespace_map.get(move.seg.dbid, []): + tblspace_by_host[move.srcHost.address].append((move.seg.dbid, tbl_dir)) # Process each host for host_addr, host_tablespaces in tblspace_by_host.items(): @@ -1025,198 +1432,189 @@ def _estimate_tablespace_sizes(self, self.logger.warning(f"Failed to get tablespace disk usage for host {host_addr}: {e}") # Continue without tablespace sizes - def _validate_target_space(self, moves: List[LogicalMove]) -> None: + def _validate_and_build_allocations(self, moves: List[LogicalMove]) -> None: """ - Validate that target hosts have sufficient disk space + Check that every target filesystem has enough free space. - Groups all space requirements by filesystem to handle cases where - datadirs and tablespaces share the same underlying filesystem. + Builds filesystem_allocations as a side effect so that + reserve_space() can later account for committed space. + Raises ResourceError listing all filesystems with insufficient space. """ - self.logger.info("Validating available disk space on target hosts...") - - # Collect all directories we need to check - dirs_by_host = defaultdict(set) - - # Track what each directory is used for (for better error reporting) - # (hostname, dir) -> moves in 'datadirs' if dir is datadir - # moves moves in 'tablespaces' if dir is tablespace - dir_usage = defaultdict(lambda: {'datadirs': [], 'tablespaces': []}) + self._fetch_target_space_info(moves) + self._build_filesystem_allocations(moves) + issues = self._find_space_issues() + if issues: + details = ''.join(self._format_space_issue(i) for i in issues) + raise ResourceError( + f"Insufficient disk space for rebalance operation:\n{details}" + f"\nNote: Estimates include {int(DISK_SPACE_SAFETY_MARGIN * 100)}% safety margin" + ) + self.logger.info("Disk space validation completed successfully") + + def _fetch_target_space_info(self, moves: List[LogicalMove]) -> None: + """ + Query free space for every target directory referenced by the moves. + """ + dirs_by_host: Dict[str, Set[str]] = defaultdict(set) for move in moves: if not move.segment_size: continue + dirs_by_host[move.dstHost.address].add(move.target_datadir) + for tbl_path in (move.segment_size.tablespace_usage or {}): + dirs_by_host[move.dstHost.address].add(os.path.dirname(tbl_path)) - target_host = move.dstHost.hostname - target_addr = move.dstHost.address - target_datadir = move.target_datadir - - # Track datadir - dirs_by_host[target_addr].add(target_datadir) - dir_usage[(target_host, target_datadir)]['datadirs'].append(move) - - # Track tablespace directories - if move.segment_size.tablespace_usage: - for tblspace_path, _ in move.segment_size.tablespace_usage.items(): - # Extract base path (remove /dbid suffix) - base_tblspace_path = os.path.dirname(tblspace_path) - - dirs_by_host[target_addr].add(base_tblspace_path) - dir_usage[(target_host, base_tblspace_path)]['tablespaces'].append(move) - - # Convert sets to lists for DiskFree - dirs_by_host = {host: list(dirs) for host, dirs in dirs_by_host.items()} - - # Check available space in batch try: - space_info_by_host = self.disk_checker.check_batch_available_space(dirs_by_host) + self.space_info_by_host = self.disk_checker.check_batch_available_space( + {host: list(dirs) for host, dirs in dirs_by_host.items()} + ) except Exception as e: raise ResourceError(f"Failed to check available disk space: {e}") - - # Group directories by filesystem and aggregate space requirements - self.logger.info("Aggregating space requirements by filesystem...") - issues = self._validate_filesystem_requirements( - dir_usage=dir_usage, - space_info_by_host=space_info_by_host - ) - - # Report all issues - if issues: - error_lines = ["Insufficient disk space for rebalance operation:\n"] - - for issue in issues: - error_lines.append(self._format_space_issue(issue)) - - error_lines.append( - f"\nNote: Estimates include {int(DISK_SPACE_SAFETY_MARGIN * 100)}% safety margin" - ) - raise ResourceError(''.join(error_lines)) - - self.logger.info("Disk space validation completed successfully") - - def _validate_filesystem_requirements(self, - dir_usage: Dict[Tuple[str, str], Dict], - space_info_by_host: Dict[str, Dict[str, DiskSpaceInfo]]) -> List[Dict]: + + def _build_filesystem_allocations(self, moves: List[LogicalMove]) -> None: """ - Validate space requirements aggregated by filesystem + Populate self.filesystem_allocations from the planned moves. - This correctly handles cases where datadirs and tablespaces share the same filesystem. - - Args: - dir_usage: Dict mapping (hostname, directory) -> {'datadirs': [moves], 'tablespaces': [moves]} - space_info_by_host: Space information from DiskSpaceChecker + Each move contributes its datadir size to the filesystem that hosts + target_datadir, and each tablespace path to the filesystem that hosts + that tablespace. + """ + self.filesystem_allocations = {} + self.logger.debug("Aggregating space requirements by filesystem...") - Returns: - List of issue dicts for filesystems with insufficient space - """ - # Group by filesystem - filesystem_requirements = defaultdict(lambda: { - 'required_kb': 0, - 'datadir_moves': set(), - 'tablespace_moves': set(), - 'datadirs': set(), - 'tablespaces': set(), - 'space_info': None - }) - - for (hostname, directory), usage in dir_usage.items(): - # Find the host address for this hostname - host_address = None - for moves_list in [usage['datadirs'], usage['tablespaces']]: - if moves_list: - host_address = moves_list[0].dstHost.address - break - - if not host_address: + for move in moves: + if not move.segment_size: continue - if host_address not in space_info_by_host: - raise ResourceError(f"No disk space information for host {hostname}") - - space_info = space_info_by_host[host_address].get(directory) - if not space_info: - raise ResourceError( - f"No disk space information for {hostname}:{directory}" - ) - - # Group by filesystem - fs_key = (hostname, space_info.filesystem) - fs_data = filesystem_requirements[fs_key] - fs_data['space_info'] = space_info - - # Process datadir moves - for move in usage['datadirs']: - if not move.segment_size: - continue - - # Only count each move once per filesystem - if move.seg.dbid not in fs_data['datadir_moves']: - fs_data['datadir_moves'].add(move.seg.dbid) - fs_data['datadirs'].add(directory) - size_kb = move.segment_size.datadir_size_kb - fs_data['required_kb'] += int(size_kb * (1 + DISK_SPACE_SAFETY_MARGIN)) - - # Process tablespace moves - for move in usage['tablespaces']: - if not move.segment_size or not move.segment_size.tablespace_usage: - continue - - # Extract tablespace sizes that belong to this directory - for tbl_path, size_kb in move.segment_size.tablespace_usage.items(): - tbl_base_path = os.path.dirname(tbl_path) + hostname = move.dstHost.hostname + host_address = move.dstHost.address - if tbl_base_path == directory: - # Use a unique key to avoid double-counting the same tablespace - tbl_key = (move.seg.dbid, tbl_path) + # datadir + datadir_fs = self._require_space_info(hostname, host_address, move.target_datadir) + fs_req = self._get_or_create_fs_requirement(hostname, datadir_fs) + fs_req.add_datadir(move.seg.dbid, move.target_datadir, move.segment_size.datadir_size_kb) - if tbl_key not in fs_data['tablespace_moves']: - fs_data['tablespace_moves'].add(tbl_key) - fs_data['tablespaces'].add(directory) - fs_data['required_kb'] += int(size_kb * (1 + DISK_SPACE_SAFETY_MARGIN)) + # tablespace + for tbl_path, size_kb in (move.segment_size.tablespace_usage or {}).items(): + tbl_base = os.path.dirname(tbl_path) + tbl_fs = self._require_space_info(hostname, host_address, tbl_base) + fs_req = self._get_or_create_fs_requirement(hostname, tbl_fs) + fs_req.add_tablespace(move.seg.dbid, tbl_path, tbl_base, size_kb) - # Validate each filesystem + def _find_space_issues(self) -> List[Dict]: issues = [] - - for (hostname, filesystem), fs_data in filesystem_requirements.items(): - required_kb = fs_data['required_kb'] - available_kb = fs_data['space_info'].available_kb - - required_gb = required_kb / 1024 / 1024 - available_gb = available_kb / 1024 / 1024 - - # Count unique segments - unique_segments = len(fs_data['datadir_moves']) + len({dbid for dbid, _ in fs_data['tablespace_moves']}) + for (hostname, filesystem), fs_req in self.filesystem_allocations.items(): + required_gb = fs_req.required_kb / 1024 / 1024 + available_gb = fs_req.available_kb / 1024 / 1024 self.logger.debug( f"Filesystem {filesystem} on {hostname}: " - f"Required {required_gb:.2f} GB, Available {available_gb:.2f} GB " - f"({unique_segments} segments, " - f"{len(fs_data['datadirs'])} datadirs, " - f"{len(fs_data['tablespaces'])} tablespace dirs)" - ) - - if available_kb < required_kb: - all_dirs = sorted(fs_data['datadirs'].union(fs_data['tablespaces'])) + f"required {required_gb:.2f} GB, available {available_gb:.2f} GB ") + if fs_req.available_kb < fs_req.required_kb: issues.append({ - 'type': 'filesystem', 'hostname': hostname, 'filesystem': filesystem, - 'target_dirs': all_dirs, - 'num_datadirs': len(fs_data['datadirs']), - 'num_tablespaces': len(fs_data['tablespaces']), - 'num_segments': unique_segments, + 'target_dirs': sorted(fs_req.datadir_paths | fs_req.tablespace_paths), + 'num_datadirs': len(fs_req.datadir_paths), + 'num_tablespaces': len(fs_req.tablespace_paths), + 'num_segments': fs_req.unique_segment_count, 'required_gb': required_gb, 'available_gb': available_gb, }) - return issues + def _get_or_create_fs_requirement(self, hostname: str, space_info: DiskSpaceInfo) -> FilesystemRequirement: + fs_key = (hostname, space_info.filesystem) + if fs_key not in self.filesystem_allocations: + self.filesystem_allocations[fs_key] = FilesystemRequirement(space_info=space_info) + return self.filesystem_allocations[fs_key] + + def _require_space_info(self, hostname: str, host_address: str, directory: str) -> DiskSpaceInfo: + """ + Return DiskSpaceInfo for directory, raising ResourceError if not found. + """ + info = self._get_or_fetch_space_info(hostname, host_address, directory) + if not info: + raise ResourceError(f"No disk space information for {hostname}:{directory}") + return info + + def _get_or_fetch_space_info(self, hostname: str, host_address: str, directory: str) -> Optional[DiskSpaceInfo]: + """ + Return cached DiskSpaceInfo for directory (or its parent). + On a cache miss, query the host and cache the result. + """ + # Check cache + host_cache = self.space_info_by_host.get(host_address, {}) + info = host_cache.get(directory) or host_cache.get( + TemplateParser.extract_parent_directory(directory) + ) + if info: + return info + + # Cache miss - query the host + try: + fetched = self.disk_checker.check_batch_available_space( + {host_address: [directory]} + ) + info = fetched.get(host_address, {}).get(directory) + if info: + self.space_info_by_host.setdefault(host_address, {})[directory] = info + return info + except Exception as e: + self.logger.warning(f"Could not check space for {hostname}:{directory}: {e}") + return None + + def _reserve_space(self, + fs_key: Tuple[str, str], + space_info: DiskSpaceInfo, + directory: str, + required_kb: int, + dbid: int, + is_tablespace: bool) -> None: + hostname, filesystem = fs_key + if fs_key not in self.filesystem_allocations: + self.filesystem_allocations[fs_key] = FilesystemRequirement(space_info=space_info) + if is_tablespace: + self.filesystem_allocations[fs_key].add_tablespace(dbid, directory + '/' + str(dbid), directory, required_kb) + else: + self.filesystem_allocations[fs_key].add_datadir(dbid, directory, required_kb) + total_allocated = self.filesystem_allocations[fs_key].required_kb + self.logger.debug( + f"Reserved {required_kb / 1024 / 1024:.2f} GB on {hostname}:{filesystem} - " + f"total allocated: {total_allocated / 1024 / 1024:.2f} GB" + ) + + def reserve_space(self, + hostname: str, + host_address: str, + data_directory: str, + dbid: int, + required_size: SegmentSize) -> None: + + dirs_to_reserve = [(data_directory, required_size.datadir_size_kb)] + tablespace_dirs = {os.path.dirname(dir) for dir in (required_size.tablespace_usage or {})} + for tbl_dir in tablespace_dirs: + # Sum all tablespace paths that live under this dir + tbl_size_kb = sum(size_kb + for tbl_path, size_kb in required_size.tablespace_usage.items() + if os.path.dirname(tbl_path) == tbl_dir) + dirs_to_reserve.append((tbl_dir, tbl_size_kb)) + for directory, size_kb in dirs_to_reserve: + space_info = self._get_or_fetch_space_info(hostname, host_address, directory) + self._reserve_space( + fs_key=(hostname, space_info.filesystem), + space_info=space_info, + directory=directory, + required_kb=size_kb, + dbid=dbid, + is_tablespace=(directory in tablespace_dirs) + ) + def _format_space_issue(self, issue: Dict) -> str: """ Format a space issue for error reporting """ - issue_type = issue.get('type', 'unknown').upper() - # Build directory breakdown dir_info = [] if issue.get('num_datadirs', 0) > 0: @@ -1227,7 +1625,7 @@ def _format_space_issue(self, issue: Dict) -> str: dir_breakdown = ', '.join(dir_info) if dir_info else 'unknown' return ( - f"\n [{issue_type}] Host: {issue['hostname']}\n" + f"\n Host: {issue['hostname']}\n" f" Filesystem: {issue['filesystem']}\n" f" Directories: {dir_breakdown}\n" f" Paths: {', '.join(issue['target_dirs'])}\n" @@ -1235,3 +1633,72 @@ def _format_space_issue(self, issue: Dict) -> str: f" Required: {issue['required_gb']:.2f} GB\n" f" Available: {issue['available_gb']:.2f} GB\n" ) + + def get_allocated_space_for_filesystem(self, hostname: str, filesystem: str) -> int: + """ + Get the space already allocated to a specific filesystem + + Returns: + Allocated space in KB, or 0 if no allocations found + """ + fs_key = (hostname, filesystem) + if fs_key in self.filesystem_allocations: + return self.filesystem_allocations[fs_key]['required_kb'] + return 0 + + def check_space(self, + hostname: str, + host_address: str, + data_directory: str, + required_size: SegmentSize) -> Tuple[bool, int, str]: + """ + Check if a data and tablespaces directories have enough space. + + Args: + hostname: Target hostname + host_address: Target host address + data_directory: Target directory path + requied_size: Required space in SegmentSize + + Returns: + (has_space, min_available_kb, comma_separated_filesystems) + """ + dirs_to_check = [(data_directory, required_size.datadir_size_kb)] + tablespace_dirs = {os.path.dirname(dir) for dir in (required_size.tablespace_usage or {})} + for tbl_dir in tablespace_dirs: + # Sum all tablespace paths that live under this dir + tbl_size_kb = sum(size_kb + for tbl_path, size_kb in required_size.tablespace_usage.items() + if os.path.dirname(tbl_path) == tbl_dir) + dirs_to_check.append((tbl_dir, tbl_size_kb)) + + # Resolve each directory to a filesystem and accumulate per-filesystem + # requirements. + fs_requirements: Dict[str, int] = defaultdict(int) + fs_space_info: Dict[str, DiskSpaceInfo] = {} + + for directory, size_kb in dirs_to_check: + space_info = self._get_or_fetch_space_info(hostname, host_address, directory) + if not space_info: + return False, 0, 'unknown' + fs = space_info.filesystem + fs_requirements[fs] += int(size_kb * (1 + DISK_SPACE_SAFETY_MARGIN)) + fs_space_info[fs] = space_info + + fs_available: Dict[str, int] = {} + for fs, required_kb in fs_requirements.items(): + fs_key = (hostname, fs) + already_allocated_kb = self.filesystem_allocations[fs_key].required_kb \ + if fs_key in self.filesystem_allocations else 0 + available_kb = fs_space_info[fs].available_kb - already_allocated_kb + fs_available[fs] = available_kb + + if available_kb < required_kb: + self.logger.debug( + f"Host {hostname}: insufficient space on {fs}\n" + f" Available: {available_kb / 1024 / 1024:.2f} GB\n" + f" Required: {required_kb / 1024 / 1024:.2f} GB" + ) + return False, min(fs_available.values()), ', '.join(fs_requirements) + + return True, min(fs_available.values()), ', '.join(fs_requirements) diff --git a/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py b/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py index 878a7fcdd358..560e17f223ac 100644 --- a/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py +++ b/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py @@ -78,8 +78,9 @@ def tearDown(self): return_value=initGparrayFromFile("balanced_grouped_6")) @patch('os.path.exists', side_effect=lambda path: path not in ['/tmp/dirdoesnotexist/gparraydump']) @patch('gppylib.db.dbconn.queryRow', side_effect=check_query) + @patch('ggrebalance.check_down_segments') @rebalance_only(numsegs = 6) - def test_already_balanced_grouped(self, mockCatalog, mockOsPath, mockCursor): + def test_already_balanced_grouped(self, mockCatalog, mockOsPath, mockCursor, mock_down): with self.assertRaises(SystemExit): self.subject.main(self.options, self.args, self.parser) self.subject.logger.info.assert_any_call( @@ -89,8 +90,9 @@ def test_already_balanced_grouped(self, mockCatalog, mockOsPath, mockCursor): return_value=initGparrayFromFile("seg_down")) @patch('os.path.exists', side_effect=lambda path: path not in ['/tmp/dirdoesnotexist/gparraydump']) @patch('gppylib.db.dbconn.queryRow', side_effect=check_query) + @patch('ggrebalance.check_down_segments') @rebalance_only(numsegs = 4) - def test_segment_down(self, mockCatalog, mockOsPath, mockCursor): + def test_segment_down(self, mockCatalog, mockOsPath, mockCursor, mock_down): with self.assertRaises(SystemExit): self.subject.main(self.options, self.args, self.parser) self.subject.logger.error.assert_any_call( @@ -100,8 +102,9 @@ def test_segment_down(self, mockCatalog, mockOsPath, mockCursor): return_value=initGparrayFromFile("balanced_spread_24")) @patch('os.path.exists', side_effect=lambda path: path not in ['/tmp/dirdoesnotexist/gparraydump']) @patch('gppylib.db.dbconn.queryRow', side_effect=check_query) + @patch('ggrebalance.check_down_segments') @rebalance_only(numsegs = 24) - def test_invalid_target_datadir(self, mockCatalog, mockOsPath, mockCursor): + def test_invalid_target_datadir(self, mockCatalog, mockOsPath, mockCursor, mock_down): self.options.target_hosts = "sdw1, sdw2, sdw3" self.options.target_datadirs = '/data/primary/gpseg{content}' with self.assertRaises(SystemExit): @@ -115,8 +118,9 @@ def test_invalid_target_datadir(self, mockCatalog, mockOsPath, mockCursor): return_value=initGparrayFromFile("role_mismatch")) @patch('os.path.exists', side_effect=lambda path: path not in ['/tmp/dirdoesnotexist/gparraydump']) @patch('gppylib.db.dbconn.queryRow', side_effect=check_query) + @patch('ggrebalance.check_down_segments') @rebalance_only(numsegs = 4) - def test_role_mistmatch(self, mockCatalog, mockOsPath, mockCursor): + def test_role_mistmatch(self, mockCatalog, mockOsPath, mockCursor, mock_down): with self.assertRaises(SystemExit): self.subject.main(self.options, self.args, self.parser) self.subject.logger.error.assert_any_call( diff --git a/gpMgmt/bin/gprebalance_modules/test/test_unit_resources.py b/gpMgmt/bin/gprebalance_modules/test/test_unit_resources.py index f4a340b1d51e..fd12f321aca8 100644 --- a/gpMgmt/bin/gprebalance_modules/test/test_unit_resources.py +++ b/gpMgmt/bin/gprebalance_modules/test/test_unit_resources.py @@ -385,7 +385,7 @@ def test_validate_target_space_sufficient(self): # Should not raise exception try: - estimator._validate_target_space(moves) + estimator._validate_and_build_allocations(moves) except ResourceError: self.fail("ResourceError raised when space is sufficient") @@ -430,7 +430,7 @@ def test_validate_target_space_insufficient(self): # Should raise ResourceError with self.assertRaises(ResourceError) as context: - estimator._validate_target_space(moves) + estimator._validate_and_build_allocations(moves) error_msg = str(context.exception) self.assertIn("Insufficient disk space", error_msg) @@ -484,7 +484,7 @@ def test_validate_multiple_moves_same_filesystem_insufficient(self): }) with self.assertRaises(ResourceError) as context: - estimator._validate_target_space(moves) + estimator._validate_and_build_allocations(moves) error_msg = str(context.exception) self.assertIn("Insufficient disk space", error_msg) @@ -543,7 +543,7 @@ def test_validate_multiple_moves_same_filesystem_sufficient(self): # Should not raise try: - estimator._validate_target_space(moves) + estimator._validate_and_build_allocations(moves) except ResourceError: self.fail("ResourceError raised when space is sufficient") @@ -572,9 +572,9 @@ def test_validate_target_space_no_space_info(self): estimator.disk_checker.check_batch_available_space = Mock(return_value={}) with self.assertRaises(ResourceError) as context: - estimator._validate_target_space(moves) + estimator._validate_and_build_allocations(moves) - self.assertIn("No disk space information for host sdw2", str(context.exception)) + self.assertIn("No disk space information for sdw2:/data/primary0", str(context.exception)) def test_validate_tablespace_space_sufficient(self): """Test validation succeeds when tablespace has sufficient space on separate filesystem""" @@ -618,7 +618,7 @@ def test_validate_tablespace_space_sufficient(self): # Should not raise exception try: - estimator._validate_target_space(moves) + estimator._validate_and_build_allocations(moves) except ResourceError: self.fail("ResourceError raised when space is sufficient") @@ -663,7 +663,7 @@ def test_validate_tablespace_space_insufficient(self): }) with self.assertRaises(ResourceError) as context: - estimator._validate_target_space(moves) + estimator._validate_and_build_allocations(moves) error_msg = str(context.exception) self.assertIn("Insufficient disk space", error_msg) @@ -718,7 +718,7 @@ def test_validate_multiple_tablespaces_different_filesystems(self): }) with self.assertRaises(ResourceError) as context: - estimator._validate_target_space(moves) + estimator._validate_and_build_allocations(moves) error_msg = str(context.exception) self.assertIn("Insufficient disk space", error_msg) @@ -768,7 +768,7 @@ def test_validate_tablespace_same_filesystem_as_datadir_insufficient(self): }) with self.assertRaises(ResourceError) as context: - estimator._validate_target_space(moves) + estimator._validate_and_build_allocations(moves) error_msg = str(context.exception) self.assertIn("Insufficient disk space", error_msg) @@ -821,7 +821,7 @@ def test_validate_tablespace_same_filesystem_as_datadir_sufficient(self): # Should not raise exception try: - estimator._validate_target_space(moves) + estimator._validate_and_build_allocations(moves) except ResourceError: self.fail("ResourceError raised when space is sufficient") @@ -889,7 +889,7 @@ def test_validate_complex_overlapping_filesystems(self): # Should not raise try: - estimator._validate_target_space(moves) + estimator._validate_and_build_allocations(moves) except ResourceError: self.fail("ResourceError raised when space is sufficient") @@ -955,7 +955,7 @@ def test_validate_complex_overlapping_filesystems_insufficient(self): }) with self.assertRaises(ResourceError) as context: - estimator._validate_target_space(moves) + estimator._validate_and_build_allocations(moves) error_msg = str(context.exception) self.assertIn("Insufficient disk space", error_msg) @@ -967,60 +967,6 @@ def test_validate_complex_overlapping_filesystems_insufficient(self): self.assertIn("11", error_msg) self.assertIn("9", error_msg) - def test_validate_no_double_counting_same_segment(self): - """Test that the same segment is not double-counted on same filesystem""" - seg0 = None - for seg in self.gparray.getDbList(): - if seg.content == 0 and seg.isSegmentPrimary(): - seg0 = seg - break - - # Create two moves for the same segment (shouldn't happen in reality, - # but tests deduplication logic) - moves = [ - LogicalMove( - seg=seg0, - srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), - dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), - target_datadir='/data/primary0', - target_port=7000, - segment_size=SegmentSize(datadir_size_kb=5242880) # 5GB - ), - LogicalMove( - seg=seg0, # Same segment! - srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), - dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), - target_datadir='/data/primary0_copy', - target_port=7001, - segment_size=SegmentSize(datadir_size_kb=5242880) # 5GB - ) - ] - - estimator = ResourceEstimator(self.logger, self.conn, self.gparray) - - # Both on same filesystem - # Should only count once: 5GB * 1.1 = 5.5GB, not 11GB - estimator.disk_checker.check_batch_available_space = Mock(return_value={ - '172.20.0.7': { - '/data/primary0': DiskSpaceInfo( - filesystem='/dev/sdb1', - available_kb=6291456, # 6GB - directory='/data/primary0' - ), - '/data/primary0_copy': DiskSpaceInfo( - filesystem='/dev/sdb1', # Same filesystem - available_kb=6291456, - directory='/data/primary0_copy' - ) - } - }) - - # Should NOT raise - only needs 5.5GB, has 6GB - try: - estimator._validate_target_space(moves) - except ResourceError: - self.fail("ResourceError raised - segment was double-counted!") - @patch('gprebalance_modules.planner.PortIsAvailable') @patch('gprebalance_modules.planner.DiskSpaceChecker') @patch('gprebalance_modules.planner.HostResolver.resolve_hostname') diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature index 5300a340ca10..48d7f40339ca 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature @@ -3,33 +3,48 @@ Feature: ggrebalance behave tests Scenario Outline: test 1. validate incompatible option combinations Given a standard local demo cluster is running + And the environment variable "COORDINATOR_DATA_DIRECTORY" is set from output of "echo $(dirname $(pwd))/gpAux/gpdemo/datadirs/qddir/demoDataDir-1" + And coordinator data directory is updated When the user runs "ggrebalance " Then ggrebalance should return a return code of 1 And ggrebalance should print "" to stdout Given + Then + And coordinator data directory is updated Examples: Mutually exclusive options - | options | error_message | run_status | - | --target-hosts sdw1,sdw2 --target-hosts-file /tmp/hosts.txt | Can't use together options '--target-hosts' and '--target-hosts-file' | stub | - | --target-hosts sdw1,sdw2 --add-hosts sdw3 | Can't use together options '--target-hosts' and '--add-hosts' | stub | - | --target-hosts sdw1,sdw2 --remove-hosts sdw3 | Can't use together options '--target-hosts' and '--remove-hosts' | stub | - | --add-hosts sdw3 --add-hosts-file /tmp/add.txt | Can't use together options '--add-hosts' and '--add-hosts-file' | stub | - | --remove-hosts sdw3 --remove-hosts-file /tmp/rm.txt | Can't use together options '--remove-hosts' and '--remove-hosts-file' | stub | - | --add-hosts sdw3 --remove-hosts sdw3 | Can't use together options '--add-hosts' and '--remove-hosts' | stub | - | --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' --target-datadirs-file /tmp/datadirs.txt | Can't use together options '--target-datadirs' and '--target-datadirs-file' | stub | - | --mirror-mode grouped --skip-rebalance | Can't use together options '--skip-rebalance' and '--mirror-mode' | stub | - | -m spread --skip-rebalance | Can't use together options '--skip-rebalance' and '--mirror-mode' | stub | - | -c --target-hosts sdw1,sdw2 | Can't use together options '--clean-required' and '--target-hosts' | stub | - | -c --add-hosts sdw3 | Can't use together options '--clean-required' and '--add-hosts' | stub | - | -c --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' | Can't use together options '--clean-required' and '--target-datadirs' | stub | - | -c -x 2 | Can't use together options '--clean-required' and '--target-segment-count' | stub | - | -c --mirror-mode grouped | Can't use together options '--clean-required' and '--mirror-mode' | stub | - | -c --skip-rebalance | Can't use together options '--clean-required' and '--skip-rebalance' | stub | - | -c --show-plan | Can't use together options '--clean-required' and '--show-plan' | stub | - | -c --analyze | Can't use together options '--clean-required' and '--analyze' | stub | - | -c --replay-lag 1 | Can't use together options '--clean-required' and '--replay-lag' | stub | - | -c --hba-hostnames | Can't use together options '--clean-required' and '--hba-hostnames' | stub | - | -c --skip-resource-estimation | Can't use together options '--clean-required' and '--skip-resource-estimation' | the database is not running | + | options | error_message | run_status | restore_env| + | --target-hosts sdw1,sdw2 --target-hosts-file /tmp/hosts.txt | Can't use together options '--target-hosts' and '--target-hosts-file' | stub |stub| + | --target-hosts sdw1,sdw2 --add-hosts sdw3 | Can't use together options '--target-hosts' and '--add-hosts' | stub |stub| + | --target-hosts sdw1,sdw2 --remove-hosts sdw3 | Can't use together options '--target-hosts' and '--remove-hosts' | stub |stub| + | --add-hosts sdw3 --add-hosts-file /tmp/add.txt | Can't use together options '--add-hosts' and '--add-hosts-file' | stub |stub| + | --remove-hosts sdw3 --remove-hosts-file /tmp/rm.txt | Can't use together options '--remove-hosts' and '--remove-hosts-file' | stub |stub| + | --add-hosts sdw3 --remove-hosts sdw3 | Can't use together options '--add-hosts' and '--remove-hosts' | stub |stub| + | --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' --target-datadirs-file /tmp/datadirs.txt | Can't use together options '--target-datadirs' and '--target-datadirs-file' | stub |stub| + | --mirror-mode grouped --skip-rebalance | Can't use together options '--skip-rebalance' and '--mirror-mode' | stub |stub| + | -m spread --skip-rebalance | Can't use together options '--skip-rebalance' and '--mirror-mode' | stub |stub| + | --skip-rebalance --inplace-swap-roles | Can't use together options '--skip-rebalance' and '--inplace-swap-roles' | stub |stub| + | -c --target-hosts sdw1,sdw2 | Can't use together options '--clean-required' and '--target-hosts' | stub |stub| + | -c --add-hosts sdw3 | Can't use together options '--clean-required' and '--add-hosts' | stub |stub| + | -c --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' | Can't use together options '--clean-required' and '--target-datadirs' | stub |stub| + | -c -x 2 | Can't use together options '--clean-required' and '--target-segment-count' | stub |stub| + | -c --mirror-mode grouped | Can't use together options '--clean-required' and '--mirror-mode' | stub |stub| + | -c --skip-rebalance | Can't use together options '--clean-required' and '--skip-rebalance' | stub |stub| + | -c --show-plan | Can't use together options '--clean-required' and '--show-plan' | stub |stub| + | -c --analyze | Can't use together options '--clean-required' and '--analyze' | stub |stub| + | -c --replay-lag 1 | Can't use together options '--clean-required' and '--replay-lag' | stub |stub| + | -c --hba-hostnames | Can't use together options '--clean-required' and '--hba-hostnames' | stub |stub| + | -c --inplace-swap-roles | Can't use together options '--clean-required' and '--inplace-swap-roles' | stub |stub| + | -c --skip-resource-estimation | Can't use together options '--clean-required' and '--skip-resource-estimation' | stub |stub| + | -r --target-hosts sdw1,sdw2 | Can't use together options '--rollback-required' and '--target-hosts' | stub |stub| + | -r --add-hosts sdw3 | Can't use together options '--rollback-required' and '--add-hosts' | stub |stub| + | -r --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' | Can't use together options '--rollback-required' and '--target-datadirs' | stub |stub| + | -r -x 2 | Can't use together options '--rollback-required' and '--target-segment-count' | stub |stub| + | -r --mirror-mode grouped | Can't use together options '--rollback-required' and '--mirror-mode' | stub |stub| + | -r --skip-rebalance | Can't use together options '--rollback-required' and '--skip-rebalance' | stub |stub| + | -r --show-plan | Can't use together options '--rollback-required' and '--show-plan' | stub |stub| + | -r --inplace-swap-roles | Can't use together options '--rollback-required' and '--inplace-swap-roles' | stub |stub| + | -r --skip-resource-estimation | Can't use together options '--rollback-required' and '--skip-resource-estimation' |the database is not running |"COORDINATOR_DATA_DIRECTORY" environment variable should be restored| Scenario: test 2. ggrebalance simple scenarios Given the database is not running diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature index a66020dc26b0..ac09d8017971 100755 --- a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -251,3 +251,130 @@ Feature: ggrebalance behave tests (rebalance scenarios) And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 + + Scenario Outline: 6. rebalance - case when mirror and primary swap their hosts. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'mkdir -p /data/gpdata/ggrebalance/primary'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'mkdir -p /data/gpdata/ggrebalance/mirror'" + And the temporary file "/tmp/rebalance_s6" is created with content + """ + TRUSTED_SHELL=ssh + ENCODING=UNICODE + SEG_PREFIX=gpseg + HEAP_CHECKSUM=on + HBA_HOSTNAMES=0 + QD_PRIMARY_ARRAY=cdw~cdw~7000~/data/gpdata/ggrebalance/gpseg-1~1~-1~0 + declare -a PRIMARY_ARRAY=( + sdw1~sdw1~7002~/data/gpdata/ggrebalance/primary/gpseg0~2~0~11100 + sdw1~sdw1~7003~/data/gpdata/ggrebalance/primary/gpseg1~3~1~11110 + sdw1~sdw1~7004~/data/gpdata/ggrebalance/primary/gpseg2~4~2~11220 + sdw2~sdw2~7003~/data/gpdata/ggrebalance/primary/gpseg3~5~3~11350 + sdw3~sdw3~7004~/data/gpdata/ggrebalance/primary/gpseg4~6~4~11360 + sdw3~sdw3~7005~/data/gpdata/ggrebalance/primary/gpseg5~7~5~11370 + ) + declare -a MIRROR_ARRAY=( + sdw3~sdw3~7050~/data/gpdata/ggrebalance/mirror/gpseg0~8~0~51130 + sdw3~sdw3~7051~/data/gpdata/ggrebalance/mirror/gpseg1~9~1~51140 + sdw2~sdw2~7052~/data/gpdata/ggrebalance/mirror/gpseg2~10~2~51160 + sdw1~sdw1~7053~/data/gpdata/ggrebalance/mirror/gpseg3~11~3~51160 + sdw2~sdw2~7054~/data/gpdata/ggrebalance/mirror/gpseg4~12~4~51200 + sdw2~sdw2~7055~/data/gpdata/ggrebalance/mirror/gpseg5~13~5~51136 + ) + """ + When initialize a cluster using "/tmp/rebalance_s6" + Then the temporary file "/tmp/rebalance_s6" is removed + Given the environment variable "COORDINATOR_DATA_DIRECTORY" is set to "/data/gpdata/ggrebalance/gpseg-1" + And coordinator data directory is updated + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 1 segments where "hostname='sdw1' and content = 2 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw2' and content = 2 and role = 'p' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + Given the database is not running + Given the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /data/gpdata/ggrebalance/primary'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /data/gpdata/ggrebalance/mirror'" + Then + And coordinator data directory is updated + + Examples: inplace swap and using 3rd host + | command | restore_env| + |"ggrebalance -x 6 -d '/data/gpdata/ggrebalance/primary, /data/gpdata/ggrebalance/mirror'"|stub| + |"ggrebalance -x 6 -d '/data/gpdata/ggrebalance/primary, /data/gpdata/ggrebalance/mirror' --inplace-swap-roles"|"COORDINATOR_DATA_DIRECTORY" environment variable should be restored| + + Scenario: 7. rebalance - case with multiple swaps. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'mkdir -p /data/gpdata/ggrebalance/primary'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'mkdir -p /data/gpdata/ggrebalance/mirror'" + And the temporary file "/tmp/rebalance_s6" is created with content + """ + TRUSTED_SHELL=ssh + ENCODING=UNICODE + SEG_PREFIX=gpseg + HEAP_CHECKSUM=on + HBA_HOSTNAMES=0 + QD_PRIMARY_ARRAY=cdw~cdw~7000~/data/gpdata/ggrebalance/gpseg-1~1~-1~0 + declare -a PRIMARY_ARRAY=( + sdw1~sdw1~7002~/data/gpdata/ggrebalance/primary/gpseg0~2~0~11100 + sdw1~sdw1~7003~/data/gpdata/ggrebalance/primary/gpseg1~3~1~11110 + sdw1~sdw1~7004~/data/gpdata/ggrebalance/primary/gpseg2~4~2~11220 + sdw1~sdw1~7005~/data/gpdata/ggrebalance/primary/gpseg3~5~3~11350 + sdw2~sdw2~7006~/data/gpdata/ggrebalance/primary/gpseg4~6~4~11360 + sdw2~sdw2~7007~/data/gpdata/ggrebalance/primary/gpseg5~7~5~11370 + ) + declare -a MIRROR_ARRAY=( + sdw2~sdw2~7050~/data/gpdata/ggrebalance/mirror/gpseg0~8~0~51130 + sdw2~sdw2~7051~/data/gpdata/ggrebalance/mirror/gpseg1~9~1~51140 + sdw3~sdw3~7052~/data/gpdata/ggrebalance/mirror/gpseg2~10~2~51160 + sdw3~sdw3~7053~/data/gpdata/ggrebalance/mirror/gpseg3~11~3~51160 + sdw3~sdw3~7054~/data/gpdata/ggrebalance/mirror/gpseg4~12~4~51200 + sdw3~sdw3~7055~/data/gpdata/ggrebalance/mirror/gpseg5~13~5~51136 + ) + """ + When initialize a cluster using "/tmp/rebalance_s6" + Then the temporary file "/tmp/rebalance_s6" is removed + Given the environment variable "COORDINATOR_DATA_DIRECTORY" is set to "/data/gpdata/ggrebalance/gpseg-1" + And coordinator data directory is updated + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6 -d '/data/gpdata/ggrebalance/primary, /data/gpdata/ggrebalance/mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 1 segments where "hostname='sdw1' and content = 2 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content = 2 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw1' and content = 3 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content = 3 and role = 'p' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + Given the database is not running + Given the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /data/gpdata/ggrebalance/primary'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /data/gpdata/ggrebalance/mirror'" + Then "COORDINATOR_DATA_DIRECTORY" environment variable should be restored + And coordinator data directory is updated diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 3d6f3936aa3f..c93909102d57 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -4608,5 +4608,25 @@ def impl(context): os.environ[fault_injection.GPMGMT_FAULT_DELAY_MS] = "" @given('stub') +@then('stub') +@when('stub') def impl(context): pass + +@given('the temporary file "{filename}" is created with content') +@then('the temporary file "{filename}" is created with content') +@when('the temporary file "{filename}" is created with content') +def impl(context, filename): + with open(filename, 'w') as f: + f.write(context.text + '\n') + +@given('the environment variable "{var}" is set from output of "{command}"') +def impl(context, var, command): + run_command(context, command) + context.execute_steps(f'Given the environment variable "{var}" is set to "{context.stdout_message.rstrip()}"') + +@given('coordinator data directory is updated') +@then('coordinator data directory is updated') +def impl(context): + global coordinator_data_dir + coordinator_data_dir = os.environ.get('COORDINATOR_DATA_DIRECTORY')