From 8b95bd2fdefee5efe129d2ac4392e58a41310c68 Mon Sep 17 00:00:00 2001 From: Ioana Meirosu Date: Thu, 28 Feb 2019 18:27:37 +0200 Subject: [PATCH 1/2] Added the functionality to run TD distributed on Kubernetes. Enabled to possibility of deploying more than one driver in order to avoid having a single point of failure in the cluster. The drivers can have one of the following roles: Master: is responsible for coordinating the entire execution in the cluster (sends Heartbeat messages to periodically check that the agents are still up and running, divides the phases from the initial configuration into multiple tasks to be distributed between the agents, etc). Candidate: it can replace the current Master in case of failure. Candidates are responsible for periodically sending Heartbeat messages to the current Master to check that it is still running properly. --- agent-job.yaml | 22 + driver_deployment.yaml | 58 +++ toughday/pom.xml | 6 + .../main/java/com/adobe/qe/toughday/Main.java | 27 +- .../qe/toughday/internal/core/TestSuite.java | 20 +- .../internal/core/config/ConfigParams.java | 8 + .../internal/core/config/Configuration.java | 110 +++-- .../internal/core/config/GlobalArgs.java | 11 +- .../core/config/parsers/cli/CliParser.java | 70 +-- .../yaml/GenerateYamlConfiguration.java | 87 +--- .../core/config/parsers/yaml/YamlBuilder.java | 99 +++++ .../parsers/yaml/YamlConfiguration.java | 3 + .../core/config/parsers/yaml/YamlParser.java | 36 +- .../DistributedPhaseMonitor.java | 186 ++++++++ .../core/distributedtd/ExecutionTrigger.java | 47 ++ .../core/distributedtd/HttpUtils.java | 100 +++++ ...amlDumpConfigurationAsTaskForTDAgents.java | 175 ++++++++ .../core/distributedtd/cluster/Agent.java | 303 +++++++++++++ .../cluster/DistributedConfig.java | 98 +++++ .../distributedtd/cluster/driver/Driver.java | 401 ++++++++++++++++++ .../cluster/driver/DriverState.java | 214 ++++++++++ .../cluster/driver/DriverUpdateInfo.java | 135 ++++++ .../cluster/driver/MasterElection.java | 264 ++++++++++++ .../requests/AbstractRequestProcessor.java | 200 +++++++++ .../requests/CandidateRequestProcessor.java | 94 ++++ .../requests/MasterRequestProcessor.java | 168 ++++++++ .../driver/requests/RequestProcessor.java | 67 +++ .../requests/RequestProcessorDispatcher.java | 38 ++ .../RedistributionInstructions.java | 45 ++ .../RedistributionInstructionsProcessor.java | 75 ++++ .../redistribution/TaskBalancer.java | 313 ++++++++++++++ .../runmodes/AbstractRunModeBalancer.java | 71 ++++ .../runmodes/ConstantLoadRunModeBalancer.java | 76 ++++ .../runmodes/NormalRunModeBalancer.java | 88 ++++ .../runmodes/RunModeBalancer.java | 47 ++ .../splitters/PhaseSplitter.java | 108 +++++ .../runmodes/ConstantLoadRunModeSplitter.java | 106 +++++ .../runmodes/NormalRunModeSplitter.java | 113 +++++ .../splitters/runmodes/RunModeSplitter.java | 31 ++ .../distributedtd/tasks/HeartbeatTask.java | 84 ++++ .../tasks/MasterHeartbeatTask.java | 146 +++++++ .../core/engine/AsyncTimeoutChecker.java | 2 +- .../toughday/internal/core/engine/Engine.java | 19 +- .../toughday/internal/core/engine/Phase.java | 15 +- .../internal/core/engine/RunMode.java | 15 + .../core/engine/runmodes/ConstantLoad.java | 228 +++++++--- .../internal/core/engine/runmodes/Normal.java | 215 ++++++---- .../adobe/qe/toughday/metrics/Percentile.java | 8 +- .../adobe/qe/toughday/TestConfiguration.java | 10 +- .../qe/toughday/TestConstantLoadMode.java | 2 +- .../com/adobe/qe/toughday/TestNormalMode.java | 2 +- .../AbstractRunModeBalancerTest.java | 59 +++ .../ConstantLoadRunModeSplitterTest.java | 98 +++++ .../DistributedPhaseMonitorTest.java | 119 ++++++ .../core/distributedtd/DummyRunMode.java | 77 ++++ .../distributedtd/DummyRunModeBalancer.java | 16 + .../distributedtd/MasterElectionTest.java | 122 ++++++ .../NormalRunModeSplitterTest.java | 111 +++++ .../core/distributedtd/PhaseSplitterTest.java | 103 +++++ ...distributionInstructionsProcessorTest.java | 86 ++++ ...umpConfigurationAsTaskForTDAgentsTest.java | 152 +++++++ .../CandidateRequestProcessorTest.java | 146 +++++++ .../requests/MasterRequestProcessorTest.java | 135 ++++++ .../RequestProcessorDispatcherTest.java | 56 +++ .../adobe/qe/toughday/mocks/MockMetric.java | 21 + .../api/annotations/ConfigArgGet.java | 12 +- 66 files changed, 5871 insertions(+), 308 deletions(-) create mode 100644 agent-job.yaml create mode 100644 driver_deployment.yaml create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlBuilder.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitor.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/ExecutionTrigger.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/HttpUtils.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgents.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Agent.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/DistributedConfig.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/Driver.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverState.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverUpdateInfo.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/MasterElection.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/AbstractRequestProcessor.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/CandidateRequestProcessor.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/MasterRequestProcessor.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessor.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessorDispatcher.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructions.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructionsProcessor.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/TaskBalancer.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/AbstractRunModeBalancer.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/ConstantLoadRunModeBalancer.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/NormalRunModeBalancer.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/RunModeBalancer.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/PhaseSplitter.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/ConstantLoadRunModeSplitter.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/NormalRunModeSplitter.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/RunModeSplitter.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/HeartbeatTask.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/MasterHeartbeatTask.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/AbstractRunModeBalancerTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/ConstantLoadRunModeSplitterTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitorTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunMode.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunModeBalancer.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/MasterElectionTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/NormalRunModeSplitterTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/PhaseSplitterTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/RedistributionInstructionsProcessorTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgentsTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/CandidateRequestProcessorTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/MasterRequestProcessorTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/RequestProcessorDispatcherTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/mocks/MockMetric.java diff --git a/agent-job.yaml b/agent-job.yaml new file mode 100644 index 0000000..78ac618 --- /dev/null +++ b/agent-job.yaml @@ -0,0 +1,22 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: agent +spec: + parallelism: 2 + ttlSecondsAfterFinished: 0 + template: + metadata: + labels: + app: agent + spec: + containers: + - name: agent-container + image: + livenessProbe: + httpGet: + path: /health + port: 4567 + initialDelaySeconds: 20 + periodSeconds: 3 + restartPolicy: Never diff --git a/driver_deployment.yaml b/driver_deployment.yaml new file mode 100644 index 0000000..cccf5a9 --- /dev/null +++ b/driver_deployment.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: driver +spec: + selector: + matchLabels: + app: driver + template: + metadata: + labels: + app: driver + spec: + containers: + - image: + name: driver-container + ports: + - containerPort: 4567 + - containerPort: 80 + livenessProbe: + httpGet: + path: /health + port: 4567 + initialDelaySeconds: 20 + periodSeconds: 3 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: driver +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 4567 + name: http + selector: + app: driver + +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: driver + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" +spec: + rules: + - http: + paths: + - backend: + serviceName: driver + servicePort: 80 + path: /config \ No newline at end of file diff --git a/toughday/pom.xml b/toughday/pom.xml index 4137f5e..f9182fb 100644 --- a/toughday/pom.xml +++ b/toughday/pom.xml @@ -271,5 +271,11 @@ test + + com.sparkjava + spark-core + 2.7.2 + + diff --git a/toughday/src/main/java/com/adobe/qe/toughday/Main.java b/toughday/src/main/java/com/adobe/qe/toughday/Main.java index 7b57ebb..e2243b4 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/Main.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/Main.java @@ -11,14 +11,15 @@ */ package com.adobe.qe.toughday; - import com.adobe.qe.toughday.internal.core.engine.Engine; import com.adobe.qe.toughday.internal.core.config.parsers.cli.CliParser; import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.ExecutionTrigger; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - /** * Main class. Creates a Configuration and an engine and runs the tests. */ @@ -31,8 +32,6 @@ public class Main { } public static void main(String[] args) { - - CliParser cliParser = new CliParser(); System.out.println(); @@ -51,15 +50,27 @@ public static void main(String[] args) { System.exit(1); } - Engine engine = new Engine(configuration); - engine.runTests(); + /* check if we should trigger an execution query in the cluster. */ + if (configuration.executeInDitributedMode()) { + new ExecutionTrigger(configuration).triggerDistributedExecution(); + System.exit(0); + } else if (configuration.getDistributedConfig().getAgent()) { + Agent agent = new Agent(); + agent.start(); + } else if (configuration.getDistributedConfig().getDriver()) { + Driver driver = new Driver(configuration); + driver.run(); + } else { + Engine engine = new Engine(configuration); + engine.runTests(); - System.exit(0); + System.exit(0); + } } catch (Throwable t) { LOG.error("Error encountered: " + (t.getMessage() != null ? t.getMessage() : "Please check toughday.log for more information.")); LogManager.getLogger(Engine.class).error("Error encountered", t); + System.exit(-1); } - System.exit(0); } } diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/TestSuite.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/TestSuite.java index 80ab294..88b7063 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/TestSuite.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/TestSuite.java @@ -21,7 +21,7 @@ /** * Test suite class. */ -public class TestSuite { +public class TestSuite implements Cloneable { private List setupStep; private String description = ""; private List tags = new ArrayList<>(); @@ -39,6 +39,24 @@ public TestSuite() { totalWeight = 0; } + /** + * Creates a copy of the current test suite. All tests contained by the test suite are cloned. + * @throws CloneNotSupportedException if the object to be cloned does not implement the Cloneable interface. + */ + public TestSuite clone() throws CloneNotSupportedException { + TestSuite newInstance = (TestSuite) super.clone(); + + /* clone all the tests in the TestSuite */ + newInstance.orderedTests = new ArrayList<>(); + newInstance.nameMap = new HashMap<>(); + + for (AbstractTest test : this.getTests()) { + newInstance.add(test.clone()); + } + + return newInstance; + } + /** * Method for adding a test. * @param test diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/ConfigParams.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/ConfigParams.java index e47fd3f..b59b1af 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/ConfigParams.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/ConfigParams.java @@ -26,6 +26,7 @@ public class ConfigParams implements Serializable { private List phasesParams = new ArrayList<>(); private Map globalParams = new HashMap<>(); + private Map distributedConfigParams = new HashMap<>(); private Map publishModeParams = new HashMap<>(); private Map runModeParams = new HashMap<>(); private List> items = new ArrayList<>(); @@ -94,6 +95,10 @@ public void setGlobalParams(Map globalParams) { this.globalParams = globalParams; } + public void setDistributedConfigParams(Map distributedConfigParams) { + this.distributedConfigParams = distributedConfigParams; + } + public void setPhasesParams(List phasesParams) { this.phasesParams = phasesParams; } @@ -163,12 +168,15 @@ public Map getGlobalParams(){ return globalParams; } + public Map getDistributedConfigParams() { return distributedConfigParams; } + public Map getPublishModeParams() { return publishModeParams; } public Map getRunModeParams() { return runModeParams; } public void merge(ConfigParams other) { globalParams.putAll(other.getGlobalParams()); + distributedConfigParams.putAll(other.distributedConfigParams); items.addAll(other.items); phasesParams.addAll(other.phasesParams); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/Configuration.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/Configuration.java index 4523850..285a6a9 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/Configuration.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/Configuration.java @@ -24,6 +24,7 @@ import com.adobe.qe.toughday.internal.core.engine.Phase; import com.adobe.qe.toughday.internal.core.engine.PublishMode; import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.DistributedConfig; import com.adobe.qe.toughday.metrics.Metric; import com.adobe.qe.toughday.publishers.CSVPublisher; import com.adobe.qe.toughday.publishers.ConsolePublisher; @@ -36,9 +37,7 @@ import org.apache.logging.log4j.core.config.LoggerConfig; import org.reflections.Reflections; -import java.io.File; -import java.io.IOException; -import java.lang.ref.WeakReference; +import java.io.*; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -63,6 +62,7 @@ public class Configuration { private PredefinedSuites predefinedSuites = new PredefinedSuites(); private GlobalArgs globalArgs; + private DistributedConfig distributedConfig; private RunMode runMode; private PublishMode publishMode; private TestSuite globalSuite; @@ -70,6 +70,7 @@ public class Configuration { private Map globalMetrics = new LinkedHashMap<>(); private List phases = new ArrayList<>(); private Set phasesWithoutDuration = new HashSet<>(); + private ConfigParams configParams; private boolean defaultSuiteAddedFromConfigExclude = false; private boolean anyMetricAdded = false; private boolean anyPublisherAdded = false; @@ -77,6 +78,37 @@ public class Configuration { private Map feeders = new LinkedHashMap<>(); private Map objects = new HashMap<>(); + public Configuration(String yamlConfig) + throws InvocationTargetException, NoSuchMethodException, InstantiationException, IOException, IllegalAccessException { + ConfigParams configParams = new YamlParser().parse(yamlConfig); + buildConfiguration(configParams); + } + + public Configuration(String[] cmdLineArgs) + throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException, IOException { + ConfigParams configParams = collectConfigurations(cmdLineArgs); + buildConfiguration(configParams); + + } + + /** + * Method for getting the property from a setter method + * + * @param methodName + * @return + */ + public static String propertyFromMethod(String methodName) { + return methodName.startsWith("set") || methodName.startsWith("get") ? StringUtils.lowerCase(methodName.substring(3)) : StringUtils.lowerCase(methodName); + } + + public static Map> getRequiredFieldsForClassAdded() { + return requiredFieldsForClassAdded; + } + + public ConfigParams getConfigParams() { + return this.configParams; + } + private void handleExtensions(ConfigParams configParams) { List extensionList = new ArrayList<>(); @@ -121,7 +153,8 @@ private void handleExtensions(ConfigParams configParams) { } /** - * Creates a jar file for each extension file that should be loaded. + * Creates a jar file for each extension file that should be loaded. + * * @param extensionList A list of names representing the jar files that should be loaded. */ private List createJarFiles(List extensionList) { @@ -139,7 +172,8 @@ private List createJarFiles(List extensionList) { } /** - * Creates an URL for each jar file, using its filename. + * Creates an URL for each jar file, using its filename. + * * @param extensionsFileNames * @return */ @@ -157,7 +191,6 @@ private URL[] formJarURLs(List extensionsFileNames) { } // loads all classes from the extension jar files using a new class loader. - private ClassLoader processJarFiles(List jarFiles, URL[] urls) throws MalformedURLException { ToughdayExtensionClassLoader classLoader = new ToughdayExtensionClassLoader(urls, Thread.currentThread().getContextClassLoader()); Map newClasses = new HashMap<>(); @@ -194,10 +227,14 @@ private ClassLoader processJarFiles(List jarFiles, URL[] urls) throws M return classLoader; } + public boolean executeInDitributedMode() { + return !configParams.getDistributedConfigParams().isEmpty() && + !this.getDistributedConfig().getAgent() && + !this.getDistributedConfig().getDriver(); + } - public Configuration(String[] cmdLineArgs) - throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException, IOException { - ConfigParams configParams = collectConfigurations(cmdLineArgs); + private void buildConfiguration(ConfigParams configParams) throws NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException, IOException { + this.configParams = ConfigParams.deepClone(configParams); ConfigParams copyOfConfigParams = ConfigParams.deepClone(configParams); Map items = new HashMap<>(); @@ -206,11 +243,12 @@ public Configuration(String[] cmdLineArgs) Map globalArgsMeta = configParams.getGlobalParams(); for (String helpOption : CliParser.availableHelpOptions) { - if (globalArgsMeta.containsKey(helpOption)) { - return; - } + if (globalArgsMeta.containsKey(helpOption)) { + return; + } } + this.distributedConfig = createObject(DistributedConfig.class, configParams.getDistributedConfigParams()); this.globalArgs = createObject(GlobalArgs.class, globalArgsMeta); configureLogPath(globalArgs.getLogPath()); @@ -232,6 +270,7 @@ public Configuration(String[] cmdLineArgs) GenerateYamlConfiguration generateYaml = new GenerateYamlConfiguration(copyOfConfigParams, items); generateYaml.createYamlConfigurationFile(); } + objects = null; } @@ -315,8 +354,8 @@ private Phase createPhase(ConfigParams configParams, PhaseParams phaseParams, Te phase.setPublishMode(publishMode); // compute the minimum timeout of the phase - phase.getTestSuite().setMinTimeout(globalArgs.getTimeout()); - for(AbstractTest test : phase.getTestSuite().getTests()) { + phase.getTestSuite().setMinTimeout(globalArgs.getTimeoutInSeconds()); + for (AbstractTest test : phase.getTestSuite().getTests()) { // set the count (the number of executions since the beginnin of the run) of each test to 0 phase.getCounts().put(test, new AtomicLong(0)); @@ -325,7 +364,7 @@ private Phase createPhase(ConfigParams configParams, PhaseParams phaseParams, Te items.put(test.getName(), test.getClass()); - if(test.getTimeout() < 0) { + if (test.getTimeout() < 0) { continue; } @@ -373,7 +412,7 @@ private void getConfigurationFromAnotherPhase(PhaseParams phaseParams) { // merge the current phase with the one whose name is the value of 'useconfig' phaseParams.merge(PhaseParams.namedPhases.get(useconfig), - new HashSet<>(Arrays.asList(name, useconfig))); + new HashSet<>(Arrays.asList(name, useconfig))); } } @@ -390,7 +429,7 @@ private void convertActionItems(List configItem((ConfigParams.NamedMetaObject) item.getValue(), items, testSuite, publishers, metrics); break; case EXCLUDE: - excludeItem(((ConfigParams.NamedMetaObject)item.getValue()).getName(), testSuite, publishers, metrics); + excludeItem(((ConfigParams.NamedMetaObject) item.getValue()).getName(), testSuite, publishers, metrics); break; } } @@ -405,7 +444,7 @@ private void configureDurationForPhases() { if (phase.getDuration() == null) { phasesWithoutDuration.add(phase); } else { - durationLeft -= phase.getDuration(); + durationLeft -= GlobalArgs.parseDurationToSeconds(phase.getDuration()); } } @@ -613,16 +652,6 @@ private void configureLogPath(String logPath) throws IOException { } } - /** - * Method for getting the property from a setter method - * - * @param methodName - * @return - */ - public static String propertyFromMethod(String methodName) { - return methodName.startsWith("set") || methodName.startsWith("get") ? StringUtils.lowerCase(methodName.substring(3)) : StringUtils.lowerCase(methodName); - } - /** * Method for setting an object properties annotated with ConfigArgSet using reflection * @@ -635,9 +664,9 @@ public static String propertyFromMethod(String methodName) { * @throws IllegalAccessException caused by reflection */ //TODO figure out if we can make this public static again - public T setObjectProperties(T object, Map args, boolean applyDefaults, Map feedersContext) throws InvocationTargetException, IllegalAccessException { + public T setObjectProperties(T object, Map args, boolean applyDefaults, Map feedersContext) throws InvocationTargetException, IllegalAccessException { Class classObject = object.getClass(); - LOGGER.info("Configuring object of class: " + classObject.getSimpleName()+" ["+classObject.getName()+"]"); + LOGGER.info("Configuring object of class: " + classObject.getSimpleName() + " [" + classObject.getName() + "]"); for (Method method : classObject.getMethods()) { callConfigArgSet(method, object, args, applyDefaults); FeederInjector.injectFeeder(method, object, args, applyDefaults, feedersContext, objects); @@ -683,7 +712,8 @@ private static void callConfigArgSet(Method method, Object object, Map T createObject(Class classObject, Map args) + + public T createObject(Class classObject, Map args) throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { return createObject(classObject, args, null); } @@ -700,7 +730,7 @@ public T createObject(Class classObject, Map a * @throws InstantiationException * @throws NoSuchMethodException */ - public T createObject(Class classObject, Map args, Map feederContext) + public T createObject(Class classObject, Map args, Map feederContext) throws IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException { Constructor constructor = null; @@ -837,6 +867,14 @@ public GlobalArgs getGlobalArgs() { return globalArgs; } + /** + * Getter for kubernetes config args + * @return + */ + public DistributedConfig getDistributedConfig() { + return this.distributedConfig; + } + /** * Getter for the run mode * @@ -870,8 +908,8 @@ public List getPhases() { return phases; } - public static Map> getRequiredFieldsForClassAdded() { - return requiredFieldsForClassAdded; + public void setPhases(List phases) { + this.phases = phases; } public TestSuite getTestSuite() { @@ -882,5 +920,7 @@ public Set getPhasesWithoutDuration() { return phasesWithoutDuration; } - public Collection getFeeders() { return feeders.values(); } + public Collection getFeeders() { + return feeders.values(); + } } diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/GlobalArgs.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/GlobalArgs.java index a58879e..dc3d7e8 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/GlobalArgs.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/GlobalArgs.java @@ -59,7 +59,11 @@ public class GlobalArgs implements com.adobe.qe.toughday.api.core.config.GlobalA private boolean saveConfig = Boolean.parseBoolean(DEFAULT_SAVE_CONFIG); private boolean showSteps = false; private boolean hostValidationEnabled = true; + private boolean k8sRun = false; + private boolean k8sAgent = false; + private boolean k8sdriver = false; private String logPath; + private String driverIp = null; /** * Constructor @@ -106,7 +110,6 @@ public static long parseDurationToSeconds(String duration) { long finalDuration = 0l; long intermDuration = 0l; - //if time unit is not specified, consider it seconds by default. if (duration.matches("^[0-9]+$")) { throw new IllegalArgumentException("Time unit is not specified"); } @@ -226,7 +229,11 @@ public Collection getPublishers() { @ConfigArgGet public long getTimeout() { - return timeout; + return timeout / 1000; + } + + public long getTimeoutInSeconds() { + return this.timeout; } @ConfigArgSet(required = false, desc = "How long a test will run before it will be interrupted and marked as failed. Expressed in seconds", diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/cli/CliParser.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/cli/CliParser.java index 498d5c3..f772b2d 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/cli/CliParser.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/cli/CliParser.java @@ -29,6 +29,7 @@ import com.adobe.qe.toughday.internal.core.engine.Engine; import com.adobe.qe.toughday.internal.core.engine.PublishMode; import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.DistributedConfig; import com.adobe.qe.toughday.metrics.Metric; import com.google.common.base.Joiner; import net.jodah.typetools.TypeResolver; @@ -53,9 +54,9 @@ public class CliParser implements ConfigurationParser { private static final String TEST_CLASS_HELP_HEADER = String.format(HELP_HEADER_FORMAT_WITH_TAGS, "Class", "Fully qualified domain name", "Tags", "Description"); private static final String PUBLISH_CLASS_HELP_HEADER = String.format(HELP_HEADER_FORMAT_NO_TAGS, "Class", "Fully qualified domain name", "Description"); private static final String METRIC_CLASS_HELP_HEADER = PUBLISH_CLASS_HELP_HEADER; - private static Method[] globalArgMethods = GlobalArgs.class.getMethods(); private static final String SUITE_HELP_HEADER = String.format(" %-40s %-40s %s", "Suite", "Tags", "Description"); private static Map> availableGlobalArgs = new HashMap<>(); + private static Map> availableDistributedConfigArgs = new HashMap<>(); private static List parserArgHelps = new ArrayList<>(); @@ -73,18 +74,24 @@ public class CliParser implements ConfigurationParser { add("tag"); }}); + private static void collectAvailableConfigurationOptions(Class type, Map> availableArgs) { + Arrays.stream(type.getMethods()) + .filter(method -> method.isAnnotationPresent(ConfigArgSet.class)) + .forEach(method -> { + ConfigArgSet annotation = method.getAnnotation(ConfigArgSet.class); + int order = annotation.order(); + if (!availableArgs.containsKey(order)) { + availableArgs.put(order, new HashMap<>()); + } + + availableArgs.get(order) + .put(Configuration.propertyFromMethod(method.getName()), annotation); + }); + } + static { - for (Method method : globalArgMethods) { - if (method.getAnnotation(ConfigArgSet.class) != null) { - ConfigArgSet annotation = method.getAnnotation(ConfigArgSet.class); - int order = annotation.order(); - if (null == availableGlobalArgs.get(order)) { - availableGlobalArgs.put(order, new HashMap()); - } - Map globalArgMap = availableGlobalArgs.get(order); - globalArgMap.put(Configuration.propertyFromMethod(method.getName()), annotation); - } - } + collectAvailableConfigurationOptions(GlobalArgs.class, availableGlobalArgs); + collectAvailableConfigurationOptions(DistributedConfig.class, availableDistributedConfigArgs); for (Class parserClass : ReflectionsContainer.getSubTypesOf(ConfigurationParser.class)) { for (Field field : parserClass.getDeclaredFields()) { @@ -230,6 +237,10 @@ private int parseObjectProperties(int startIndex, String[] cmdLineArgs, HashMap< return j - startIndex; } + private boolean isGlobalArg(String paramName) { + return availableGlobalArgs.values().stream().anyMatch(map -> map.containsKey(paramName)); + } + /** * Implementation of parser interface * @param cmdLineArgs command line arguments @@ -261,6 +272,9 @@ public ConfigParams parse(String[] cmdLineArgs) { } else if (arg.equals("runmode")) { skip = parseObjectProperties(i + 1, cmdLineArgs, args); configParams.setRunModeParams(args); + } else if (arg.equals("distributedconfig")) { + skip = parseObjectProperties(i + 1, cmdLineArgs, args); + configParams.setDistributedConfigParams(args); } else if (arg.equals("help")) { skip = 1; globalArgs.put("host", "N/A"); //TODO remove ugly hack @@ -268,14 +282,9 @@ public ConfigParams parse(String[] cmdLineArgs) { String[] res = parseProperty(arg); String key = res[0]; Object val = getObjectFromString(res[1]); - // if global param does not exist - boolean found = false; - for (Map argz : availableGlobalArgs.values()) { - if (argz.containsKey(key)) { - found = true; - break; - } - } + // if global param or distributed td config param does not exist + boolean found = isGlobalArg(key); + if (!found && !parserArgs.contains(key) && !availableHelpOptions.contains(key) && !helpOptionsParameters.contains(key) && !key.equals("suite") && !key.equals("suitesetup")) { throw new IllegalArgumentException("Unrecognized argument --" + key); @@ -468,24 +477,27 @@ public void printShortHelp(boolean printSuitesTests) { System.out.println("\t java -jar toughday.jar --host=localhost --add extension.jar --add com.adobe.qe.toughday.tests.extensionTest"); System.out.println("\t java -jar toughday.jar --suite=toughday --add BASICMetrics --add Average decimals=3 --exclude Failed"); - System.out.println("\r\nGlobal arguments:"); + System.out.println("\r\nExamples for running TD distributed: \r\n"); + System.out.println("\t java -jar toughday.jar --host=localhost --distributedconfig driverip=1.1.1.1"); + System.out.println("\t java -jar toughday.jar --host=localhost --distributedconfig driverip=1.1.1.1 heartbeatinterval=10s --suite=toughday"); - for (Integer order : availableGlobalArgs.keySet()) { - Map paramGroup = availableGlobalArgs.get(order); - for (String param : paramGroup.keySet()) { - System.out.printf("\t--%-32s\t Default: %s - %s\r\n", - param + "=val", paramGroup.get(param).defaultValue(), paramGroup.get(param).desc()); - } - } + System.out.println("\r\nGlobal arguments:"); + availableGlobalArgs.forEach((order, paramGroup) -> + paramGroup.forEach((key, value) -> System.out.printf("\t--%-32s\t Default: %s - %s\r\n", + key + "=val", value.defaultValue(), value.desc()))); for (ParserArgHelp parserArgHelp : parserArgHelps) { System.out.printf("\t--%-32s\t Default: %s - %s\r\n", parserArgHelp.name() + "=val", parserArgHelp.defaultValue(), parserArgHelp.description()); } - //System.out.printf("\t%-32s\t %s\r\n", "--suitesetup=val", getSuiteSetupDescription()); System.out.printf("\t%-32s\t %s\r\n", "--suite=val", "Default: toughday - Where \"val\" can be one or a list (separated by commas) of the predefined suites"); + System.out.println("\r\n Distributed run arguments (--distributedconfig):"); + availableDistributedConfigArgs.forEach((order, paramGroup) -> + paramGroup.forEach((key, value) -> System.out.printf("\t%-32s\t Default: %s - %s\r\n", + key + "=val", value.defaultValue(), value.desc()))); + System.out.println("\r\nAvailable run modes (--runmode):"); for(Map.Entry> runMode : ReflectionsContainer.getInstance().getRunModeClasses().entrySet()) { Description description = runMode.getValue().getAnnotation(Description.class); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/GenerateYamlConfiguration.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/GenerateYamlConfiguration.java index dec7b78..1216be3 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/GenerateYamlConfiguration.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/GenerateYamlConfiguration.java @@ -17,17 +17,11 @@ import com.adobe.qe.toughday.internal.core.Timestamp; import com.adobe.qe.toughday.internal.core.config.ConfigParams; import com.adobe.qe.toughday.internal.core.config.PhaseParams; -import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml; -import org.yaml.snakeyaml.introspector.Property; -import org.yaml.snakeyaml.nodes.NodeTuple; -import org.yaml.snakeyaml.nodes.Tag; -import org.yaml.snakeyaml.representer.Representer; import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; -import java.lang.reflect.Method; import java.util.*; public class GenerateYamlConfiguration { @@ -43,6 +37,7 @@ public class GenerateYamlConfiguration { private static final String DEFAULT_YAML_CONFIGURATION_FILENAME = "toughday_"; private static final String DEFAULT_YAML_EXTENSION = ".yaml"; private static final String TIMESTAMP = Timestamp.START_TIME; + public static final String yamlConfigFilename = DEFAULT_YAML_CONFIGURATION_FILENAME + TIMESTAMP + DEFAULT_YAML_EXTENSION; public GenerateYamlConfiguration(ConfigParams configParams, Map items) { this.configParams = configParams; @@ -61,6 +56,13 @@ public Map getGlobals() { return globals; } + public Map getDistributedConfig() { + Map distributedConfigParms = configParams.getDistributedConfigParams(); + // remove this so that the driver won't try to trigger the execution + distributedConfigParms.remove("driverip"); + return distributedConfigParms; + } + public Map getPublishmode() { return configParams.getPublishModeParams(); } @@ -185,85 +187,28 @@ private void excludeAction(String item, int index) { } } - // Configure yaml representer to exclude class tags when dumping an object. - private void configureYamlRepresenterToExcludeClassTags(Representer representer) { - // Tag.MAP is by default ignored when dumping an object - representer.addClassTag(GenerateYamlConfiguration.class, Tag.MAP); - for (Class klass : ReflectionsContainer.getSubTypesOf(YamlDumpAction.class)) { - representer.addClassTag(klass, Tag.MAP); - } + public String createYamlStringRepresentation() { + Yaml yaml = YamlBuilder.getYamlInstance(); + return yaml.dump(this); } /** * Creates a YAML configuration file. */ public void createYamlConfigurationFile() { + String yamlStringRepresentation = createYamlStringRepresentation(); - final String filename = DEFAULT_YAML_CONFIGURATION_FILENAME + TIMESTAMP + DEFAULT_YAML_EXTENSION; - - FileWriter fileWriter = null; try { - fileWriter = new FileWriter(filename); - } catch (IOException e) { - e.printStackTrace(); - } - - BufferedWriter bufferedWriter = new BufferedWriter(fileWriter); - org.yaml.snakeyaml.constructor.Constructor constructor = new org.yaml.snakeyaml.constructor.Constructor - (GenerateYamlConfiguration.class); - - DumperOptions dumperOptions = new DumperOptions(); - dumperOptions.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - dumperOptions.setAllowReadOnlyProperties(true); - - // Configure the representer to ignore empty fields when dumping the object. By default, each empty filed is represented as {}. - - Representer representer = new Representer() { - @Override - protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) { - - Method method = null; - try { - if (propertyValue == null) { - propertyValue = ""; - } - method = propertyValue.getClass().getMethod("isEmpty"); - } catch (NoSuchMethodException e) { } - - if (method == null) { - return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); - } else { - - try { - if (Boolean.valueOf(method.invoke(propertyValue).toString())) { - return null; - } - } catch (Throwable t) { - t.printStackTrace(); - } - - return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); - } - } - }; + FileWriter fileWriter = new FileWriter(yamlConfigFilename); + BufferedWriter bufferedWriter = new BufferedWriter(fileWriter); - configureYamlRepresenterToExcludeClassTags(representer); - - // dump configuration - Yaml yaml = new Yaml(constructor, representer, dumperOptions); - String yamlObjectRepresentation = yaml.dump(this); - - try { - bufferedWriter.write(yamlObjectRepresentation); + bufferedWriter.write(yamlStringRepresentation); bufferedWriter.flush(); - } catch (IOException e) { - e.printStackTrace(); - } - try { fileWriter.close(); bufferedWriter.close(); + } catch (IOException e) { e.printStackTrace(); } diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlBuilder.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlBuilder.java new file mode 100644 index 0000000..93345db --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlBuilder.java @@ -0,0 +1,99 @@ +package com.adobe.qe.toughday.internal.core.config.parsers.yaml; + +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; + +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.introspector.Property; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Representer; + +import java.lang.reflect.Method; + +/** + * Class responsible for building and configuring a Yaml instance that knows how to dump a configuration + * in the desired format(without class tags, empty objects etc.) + */ +public class YamlBuilder { + + private static Yaml instance = null; + + // Configure yaml representer to exclude class tags when dumping an object. + private static void configureYamlRepresenterToExcludeClassTags(Representer representer) { + // Tag.MAP is by default ignored when dumping an object + representer.addClassTag(GenerateYamlConfiguration.class, Tag.MAP); + representer.addClassTag(YamlDumpConfigurationAsTaskForTDAgents.class, Tag.MAP); + for (Class type : ReflectionsContainer.getSubTypesOf(YamlDumpAction.class)) { + representer.addClassTag(type, Tag.MAP); + } + + } + + private static void buildInstance() { + DumperOptions dumperOptions = getDumperOptions(); + + // Configure the representer to ignore empty fields when dumping the object. By default, each empty filed is represented as {}. + Representer representer = getRepresenter(); + configureYamlRepresenterToExcludeClassTags(representer); + + Constructor constructor = new Constructor(GenerateYamlConfiguration.class); + instance = new Yaml(constructor, representer, dumperOptions); + } + + /** + * Getter for the Yaml instance. + */ + public static Yaml getYamlInstance() { + if (instance == null) { + buildInstance(); + } + + return instance; + } + + + private static Representer getRepresenter() { + return new Representer() { + @Override + protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) { + + Method method = null; + try { + if (propertyValue == null) { + propertyValue = ""; + } + method = propertyValue.getClass().getMethod("isEmpty"); + } catch (NoSuchMethodException ignored) { } + + if (method == null) { + return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); + } else { + + try { + if (Boolean.valueOf(method.invoke(propertyValue).toString())) { + return null; + } + } catch (Throwable t) { + /* no action required here. The only implication is that the empty collection will be included + * in the Yaml representation when dumping an object. + */ + } + + return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); + } + } + }; + } + + private static DumperOptions getDumperOptions() { + DumperOptions dumperOptions = new DumperOptions(); + + dumperOptions.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + dumperOptions.setAllowReadOnlyProperties(true); + + return dumperOptions; + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlConfiguration.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlConfiguration.java index 46f3202..4d40077 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlConfiguration.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlConfiguration.java @@ -21,6 +21,9 @@ public class YamlConfiguration { private ConfigParams configParams = new ConfigParams(); + private void setDistributedConfig(Map distributedConfig) { + this.configParams.setDistributedConfigParams(distributedConfig); + } public void setGlobals(Map globals) { this.configParams.setGlobalParams(globals); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlParser.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlParser.java index b511938..a5cef99 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlParser.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlParser.java @@ -43,6 +43,31 @@ public String description() { } }; + private Constructor getYamlConstructor() { + Constructor constructor = new Constructor(YamlConfiguration.class); + TypeDescription yamlParserDesc = new TypeDescription(YamlConfiguration.class); + + yamlParserDesc.putListPropertyType("tests", YamlParseAction.class); + yamlParserDesc.putListPropertyType("phases", YamlParsePhase.class); + yamlParserDesc.putListPropertyType("publishers", YamlParseAction.class); + yamlParserDesc.putListPropertyType("metrics", YamlParseAction.class); + yamlParserDesc.putListPropertyType("extensions", YamlParseAction.class); + yamlParserDesc.putListPropertyType("feeders", YamlParseAction.class); + + constructor.addTypeDescription(yamlParserDesc); + + return constructor; + } + + public ConfigParams parse(String stringYaml) { + Constructor constructor = getYamlConstructor(); + + Yaml yaml = new Yaml(constructor); + YamlConfiguration yamlConfig = (YamlConfiguration) yaml.load(stringYaml); + + return yamlConfig.getConfigParams(); + } + @Override public ConfigParams parse(String[] cmdLineArgs) { String configFilePath = null; @@ -55,16 +80,7 @@ public ConfigParams parse(String[] cmdLineArgs) { if(configFilePath != null) { try { - Constructor constructor = new Constructor(YamlConfiguration.class); - TypeDescription yamlParserDesc = new TypeDescription(YamlConfiguration.class); - yamlParserDesc.putListPropertyType("tests", YamlParseAction.class); - yamlParserDesc.putListPropertyType("phases", YamlParsePhase.class); - yamlParserDesc.putListPropertyType("publishers", YamlParseAction.class); - yamlParserDesc.putListPropertyType("metrics", YamlParseAction.class); - yamlParserDesc.putListPropertyType("extensions", YamlParseAction.class); - yamlParserDesc.putListPropertyType("feeders", YamlParseAction.class); - - constructor.addTypeDescription(yamlParserDesc); + Constructor constructor = getYamlConstructor(); Yaml yaml = new Yaml(constructor); YamlConfiguration yamlConfig = (YamlConfiguration) yaml.load(new FileInputStream(configFilePath)); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitor.java new file mode 100644 index 0000000..5e29cd9 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitor.java @@ -0,0 +1,186 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; + +/** + * Class responsible for monitoring the execution of a distributed phase. + */ +public class DistributedPhaseMonitor { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private final ExecutorService executorService = Executors.newFixedThreadPool(1); + private final List agentsRunningTD = new ArrayList<>(); + private final List agentsWhichCompletedCurrentPhase = new ArrayList<>(); + // key = name of the test; value = map(key = name of the agent, value = nr of tests executed) + private Map> executions = new HashMap<>(); + private Phase phase; + private long phaseStartTime = 0; + + /** + * Returns true is the exections of the phase has started. + */ + public boolean isPhaseExecuting() { + return this.phase != null && !this.agentsRunningTD.isEmpty(); + } + + public List getAgentsRunningTD() { + return this.agentsRunningTD; + } + + public List getAgentsWhichCompletedCurrentPhase() { + return this.agentsWhichCompletedCurrentPhase; + } + + public void setExecutions(Map> executions) { + this.executions = executions; + } + + /** + * Setter for the execution start time of the phase. + */ + public void setPhaseStartTime(long phaseStartTime) { + this.phaseStartTime = phaseStartTime; + } + + /** + * Getter for the execution start time of the phase + */ + public long getPhaseStartTime() { + return this.phaseStartTime; + } + + /** + * Setter for the phase being monitored + */ + public void setPhase(Phase phase) { + this.phase = phase; + this.phase.getTestSuite().getTests().forEach(test -> executions.put(test.getName(), new HashMap<>())); + + // reset agents which completed the current phase + this.agentsWhichCompletedCurrentPhase.clear(); + } + + /** + * Adds the agent received as a parameter to the list of agents which are currently executing the + * phase being monitored. + * @param agentIdentifier : ip address that identifies the agent inside the cluster + */ + public void registerAgentRunningTD(String agentIdentifier) { + if (!this.agentsRunningTD.contains(agentIdentifier)) { + this.agentsRunningTD.add(agentIdentifier); + } + + } + + public void addAgentWhichCompletedTheCurrentPhase(String agentIdentifier) { + if (!this.agentsWhichCompletedCurrentPhase.contains(agentIdentifier)) { + this.agentsWhichCompletedCurrentPhase.add(agentIdentifier); + } + } + + /** + * Removes tha agent received as a parameter from the list of agents which are currently executing + * the phase being monitored. + * @param agentIdentifier : ip address that identifies the agent inside the cluster + */ + public void removeAgentFromActiveTDRunners(String agentIdentifier) { + this.agentsRunningTD.remove(agentIdentifier); + + if (agentsWhichCompletedCurrentPhase.contains(agentIdentifier)) { + this.agentsWhichCompletedCurrentPhase.remove(agentIdentifier); + } + + // remove agent from executions map + this.executions.forEach((test, executionsPerAgent) -> this.executions.get(test).remove(agentIdentifier)); + } + + /** + * Method for waiting until all agents finish executing the phase being monitored. + */ + public boolean waitForPhaseCompletion(int retries) { + while (retries > 0) { + Future waitPhaseCompletion = executorService.submit(() -> { + + while (this.agentsRunningTD.size() != this.agentsWhichCompletedCurrentPhase.size()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // this will not cause further problems => it can be ignored + } + } + }); + + try { + waitPhaseCompletion.get(); + return true; + } catch (InterruptedException | ExecutionException e) { + retries--; + } + } + + LOG.warn("Exception occurred while waiting for the completion of the current phase. The rest " + + "of the phases will no longer be executed"); + return false; + + } + + /** + * Updates the number of executions/test that must be executed by the agents running in the cluster. This method is + * called during the work redistribution process. + */ + public void updateCountPerTest() { + phase.getTestSuite().getTests().forEach(test -> { + long remained = test.getCount() - this.getExecutionsPerTest().get(test.getName()); + if (remained < 0) { + // set this to 0 so that the agents will know to delete the test from the test suite + test.setCount("0"); + } else { + test.setCount(String.valueOf(remained)); + } + }); + } + + /** + * Returns the number of executions/test/agent. + */ + public Map> getExecutions() { + return this.executions; + } + + /** + * Resets the number of execution/test for each agent executing the phase being monitored. + */ + public void resetExecutions() { + this.executions.forEach((testName, executionsPerAgent) -> + executionsPerAgent.keySet().forEach(agentName -> executionsPerAgent.put(agentName, 0L))); + } + + /** + * Returns the number of executions/test executed by all the agents running in the cluster. + */ + public Map getExecutionsPerTest() { + Map executionsPerTest = new HashMap<>(); + + this.executions.forEach((testName, executionsPerAgent) -> + executionsPerTest.put(testName, executionsPerAgent.values().stream().mapToLong(x -> x).sum())); + + return executionsPerTest; + } + + /** + * Getter for the phase being monitored by this class. + */ + public Phase getPhase() { + return this.phase; + } + +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/ExecutionTrigger.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/ExecutionTrigger.java new file mode 100644 index 0000000..064a415 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/ExecutionTrigger.java @@ -0,0 +1,47 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.GenerateYamlConfiguration; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import org.apache.http.HttpResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashMap; + + +/** + * Class responsible for sending a request to the driver component running in the cluster. This + * request will trigger the distributed execution of TD. + * */ +public class ExecutionTrigger { + + protected static final Logger LOG = LogManager.getLogger(ExecutionTrigger.class); + private final Configuration configuration; + + + public ExecutionTrigger(Configuration configuration) { + this.configuration = configuration; + // sanity check + if (configuration.getDistributedConfig().getDriverIp() == null || configuration.getDistributedConfig().getDriverIp().isEmpty()) { + throw new IllegalStateException("The public ip address at which the driver's service is accessible " + + " is required when running TD in distributed mode."); + } + } + + /** + * Method used for triggering the execution of TD in distributed mode. + */ + public void triggerDistributedExecution() { + GenerateYamlConfiguration generateYaml = new GenerateYamlConfiguration(this.configuration.getConfigParams(), new HashMap<>()); + String yamlConfig = generateYaml.createYamlStringRepresentation(); + HttpUtils httpUtils = new HttpUtils(); + + HttpResponse response = httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, yamlConfig, + Driver.getExecutionPath(this.configuration.getDistributedConfig().getDriverIp(), HttpUtils.SVC_PORT, true), 3); + if (response == null) { + LOG.warn("TD execution request could not be sent to driver. Make sure that driver is up" + + " and ready to process requests."); + } + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/HttpUtils.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/HttpUtils.java new file mode 100644 index 0000000..91917b2 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/HttpUtils.java @@ -0,0 +1,100 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.api.annotations.labels.Nullable; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.io.IOException; + +/** + * Class responsible for handling the communication between the agents and the drivers running in the cluster. + */ +public class HttpUtils { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + public static final String POST_METHOD = "POST"; + public static final String GET_METHOD = "GET"; + public static final String URL_PREFIX = "http://"; + public static final int HTTP_REQUEST_RETRIES = 3; + public static final String SVC_PORT = "80"; + public static final String SPARK_PORT = "4567"; + public static final String FORWARD_QUERY_PARAM = "?forward="; + + + + private HttpResponse sendGetRequest(String URI) { + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(10 * 1000) // 5 seconds + .build(); + HttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(config).build(); + HttpGet request = new HttpGet(URI); + + try { + return httpClient.execute(request); + } catch (IOException e) { + LOG.warn("Http request could not be sent to " + URI + ". Received error " + e.getMessage()); + } + + return null; + } + + private HttpResponse sendPostRequest(String requestContent, String URI) { + HttpClient httpClient = HttpClientBuilder.create().build(); + HttpPost request = new HttpPost(URI); + + try { + StringEntity params = new StringEntity(requestContent); + request.setEntity(params); + request.setHeader("Content-type", "text/plain"); + + // submit request and wait for ack from agent + return httpClient.execute(request); + } catch (IOException e) { + LOG.warn("Http request could not be sent to " + URI + ". Received error " + e.getMessage()); + } + + return null; + } + + private boolean checkSuccessfulRequest(HttpResponse response) { + return response != null && response.getStatusLine().getStatusCode() >= 200 && + response.getStatusLine().getStatusCode() < 300; + } + + /** + * Method used for sending http requests. + * @param requestContent content of the request + * @param URI the path that uniquely identifies the component in the cluster and the http endpoint to be used + * when sending the request + * @param retries how many times to retry sending the request in case of failure + * @return true is the the request is successfully sent, false otherwise. + */ + @Nullable + public HttpResponse sendHttpRequest(String requestType, String requestContent, String URI, int retries) { + HttpResponse response = null; + + while (retries > 0) { + if (requestType.equals(POST_METHOD)) { + response = sendPostRequest(requestContent, URI); + } else if (requestType.equals(GET_METHOD)) { + response = sendGetRequest(URI); + } + + if (checkSuccessfulRequest(response)) { + return response; + } + + retries--; + } + + return null; + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgents.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgents.java new file mode 100644 index 0000000..9c3ba5f --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgents.java @@ -0,0 +1,175 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; +import com.adobe.qe.toughday.api.annotations.ConfigArgSet; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.*; + +import com.adobe.qe.toughday.internal.core.config.parsers.cli.CliParser; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.YamlBuilder; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.YamlDumpAddAction; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.YamlDumpPhase; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.DistributedConfig; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.yaml.snakeyaml.Yaml; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.*; + +/** + * Knows how to dump a Configuration object in yaml format in order to send it to the agents running in the cluster. + * This class assumes that it is not necessary to dump the run mode and the publish mode fields of the Configuration + * class since each Phase contained by this configuration already defines them. + * + * It is also assumed that the only action to be taken into consideration when dumping the configuration is Actions.ADD + * since all the other actions were already processed before sending the execution query to the driver. + */ +public class YamlDumpConfigurationAsTaskForTDAgents { + private static final Logger LOG= LogManager.getLogger(CliParser.class); + + private Configuration configuration; + private List phases = new ArrayList<>(); + + public List getPhases() { + return this.phases; + } + + /** + * This method is used for collecting all configurable properties that were assigned a new value(different than the + * default one) from the command line or the yaml configuration file used for running ToughDay. + * @param type class of the object whose properties are collected + * @param object instance + */ + protected Map collectConfigurableProperties(Class type, Object object) { + if (!type.isAssignableFrom(object.getClass())) { + throw new IllegalArgumentException("The object must have the specified type."); + } + + Map configurableArgs = new HashMap<>(); + + /* add all inherited configurable properties */ + if (type.getSuperclass() != Object.class) { + configurableArgs.putAll(collectConfigurableProperties(type.getSuperclass(), object)); + } + + Arrays.stream(type.getDeclaredMethods()) + .filter(method -> method.isAnnotationPresent(ConfigArgGet.class)) + .forEach(method -> { + String property = Configuration.propertyFromMethod(method.getName()); + try { + Object value = method.invoke(object); + + Class[] parametersType = {String.class}; + Method m = type.getMethod(method.getName().replace("get", "set"), parametersType); + String defaultValue = m.getAnnotation(ConfigArgSet.class).defaultValue(); + + // skip all default values + if (!String.valueOf(value).equals(defaultValue) && value != null) { + if (value instanceof Level) { + configurableArgs.put(property, ((Level) value).name()); + } else { + configurableArgs.put(property, value); + } + } + } catch (IllegalAccessException | InvocationTargetException e) { + LOG.warn("Configurable property " + property + " could not be collected when dumping the " + + "configuration. Please check that public methods with @ConfigArgSet and @ConfigArgGet " + + "were defined for this property."); + } catch ( NoSuchMethodException e) { + // skip for now + LOG.warn("No @ConfigArgGet method was defined for property " + property + ". The configuration " + + "will be dumped without this property."); + } + }); + + return configurableArgs; + } + + private void addAction(ConfigParams.ClassMetaObject item, YamlDumpPhase yamlDumpPhase) { + YamlDumpAddAction addAction = new YamlDumpAddAction(item.getClassName(), item.getParameters()); + if (ReflectionsContainer.getInstance().isTestClass(item.getClassName())) { + yamlDumpPhase.getTests().add(addAction); + } else if (ReflectionsContainer.getInstance().isMetricClass(item.getClassName())) { + yamlDumpPhase.getMetrics().add(addAction); + } else if (ReflectionsContainer.getInstance().isPublisherClass(item.getClassName())) { + yamlDumpPhase.getPublishers().add(addAction); + } + } + + private void createItem(Object object, List> items ) { + Map parameters = collectConfigurableProperties(object.getClass(), object); + ConfigParams.MetaObject metaObject = new ConfigParams.ClassMetaObject(object.getClass().getSimpleName(), parameters); + items.add(new AbstractMap.SimpleEntry<>(Actions.ADD, metaObject)); + } + + private void buildYamlDumpPhases() { + configuration.getPhases().forEach(phase -> { + // collect all configurable properties + Map properties = collectConfigurableProperties(Phase.class, phase); + Map runMode = collectConfigurableProperties(phase.getRunMode().getClass(), phase.getRunMode()); + Map publishMode = collectConfigurableProperties(phase.getPublishMode().getClass(), phase.getPublishMode()); + + // add required type parameter for run mode and publish mode + runMode.put("type", phase.getRunMode().getClass().getSimpleName().toLowerCase()); + publishMode.put("type", phase.getPublishMode().getClass().getSimpleName().toLowerCase()); + + List> items = new ArrayList<>(); + + // create tests + phase.getTestSuite().getTests().forEach(test -> createItem(test, items)); + // create metrics + phase.getMetrics().forEach(metric -> createItem(metric, items)); + // create publishers + phase.getPublishers().forEach(publisher -> createItem(publisher, items)); + + YamlDumpPhase yamlDumpPhase = new YamlDumpPhase(properties, runMode, publishMode); + items.forEach(entry -> addAction((ConfigParams.ClassMetaObject)entry.getValue(), yamlDumpPhase)); + + this.phases.add(yamlDumpPhase); + }); + } + + public YamlDumpConfigurationAsTaskForTDAgents(Configuration configuration) { + if (configuration == null) { + throw new IllegalArgumentException("Configuration must not be null."); + } + + this.configuration = configuration; + } + + /** + * Getter for global params. + */ + public Map getGlobals() { + Map globals = + collectConfigurableProperties(GlobalArgs.class, configuration.getGlobalArgs()); + /* this is required because duration is internally converted from string(which includes the unit of measure) + to long value(duration in seconds) so we need to manually add the unit of measure when dumping it. */ + globals.put("duration", String.valueOf(globals.get("duration")) + 's'); + + return globals; + } + + + /** + * Getter for the distributed configuration. + */ + public Map getDistributedConfig() { + return collectConfigurableProperties(DistributedConfig.class, configuration.getDistributedConfig()); + } + + /** + * Dumps the configuration received in the constructor. + */ + public String generateConfigurationObject() { + buildYamlDumpPhases(); + Yaml yaml = YamlBuilder.getYamlInstance(); + + return yaml.dump(this); + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Agent.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Agent.java new file mode 100644 index 0000000..edd6070 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Agent.java @@ -0,0 +1,303 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster; + +import com.adobe.qe.toughday.api.core.AbstractTest; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.config.PhaseParams; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructionsProcessor; +import com.google.gson.Gson; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.net.*; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.URL_PREFIX; +import static com.adobe.qe.toughday.internal.core.engine.Engine.installToughdayContentPackage; +import static spark.Spark.*; + +/** + * Agent component for running TD distributed. + */ +public class Agent { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + private final ExecutorService tdExecutorService = Executors.newFixedThreadPool(1); + private final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(1); + + public enum Status { + IDLE, /* Agent is waiting to receive tasks from the driver */ + BUILDING_CONFIG, /* Agent has received one task from the driver and it is building the TD configuration */ + RUNNING, /* Agent is running TD tests */ + PHASE_COMPLETED /* Agent finished executing the current phase */ + } + + // available routes + private static final String INSTALL_SAMPLE_CONTENT_PATH = "/sampleContent"; + private static final String SUBMIT_TASK_PATH = "/submitTask"; + private static final String FINISH_PATH = "/finish"; + private static final String HEARTBEAT_PATH = "/heartbeat"; + private static final String REBALANCE_PATH = "/rebalance"; + private static final String HEALTH_PATH = "/health"; + private static final String GET_STATUS_PATH = "/status"; + + private HttpUtils httpUtils = new HttpUtils(); + private Status status = Status.IDLE; + private final Lock statusLock = new ReentrantLock(); + + + /** + * Returns the http URL that the driver should use for finished the execution of the agent. + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getFinishPath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + HttpUtils.SPARK_PORT + FINISH_PATH; + } + + /** + * Returns the http URL that the driver should use for sending heartbeat messages to the agent. + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getHeartbeatPath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + HttpUtils.SPARK_PORT + HEARTBEAT_PATH; + } + + /** + * Returns the http URL that the driver should use for sending a task to the agent. + * @param agentIpAdress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getSubmissionTaskPath(String agentIpAdress) { + return URL_PREFIX + agentIpAdress + ":" + HttpUtils.SPARK_PORT + SUBMIT_TASK_PATH; + } + + /** + * Returns the http URL that the driver should use for sending redistribution instructions to the agent. + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getRebalancePath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + HttpUtils.SPARK_PORT + REBALANCE_PATH; + } + + /** + * Returns the http URL that the driver should use for requiring the agent to install the TD sample content package. + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getInstallSampleContentPath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + HttpUtils.SPARK_PORT + INSTALL_SAMPLE_CONTENT_PATH; + } + + /** + * Returns the http URL that the driver should use for getting the current status of the agent + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getGetStatusPath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + HttpUtils.SPARK_PORT + GET_STATUS_PATH; + } + + private Engine engine; + private static String ipAddress; + + static { + try { + ipAddress = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + System.exit(-1); + } + } + + private final RedistributionInstructionsProcessor redistributionInstructionsProcessor = new RedistributionInstructionsProcessor(); + + public boolean announcePhaseCompletion() { + /* the current master might be dead so we should retry this for a certain amount of time before shutting + * down the execution. + */ + + HttpUtils httpUtils = new HttpUtils(); + HttpResponse response = null; + long duration = GlobalArgs.parseDurationToSeconds("60s"); + + while (duration > 0 && response == null) { + response = httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, ipAddress, + Driver.getPhaseFinishedByAgentPath("driver", HttpUtils.SVC_PORT, true), HTTP_REQUEST_RETRIES); + + try { + Thread.sleep(10 * 1000L); // try again in 10 seconds + } catch (InterruptedException e) { + // skip this since this thread is not interrupted by anyone + } finally { + duration -= GlobalArgs.parseDurationToSeconds("10s"); + } + } + + return response != null; + } + + public void start() { + register(); + + /* expose http endpoint to allow the driver to ask for ToughDay sample content package to be installed */ + post(INSTALL_SAMPLE_CONTENT_PATH, ((request, response) -> { + String yamlConfig = request.body(); + Configuration configuration = new Configuration(yamlConfig); + + this.tdExecutorService.submit(() -> { + boolean installed = true; + + try { + installToughdayContentPackage(configuration.getGlobalArgs()); + } catch (Exception e) { + installed = false; + LOG.error("Error encountered when installing TD sample content", e); + } + + HttpResponse driverResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, String.valueOf(installed), + Driver.getSampleContentAckPath("driver", HttpUtils.SVC_PORT), HTTP_REQUEST_RETRIES); + if (driverResponse == null) { + LOG.error("Agent " + ipAddress + " could not announce the driver that Toughday sample content" + + " package was installed."); + System.exit(-1); + } + + }); + + // clear all phases + PhaseParams.namedPhases.clear(); + return ""; + })); + + /* Expose http endpoint for receiving ToughDay execution request from the driver */ + post(SUBMIT_TASK_PATH, ((request, response) -> { + try { + this.statusLock.lock(); + if (this.status != Status.IDLE) { + LOG.info("Agent is currently executing a different task. Request will be ignored."); + + // set status code to Conflict + response.status(HttpStatus.SC_CONFLICT); + return "Agent is already running a task received from the Driver."; + } + + // update current status + this.status = Status.BUILDING_CONFIG; + } finally { + this.statusLock.unlock(); + } + + String yamlTask = request.body(); + LOG.info("[Agent] Received task:\n" + yamlTask); + + Configuration configuration = new Configuration(yamlTask); + this.engine = new Engine(configuration); + this.engine.setAgent(this); + + tdExecutorService.submit(() -> { + this.statusLock.lock(); + this.status = Status.RUNNING; + this.statusLock.unlock(); + + this.engine.runTests(); + + if (!announcePhaseCompletion()) { + LOG.error("Agent " + ipAddress + " could not inform driver that phase was executed."); + System.exit(-1); + } + + this.statusLock.lock(); + this.status = Status.PHASE_COMPLETED; + this.statusLock.unlock(); + + }); + + return ""; + })); + + /* Expose http endpoint to be used by the driver for heartbeat messages */ + get(HEARTBEAT_PATH, ((request, response) -> + { + // send to driver the total number of executions/test + Gson gson = new Gson(); + Map currentCounts = new HashMap<>(); + + // check if execution has started + if (this.status != Status.RUNNING) { + return gson.toJson(currentCounts); + } + + Map phaseCounts = engine.getCurrentPhase().getCounts(); + phaseCounts.forEach((test, count) -> currentCounts.put(test.getName(), count.get())); + + return gson.toJson(currentCounts); + })); + + + + /* expose http endpoint for receiving redistribution requests from the driver */ + post(REBALANCE_PATH, (((request, response) -> { + try { + statusLock.lock(); + if (this.status == Status.IDLE) { + LOG.warn("Rebalance was requested before submitting a task. Request will be ignored."); + return ""; + } + } finally { + statusLock.unlock(); + } + + String instructionsMessage = request.body(); + LOG.info("[Agent] Received " + instructionsMessage + " from driver"); + this.redistributionInstructionsProcessor.processInstructions(instructionsMessage, this.engine.getConfiguration().getPhases().get(0)); + + return ""; + }))); + + + /* expose http endpoint for health checks */ + get(HEALTH_PATH, ((request, response) -> "Healthy")); + + /* expose http endpoint for getting the current status of the agent */ + get(GET_STATUS_PATH, ((request, response) -> { + try { + this.statusLock.lock(); + return this.status; + } finally { + this.statusLock.unlock(); + } + })); + + /* expose http endpoint for finishing the execution of the agent */ + post(FINISH_PATH, ((request, response) -> { + scheduledExecutorService.schedule(() -> { + this.tdExecutorService.shutdown(); + this.scheduledExecutorService.shutdown(); + + System.exit(0); + }, GlobalArgs.parseDurationToSeconds("3s"), TimeUnit.SECONDS); + + return ""; + })); + } + + /* Method responsible for registering the current agent to the driver. It should be the + * first method executed. + */ + private void register() { + LOG.info("Registering..."); + HttpResponse response = + this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, ipAddress, + Driver.getAgentRegisterPath("driver", HttpUtils.SVC_PORT, true), HTTP_REQUEST_RETRIES); + if (response == null) { + System.exit(-1); + } + + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/DistributedConfig.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/DistributedConfig.java new file mode 100644 index 0000000..62ecf1a --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/DistributedConfig.java @@ -0,0 +1,98 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster; + +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; +import com.adobe.qe.toughday.api.annotations.ConfigArgSet; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; + +/** + * Contains all the configurable arguments when running TD distributed. + */ +public class DistributedConfig { + private static final String DEFAULT_HEARTBEAT_INTERVAL = "5s"; + private static final String DEFAULT_REDISTRIBUTION_WAIT_TIME = "3s"; + private static final String DEFAULT_CLUSTER_NAMESPACE = "default"; + + private boolean agent = false; + private boolean driver = false; + private String driverIp = null; + private String heartbeatInterval = DEFAULT_HEARTBEAT_INTERVAL; + private String redistributionWaitTime = DEFAULT_REDISTRIBUTION_WAIT_TIME; + private String clusterNamespace = DEFAULT_CLUSTER_NAMESPACE; + + @ConfigArgSet(required = false, desc = "The public ip address of the cluster. The driver" + + " service must be accessible at this address. This property is required when running in distributed mode.") + public void setDriverIp(String driverIp) { + this.driverIp = driverIp; + } + + @ConfigArgGet + public String getDriverIp() { + return this.driverIp; + } + + @ConfigArgGet + public boolean getAgent() { return this.agent; } + + @ConfigArgSet(required = false, defaultValue = "false", desc = "If true, TD runs as a cluster agent, waiting to receive" + + " a task from the driver.") + public void setAgent(String agent) { + this.agent = Boolean.parseBoolean(agent); + } + + @ConfigArgGet + public boolean getDriver() { + return this.driver; + } + + @ConfigArgSet(required = false, defaultValue = "false", desc = "If true, TD runs as a driver in the cluster," + + " distributing the work between the agents.") + public void setDriver(String driver) { + this.driver = Boolean.parseBoolean(driver); + } + + @ConfigArgSet(required = false, defaultValue = DEFAULT_HEARTBEAT_INTERVAL, desc = "Period of time for sending " + + "heartbeat messages to the agents in the cluster.") + public void setHeartbeatInterval(String heartbeatInterval) { + this.heartbeatInterval = heartbeatInterval; + } + + @ConfigArgGet + public String getHeartbeatInterval() { + return this.heartbeatInterval; + } + + @ConfigArgSet(required = false, defaultValue = DEFAULT_REDISTRIBUTION_WAIT_TIME, desc = "The minimum amount of time " + + " to wait before scheduling work redistribution if required.") + public void setRedistributionWaitTime(String redistributionWaitTime) { + this.redistributionWaitTime = redistributionWaitTime; + } + + @ConfigArgGet + public String getRedistributionWaitTime() { + return this.redistributionWaitTime; + } + + public long getRedistributionWaitTimeInSeconds() { + return GlobalArgs.parseDurationToSeconds(this.redistributionWaitTime); + } + + public long getHeartbeatIntervalInSeconds() { + return GlobalArgs.parseDurationToSeconds(this.heartbeatInterval); + } + + @ConfigArgSet(required = false, defaultValue = DEFAULT_CLUSTER_NAMESPACE, desc = "The namespace in the cluster " + + " where the resources(agents and drivers) were deployed.") + public void setClusterNamespace(String clusterNamespace) { + this.clusterNamespace = clusterNamespace; + } + + @ConfigArgGet + public String getClusterNamespace() { + return this.clusterNamespace; + } + + public void merge(DistributedConfig other) { + this.setHeartbeatInterval(other.getHeartbeatInterval()); + this.setRedistributionWaitTime(other.redistributionWaitTime); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/Driver.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/Driver.java new file mode 100644 index 0000000..138bd10 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/Driver.java @@ -0,0 +1,401 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.RequestProcessorDispatcher; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.PhaseSplitter; +import com.adobe.qe.toughday.internal.core.distributedtd.tasks.MasterHeartbeatTask; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.distributedtd.DistributedPhaseMonitor; +import com.adobe.qe.toughday.internal.core.distributedtd.tasks.HeartbeatTask; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.TaskBalancer; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import org.apache.http.HttpResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.*; +import java.util.concurrent.*; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.*; +import static spark.Spark.*; + +/** + * Driver component for the cluster. + */ +public class Driver { + // routes + public static final String EXECUTION_PATH = "/config"; + private static final String REGISTER_PATH = "/registerAgent"; + private static final String PHASE_FINISHED_BY_AGENT_PATH = "/phaseFinished"; + private static final String HEALTH_PATH = "/health"; + private static final String SAMPLE_CONTENT_ACK_PATH = "/contentAck"; + private static final String MASTER_ELECTION_PATH = "/masterElection"; + private static final String ASK_FOR_UPDATES_PATH = "/driverUpdates"; + private static final String GET_NR_DRIVERS_PATH = "/getNrDrivers"; + private static final String HEARTBEAT_PATH = "/heartbeat"; + private static final String AGENT_FAILURE_PATH = "/agentFailure"; + + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private final ScheduledExecutorService heartbeatScheduler = Executors.newSingleThreadScheduledExecutor(); + private final ExecutorService executorService = Executors.newFixedThreadPool(1); + private final HttpUtils httpUtils = new HttpUtils(); + private final TaskBalancer taskBalancer = TaskBalancer.getInstance(); + private DistributedPhaseMonitor distributedPhaseMonitor = new DistributedPhaseMonitor(); + private Configuration configuration; + private DriverState driverState; + private MasterElection masterElection; + private ScheduledFuture scheduledFuture = null; + + public DistributedPhaseMonitor getDistributedPhaseMonitor() { + return this.distributedPhaseMonitor; + } + + /** + * Getter for the task balancer instance. + */ + public TaskBalancer getTaskBalancer() { + return this.taskBalancer; + } + + /** + * Getter for the TD configuration which must be executed in distributed mode. + */ + public Configuration getConfiguration() { + return this.configuration; + } + + /** + * Getter for the state of the driver. + */ + public DriverState getDriverState() { + return this.driverState; + } + + /** + * Getter for master election instance. + */ + public MasterElection getMasterElection() { + return this.masterElection; + } + + /** + * Constructor + * @param configuration : configuration given when the driver was deployed into the cluster. + */ + public Driver(Configuration configuration) { + try { + String hostname = InetAddress.getLocalHost().getHostName(); + this.driverState = new DriverState(hostname, configuration); + } catch (UnknownHostException e) { + System.exit(-1); + } + + this.masterElection = MasterElection.getInstance(this.driverState.getNrDrivers()); + this.masterElection.electMasterWhenDriverJoinsTheCluster(this); + + LOG.info("Driver " + this.driverState.getHostname() + " elected as master " + this.driverState.getMasterId()); + } + + /** + * Returns the http URL that should be used for triggering the distributed execution in the cluster. + * @param driverIdentifier : the identifier of the driver that will received this execution query. + * @param port : the port on which the query must be sent. + * @param forwardReq : true if the request must be forwarded to all the other drivers running in the cluster; false + * otherwise. + */ + public static String getExecutionPath(String driverIdentifier, String port, boolean forwardReq) { + return HttpUtils.URL_PREFIX + driverIdentifier + ":" + port + Driver.EXECUTION_PATH + + HttpUtils.FORWARD_QUERY_PARAM + forwardReq; + } + + /** + * Returns the http URL that should be used by the agents whenever they finished executing the task received from + * the driver. + */ + public static String getPhaseFinishedByAgentPath(String driverIdentifier, String port, boolean forwardReq) { + return URL_PREFIX + driverIdentifier + ":" + port + PHASE_FINISHED_BY_AGENT_PATH + + HttpUtils.FORWARD_QUERY_PARAM + forwardReq; + } + + /** + * Returns the http URL that should be used by the agent chosen to install the TD sample content package for + * informing the driver that the installation was completed successfully. + */ + public static String getSampleContentAckPath(String driverIdentifier, String port) { + return URL_PREFIX + driverIdentifier + ":" + port + SAMPLE_CONTENT_ACK_PATH; + } + + /** + * Returns the http URL that should be used by the driver which detected that the current Master died to inform all + * the other drivers running in the cluster that the master election process must be triggered. + * @param driverHostname : hostname of the driver exposing this http endpoint + */ + public static String getMasterElectionPath(String driverHostname, String port) { + return URL_PREFIX + driverHostname + ":" + port + MASTER_ELECTION_PATH; + } + + /** + * Returns the http URL that should be used by the agents to register themselves after joining the cluster. + * @param driverIdentifier : specifies what kind of pods should received this request. + * @param port : the port on which the driver is listening for this type of request. + * @param forwardReq : true if this request should be forwarded to all the drivers running in the cluster; false + * otherwise + */ + public static String getAgentRegisterPath(String driverIdentifier, String port, boolean forwardReq) { + return URL_PREFIX + driverIdentifier + ":" + port + REGISTER_PATH + HttpUtils.FORWARD_QUERY_PARAM + forwardReq; + } + + + /** + * Returns the http URL that should be used by the Candidates to periodically check if the current Master is running + * properly. + * @param driverHostName : hostname that uniquely identifies the driver which should receive the heartbeat message + */ + public static String getHeartbeatPath(String driverHostName) { + return URL_PREFIX + driverHostName + ":" + HttpUtils.SPARK_PORT + HEARTBEAT_PATH; + } + + /** + * Returns the http URL that should be used by the drivers to get information about the current state of the + * execution. + * @param driverHostName : hostname of the driver component exposing this http endpoint + */ + public static String getAskForUpdatesPath(String driverHostName) { + return URL_PREFIX + driverHostName + ":" + SPARK_PORT + ASK_FOR_UPDATES_PATH; + } + + /** + * Returns the http URL that should be used by the current master to announce that a certain agent failed to respond + * to heartbeat messages. + * @param driverHostName : hostname of the driver component that should receive the http request + */ + public static String getAgentFailurePath(String driverHostName) { + return URL_PREFIX + driverHostName + ":" + SPARK_PORT + AGENT_FAILURE_PATH; + } + + /** + * Getter for the executor service used to submit the task of executing ToughDay distributed. The execution request + * must be handled in a different thread in order to be able to quickly send a response for this request. + */ + public ExecutorService getExecutorService() { + return this.executorService; + } + + /** + * Setter for the TD configuration to be executed in distributed mode. + */ + public void setConfiguration(Configuration configuration) { + this.configuration = configuration; + } + + /** + * Method used for announcing the agents that the TD configuration was successfully executed and they should stop + * their execution. + */ + public void finishAgents() { + this.driverState.getRegisteredAgents().forEach(agentIp -> { + LOG.info("[Driver] Finishing agent " + agentIp); + HttpResponse response = + httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, "", Agent.getFinishPath(agentIp), HTTP_REQUEST_RETRIES); + if (response == null) { + // the assumption is that the agent will be killed when he fails to respond to heartbeat request + LOG.warn("Driver could not finish the execution on agent " + agentIp + "."); + } + }); + + this.driverState.getRegisteredAgents().clear(); + } + + /** + * Method used to finish the distributed execution. The agents will be informed that there are no more phases to be + * executed and they will stop running. + */ + public void finishDistributedExecution() { + finishAgents(); + + this.executorService.shutdownNow(); + // finish tasks + this.heartbeatScheduler.shutdownNow(); + } + + /** + * Method used for scheduling the periodic task of sending heartbeat messages to all the agents running in the + * cluster to be executed at every 'heartbeatInterval' seconds. + */ + public void scheduleHeartbeatTask() { + // we should periodically send heartbeat messages from driver to all the agents + heartbeatScheduler.scheduleAtFixedRate(new HeartbeatTask(this), + 0, this.driverState.getDriverConfig().getDistributedConfig().getHeartbeatIntervalInSeconds(), TimeUnit.SECONDS); + } + + /** + * Method used for scheduling the periodic task of sending heartbeat messages to the driver playing the role of the + * Master to be executed at every 10 seconds. + */ + public void scheduleMasterHeartbeatTask() { + // we should periodically send heartbeat messages from candidates to check id the master is still running + this.scheduledFuture = this.heartbeatScheduler.scheduleAtFixedRate(new MasterHeartbeatTask(this), 0, + GlobalArgs.parseDurationToSeconds("10s"), TimeUnit.SECONDS); + } + + /** + * Method used for cancelling the periodic task of sending heartbeat messages to the driver running as Master in the + * cluster. + */ + public void cancelMasterHeartBeatTask() { + if (this.scheduledFuture != null) { + if(!this.scheduledFuture.cancel(true) && !this.scheduledFuture.isDone()) { + LOG.warn("Could not cancel task used to periodically send heartbeat messages to the Master."); + } + } + } + + /** + * Method used for cancelling the periodic task of sending heartbeat messaged to the agents running in the cluster. + */ + public void cancelHeartbeatTask() { + this.heartbeatScheduler.shutdownNow(); + } + + /** + * Method used to resume the distributed execution of ToughDay whenever a new master is elected in the cluster. + */ + public void resumeExecution() { + if (this.configuration == null) { + return; + } + + LOG.info("Resuming execution..."); + + // wait until current phase is successfully finished + if (!distributedPhaseMonitor.waitForPhaseCompletion(3)) { + finishDistributedExecution(); + return; + } + + LOG.info("Phase " + distributedPhaseMonitor.getPhase().getName() + " finished execution successfully."); + List remainingPhases = new ArrayList<>(configuration.getPhases()); + remainingPhases.remove(remainingPhases.get(0)); + + LOG.info("Phases left to be executed: " + remainingPhases.toString()); + this.configuration.setPhases(remainingPhases); + + // continue executing all the other phases + executePhases(); + } + + + /** + * Method used for running the phases of the TD configuration received to be executed distributed. This method + * should only be called once, when the master receives the query to execute TD. Whenever a new master is elected, + * the {@link #resumeExecution()} method should be used. + */ + public void executePhases() { + PhaseSplitter phaseSplitter = new PhaseSplitter(); + + LOG.info("[Driver] Executing phases + " + this.configuration.getPhases().toString()); + for (Phase phase : configuration.getPhases()) { + try { + Map tasks = phaseSplitter.splitPhase(phase, new ArrayList<>(this.driverState.getRegisteredAgents())); + this.distributedPhaseMonitor.setPhase(phase); + + for (String agentIp : this.driverState.getRegisteredAgents()) { + configuration.setPhases(Collections.singletonList(tasks.get(agentIp))); + + // convert configuration to yaml representation + YamlDumpConfigurationAsTaskForTDAgents dumpConfig + = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + String yamlTask = dumpConfig.generateConfigurationObject(); + + /* send query to agent and register running task */ + String URI = Agent.getSubmissionTaskPath(agentIp); + HttpResponse response = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, yamlTask, URI, HTTP_REQUEST_RETRIES); + + if (response != null) { + this.distributedPhaseMonitor.registerAgentRunningTD(agentIp); + LOG.info("Task was submitted to agent " + agentIp); + } else { + /* the assumption is that the agent is no longer active in the cluster and he will fail to respond + * to the heartbeat request sent by the driver. This will automatically trigger process of + * redistributing the work + * */ + LOG.info("Task\n" + yamlTask + " could not be submitted to agent " + agentIp + + ". Work will be rebalanced once the agent fails to respond to heartbeat request."); + } + } + + // al execution queries were sent => set phase execution start time + this.distributedPhaseMonitor.setPhaseStartTime(System.currentTimeMillis()); + + // we should wait until all agents complete the current tasks in order to execute phases sequentially + if (!this.distributedPhaseMonitor.waitForPhaseCompletion(3)) { + break; + } + + LOG.info("Phase " + phase.getName() + " finished execution successfully."); + + } catch (CloneNotSupportedException e) { + LOG.error("Phase " + phase.getName() + " could not de divided into tasks to be sent to the agents.", e); + + LOG.info("Finishing agents"); + finishAgents(); + + System.exit(-1); + } + } + + finishDistributedExecution(); + } + + /** + * Starts the execution of the driver. + */ + public void run() { + RequestProcessorDispatcher dispatcher = RequestProcessorDispatcher.getInstance(); + /* expose http endpoint for running TD with the given configuration */ + post(EXECUTION_PATH, ((request, response) -> dispatcher.getRequestProcessor(this) + .processExecutionRequest(request, this) + )); + + /* health check http endpoint */ + get(HEALTH_PATH, ((request, response) -> "Healthy")); + + /* expose http endpoint to allow agents to get the number of drivers running in the cluster */ + get(GET_NR_DRIVERS_PATH, ((request, response) -> String.valueOf(this.getDriverState().getNrDrivers()))); + + /* http endpoint used by the agent installing the sample content to announce if the installation was + * successful or not. */ + post(SAMPLE_CONTENT_ACK_PATH, ((request, response) -> + dispatcher.getRequestProcessor(this) + .acknowledgeSampleContentSuccessfulInstallation(request, this, response))); + + /* expose http endpoint to allow agents to announce when they finished executing the current phase */ + post(PHASE_FINISHED_BY_AGENT_PATH, ((request, response) -> + dispatcher.getRequestProcessor(this).processPhaseCompletionAnnouncement(request))); + + /* expose http endpoint for triggering the master election process */ + post(MASTER_ELECTION_PATH, ((request, response) -> + dispatcher.getRequestProcessor(this).processMasterElectionRequest(request, this))); + + /* expose http endpoint for sending information about the current state of the distributed execution */ + get(ASK_FOR_UPDATES_PATH, ((request, response) -> dispatcher.getRequestProcessor(this) + .processUpdatesRequest(request, this))); + + get(HEARTBEAT_PATH, ((request, response) -> dispatcher.getRequestProcessor(this) + .processHeartbeatRequest(request, this))); + + post(AGENT_FAILURE_PATH, ((request, response) -> dispatcher.getRequestProcessor(this) + .processAgentFailureAnnouncement(request, this))); + + /* expose http endpoint for registering new agents in the cluster */ + post(REGISTER_PATH, (request, response) -> dispatcher.getRequestProcessor(this) + .processRegisterRequest(request, this)); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverState.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverState.java new file mode 100644 index 0000000..2a868ea --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverState.java @@ -0,0 +1,214 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.regex.Pattern; + +/** + * Class responsible for encapsulating all the information required for describing the state of a Driver component. + */ +public class DriverState { + // Documentation says that each service has the following DNS A record: my-svc.my-namespace.svc.cluster.local + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private final String hostname; + private static final String SVC_NAME = "driver"; + private final Queue agents = new ConcurrentLinkedQueue<>(); + private final int id; + private int masterId = -1; + private int nrDrivers; + private Configuration driverConfig; + private Role currentState; + private final ReadWriteLock masterIdLock = new ReentrantReadWriteLock(); + + public enum Role { + CANDIDATE, + MASTER + } + + /** + * Returns the URI to be used for identifying a driver component using its id. + * @param id : the unique identifier associated with the driver component. + */ + public String getPathForId(int id) { + return SVC_NAME + "-" + id + "." + SVC_NAME + "." + + this.driverConfig.getDistributedConfig().getClusterNamespace() + ".svc.cluster.local"; + } + + /** + * Returns the lock used for synchronised access to the id of the driver component considered to be the Master. + */ + public ReadWriteLock getMasterIdLock() { + return this.masterIdLock; + } + + /** + * Constructor. + * @param hostname : hostname used to access the driver service inside the cluster. + * @param driverConfig : TD configuration used for deploying the driver. + */ + public DriverState(String hostname, Configuration driverConfig) { + this.driverConfig = driverConfig; + this.hostname = hostname; + + String name_pattern = "driver-[0-9]+\\." + SVC_NAME + "\\." + this.driverConfig.getDistributedConfig().getClusterNamespace() + + "\\.svc\\.cluster\\.local"; + + if (!Pattern.matches(name_pattern, hostname)) { + throw new IllegalStateException("Driver's name should respect the following format: driver-...svc.cluster.local"); + } + + String podName = hostname.substring(0, hostname.indexOf("." + SVC_NAME + "." + + this.driverConfig.getDistributedConfig().getClusterNamespace())); + + this.id = Integer.parseInt(podName.substring(podName.lastIndexOf("-") + 1)); + this.nrDrivers = Integer.parseInt(System.getenv("NR_DRIVERS")); + this.currentState = Role.CANDIDATE; + } + + /** + * Getter for the hostname. + */ + public String getHostname() { + return this.hostname; + } + + /** + * Getter for the id associated with the driver. + */ + public int getId() { + return this.id; + } + + /** + * Getter for the id of the driver considered to be the current master. + */ + public int getMasterId() { + return this.masterId; + } + + /** + * Getter for the number of driver components deployed in the cluster. + */ + public int getNrDrivers() { + return this.nrDrivers; + } + + /** + * Returns the list of active agents running in the cluster. + */ + public Queue getRegisteredAgents() { + return this.agents; + } + + /** + * Adds the agent to the list of agents running in the cluster. + * @param agentIdentifier : ipAddress used to uniquely identify the agent inside the cluster. + */ + public void registerAgent(String agentIdentifier) { + if (!this.agents.contains(agentIdentifier)) { + this.agents.add(agentIdentifier); + } + } + + /** + * Removes an agent from the list of active agents running in the cluster. + * @param agentIdentifier : ipAddress that uniquely identifies the agent to be removed. + */ + public void removeAgent(String agentIdentifier) { + this.agents.remove(agentIdentifier); + } + + /** + * Getter for the TD configuration of the driver component. + */ + public Configuration getDriverConfig() { + return this.driverConfig; + } + + /** + * Getter for the current status of the driver component. (Master or Candidate) + */ + public Role getCurrentState() { + return this.currentState; + } + + /** + * Setter the current status of the driver component. + */ + public void setCurrentState(Role role) { + this.currentState = role; + } + + /** + * Setter for the id of the driver running as master in the cluster. + */ + public void setMasterId(int id) { + this.masterId = id; + } + + /** + * Method used for updating the state of the cluster when information is received from another drivers. This method + * is called usually when a new driver joins the cluster and needs to know the current state of the execution (for + * example how many active agents are registered in the cluster) + * @param updates : information received from other drivers running in the cluster + * @param driverInstance : the Driver instance which received the updates. + */ + public void updateDriverState(DriverUpdateInfo updates, Driver driverInstance) { + // excludes candidates which are not allowed to become master + updates.getInvalidCandidates().forEach(driverInstance.getMasterElection()::markCandidateAsInvalid); + + // add all agents + updates.getRegisteredAgents().forEach(driverInstance.getDriverState()::registerAgent); + LOG.info("After processing update instructions, registered agents is " + + driverInstance.getDriverState().getRegisteredAgents().toString()); + LOG.info("After processing update instructions, invalid candidates are: " + + updates.getInvalidCandidates().toString()); + + // build TD configuration to be executed distributed + if (!updates.getYamlConfig().isEmpty() && driverInstance.getConfiguration() == null) { + LOG.info("Building configuration received from the other drivers running in the cluster."); + try { + Configuration configuration = new Configuration(updates.getYamlConfig()); + driverInstance.setConfiguration(configuration); + } catch (InvocationTargetException | NoSuchMethodException | InstantiationException | IOException | + IllegalAccessException e) { + /* if driver can't build the configuration executed distributed it will not be able to become the master + * and to coordinate the entire execution + */ + LOG.warn("Exception occurred when building ToughDay configuration to be executed distributed. Driver" + + " will leave the cluster"); + System.exit(-1); + } + + // delete phases that were previously executed + List phases = driverInstance.getConfiguration().getPhases(); + List previouslyExecutedPhases = new ArrayList<>(); + for (Phase phase : phases) { + if (phase.getName().equals(updates.getCurrentPhaseName())) { + break; + } + + previouslyExecutedPhases.add(phase); + } + LOG.info("phases that will be removed " + previouslyExecutedPhases.toString()); + phases.removeAll(previouslyExecutedPhases); + + // set current phase to be monitored + driverInstance.getDistributedPhaseMonitor().setPhase(phases.get(0)); + LOG.info("Current phase being executed is " + driverInstance.getDistributedPhaseMonitor().getPhase().getName()); + } + } +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverUpdateInfo.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverUpdateInfo.java new file mode 100644 index 0000000..958d8bc --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverUpdateInfo.java @@ -0,0 +1,135 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver; + +import com.adobe.qe.toughday.internal.core.engine.Engine; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.Queue; + +/** + * Contains all the information needed for updating the state of a new driver when joining the cluster. + */ +public class DriverUpdateInfo { + private int driverId; + private DriverState.Role role; + private Queue invalidCandidates; + private Queue registeredAgents; + private String yamlConfig; + private String currentPhaseName; + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + + // TODO: add registeredAgents currently running tasks + // TODO: add registeredAgents which finished running the current phase + + + // dummy constructor used to dump the class + public DriverUpdateInfo() {} + + /** + * Constructor. + * @param driverId : the id of the driver which sent this updates. + * @param role : the status of the driver which sent this updates (Master or Candidate). + * @param invalidCandidates : list of drivers that are not eligible to become the new master if the current one dies + * @param registeredAgents : list of active agents running in the cluster. + * @param yamlConfig : TD configuration to be executed in distributed mode. This string is empty if the + * configuration was not received yet. + * @param currentPhaseName : the name of the current phase being executed by the agents running in the cluster. This + * is empty if the execution did not started yet. + */ + public DriverUpdateInfo(int driverId, DriverState.Role role, Queue invalidCandidates, + Queue registeredAgents, String yamlConfig, String currentPhaseName) { + this.driverId = driverId; + this.role = role; + this.invalidCandidates = invalidCandidates; + this.registeredAgents = registeredAgents; + this.yamlConfig = yamlConfig; + this.currentPhaseName = currentPhaseName; + } + + /** + * Getter for the id of the driver sending the updates. + */ + public int getDriverId() { + return this.driverId; + } + + /** + * Setter for the driver sending the updates. + */ + public void setDriverId(int driverId) { + this.driverId = driverId; + } + + /** + * Setter for the status of the driver sending the updates. + */ + public void setRole(DriverState.Role driverState) { + this.role = driverState; + } + + /** + * Getter for the status of the driver sending the updates. + * @return + */ + public DriverState.Role getRole() { + return this.role; + } + + /** + * Setter for the list of candidates to be excluded when electing a new master. + */ + public void setInvalidCandidates(Queue invalidCandidates) { + this.invalidCandidates = invalidCandidates; + } + + /** + * Getter for the list of candidates to be excluded when electing a new master. + * @return + */ + public Queue getInvalidCandidates() { + return this.invalidCandidates; + } + + /** + * Setter for the list of agents which are currently running in the cluster. + */ + public void setRegisteredAgents(Queue registeredAgents) { + this.registeredAgents = registeredAgents; + } + + /** + * Getter for the list of agents which are currently running in the cluster. + */ + public Queue getRegisteredAgents() { + return this.registeredAgents; + } + + /** + * Setter for the YAML representation of the TD configuration to be executed in distributed mode. + */ + public void setYamlConfig(String yamlConfig) { + this.yamlConfig = yamlConfig; + } + + /** + * Getter for the YAML representation of the TD configuration to be executed in distributed mode. + */ + public String getYamlConfig() { + return this.yamlConfig; + } + + /** + * Getter for the name of the phase being executed by the agents. + */ + public String getCurrentPhaseName() { + return this.currentPhaseName; + } + + /** + * Setter for the name of the phase being executed by the agents. + */ + public void setCurrentPhaseName(String currentPhaseName) { + this.currentPhaseName = currentPhaseName; + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/MasterElection.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/MasterElection.java new file mode 100644 index 0000000..c1c2945 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/MasterElection.java @@ -0,0 +1,264 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver; + +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpResponse; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Class responsible for implementing the master election process to be executed whenever a new Master must be elected + * in the cluster. + */ +public class MasterElection { + private int nrDrivers; + private Queue candidates; + protected static final Logger LOG = LogManager.getLogger(Engine.class); + private static MasterElection instance = null; + + private MasterElection(int nrDrivers) { + this.nrDrivers = nrDrivers; + this.candidates = IntStream.rangeClosed(0, nrDrivers - 1).boxed() + .collect(Collectors.toCollection(ConcurrentLinkedQueue::new)); + } + + /** + * Returns a Singleton instance of this class. This method is not thread safe and it should not be called from + * multiple threads. + * @param nrDrivers : number of driver components deployed in the cluster. + */ + public static MasterElection getInstance(int nrDrivers) { + if (instance == null) { + instance = new MasterElection(nrDrivers); + } + + return instance; + } + + /** + * Mark the given candidate as unable to become the new master, in case the current one fails. + * @param candidateId : id used to identify the driver component which should no be taken into consideration when + * electing a new master. + */ + public void markCandidateAsInvalid(int candidateId) { + this.candidates.remove(candidateId); + } + + /** + * Return true if the given candidate should not be chosen as the new Master; false otherwise. + */ + public boolean isCandidateInvalid(int candidateId) { + return !this.candidates.contains(candidateId); + } + + /** + * Return a list with all the ids of the candidates that should not be taken into consideration when electing a new + * master in the cluster. + */ + public Queue getInvalidCandidates() { + Queue invalidCandidates = IntStream.rangeClosed(0, nrDrivers - 1).boxed() + .collect(Collectors.toCollection(LinkedList::new)); + invalidCandidates.removeAll(this.candidates); + + return invalidCandidates; + } + + private void before() { + // restore all valid candidates in case there is no option; + if (candidates.isEmpty()) { + resetInvalidCandidates(); + } + } + + /** + * Resets the list of candidates considered invalid. After this method is called, all the drivers will be considered + * eligible to be elected as the new master. + */ + public void resetInvalidCandidates() { + LOG.info("Resetting list of candidates to be considered for master election"); + this.candidates = IntStream.rangeClosed(0, nrDrivers - 1).boxed().collect(Collectors.toCollection(ConcurrentLinkedQueue::new)); + } + + /** + * Method used for electing a new master when a Driver component is joining the cluster. + * @param newDriver : driver instance which joined the cluster recently and which is currently not aware if there is + * already a driver which plays the role of the Master. + */ + public void electMasterWhenDriverJoinsTheCluster(Driver newDriver) { + // check if there is already a master running in the cluster + collectUpdatesFromAllDrivers(newDriver); + DriverState driverState = newDriver.getDriverState(); + + // if no master was detected, trigger the master election process + driverState.getMasterIdLock().readLock().lock(); + if (driverState.getMasterId() == -1) { + driverState.getMasterIdLock().readLock().unlock(); + electMaster(newDriver); + return; + } + + driverState.getMasterIdLock().readLock().unlock(); + after(newDriver); + } + + /** + * Method used for electing a new Master whenever the previous one fails to respond to heartbeat messages. + * @param driver : the driver instance electing a new master. + */ + public void electMaster(Driver driver) { + before(); + + // set new master + LOG.info("Electing new master " + candidates.stream().min(Integer::compareTo).get()); + + driver.getDriverState().getMasterIdLock().writeLock().lock(); + driver.getDriverState().setMasterId(candidates.stream().min(Integer::compareTo).get()); + driver.getDriverState().getMasterIdLock().writeLock().unlock(); + + after(driver); + } + + private List getIdleAgents(Driver driver) { + HttpUtils httpUtils = new HttpUtils(); + List idleAgents = new ArrayList<>(); + + driver.getDriverState().getRegisteredAgents() + .forEach(agentIp -> { + HttpResponse agentResponse = httpUtils.sendHttpRequest(HttpUtils.GET_METHOD, "", + Agent.getGetStatusPath(agentIp), HttpUtils.HTTP_REQUEST_RETRIES); + if (agentResponse != null) { + try { + String status = EntityUtils.toString(agentResponse.getEntity()); + if (status.equals(Agent.Status.IDLE.toString())) { + LOG.info("Agent " + agentIp + " is idle."); + idleAgents.add(agentIp); + } else if (status.equals(Agent.Status.PHASE_COMPLETED.toString())) { + LOG.info(("Agent " + agentIp + " finished executing the current phase.")); + driver.getDistributedPhaseMonitor().registerAgentRunningTD(agentIp); + driver.getDistributedPhaseMonitor().addAgentWhichCompletedTheCurrentPhase(agentIp); + } else { + LOG.info("Agent " + agentIp + " is running TD."); + driver.getDistributedPhaseMonitor().registerAgentRunningTD(agentIp); + } + } catch (IOException e) { + LOG.warn("Could not check status of agent. Current phase might not be executed with the " + + "desired configuration."); + } + } + }); + + return idleAgents; + } + + private void after(Driver driver) { + // cancel heartbeat task for the diver elected as the new master + driver.getDriverState().getMasterIdLock().readLock().lock(); + + // running as master + if (driver.getDriverState().getMasterId() == driver.getDriverState().getId()) { + LOG.info("Running as MASTER"); + + driver.getDriverState().setCurrentState(DriverState.Role.MASTER); + LOG.info("Stopping master heartbeat periodic task since this driver was elected as master."); + driver.cancelMasterHeartBeatTask(); + + /* check if work redistribution is required */ + List idleAgents = getIdleAgents(driver); + + /* the assumption is that all the registered agents are currently executing TD test or they've completed + * the execution of the current phase + */ + if (driver.getConfiguration() != null && !idleAgents.isEmpty()) { + LOG.info("New master must redistribute the work between the agents because of idle agents " + + idleAgents.toString()); + idleAgents.forEach(idleAgent -> + driver.getTaskBalancer().scheduleWorkRedistributionProcess(driver, idleAgent, true)); + } + + // schedule heartbeat task for periodically checking agents + LOG.info("Scheduling heartbeat task for monitoring agents..."); + driver.scheduleHeartbeatTask(); + + // resume execution + driver.getExecutorService().submit(driver::resumeExecution); + } else { + LOG.info("Running as CANDIDATE"); + + // running as candidate + driver.getDriverState().setCurrentState(DriverState.Role.CANDIDATE); + + // schedule heartbeat task to periodically monitor the agents + driver.scheduleMasterHeartbeatTask(); + } + + driver.getDriverState().getMasterIdLock().readLock().unlock(); + } + + private void processUpdatesFromDriver(Driver currentDriver, String yamlUpdates) { + LOG.info("Current driver " + currentDriver.getDriverState().getId() + " is processing updates:\n" + yamlUpdates); + ObjectMapper objectMapper = new ObjectMapper(); + DriverUpdateInfo updates = null; + + try { + updates = objectMapper.readValue(yamlUpdates, DriverUpdateInfo.class); + } catch (IOException e) { + LOG.info("Unable to process updates about the cluster state. Driver will restart now..."); + System.exit(-1); + } + + currentDriver.getDriverState().updateDriverState(updates, currentDriver) + ; + + // set master if updates were received from the current master running in the cluster + if (updates.getRole() == DriverState.Role.MASTER) { + LOG.info("Received instructions that " + currentDriver.getDriverState().getPathForId(updates.getDriverId()) + "is the " + + "current master running in the cluster."); + currentDriver.getDriverState().getMasterIdLock().writeLock().lock(); + currentDriver.getDriverState().setMasterId(updates.getDriverId()); + currentDriver.getDriverState().getMasterIdLock().writeLock().unlock(); + } + } + + private void collectUpdatesFromAllDrivers(Driver currentDriver) { + HttpUtils httpUtils = new HttpUtils(); + List ids = IntStream.rangeClosed(0, nrDrivers - 1).boxed().collect(Collectors.toList()); + + // send request to all drivers, except the current one + List paths = ids.stream() + .filter(id -> id != currentDriver.getDriverState().getId()) + .map(id -> currentDriver.getDriverState().getPathForId(id)) + .map(Driver::getAskForUpdatesPath) + .collect(Collectors.toList()); + + for (String URI : paths) { + HttpResponse driverResponse = httpUtils.sendHttpRequest(HttpUtils.GET_METHOD, "", URI, + HttpUtils.HTTP_REQUEST_RETRIES); + if (driverResponse != null) { + try { + String yamlUpdates = EntityUtils.toString(driverResponse.getEntity()); + LOG.info("Received updates: " + yamlUpdates); + + processUpdatesFromDriver(currentDriver, yamlUpdates); + + // if updates were received from the master -> finish process + if (currentDriver.getDriverState().getMasterId() != -1) { + break; + } + } catch (IOException e) { + LOG.info("Unable to process updates about the cluster state. Driver will restart now..."); + System.exit(-1); + } + } + } + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/AbstractRequestProcessor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/AbstractRequestProcessor.java new file mode 100644 index 0000000..b382bed --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/AbstractRequestProcessor.java @@ -0,0 +1,200 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.GenerateYamlConfiguration; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverUpdateInfo; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.MasterElection; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import spark.Request; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; + +/** + * Base implementation for the RequestProcessor interface. Implements the common actions that should be taken by both + * driver components (master and candidates) when receiving certain types of HTTP requests. + */ +public abstract class AbstractRequestProcessor implements RequestProcessor { + protected final HttpUtils httpUtils = new HttpUtils(); + protected static final Logger LOG = LogManager.getLogger(Engine.class); + protected Driver driverInstance; + + /** + * Constructor. + * @param driverInstance : the driver Instance which will receive the http requests to be processed. + */ + public AbstractRequestProcessor(Driver driverInstance) { + this.driverInstance = driverInstance; + } + + /** + * Returns a list containing the URIs that should be used for forwarding a request to all the other drivers running + * in the cluster. + * @param driverInstance : the driver instance that must forward the request. + */ + protected List getDriverPathsForRedirectingRequests(Driver driverInstance) { + return IntStream.rangeClosed(0, driverInstance.getDriverState().getNrDrivers() - 1).boxed() + .filter(id -> id != driverInstance.getDriverState().getId()) // exclude current driver + .map(id -> driverInstance.getDriverState().getPathForId(id)) + .collect(Collectors.toCollection(LinkedList::new)); + } + + @Override + public String processRegisterRequest(Request request, Driver driverInstance) { + String agentIp = request.body(); + DriverState driverState = this.driverInstance.getDriverState(); + + if (request.queryParams("forward").equals("true")) { + /* register new agents to all the drivers running in the cluster */ + for (int i = 0; i < driverState.getNrDrivers(); i++) { + /* skip current driver */ + if (i == driverState.getId()) { + continue; + } + + LOG.info(this.driverInstance.getDriverState().getHostname() + ": sending agent register request for agent " + agentIp + "" + + "to driver " + this.driverInstance.getDriverState().getPathForId(i)); + HttpResponse regResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, agentIp, + Driver.getAgentRegisterPath(driverState.getPathForId(i), HttpUtils.SPARK_PORT, false), HTTP_REQUEST_RETRIES); + if (regResponse == null) { + // the assumption is that the new driver will receive the full list of active agents after being restarted + LOG.info("Driver " + driverState.getHostname() + "failed to send register request for agent " + agentIp + + "to driver " + driverState.getPathForId(i)); + } + } + } + + return ""; + } + + @Override + public String processUpdatesRequest(Request request, Driver driverInstance) throws JsonProcessingException { + LOG.info("Driver has requested updates about the state of the cluster."); + String currentPhaseName = ""; + String yamlConfig = ""; + DriverState driverState = this.driverInstance.getDriverState(); + + /* send configuration received to be executed in distributed mode and the phase being executed at this moment, + * if applicable. + */ + if (driverInstance.getConfiguration() != null) { + GenerateYamlConfiguration generateYaml = + new GenerateYamlConfiguration(driverInstance.getConfiguration().getConfigParams(), new HashMap<>()); + yamlConfig = generateYaml.createYamlStringRepresentation(); + if (driverInstance.getDistributedPhaseMonitor().isPhaseExecuting()) { + currentPhaseName = this.driverInstance.getDistributedPhaseMonitor().getPhase().getName(); + } + } + + // build information to send to the driver that recently joined the cluster + DriverUpdateInfo driverUpdateInfo = new DriverUpdateInfo(driverState.getId(), + driverState.getCurrentState(), this.driverInstance.getMasterElection().getInvalidCandidates(), + driverState.getRegisteredAgents(), yamlConfig, currentPhaseName); + + ObjectMapper objectMapper = new ObjectMapper(); + String yamlUpdateInfo = objectMapper.writeValueAsString(driverUpdateInfo); + LOG.info("Create YAML update info: " + yamlUpdateInfo); + + // set response + return yamlUpdateInfo; + } + + @Override + public String processMasterElectionRequest(Request request, Driver driverInstance) { + int failedDriverId = Integer.parseInt(request.body()); + MasterElection masterElection = this.driverInstance.getMasterElection(); + // check if this news was already processed + if (masterElection.isCandidateInvalid(failedDriverId)) { + return ""; + } + + LOG.info("Driver was informed that the current master (id: " + failedDriverId + ") died"); + masterElection.markCandidateAsInvalid(failedDriverId); + + // pick a new leader + masterElection.electMaster(driverInstance); + LOG.info("New master was elected: " + this.driverInstance.getDriverState().getMasterId()); + + return ""; + } + + @Override + public String processPhaseCompletionAnnouncement(Request request) { + String agentIp = request.body(); + DriverState driverState = this.driverInstance.getDriverState(); + + LOG.info("Agent " + agentIp + " finished executing the current phase."); + this.driverInstance.getDistributedPhaseMonitor().addAgentWhichCompletedTheCurrentPhase(agentIp); + + /* if this is the first driver receiving this type of request, forward it to all the other drivers running in + * the cluster. + */ + if (request.queryParams("forward").equals("true")) { + for (int i = 0; i < driverState.getNrDrivers(); i++) { + /* skip current driver and inactive drivers */ + if (i == driverState.getId()) { + continue; + } + + LOG.info(driverState.getHostname() + ": sending agent announcement for phase completion " + + agentIp + "" + "to driver " + driverState.getPathForId(i)); + HttpResponse response = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, agentIp, + Driver.getPhaseFinishedByAgentPath(driverState.getPathForId(i), HttpUtils.SPARK_PORT, false), + HTTP_REQUEST_RETRIES); + + if (response == null) { + // the assumption is that the new driver will receive the full list of active agents after being restarted + LOG.info("Driver " + driverState.getHostname() + "failed to send announcement for phase " + + "of agent " + agentIp + "to driver " + driverState.getPathForId(i)); + } + + } + + } + + // TODO: update current phase for stand-by drivers if the phase was successfully finished + return ""; + } + + + @Override + public String processExecutionRequest(Request request, Driver driverInstance) throws Exception { + String yamlConfiguration = request.body(); + LOG.info("Received execution request for TD configuration:\n"); + LOG.info(yamlConfiguration); + + // save TD configuration which must be executed in distributed mode + driverInstance.setConfiguration(new Configuration(yamlConfiguration)); + + // send TD configuration to all the other drivers running in the cluster + if (request.queryParams("forward").equals("true")) { + List forwardPaths = this.getDriverPathsForRedirectingRequests(driverInstance); + forwardPaths.forEach(forwardPath -> { + LOG.info("Forwarding execution request to driver " + forwardPath); + HttpResponse driverResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, request.body(), + Driver.getExecutionPath(forwardPath, HttpUtils.SPARK_PORT, false), HttpUtils.HTTP_REQUEST_RETRIES); + if (driverResponse == null) { + /* the assumption is that the driver will fail to respond to heartbeat request and will receive this + * information after being restarted. + */ + LOG.warn("Unable to forward execution request to " + forwardPath); + } + }); + } + + return ""; + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/CandidateRequestProcessor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/CandidateRequestProcessor.java new file mode 100644 index 0000000..0d30611 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/CandidateRequestProcessor.java @@ -0,0 +1,94 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests; + +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import org.apache.http.HttpResponse; +import spark.Request; +import spark.Response; + +/** + * Specifies how a candidate will process the HTTP requests received from the other drivers or the agents running in the + * cluster. + */ +public class CandidateRequestProcessor extends AbstractRequestProcessor { + private static CandidateRequestProcessor instance = null; + + /** + * Returns an instance of this class. + * @param driver : the driver instance that will use this class for processing HTTP requests. + */ + public static CandidateRequestProcessor getInstance(Driver driver) { + if (instance == null || !instance.driverInstance.equals(driver)) { + instance = new CandidateRequestProcessor(driver); + } + + return instance; + } + + private CandidateRequestProcessor(Driver driverInstance) { + super(driverInstance); + } + + @Override + public String processRegisterRequest(Request request, Driver driverInstance) { + super.processRegisterRequest(request, driverInstance); + + String agentIp = request.body(); + this.driverInstance.getDriverState().registerAgent(agentIp); + LOG.info("[driver] Registered agent with ip " + agentIp); + LOG.info("[driver] active agents " + this.driverInstance.getDriverState().getRegisteredAgents().toString()); + + return ""; + } + + @Override + public String acknowledgeSampleContentSuccessfulInstallation(Request request, Driver driverInstance, Response response) { + DriverState driverState = driverInstance.getDriverState(); + + LOG.info("Candidate driver is redirecting sample content ack to the master..."); + // another driver should be chosen for this responsibility + if (driverState.getMasterId() == -1) { + response.status(503); + } + // request should be forwarded to the current master + HttpResponse masterResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, request.body(), + Driver.getSampleContentAckPath(driverState.getPathForId(driverState.getMasterId()), HttpUtils.SPARK_PORT), + HttpUtils.HTTP_REQUEST_RETRIES); + + if (masterResponse == null) { + response.status(503); + } + + return ""; + } + + @Override + public String processExecutionRequest(Request request, Driver driverInstance) throws Exception { + super.processExecutionRequest(request, driverInstance); + + // candidates will assume that the execution started successfully + driverInstance.getDistributedPhaseMonitor().setPhase(driverInstance.getConfiguration().getPhases().get(0)); + // the assumption is that the master is responsible for installing TD sample content + driverInstance.getConfiguration().getGlobalArgs().setInstallSampleContent("false"); + + return ""; + } + + @Override + public String processHeartbeatRequest(Request request, Driver driverInstance) { + throw new IllegalStateException("Candidates should never receive this type of requests."); + } + + @Override + public String processAgentFailureAnnouncement(Request request, Driver driverInstance) { + String agentIdentifier = request.body(); + + LOG.info("The driver was announced by the master that agent " + agentIdentifier + " failed to respond" + + " to heartbeat request."); + this.driverInstance.getDistributedPhaseMonitor().removeAgentFromActiveTDRunners(agentIdentifier); + this.driverInstance.getDriverState().removeAgent(agentIdentifier); + + return ""; + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/MasterRequestProcessor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/MasterRequestProcessor.java new file mode 100644 index 0000000..d88282c --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/MasterRequestProcessor.java @@ -0,0 +1,168 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.google.gson.Gson; +import org.apache.http.HttpResponse; +import spark.Request; +import spark.Response; + +import java.util.ArrayList; +import java.util.List; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; +import static com.adobe.qe.toughday.internal.core.engine.Engine.logGlobal; + +/** + * Specifies how the master will process the HTTP requests received from the other components running in the cluster. + */ +public class MasterRequestProcessor extends AbstractRequestProcessor { + private final Object object = new Object(); + private static MasterRequestProcessor instance = null; + + /** + * Returns an instance of this class. + * @param driver : the driver instance that will use this class for processing HTTP requests. + */ + public static MasterRequestProcessor getInstance(Driver driver) { + if (instance == null || !instance.driverInstance.equals(driver)) { + instance = new MasterRequestProcessor(driver); + } + + return instance; + } + + private MasterRequestProcessor(Driver driverInstance) { + super(driverInstance); + } + + private void waitForSampleContentToBeInstalled(Driver currentDriver) { + synchronized (object) { + try { + object.wait(); + } catch (InterruptedException e) { + LOG.error("Failed to install ToughDay sample content package. Execution will be stopped."); + currentDriver.finishDistributedExecution(); + System.exit(-1); + } + } + } + + private void installToughdayContentPackage(Configuration configuration, Driver currentDriver) { + logGlobal("Installing ToughDay 2 Content Package..."); + GlobalArgs globalArgs = configuration.getGlobalArgs(); + + if (globalArgs.getDryRun() || !globalArgs.getInstallSampleContent()) { + return; + } + + HttpResponse agentResponse = null; + List agentsCopy = new ArrayList<>(this.driverInstance.getDriverState().getRegisteredAgents()); + + while (agentResponse == null && agentsCopy.size() > 0) { + // pick one agent to install the sample content + String agentIpAddress = agentsCopy.remove(0); + String URI = Agent.getInstallSampleContentPath(agentIpAddress); + LOG.info("Installing sample content request was sent to agent " + agentIpAddress); + + YamlDumpConfigurationAsTaskForTDAgents dumpConfig + = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + String yamlTask = dumpConfig.generateConfigurationObject(); + + agentResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, yamlTask, URI, HTTP_REQUEST_RETRIES); + } + + // we should wait until the we receive confirmation that the sample content was successfully installed + waitForSampleContentToBeInstalled(currentDriver); + + logGlobal("Finished installing ToughDay 2 Content Package."); + globalArgs.setInstallSampleContent("false"); + + } + + private void mergeDistributedConfigParams(Configuration configuration, Driver currentDriver) { + DriverState driverState = currentDriver.getDriverState(); + + if (driverState.getDriverConfig().getDistributedConfig().getHeartbeatIntervalInSeconds() == + currentDriver.getConfiguration().getDistributedConfig().getHeartbeatIntervalInSeconds()) { + + driverState.getDriverConfig().getDistributedConfig().merge(configuration.getDistributedConfig()); + return; + } + + // cancel heartbeat task and reschedule it with the new period + currentDriver.cancelHeartbeatTask(); + driverState.getDriverConfig().getDistributedConfig().merge(configuration.getDistributedConfig()); + currentDriver.scheduleHeartbeatTask(); + } + + + private void handleExecutionRequest(Configuration configuration, Driver currentDriver) { + installToughdayContentPackage(configuration, currentDriver); + mergeDistributedConfigParams(configuration, currentDriver); + + currentDriver.executePhases(); + } + + @Override + public String acknowledgeSampleContentSuccessfulInstallation(Request request, Driver driverInstance, Response response) { + boolean installed = Boolean.parseBoolean(request.body()); + + if (!installed) { + LOG.error("Failed to install the ToughDay sample content package. Execution will be stopped."); + driverInstance.finishDistributedExecution(); + } + + synchronized (this.object) { + this.object.notify(); + } + + return ""; + } + + @Override + public String processExecutionRequest(Request request, Driver driverInstance) throws Exception { + super.processExecutionRequest(request, driverInstance); + + // handle execution in a different thread to be able to quickly respond to this request + driverInstance.getExecutorService().submit(() -> handleExecutionRequest(driverInstance.getConfiguration(), driverInstance)); + + return ""; + } + + @Override + public String processHeartbeatRequest(Request request, Driver driverInstance) { + // send executions/test to candidates to keep them updated + Gson gson = new Gson(); + return gson.toJson(driverInstance.getDistributedPhaseMonitor().getExecutions()); + } + + @Override + public String processAgentFailureAnnouncement(Request request, Driver driverInstance) { + throw new IllegalStateException("Master should never receive this type of request."); + } + + @Override + public String processRegisterRequest(Request request, Driver driverInstance) { + super.processRegisterRequest(request, driverInstance); + String agentIp = request.body(); + DriverState driverState = this.driverInstance.getDriverState(); + LOG.info("[driver] Registered agent with ip " + agentIp); + + if (!this.driverInstance.getDistributedPhaseMonitor().isPhaseExecuting()) { + driverState.registerAgent(agentIp); + LOG.info("[driver] active agents " + driverState.getRegisteredAgents().toString()); + return ""; + } + + // master must schedule the work redistribution process when a new agent is registering + this.driverInstance.getTaskBalancer().scheduleWorkRedistributionProcess(this.driverInstance, agentIp, true); + + return ""; + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessor.java new file mode 100644 index 0000000..63d8c88 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessor.java @@ -0,0 +1,67 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests; + +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.fasterxml.jackson.core.JsonProcessingException; +import spark.Request; +import spark.Response; + +/** + * Common interface for handling http requests received by the driver components running in the cluster. + */ +public interface RequestProcessor { + /** + * Method responsible for processing the request of registering a new Agent which joined the cluster recently. + * @param request : http request received from the Agent. + * @param driverInstance : the Driver instance processing this request + */ + String processRegisterRequest(Request request, Driver driverInstance); + + /** + * Method responsible for processing the request sent by the drivers running in the cluster to ask for updates + * regarding the current state of the cluster. This method is called whenever a new Driver component joins the + * cluster. + * @param request : http request received from a Driver + * @param driverInstance : The driver instance processing this request + * @return Yaml representation of the current state of the cluster (generated by dumping the DriverUpdateInfo class) + */ + String processUpdatesRequest(Request request, Driver driverInstance) throws JsonProcessingException; + + /** + * Method responsible for processing the request sent by a driver to announce that the master election process must + * be triggered. + * @param request : http request used for triggering the master election process. + * @param driverInstance : the Driver instance processing this request. + */ + String processMasterElectionRequest(Request request, Driver driverInstance); + + /** + * Method responsible for processing the request sent by an agent to announce that the Phase received as a task from + * the driver was successfully executed. + * @param request : http request sent by an agent which completed the execution of a phase. + */ + String processPhaseCompletionAnnouncement(Request request); + + /** + * Method responsible for processing the request sent by the agent responsible to install the TD sample content + * package to announce that the installation was successfully completed. + * @param request : http request sent by the agent. + * @param driverInstance : the Driver instance processing this request + * @param response + * @return + */ + String acknowledgeSampleContentSuccessfulInstallation(Request request, Driver driverInstance, Response response); + + /** + * Method responsible for processing the request used to trigger the distributed execution of ToughDay. + * @param request : http request + * @param driverInstance : the Driver instance processing this request + * @throws Exception : if the Configuration for running ToughDay cannot be successfully built and dumped as a YAML + * string. + */ + String processExecutionRequest(Request request, Driver driverInstance) throws Exception; + + String processHeartbeatRequest(Request request, Driver driverInstance); + + String processAgentFailureAnnouncement(Request request, Driver driverInstance); + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessorDispatcher.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessorDispatcher.java new file mode 100644 index 0000000..e03adb9 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessorDispatcher.java @@ -0,0 +1,38 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests; + +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; + +/** + * Class responsible for choosing the appropriate RequestProcessor implementation to be used for processing requests by + * the driver, depending on the role played by the Driver receiving the request (Master or Candidate). + */ +public class RequestProcessorDispatcher { + private static RequestProcessorDispatcher instance = null; + + private RequestProcessorDispatcher() { } + + /** + * Returns an instance of this class. + */ + public static RequestProcessorDispatcher getInstance() { + if (instance == null) { + instance = new RequestProcessorDispatcher(); + } + + return instance; + } + + /** + * Returns the appropriate RequestProcessor to be used for processing the requests received by the driver. + * @param driver : the Driver instance that must process the http requests. + */ + public RequestProcessor getRequestProcessor(Driver driver) { + if (driver.getDriverState().getCurrentState() == DriverState.Role.MASTER) { + return MasterRequestProcessor.getInstance(driver); + } else { + return CandidateRequestProcessor.getInstance(driver); + } + + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructions.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructions.java new file mode 100644 index 0000000..383ad67 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructions.java @@ -0,0 +1,45 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution; + +import java.util.Map; + +/** Contains all the information needed by the agents for updating their configuration when the work needs to be + * rebalanced. + */ +public class RedistributionInstructions { + private Map counts; + private Map runModeProperties; + + // dummy constructor, required for dumping the class + public RedistributionInstructions() { } + + public RedistributionInstructions(Map counts, Map runModeProperties) { + this.counts = counts; + this.runModeProperties = runModeProperties; + } + + /** + * Getter for counts. + */ + public Map getCounts() { + return this.counts; + } + + /** + * Setter for counts. + */ + public void setCounts(Map counts) { + this.counts = counts; + } + + /** + * Getter for run mode properties that must be redistributed. + */ + public Map getRunModeProperties() { return this.runModeProperties; } + + /** + * Setter for run mode properties that must be redistributed. + */ + public void setRunModeProperties(Map runModeProperties) { + this.runModeProperties = runModeProperties; + } +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructionsProcessor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructionsProcessor.java new file mode 100644 index 0000000..7147eff --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructionsProcessor.java @@ -0,0 +1,75 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution; + +import com.adobe.qe.toughday.internal.core.TestSuite; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.ConstantLoadRunModeBalancer; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Class responsible for processing the rebalancing request received by an agent from the driver when the + * work must be rebalanced. + */ +public class RedistributionInstructionsProcessor { + protected static final Logger LOG = LogManager.getLogger(ConstantLoadRunModeBalancer.class); + + private void processTestSuiteChanges(Map counts, Phase phase) { + TestSuite testSuite = phase.getTestSuite(); + long nr = testSuite.getTests().stream() + .filter(test -> !counts.containsKey(test.getName())) + .count(); + if (nr > 0) { + throw new IllegalStateException("Instructions were not received for each test in the test suite."); + } + + testSuite.getTests() + .stream() + .filter(test -> counts.containsKey(test.getName())) + .forEach(test -> { + // remove tests for which the count property was achieved + if (counts.get(test.getName()) == 0) { + phase.getCounts().remove(test); + } else { + // reset number of tests executed so far + phase.getCounts().put(test, new AtomicLong(0)); + // update number of executions left for this test + test.setCount(String.valueOf(counts.get(test.getName()))); + } + }); + } + + private void processRunModeChanges(RedistributionInstructions redistributionInstructions, RunMode runMode) { + runMode.getRunModeBalancer().before(redistributionInstructions, runMode); + + runMode.getRunModeBalancer().processRunModeInstructions(redistributionInstructions, runMode); + + runMode.getRunModeBalancer().after(redistributionInstructions, runMode); + } + + /** + * Method used for processing the rebalance request. + * @param jsonInstructions : redistribution instructions received from the driver + * @param phase : the current phase being executed by the agents. + * @throws IOException : if the instructions don't have the appropriate format. + */ + public void processInstructions(String jsonInstructions, Phase phase) throws IOException { + if (phase == null || jsonInstructions == null) { + throw new IllegalArgumentException("Phase and redistribution instructions must not be null during work " + + "redistribution process."); + } + + ObjectMapper objectMapper = new ObjectMapper(); + RedistributionInstructions redistributionInstructions = + objectMapper.readValue(jsonInstructions, RedistributionInstructions.class); + + // update values for each modified property + processRunModeChanges(redistributionInstructions, phase.getRunMode()); + processTestSuiteChanges(redistributionInstructions.getCounts(), phase); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/TaskBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/TaskBalancer.java new file mode 100644 index 0000000..b7bd012 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/TaskBalancer.java @@ -0,0 +1,313 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution; + +import com.adobe.qe.toughday.internal.core.TestSuite; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.DistributedPhaseMonitor; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.DistributedConfig; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.PhaseSplitter; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpResponse; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.*; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; + +/** + * Class responsible for balancing the work between the agents running the cluster + * whenever the number of agents is changing. + */ +public class TaskBalancer { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + private static TaskBalancer instance = null; + + private RedistributionStatus status = RedistributionStatus.UNNECESSARY; + private final PhaseSplitter phaseSplitter = new PhaseSplitter(); + private final HttpUtils httpUtils = new HttpUtils(); + private final ConcurrentLinkedQueue inactiveAgents = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue recentlyAddedAgents = new ConcurrentLinkedQueue<>(); + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + + public enum RedistributionStatus { + UNNECESSARY, // no need to redistribute the work + SCHEDULED, // redistribution is scheduled + RESCHEDULE_REQUIRED, /* nr of agents running in the cluster has changed while the work was redistributed so + * we must reschedule the process */ + EXECUTING // work redistribution process is executing + } + + /** + * Returns a list with all the agents that failed to respond to the heartbeat request sent + * by the driver. + */ + public ConcurrentLinkedQueue getInactiveAgents() { + return this.inactiveAgents; + } + + private TaskBalancer() {} + + /** + * Returns a singleton instance of this class. + */ + public static TaskBalancer getInstance() { + if (instance == null) { + instance = new TaskBalancer(); + } + + return instance; + } + + private Map getTestSuitePropertiesToRedistribute(TestSuite taskTestSuite) { + HashMap map = new HashMap<>(); + taskTestSuite.getTests().forEach(test -> map.put(test.getName(), test.getCount())); + + return map; + } + + private void addNewAgent(String agentIdentifier) { + this.recentlyAddedAgents.add(agentIdentifier); + } + + private void addInactiveAgent(String agentIdentifier) { + this.inactiveAgents.add(agentIdentifier); + } + + private void waitUntilAllAgentsAreReadyForRebalancing(List agents) { + boolean ready = false; + + while(!ready) { + ready = true; + + // get status of all new agents + for (String agentIpAddress : agents) { + HttpResponse agentResponse = this.httpUtils.sendHttpRequest(HttpUtils.GET_METHOD, "", + Agent.getGetStatusPath(agentIpAddress), HTTP_REQUEST_RETRIES); + + if (agentResponse != null) { + try { + String response = EntityUtils.toString(agentResponse.getEntity()); + if (!response.equals(Agent.Status.RUNNING.toString())) { + ready = false; + break; + } + } catch (IOException e) { + LOG.warn("Could not confirm that agent " + agentIpAddress + " is ready for processing " + + "redistribution instructions") ; + } + } + } + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // skip this and continue waiting for all agents to be ready for redistribution + } + } + } + + private void sendInstructionsToExistingAgents(Map phases, List activeAgents) { + ObjectMapper mapper = new ObjectMapper(); + + // send instructions to agents that are already running TD + activeAgents.stream() + .filter(ipAddress -> !this.inactiveAgents.contains(ipAddress)) // filter agents that become inactive + .forEach(ipAddress -> { + String agentURI = Agent.getRebalancePath(ipAddress); + TestSuite testSuite = phases.get(ipAddress).getTestSuite(); + RunMode runMode = phases.get(ipAddress).getRunMode(); + + // build redistribution instructions + Map testSuiteProperties = getTestSuitePropertiesToRedistribute(testSuite); + Map runModeProperties = runMode.getRunModeBalancer() + .getRunModePropertiesToRedistribute(runMode.getClass(), runMode); + RedistributionInstructions redistributionInstructions = + new RedistributionInstructions(testSuiteProperties, runModeProperties); + try { + String message = mapper.writeValueAsString(redistributionInstructions); + LOG.info("[Redistribution] Sending " + message + " to agent " + ipAddress); + + HttpResponse response = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, message, + agentURI, HTTP_REQUEST_RETRIES); + if (response == null) { + /* redistribution will be automatically triggered as soon as this agent fails to respond to + * heartbeat request + */ + LOG.warn("Redistribution instructions could not be sent to agent " + ipAddress + "."); + } + + } catch (JsonProcessingException e) { + LOG.error("Unexpected exception while sending redistribution instructions", e); + LOG.warn("Agent " + ipAddress + " will continue to run with the configuration it had before " + + "the process of redistribution was triggered."); + } + + }); + } + + private void sendExecutionRequestsToNewAgents(List recentlyAddedAgents, Map phases, + Configuration configuration, DriverState driverState) { + // for the recently added agents, send execution queries + recentlyAddedAgents.forEach(newAgentIpAddress -> { + configuration.setPhases(Collections.singletonList(phases.get(newAgentIpAddress))); + YamlDumpConfigurationAsTaskForTDAgents dumpConfig + = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + String yamlTask = dumpConfig.generateConfigurationObject(); + + + LOG.info("Sending execution request + " + yamlTask + " to new agent " + newAgentIpAddress); + HttpResponse response = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, yamlTask, + Agent.getSubmissionTaskPath(newAgentIpAddress), 3); + + if (response == null) { + /* the assumption is that the agent will fail to respond to heartbeat request => work will be + * automatically redistributed when this happens. + */ + LOG.info("Failed to send task to new agent " + newAgentIpAddress + "."); + } + }); + } + + private void updateStateOfAgents(Driver driverInstance, List recentlyAddedAgents) throws CloneNotSupportedException { + DistributedPhaseMonitor distributedPhaseMonitor = driverInstance.getDistributedPhaseMonitor(); + DriverState driverState = driverInstance.getDriverState(); + + if (!distributedPhaseMonitor.isPhaseExecuting()) { + return; + } + + Phase phase = distributedPhaseMonitor.getPhase(); + // update number of tests that are left to be executed by the agents + distributedPhaseMonitor.updateCountPerTest(); + Map phases = this.phaseSplitter.splitPhaseForRebalancingWork(phase, + new LinkedList<>(driverState.getRegisteredAgents()), + new ArrayList<>(recentlyAddedAgents), distributedPhaseMonitor.getPhaseStartTime()); + + // wait until all agents are able to process redistribution instructions + waitUntilAllAgentsAreReadyForRebalancing(distributedPhaseMonitor.getAgentsRunningTD()); + sendInstructionsToExistingAgents(phases, distributedPhaseMonitor.getAgentsRunningTD()); + sendExecutionRequestsToNewAgents(recentlyAddedAgents, phases, driverInstance.getConfiguration(), driverState); + + } + + private void after(Driver driverInstance, List newAgents, List inactiveAgents) { + DistributedPhaseMonitor distributedPhaseMonitor = driverInstance.getDistributedPhaseMonitor(); + DriverState driverState = driverInstance.getDriverState(); + + distributedPhaseMonitor.resetExecutions(); + + // mark recently added agents as active agents executing tasks + newAgents.forEach(driverState::registerAgent); + newAgents.forEach(distributedPhaseMonitor::registerAgentRunningTD); + newAgents.forEach(this.recentlyAddedAgents::remove); + + // inform all the other drivers about the agents which failed to respond to heartbeat request + inactiveAgents.forEach(agentIp -> { + for (int i = 0; i < driverInstance.getDriverState().getNrDrivers(); i++) { + if (i == driverInstance.getDriverState().getId()) { + continue; + } + + HttpResponse driverResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, agentIp, + Driver.getAgentFailurePath(driverInstance.getDriverState().getPathForId(i)), HTTP_REQUEST_RETRIES); + if (driverResponse == null) { + /* the assumption is that if this driver will become the master it will detect this change once the + * heartbeat task is scheduled so no further actions should be done at this moment + */ + LOG.warn("Failed to announce driver-" + i + " that agent " + agentIp + " died."); + } + + LOG.info("Successfully announced driver-" + i + " that the agent " + agentIp + " died"); + } + + }); + + LOG.info("[Redistribution] Finished redistributing the work"); + + if (this.status == RedistributionStatus.RESCHEDULE_REQUIRED) { + this.status = RedistributionStatus.SCHEDULED; + LOG.info("[driver] Scheduling delayed redistribution process for agents " + + this.recentlyAddedAgents.toString()); + this.scheduler.schedule(() -> { + LOG.info("[Redistribution] starting delayed work redistribution process"); + rebalanceWork(driverInstance); + }, driverState.getDriverConfig().getDistributedConfig().getRedistributionWaitTimeInSeconds(), TimeUnit.SECONDS); + } else { + this.status = RedistributionStatus.UNNECESSARY; + } + + + } + + private void excludeInactiveAgents(List agentsToBeExcluded, Driver driverInstance) { + agentsToBeExcluded.forEach(driverInstance.getDriverState()::removeAgent); + agentsToBeExcluded.forEach(this.inactiveAgents::remove); + + // we should not wait for task completion since the agent running it left the cluster + agentsToBeExcluded.forEach(driverInstance.getDistributedPhaseMonitor()::removeAgentFromActiveTDRunners); + } + + private void rebalanceWork(Driver driverInstance) { + this.status = RedistributionStatus.EXECUTING; + + List newAgents = new ArrayList<>(recentlyAddedAgents); + List inactiveAgents = new ArrayList<>(this.inactiveAgents); + + // remove all agents who failed answering the heartbeat request in the past + excludeInactiveAgents(inactiveAgents, driverInstance); + LOG.info("[Redistribution] Starting the process...."); + + try { + updateStateOfAgents(driverInstance, newAgents); + } catch (CloneNotSupportedException e) { + LOG.warn(""); + } + + after(driverInstance, newAgents, inactiveAgents); + } + + /** + * Method used for scheduling the process of redistributing the work between the agents running in the cluster. This + * method is called whenever the number of active agents is changing. + * @param driverInstance : the driver instance that will schedule the work redistribution process + * @param agentIdentifier : the identifier of the agent that generated this call + * @param activeAgent : true when the agent represented by agentIdentifier has joined the cluster; false if the + * agent has become inactive(he failed to respond to heartbeat request) + */ + public void scheduleWorkRedistributionProcess(Driver driverInstance, String agentIdentifier, boolean activeAgent) { + DriverState driverState = driverInstance.getDriverState(); + + if (activeAgent) { + this.addNewAgent(agentIdentifier); + } else { + this.addInactiveAgent(agentIdentifier); + } + + DistributedConfig distributedConfig = driverState.getDriverConfig().getDistributedConfig(); + if (this.status == RedistributionStatus.UNNECESSARY) { + this.status = RedistributionStatus.SCHEDULED; + LOG.info("Scheduling redistribution process to start in " + + distributedConfig.getRedistributionWaitTimeInSeconds() + " seconds."); + + // schedule work redistribution process + this.scheduler.schedule(() -> rebalanceWork(driverInstance), + distributedConfig.getRedistributionWaitTimeInSeconds(), TimeUnit.SECONDS); + } else if (this.status == RedistributionStatus.EXECUTING) { + LOG.info("Work redistribution process must be rescheduled."); + this.status = RedistributionStatus.RESCHEDULE_REQUIRED; + } + + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/AbstractRunModeBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/AbstractRunModeBalancer.java new file mode 100644 index 0000000..b6dd2a0 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/AbstractRunModeBalancer.java @@ -0,0 +1,71 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes; + +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; +import com.adobe.qe.toughday.api.annotations.ConfigArgSet; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public abstract class AbstractRunModeBalancer implements RunModeBalancer { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + @Override + public Map getRunModePropertiesToRedistribute(Class type, T object) { + final Map properties = new HashMap<>(); + + if (object == null) { + throw new IllegalArgumentException("Run mode object must not be null."); + } + + Arrays.stream(type.getDeclaredMethods()) + .filter(method -> method.isAnnotationPresent(ConfigArgGet.class)) + .filter(method -> method.getAnnotation(ConfigArgGet.class).redistribute()) + .forEach(method -> { + String propertyName = Configuration.propertyFromMethod(method.getName()); + try { + Object value = method.invoke(object); + properties.put(propertyName, String.valueOf(value)); + } catch (IllegalAccessException | InvocationTargetException e) { + LOG.warn("Property " + propertyName + " could not be collected for redistribution instructions." + + " Received error " + e.getMessage()); + } + }); + + return properties; + } + + @Override + public void processRunModeInstructions(RedistributionInstructions redistributionInstructions, T runMode) { + if (redistributionInstructions == null || runMode == null) { + throw new IllegalArgumentException("Rebalance instructions and run mode must not be null."); + } + + Map runModeProperties = redistributionInstructions.getRunModeProperties(); + if (runModeProperties == null || runModeProperties.isEmpty()) { + return; + } + + Arrays.stream(runMode.getClass().getDeclaredMethods()) + .filter(method -> runModeProperties.containsKey(Configuration.propertyFromMethod(method.getName()))) + .filter(method -> method.isAnnotationPresent(ConfigArgSet.class)) + .forEach(method -> { + String property = Configuration.propertyFromMethod(method.getName()); + LOG.info("[Agent] Setting property " + property + " to " + runModeProperties.get(property)); + + try { + method.invoke(runMode, runModeProperties.get(property)); + } catch (IllegalAccessException | InvocationTargetException e) { + LOG.warn("[Agent] Property " + property + " could not be set to " + runModeProperties.get(property)); + } + + }); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/ConstantLoadRunModeBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/ConstantLoadRunModeBalancer.java new file mode 100644 index 0000000..48b2a6d --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/ConstantLoadRunModeBalancer.java @@ -0,0 +1,76 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes; + +import com.adobe.qe.toughday.internal.core.engine.runmodes.ConstantLoad; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; + + +public class ConstantLoadRunModeBalancer extends AbstractRunModeBalancer { + protected static final Logger LOG = LogManager.getLogger(ConstantLoadRunModeBalancer.class); + private static final String CURRENT_LOAD_PROPERTY = "currentload"; + + @Override + public Map getRunModePropertiesToRedistribute(Class type, ConstantLoad runMode) { + Map runModeProps = super.getRunModePropertiesToRedistribute(type, runMode); + + if (runMode.isVariableLoad()) { + runModeProps.put(CURRENT_LOAD_PROPERTY, String.valueOf(runMode.getCurrentLoad())); + } + + return runModeProps; + } + + @Override + public void before(RedistributionInstructions redistributionInstructions, ConstantLoad runMode) { + if (runMode.isVariableLoad()) { + // We must cancel the scheduled task and reschedule it with the new values for 'period' and initial delay. + boolean cancelled = runMode.cancelPeriodicTask(); + if (!cancelled) { + LOG.warn("[Driver] Periodic task for updating the load could not be cancelled. Run mode properties " + + "might not be respected."); + } else { + LOG.info("[Driver] Periodic task for updating the load was cancelled."); + } + } + + Map runModeProperties = redistributionInstructions.getRunModeProperties(); + runModeProperties.forEach((property, propValue) -> + beforeChangingValueOfProperty(property, propValue, runMode)); + } + + private void beforeChangingValueOfProperty(String property, String newValue, ConstantLoad runMode) { + if (property.equals("load") && !runMode.isVariableLoad()) { + long newLoad = Long.parseLong(newValue); + long difference = runMode.getLoad() - newLoad; + + if (difference > 0 ) { + runMode.removeRunMaps(difference); + } else { + runMode.addRunMaps(Math.abs(difference)); + } + } + } + + @Override + public void processRunModeInstructions(RedistributionInstructions redistributionInstructions, ConstantLoad runMode) { + super.processRunModeInstructions(redistributionInstructions, runMode); + + Map runModeProps = redistributionInstructions.getRunModeProperties(); + if (runModeProps.containsKey(CURRENT_LOAD_PROPERTY)) { + LOG.info("[Driver[ Setting current load to " + runModeProps.get(CURRENT_LOAD_PROPERTY)); + runMode.setCurrentLoad(Integer.parseInt(runModeProps.get(CURRENT_LOAD_PROPERTY))); + } + } + + @Override + public void after(RedistributionInstructions redistributionInstructions, ConstantLoad runMode) { + if (runMode.isVariableLoad()) { + runMode.schedulePeriodicTask(); + LOG.info("Periodic task for updating the load was rescheduled with interval " + runMode.getInterval() + + " and initial delay " + runMode.getInitialDelay()); + } + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/NormalRunModeBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/NormalRunModeBalancer.java new file mode 100644 index 0000000..d2584a9 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/NormalRunModeBalancer.java @@ -0,0 +1,88 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes; + +import com.adobe.qe.toughday.internal.core.engine.AsyncEngineWorker; +import com.adobe.qe.toughday.internal.core.engine.AsyncTestWorker; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.runmodes.Normal; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class NormalRunModeBalancer extends AbstractRunModeBalancer { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + @Override + public void before(RedistributionInstructions redistributionInstructions, Normal runMode) { + if (runMode.isVariableConcurrency()) { + /* We must cancel the scheduled task for updating concurrency and reschedule it with the new values + * for interval and initial delay. + */ + boolean cancelled = runMode.cancelPeriodicTask(); + if (!cancelled) { + LOG.warn("Periodic task for updating the number of threads could not be cancelled. Normal run mode" + + " properties might not be respected."); + } + } + + Map runModeProperties = redistributionInstructions.getRunModeProperties(); + runModeProperties.forEach((propertyName, propValue) -> this.beforeChangingValueOfProperty(propertyName, propValue, runMode)); + } + + private void beforeChangingValueOfProperty(String property, String newValue, Normal runMode) { + if (property.equals("concurrency")) { + long currentConcurrency = runMode.isVariableConcurrency() ? runMode.getActiveThreads() : + runMode.getConcurrency(); + long newConcurrency = Long.parseLong(newValue); + long difference = currentConcurrency - newConcurrency; + + if (difference > 0) { + reduceConcurrency(difference, runMode); + } else { + increaseConcurrency(Math.abs(difference), runMode); + } + + concurrencySanityChecks(runMode, newConcurrency); + } + } + + private void concurrencySanityChecks(Normal runMode, long newConcurrency) { + // check that we have the exact number of active test workers + if (runMode.getRunContext().getTestWorkers().size() != newConcurrency) { + throw new IllegalStateException("[redistribution] TestWorkers size is " + + runMode.getRunContext().getTestWorkers().size() + " but" + " new value for concurrency is " + newConcurrency); + } + + // check that all test workers are active + if (runMode.getRunContext().getTestWorkers().stream().anyMatch(AsyncEngineWorker::isFinished)) { + throw new IllegalStateException("[redistribution] " + + "There are finished test workers in the list of active workers."); + } + } + + private void reduceConcurrency(long reduce, Normal runMode) { + List workerList = new ArrayList<>(runMode.getRunContext().getTestWorkers()); + + for (int i = 0; i < reduce; i++) { + LOG.info("[Agent - redistribution] Finished test worker " + workerList.get(i).getWorkerThread().getId()); + runMode.finishAndDeleteWorker(workerList.get(i)); + } + } + + private void increaseConcurrency(long increase, Normal runMode) { + for (int i = 0; i < increase; i++) { + runMode.createAndExecuteWorker(runMode.getEngine(), runMode.getEngine().getCurrentPhase().getTestSuite()); + } + } + + @Override + public void after(RedistributionInstructions redistributionInstructions, Normal runMode) { + if (runMode.isVariableConcurrency()) { + runMode.schedulePeriodicTask(); + LOG.info("Task responsible for updating the number threads used to run tests was rescheduled"); + } + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/RunModeBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/RunModeBalancer.java new file mode 100644 index 0000000..1a79a49 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/RunModeBalancer.java @@ -0,0 +1,47 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes; + +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; + +import java.util.Map; + +/** + * Contains all the methods that should be implemented in order to describe how a certain + * run mode must be redistributed when the number of agents running in the cluster modifies. + * @param : type of the run mode + */ +public interface RunModeBalancer { + /** + * Collects all the properties that must be taken into consideration when redistributing + * the work between the agents. + * @param type Class of the run mode + * @param runMode + * @return + */ + Map getRunModePropertiesToRedistribute(Class type, T runMode); + + /** + * This method is called before changing the actual value of the properties to be + * modified. It should be used for creating the appropriate environment for running + * with the run mode configuration. + * @param redistributionInstructions instructions for redistributing the work + * @param runMode + */ + void before(RedistributionInstructions redistributionInstructions, T runMode); + + /** + * This method is responsible for changing the values of the properties that must be + * redistributed according to the redistribution instructions received as a parameter. + * @param redistributionInstructions instructions for redistributing the work + * @param runMode + */ + void processRunModeInstructions(RedistributionInstructions redistributionInstructions, T runMode); + + /** + * The last method called when the process of redistributing the work is + * executing. + * @param redistributionInstructions instructions for redistributing the work + * @param runMode + */ + void after(RedistributionInstructions redistributionInstructions, T runMode); +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/PhaseSplitter.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/PhaseSplitter.java new file mode 100644 index 0000000..1bb1ed7 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/PhaseSplitter.java @@ -0,0 +1,108 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.splitters; + +import com.adobe.qe.toughday.internal.core.TestSuite; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.engine.RunMode; + +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Class responsible for partitioning phases into multiple phases to be distributed between the agents running in + * the cluster. + */ +public class PhaseSplitter { + + private Map distributeTestSuite(TestSuite initialTestSuite, List agents) + throws CloneNotSupportedException { + Map taskTestSuites = new HashMap<>(); + + for (int i = 0; i < agents.size(); i++) { + TestSuite clone = initialTestSuite.clone(); + + clone.getTests().forEach(test -> { + if (taskTestSuites.isEmpty()) { + test.setCount(String.valueOf(test.getCount() / agents.size() + test.getCount() % agents.size())); + } else { + test.setCount(String.valueOf(test.getCount() / agents.size())); + } + }); + + taskTestSuites.put(agents.get(i), clone); + } + + return taskTestSuites; + } + + private void sanityChecks(Phase phase, List agents) { + if (phase == null || agents == null) { + throw new IllegalArgumentException("Phase/List of agents must not be null"); + } + + if (agents.isEmpty()) { + throw new IllegalStateException("At least one agent must be running in the cluster."); + } + + } + + private Map mapPhaseToAgent(Phase phase,Map partitionRunModes, + Map partitionTestSuites, List agents) + throws CloneNotSupportedException { + Map phasePerAgent = new HashMap<>(); + + for (String agent : agents) { + Phase taskPhase = (Phase) phase.clone(); + TestSuite taskTestSuite = partitionTestSuites.get(agent); + + // set the count (the number of executions since the beginning of the run) of each test to 0 + taskTestSuite.getTests().forEach(test -> taskPhase.getCounts().put(test, new AtomicLong(0))); + + taskPhase.setTestSuite(taskTestSuite); + taskPhase.setRunMode(partitionRunModes.get(agent)); + + phasePerAgent.put(agent, taskPhase); + } + + return phasePerAgent; + } + + /** + * Knows how to divide a phase into a number of phases equal to the number of agents running in the cluster. + * @param phase the phase to be partitioned. + * @param agents list with all the agents able to receive a task and to execute it. + */ + public Map splitPhase(Phase phase, List agents) throws CloneNotSupportedException { + sanityChecks(phase, agents); + + Map partitionRunModes = phase.getRunMode().getRunModeSplitter().distributeRunMode(phase.getRunMode(), agents); + Map partitionTestSuites = distributeTestSuite(phase.getTestSuite(), agents); + + return mapPhaseToAgent(phase, partitionRunModes, partitionTestSuites, agents); + } + + /** + * Knows how to divide a phase into a number of phases equal to the number of agents running in the cluster taking + * into consideration the agents that recently joined the cluster and triggered the work redistribution process. + * @param phase : the phase to be partitioned + * @param existingAgents : agents that were running TD before redistributing the work + * @param newAgents : agents that recently joined the cluster + * @param phaseStartTime : execution start time of the phase + * @return : a map having as key the IP address of an Agent running in the cluster and the Phase to be executed by + * this agent as value + */ + public Map splitPhaseForRebalancingWork(Phase phase, List existingAgents, + List newAgents, long phaseStartTime) + throws CloneNotSupportedException { + sanityChecks(phase, existingAgents); + + List allAgents = new ArrayList<>(existingAgents); + allAgents.addAll(newAgents); + + Map partitionRunModes = + phase.getRunMode().getRunModeSplitter().distributeRunModeForRebalancingWork(phase.getRunMode(), existingAgents, + newAgents, phaseStartTime); + Map partitionTestSuites = distributeTestSuite(phase.getTestSuite(), allAgents); + + return mapPhaseToAgent(phase, partitionRunModes, partitionTestSuites, allAgents); + } +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/ConstantLoadRunModeSplitter.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/ConstantLoadRunModeSplitter.java new file mode 100644 index 0000000..ba95c53 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/ConstantLoadRunModeSplitter.java @@ -0,0 +1,106 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes; + +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.runmodes.ConstantLoad; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Knows how to split the constant load run mode into multiple run modes to be distributed to the agents + * running in the cluster. + */ +public class ConstantLoadRunModeSplitter implements RunModeSplitter { + + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private ConstantLoad setParamsForDistributedRunMode(ConstantLoad runMode, int nrAgents, int rateRemainder, + int startRemainder, int endRemainder, + int loadRemainder, int agentId) throws CloneNotSupportedException { + ConstantLoad clone = (ConstantLoad) runMode.clone(); + + + if (runMode.isVariableLoad()) { + if (runMode.getRate() >= nrAgents) { + clone.setStart(String.valueOf(runMode.getStart()/ nrAgents + startRemainder)); + clone.setEnd(String.valueOf(runMode.getEnd()/ nrAgents + endRemainder)); + clone.setRate(String.valueOf(runMode.getRate()/ nrAgents + rateRemainder)); + } else { + clone.setInitialDelay(agentId * GlobalArgs.parseDurationToSeconds(runMode.getInterval()) * 1000); + clone.setOneAgentRate(runMode.getRate()); + clone.setStart(String.valueOf(runMode.getStart()+ agentId * runMode.getRate())); + clone.setRate(String.valueOf(nrAgents * runMode.getRate())); + long interval = GlobalArgs.parseDurationToSeconds(runMode.getInterval()); + clone.setInterval(String.valueOf(interval * nrAgents) + 's'); + } + + return clone; + } + + /* we must distribute the load between the agents */ + clone.setLoad(String.valueOf(runMode.getLoad() / nrAgents + loadRemainder)); + + return clone; + } + + @Override + public Map distributeRunMode(ConstantLoad runMode, List agents) + throws CloneNotSupportedException { + int nrAgents = agents.size(); + Map runModes = new HashMap<>(); + + ConstantLoad firstTask = setParamsForDistributedRunMode(runMode, nrAgents, runMode.getRate() % nrAgents, + runMode.getStart() % nrAgents, runMode.getEnd() % nrAgents, runMode.getLoad() % nrAgents, 0); + runModes.put(agents.get(0), firstTask); + + for (int i = 1; i < nrAgents; i++) { + ConstantLoad task = setParamsForDistributedRunMode(runMode, nrAgents, 0, 0, 0, 0, i); + runModes.put(agents.get(i), task); + } + + return runModes; + } + + @Override + public Map distributeRunModeForRebalancingWork(ConstantLoad runMode, List existingAgents, + List newAgents, long phaseStartTime) throws CloneNotSupportedException { + List agents = new ArrayList<>(existingAgents); + agents.addAll(newAgents); + + Map taskRunModes = distributeRunMode(runMode, agents); + newAgents.forEach(agentName -> taskRunModes.get(agentName).setStart("0")); + + if (!runMode.isVariableLoad()) { + return taskRunModes; + } + + // compute the current load to determine the new values for start/current load + long endTime = System.currentTimeMillis(); + long diff = (endTime - phaseStartTime) / 1000; // convert to seconds + int estimatedCurrentLoad = ((int)(diff / GlobalArgs.parseDurationToSeconds(runMode.getInterval()))) + * runMode.getRate() + runMode.getStart(); + + LOG.info("Phase was executed for " + diff + " seconds"); + LOG.info("Estimated current load " + estimatedCurrentLoad); + + // set start property for new agents + newAgents.forEach(agentName -> taskRunModes.get(agentName) + .setStart(String.valueOf(estimatedCurrentLoad / agents.size()))); + + // set current load for old agents + taskRunModes.get(existingAgents.get(0)).setCurrentLoad(estimatedCurrentLoad / agents.size() + + estimatedCurrentLoad % agents.size()); + for (int i = 1; i < existingAgents.size(); i++) { + taskRunModes.get(existingAgents.get(i)).setCurrentLoad(estimatedCurrentLoad / agents.size()); + } + + return taskRunModes; + + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/NormalRunModeSplitter.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/NormalRunModeSplitter.java new file mode 100644 index 0000000..67eac1e --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/NormalRunModeSplitter.java @@ -0,0 +1,113 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes; + +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.runmodes.Normal; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Knows how to split the normal run mode into multiple normal run modes to be distributed to the agents + * running in the cluster. + */ +public class NormalRunModeSplitter implements RunModeSplitter { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private Normal setParamsForDistributedRunMode(Normal runMode, int nrAgents, int rateRemainder, int endRemainder, + int startRemainder, int concurrencyRemainder, int agentId) throws CloneNotSupportedException { + Normal clone = (Normal) runMode.clone(); + + + if (runMode.isVariableConcurrency()) { + if (runMode.getRate() >= nrAgents) { + clone.setRate(String.valueOf(runMode.getRate() / nrAgents + rateRemainder)); + clone.setStart(String.valueOf(runMode.getStart() / nrAgents + startRemainder)); + clone.setEnd(String.valueOf(runMode.getEnd() / nrAgents + endRemainder)); + } else { + clone.setInitialDelay(agentId * GlobalArgs.parseDurationToSeconds(runMode.getInterval()) * 1000); + long interval = GlobalArgs.parseDurationToSeconds(runMode.getInterval()); + clone.setInterval(String.valueOf(interval * nrAgents) + 's'); + + if (agentId > 0) { + clone.setStart("0"); + clone.setEnd(String.valueOf((runMode.getEnd() - runMode.getStart()) / nrAgents)); + } else { + long diff = runMode.getEnd() - runMode.getStart(); + clone.setEnd(String.valueOf(diff / nrAgents + diff % nrAgents + runMode.getStart())); + } + } + } else { + clone.setConcurrency(String.valueOf(runMode.getConcurrency()/ nrAgents + concurrencyRemainder)); + } + + return clone; + } + + + @Override + public Map distributeRunMode(Normal runMode, List agents) throws CloneNotSupportedException { + if (runMode == null) { + throw new IllegalArgumentException("Run mode must not be null."); + } + + Map taskRunModes = new HashMap<>(); + int nrAgents = agents.size(); + + Normal firstTask = setParamsForDistributedRunMode(runMode, nrAgents, runMode.getRate() % nrAgents, runMode.getEnd() % nrAgents, + runMode.getStart() % nrAgents, runMode.getConcurrency() % nrAgents, 0); + taskRunModes.put(agents.get(0), firstTask); + + for (int i = 1; i < nrAgents; i++) { + Normal task = setParamsForDistributedRunMode(runMode, nrAgents, 0, 0, 0, 0, i); + taskRunModes.put(agents.get(i), task); + } + + return taskRunModes; + } + + @Override + public Map distributeRunModeForRebalancingWork(Normal runMode, List existingAgents, + List newAgents, long phaseStartTime) throws CloneNotSupportedException { + if (runMode == null) { + throw new IllegalArgumentException("Run mode must not be null."); + } + + List agents = new ArrayList<>(existingAgents); + agents.addAll(newAgents); + Map taskRunModes = distributeRunMode(runMode, agents); + + // set start property to 'rate' for each new agent + if (!runMode.isVariableConcurrency()) { + return taskRunModes; + } + + // compute the expected current concurrency + long endTime = System.currentTimeMillis(); + long diff = (endTime - phaseStartTime) / 1000; + int estimatedConcurrency = ((int)(diff / GlobalArgs.parseDurationToSeconds(runMode.getInterval()))) + * runMode.getRate() + runMode.getStart(); + + LOG.info("Phase was executed for " + diff + " seconds"); + LOG.info("Estimated concurrency " + estimatedConcurrency); + + // set start property for new agents + newAgents.forEach(agentName -> + taskRunModes.get(agentName).setStart(String.valueOf(estimatedConcurrency / agents.size()))); + + // set desired active threads for old agents + taskRunModes.get(existingAgents.get(0)).setConcurrency(String.valueOf(estimatedConcurrency / agents.size() + + estimatedConcurrency % agents.size())); + for (int i = 1; i < existingAgents.size(); i++) { + taskRunModes.get(existingAgents.get(i)).setConcurrency(String.valueOf(estimatedConcurrency / agents.size())); + } + + return taskRunModes; + + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/RunModeSplitter.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/RunModeSplitter.java new file mode 100644 index 0000000..95015af --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/RunModeSplitter.java @@ -0,0 +1,31 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes; + +import com.adobe.qe.toughday.internal.core.engine.RunMode; + +import java.util.List; +import java.util.Map; + +/** + * Common interface for all run mode splitters. Implement this interface to specify how a certain type of run mode + * must be partitioned into multiple run modes to be distributed to the agents running in the cluster. + * @param Type of the run mode to be partitioned + */ +public interface RunModeSplitter { + /** + * Used for distributing the run mode when before sending TD execution requests to the agents + * running in the cluster. + * @param runMode object to be partitioned + * @param agents names of the agents waiting to receive TD execution requests from the driver. + */ + Map distributeRunMode(T runMode, List agents) throws CloneNotSupportedException; + + /** + * Used for redistributing the run mode when the process of rebalancing the work is triggered. + * @param runMode object to be partitioned + * @param existingAgents list of agents that were running TD before starting the work redistribution process. + * @param newAgents list of agents which recently joined the cluster + * @param phaseStartTime : time when the phase execution started + */ + Map distributeRunModeForRebalancingWork(T runMode, List existingAgents, List newAgents, + long phaseStartTime) throws CloneNotSupportedException; +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/HeartbeatTask.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/HeartbeatTask.java new file mode 100644 index 0000000..67ebe19 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/HeartbeatTask.java @@ -0,0 +1,84 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.tasks; + +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.TaskBalancer; +import com.google.gson.Gson; +import org.apache.http.HttpResponse; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.*; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; + +/** + * Class used for implementing the heartbeat protocol for monitoring the state of the agents running in the cluster. + */ +public class HeartbeatTask implements Runnable { + protected static final Logger LOG = LogManager.getLogger(Driver.class); + + private Driver driverInstance; + private final HttpUtils httpUtils = new HttpUtils(); + private final TaskBalancer taskBalancer = TaskBalancer.getInstance(); + + public HeartbeatTask(Driver driverInstance) { + this.driverInstance = driverInstance; + } + + private void processHeartbeatResponse(String agentIp, HttpResponse agentResponse) throws IOException { + // the agent has sent his statistic for executions/test => aggregate counts + Gson gson = new Gson(); + String yamlCounts = EntityUtils.toString(agentResponse.getEntity()); + + // gson treats numbers as double values by default + Map doubleAgentCounts = gson.fromJson(yamlCounts, Map.class); + LOG.info("[Heartbeat] Received execution state from agent " + agentIp + ": " + doubleAgentCounts.toString()); + + // recently added agents might not execute tests yet + if (doubleAgentCounts.isEmpty()) { + return; + } + + this.driverInstance.getDistributedPhaseMonitor().getExecutions().forEach((testName, executionsPerAgent) -> + executionsPerAgent.put(agentIp, doubleAgentCounts.get(testName).longValue())); + } + + /** + * Method used for sending heartbeat messages to all the agents running in the cluster. This method should be + * called periodically during the entire distributed execution of ToughDay. + */ + @Override + public void run() { + List activeAgents = new ArrayList<>(this.driverInstance.getDriverState().getRegisteredAgents()); + // remove agents which previously failed to respond to heartbeat request + this.taskBalancer.getInactiveAgents().forEach(activeAgents::remove); + + for (String agentIp : activeAgents) { + String URI = Agent.getHeartbeatPath(agentIp); + HttpResponse agentResponse = httpUtils.sendHttpRequest(HttpUtils.GET_METHOD, "", URI, HTTP_REQUEST_RETRIES); + if (agentResponse != null) { + try { + processHeartbeatResponse(agentIp, agentResponse); + } catch (IOException e) { + // the system may reach the limit of available IO resources + LOG.warn("Could not process heartbeat information received form agent " + agentIp); + } + continue; + } + + if (!this.driverInstance.getDistributedPhaseMonitor().isPhaseExecuting()) { + this.driverInstance.getDriverState().removeAgent(agentIp); + continue; + } + + this.taskBalancer.scheduleWorkRedistributionProcess(driverInstance, agentIp, false); + } + + LOG.info("Number of executions per test: " + + this.driverInstance.getDistributedPhaseMonitor().getExecutionsPerTest()); + } +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/MasterHeartbeatTask.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/MasterHeartbeatTask.java new file mode 100644 index 0000000..80647db --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/MasterHeartbeatTask.java @@ -0,0 +1,146 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.tasks; + +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.google.gson.Gson; +import org.apache.http.HttpResponse; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Class used for implementing the heartbeat protocol for monitoring the state of the Master running in the cluster. + */ +public class MasterHeartbeatTask implements Runnable { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private final Driver driver; + private final HttpUtils httpUtils = new HttpUtils(); + + /** + * Constructor. + * @param driver : the Driver instance which will execute this task periodically. + */ + public MasterHeartbeatTask(Driver driver) { + this.driver = driver; + } + + private void announceDriversThatMasterDied() { + LOG.info("Starting to announce drivers that the current master just died."); + + for (int i = 0; i < this.driver.getDriverState().getNrDrivers(); i++) { + // skip current driver and the former master(which just died) + if (i == this.driver.getDriverState().getId() || i == this.driver.getDriverState().getMasterId()) { + continue; + } + + String hostname = this.driver.getDriverState().getPathForId(i); + HttpResponse driverResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, + String.valueOf(this.driver.getDriverState().getMasterId()), Driver.getMasterElectionPath(hostname, HttpUtils.SPARK_PORT), + HttpUtils.HTTP_REQUEST_RETRIES); + + if (driverResponse == null) { + /* the assumption is that the driver failed to respond to heartbeat request and he will receive the + * updates after rejoining the cluster. + */ + LOG.warn("Failed to announce driver " + hostname + " that the master failed."); + } + + LOG.info("Successfully announced driver " + hostname + " that the current master died."); + } + + } + + /* Function responsible for determining if the number of tests left to executed by the agents must be updated. This + * operation is required whenever the work is rebalanced by the master. + */ + private void checkIfCountsPerTestMustBeUpdated(Map> masterData) { + Map> executions = driver.getDistributedPhaseMonitor().getExecutions(); + + executions.forEach((testName, execPerAgent) -> { + execPerAgent.forEach((agentIp, counts) -> { + // check if one agent is no longer running TD tests + if (!masterData.get(testName).containsKey(agentIp)) { + LOG.info("Counts must be updated because one agent is no longer running TD tests."); + this.driver.getDistributedPhaseMonitor().updateCountPerTest(); + this.driver.getDistributedPhaseMonitor().resetExecutions(); + return; + } + + // check if work was rebalanced and executions must be reset + if (masterData.get(testName).get(agentIp).longValue() < counts) { + LOG.info("Work was rebalanced so counts must be updated"); + this.driver.getDistributedPhaseMonitor().updateCountPerTest(); + this.driver.getDistributedPhaseMonitor().resetExecutions(); + return; + } + }); + }); + + } + + @Override + public void run() { + DriverState driverState = this.driver.getDriverState(); + Gson gson = new Gson(); + + LOG.info(driverState.getHostname() + ": sending heartbeat message to master: " + + Driver.getHeartbeatPath(driverState.getPathForId(driverState.getMasterId()))); + + /* acquire read lock to prevent modification of the current master when receiving information from a different + * driver while the current one is informing others to trigger the master election process. + * */ + + this.driver.getDriverState().getMasterIdLock().readLock().lock(); + HttpResponse driverResponse = this.httpUtils.sendHttpRequest(HttpUtils.GET_METHOD, "", + Driver.getHeartbeatPath(driverState.getPathForId(driverState.getMasterId())), + HttpUtils.HTTP_REQUEST_RETRIES); + + if (driverResponse != null) { + this.driver.getDriverState().getMasterIdLock().readLock().unlock(); + + /* process the information received from the master regarding the number of tests that were executed by the + * agents running TD. + */ + + try { + String jsonExecutions = EntityUtils.toString(driverResponse.getEntity()); + Map> executions = gson.fromJson(jsonExecutions, Map.class); + + LOG.info("Received execution state from master " + executions.toString()); + checkIfCountsPerTestMustBeUpdated(executions); + + executions.forEach((testName, executionsPerAgent) -> { + Map execPerAgent = new HashMap<>(); + executionsPerAgent.forEach((agentIp, counts) -> execPerAgent.put(agentIp, counts.longValue())); + this.driver.getDistributedPhaseMonitor().getExecutions().put(testName, execPerAgent); + }); + + LOG.info("Successfully updated executions/agent"); + + } catch (IOException e) { + LOG.info("Could not process the updates from the driver regarding the number of executions/test. " + + "Count property will no longer be respected if the current master dies."); + } + return; + } + + LOG.info(driverState.getHostname() + ": master failed to respond to heartbeat message. Master " + + "election process will be triggered soon."); + /* mark candidate as inactive */ + this.driver.getMasterElection().markCandidateAsInvalid(driverState.getMasterId()); + + /* announce all drivers that the current master died */ + announceDriversThatMasterDied(); + + this.driver.getDriverState().getMasterIdLock().readLock().unlock(); + /* elect a new master */ + this.driver.getMasterElection().electMaster(driver); + } +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/AsyncTimeoutChecker.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/AsyncTimeoutChecker.java index bda8e69..0b867a2 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/AsyncTimeoutChecker.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/AsyncTimeoutChecker.java @@ -44,7 +44,7 @@ private void interruptWorkerIfTimeout(AsyncTestWorker worker) { return; Long testTimeout = currentTest.getTimeout(); - long timeout = testTimeout >= 0 ? testTimeout : engine.getGlobalArgs().getTimeout(); + long timeout = testTimeout >= 0 ? testTimeout : engine.getGlobalArgs().getTimeoutInSeconds(); if (!worker.getMutex().tryLock()) { /* nothing to interrupt. if the test was running diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Engine.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Engine.java index bcdda9b..48c1c90 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Engine.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Engine.java @@ -23,6 +23,7 @@ import com.adobe.qe.toughday.api.annotations.ConfigArgGet; import com.adobe.qe.toughday.internal.core.config.Configuration; import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; import com.adobe.qe.toughday.metrics.Metric; import com.adobe.qe.toughday.tests.sequential.AEMTestBase; import com.adobe.qe.toughday.tests.utils.PackageManagerClient; @@ -54,6 +55,7 @@ public class Engine { protected static Random _rnd = new Random(); private final Configuration configuration; + private Agent agent; private GlobalArgs globalArgs; private ExecutorService engineExecutorService = Executors.newFixedThreadPool(2); private final ReentrantReadWriteLock engineSync = new ReentrantReadWriteLock(); @@ -92,6 +94,10 @@ public GlobalArgs getGlobalArgs() { return globalArgs; } + public void setAgent(Agent agent) { + this.agent = agent; + } + public boolean areTestsRunning() { return testsRunning; } /** @@ -370,6 +376,7 @@ public void run() { } finally { currentPhaseLock.readLock().unlock(); } + Engine.logGlobal("Test execution finished at: " + Engine.getCurrentDateTime()); LogManager.shutdown(); } @@ -411,7 +418,7 @@ public void run() { } RunMode currentRunmode = phase.getRunMode(); - Long currentDuration = phase.getDuration(); + Long currentDuration = GlobalArgs.parseDurationToSeconds(phase.getDuration()); currentRunmode.runTests(this); long start = System.currentTimeMillis(); @@ -423,6 +430,10 @@ public void run() { LOG.info("Phase interrupted."); long elapsed = System.currentTimeMillis() - start; + if (this.agent != null) { + agent.announcePhaseCompletion(); + } + // if the phase finishes sooner than its duration, // the remainder is split equally between the remaining phases // whose duration was not given as input @@ -532,7 +543,11 @@ protected void shutdownAndAwaitTermination(ExecutorService pool) { } public Phase getCurrentPhase() { - return currentPhase; + currentPhaseLock.readLock().lock(); + Phase phase = currentPhase; + currentPhaseLock.readLock().unlock(); + + return phase; } public ReadWriteLock getCurrentPhaseLock() { diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Phase.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Phase.java index 53f8385..a0d9a1c 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Phase.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Phase.java @@ -24,7 +24,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicLong; -public class Phase { +public class Phase implements Cloneable { private static final String DEFAULT_MEASURABILITY = "true"; private String name; @@ -39,6 +39,15 @@ public class Phase { private Map metrics = new LinkedHashMap<>(); private Map counts = new HashMap<>(); + /** + * Creates shallow copy. + * @throws CloneNotSupportedException if the object to be cloned does not implement the Cloneable interface. + */ + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } + @ConfigArgGet public String getName() { return name; @@ -71,8 +80,8 @@ public void setUseconfig(String useconfig) { } @ConfigArgGet - public Long getDuration() { - return duration; + public String getDuration() { + return duration == null ? null : String.valueOf(duration) + 's'; } @ConfigArgSet(required = false, desc = "The duration of the current phase.") diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/RunMode.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/RunMode.java index d29f263..3450a99 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/RunMode.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/RunMode.java @@ -12,6 +12,8 @@ package com.adobe.qe.toughday.internal.core.engine; import com.adobe.qe.toughday.api.core.RunMap; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; import java.util.Collection; import java.util.concurrent.ExecutorService; @@ -22,6 +24,19 @@ public interface RunMode { ExecutorService getExecutorService(); RunContext getRunContext(); + /** + * Returns the RunModeSplitter instance which knows how to split the run mode for running ToughDay distributed. + * @param type of the run mode. + */ + RunModeSplitter getRunModeSplitter(); + + /** + * Returns the RunModeBalancer instance which knows how to process the redistribution instructions received from + * the driver when the number of agents running in the cluster modifies and the work must be redistributed. + * @param type of the run node. + */ + RunModeBalancer getRunModeBalancer(); + interface RunContext { Collection getTestWorkers(); Collection getRunMaps(); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/ConstantLoad.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/ConstantLoad.java index 36318d5..f2df205 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/ConstantLoad.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/ConstantLoad.java @@ -18,21 +18,26 @@ import com.adobe.qe.toughday.api.annotations.ConfigArgGet; import com.adobe.qe.toughday.api.annotations.ConfigArgSet; import com.adobe.qe.toughday.internal.core.TestSuite; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.ConstantLoadRunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; import com.adobe.qe.toughday.internal.core.engine.*; import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.ConstantLoadRunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.ConstantLoadRunModeSplitter; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; import org.apache.commons.lang3.mutable.MutableLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; @Description(desc = "Generates a constant load of test executions, regardless of their execution time.") -public class ConstantLoad implements RunMode { +public class ConstantLoad implements RunMode, Cloneable { private static final Logger LOG = LoggerFactory.getLogger(ConstantLoad.class); private static final String DEFAULT_LOAD_STRING = "50"; @@ -52,22 +57,32 @@ public class ConstantLoad implements RunMode { private long interval = DEFAULT_INTERVAL; private int rate; private int currentLoad; + //private int currentLoad; + private long initialDelay = 0; + /* field used for checking the finish condition when running TD distributed with the rate + * smaller than number of agents in the cluster. */ + private int oneAgentRate = 0; + private ScheduledExecutorService runRoundScheduler = Executors.newScheduledThreadPool(1); private TestCache testCache; private Phase phase; + private RunModeSplitter runModeSplitter = new ConstantLoadRunModeSplitter(); + private ScheduledFuture scheduledFuture = null; + private final ConstantLoadRunModeBalancer runModeBalancer = new ConstantLoadRunModeBalancer(); private Boolean measurable = true; - @ConfigArgSet(required = false, defaultValue = DEFAULT_LOAD_STRING, desc = "Set the load, in requests per second for the \"constantload\" runmode.") + @ConfigArgSet(required = false, defaultValue = DEFAULT_LOAD_STRING, + desc = "Set the load, in requests per second for the \"constantload\" runmodes.") public void setLoad(String load) { checkNotNegative(Long.parseLong(load), "load"); this.load = Integer.parseInt(load); } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getLoad() { return this.load; } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getStart() { return start; } @@ -81,12 +96,13 @@ public void setStart(String start) { this.start = Integer.valueOf(start); } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getRate() { return rate; } - @ConfigArgSet(required = false, desc = "The increase in load per time unit. When it equals -1, it means it is not set.", defaultValue = "-1") + @ConfigArgSet(required = false, desc = "The increase in load per time unit. When it equals -1, it means it is not set.", + defaultValue = "-1") public void setRate(String rate) { if (!rate.equals("-1")) { checkNotNegative(Long.parseLong(rate), "rate"); @@ -94,17 +110,18 @@ public void setRate(String rate) { this.rate = Integer.valueOf(rate); } - @ConfigArgGet - public long getInterval() { - return interval; + @ConfigArgGet(redistribute = true) + public String getInterval() { + return String.valueOf(interval / 1000) + 's'; } - @ConfigArgSet(required = false, desc = "Used with rate to specify the time interval to add increase the load.", defaultValue = DEFAULT_INTERVAL_STRING) + @ConfigArgSet(required = false, desc = "Used with rate to specify the time interval to add increase the load.", + defaultValue = DEFAULT_INTERVAL_STRING) public void setInterval(String interval) { - this.interval = GlobalArgs.parseDurationToSeconds(interval); + this.interval = GlobalArgs.parseDurationToSeconds(interval) * 1000; } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getEnd() { return end; } @@ -117,6 +134,31 @@ public void setEnd(String end) { this.end = Integer.valueOf(end); } + public ConstantLoad() { + /* this is required when running TD distributed because scheduled task might be cancelled and + * rescheduled when rebalancing the work between the agents. + */ + ScheduledThreadPoolExecutor scheduledPoolExecutor = (ScheduledThreadPoolExecutor) runRoundScheduler; + scheduledPoolExecutor.setRemoveOnCancelPolicy(true); + } + + public int getOneAgentRate() { + return this.oneAgentRate; + } + + public void setCurrentLoad(int currentLoad) { + this.currentLoad = currentLoad; + } + + public int getCurrentLoad() { + return this.currentLoad; + } + + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } + private static class TestCache { public Map> cache = new HashMap<>(); @@ -135,7 +177,7 @@ public void add(@NotNull AbstractTest test) { } } - private boolean isVariableLoad() { + public boolean isVariableLoad() { return start != -1 && end != -1; } @@ -155,6 +197,22 @@ private void checkInvalidArgs() { } } + public void addRunMaps(long nr) { + for (long i = 0; i < nr; i++) { + synchronized (runMaps) { + runMaps.add(phase.getPublishMode().getRunMap().newInstance()); + } + } + } + + public void removeRunMaps(long nr) { + for (long i = 0; i < nr; i++) { + synchronized (runMaps) { + this.runMaps.remove(0); + } + } + } + @Override public void runTests(Engine engine) { checkInvalidArgs(); @@ -169,11 +227,7 @@ public void runTests(Engine engine) { load = Math.max(start, end); } - for(int i = 0; i < load; i++) { - synchronized (runMaps) { - runMaps.add(phase.getPublishMode().getRunMap().newInstance()); - } - } + addRunMaps(load); this.scheduler = new AsyncTestWorkerScheduler(engine); executorService.execute(scheduler); @@ -193,11 +247,29 @@ public Collection getRunMaps() { @Override public boolean isRunFinished() { - return scheduler.isFinished(); + return scheduler != null && scheduler.isFinished(); } }; } + @Override + public RunModeBalancer getRunModeBalancer() { + return this.runModeBalancer; + } + + public void setInitialDelay(long initialDelay) { + this.initialDelay = initialDelay; + } + + public void setOneAgentRate(int oneAgentRate) { + this.oneAgentRate = oneAgentRate; + } + + @Override + public RunModeSplitter getRunModeSplitter() { + return this.runModeSplitter; + } + @Override public void finishExecutionAndAwait() { scheduler.finishExecution(); @@ -231,6 +303,26 @@ public void finishExecutionAndAwait() { } + /** + * Method used for cancelling the task responsible for updating the current load. + * @return true if the task is successfully cancelled; false otherwise + */ + public boolean cancelPeriodicTask() { + return this.scheduledFuture.cancel(true); + } + + + /** + * Method used for scheduling the task responsible for updating the current load to be executed every 'interval' + * milliseconds. + */ + public void schedulePeriodicTask() { + MutableLong secondsLeft = new MutableLong((initialDelay + interval) / 1000); + + this.runRoundScheduler.scheduleAtFixedRate(this.scheduler.getRunnableToSchedule(secondsLeft), 0, + GlobalArgs.parseDurationToSeconds("1s"), TimeUnit.SECONDS); + } + private class AsyncTestWorkerImpl extends AsyncTestWorker { private AbstractTest test; private RunMap runMap; @@ -269,98 +361,107 @@ public boolean hasExited() { } } - private class AsyncTestWorkerScheduler extends AsyncEngineWorker { private Engine engine; + public AsyncTestWorkerScheduler(Engine engine) { this.engine = engine; } - private void configureRateAndInterval(MutableLong secondsLeft) { + private void configureRateAndInterval() { //the difference from the beginning load to the end one int loadDifference = Math.abs(end - start); // suppose load will increase by second - secondsLeft.setValue(1); - rate = (int)Math.floor(1.0 * secondsLeft.getValue() * loadDifference - / phase.getDuration()); + long newInterval = 1; + rate = (int)Math.floor(1.0 * newInterval* loadDifference + / GlobalArgs.parseDurationToSeconds(phase.getDuration())); // if the rate becomes too small, increase the interval at which the load is increased while (rate < 1) { - secondsLeft.increment(); - rate = (int)Math.floor(1.0 * secondsLeft.getValue() * loadDifference - / phase.getDuration()); + newInterval += 1; + rate = (int)Math.floor(1.0 * newInterval * loadDifference + / GlobalArgs.parseDurationToSeconds(phase.getDuration())); } - interval = secondsLeft.getValue(); + interval = newInterval * 1000; // set interval in milliseconds } - @Override - public void run() { - try { - currentLoad = load; - MutableLong secondsLeft = new MutableLong(interval); + private Runnable getRunnableToSchedule(MutableLong secondsLeft) { + return () -> { + if (!isFinished()) { + try { + // run the current run with the current load + runRound(); + secondsLeft.decrement(); - // if the rate was not specified and start and end were - if (isVariableLoad()) { - if (rate == -1) { - configureRateAndInterval(secondsLeft); - } + rampUp(secondsLeft); - currentLoad = start; - } + rampDown(secondsLeft); - while (!isFinished()) { - // run the current run with the current load - runRound(); + } catch (InterruptedException e) { + finishExecution(); + LOG.warn("Constant load scheduler thread was interrupted."); - secondsLeft.decrement(); + // gracefully shut down scheduler + runRoundScheduler.shutdownNow(); + } + } + }; + } - // ramp up the load if 'start' was specified - rampUp(secondsLeft); + @Override + public void run() { + currentLoad = load; - // ramp down the load if 'end' was specified - rampDown(secondsLeft); + // if the rate was not specified and start and end were + if (isVariableLoad()) { + if (rate == -1) { + configureRateAndInterval(); } - } catch (InterruptedException e) { - finishExecution(); - LOG.warn("Constant load scheduler thread was interrupted."); + + currentLoad = start; } + + MutableLong secondsLeft = new MutableLong((interval + initialDelay) / 1000); + scheduledFuture = runRoundScheduler.scheduleAtFixedRate(getRunnableToSchedule(secondsLeft), + 0, GlobalArgs.parseDurationToSeconds("1s"), TimeUnit.SECONDS); + } private void rampUp(MutableLong secondsLeft) { - if (currentLoad == end) { + if (currentLoad == end || ((oneAgentRate > 0) && (currentLoad + rate >= end + oneAgentRate))) { finishExecution(); } // if 'interval' has passed and the current load is still below 'end', // increase the current load if (secondsLeft.getValue() == 0 && end != -1 && currentLoad < end) { - secondsLeft.setValue(interval); currentLoad += rate; - LOG.debug("Current load: " + currentLoad); if (currentLoad > end) { currentLoad = end; } + + secondsLeft.setValue(interval / 1000); } } private void rampDown(MutableLong secondsLeft) { - if (currentLoad == end) { + if (currentLoad == end || ((oneAgentRate > 0) && (currentLoad - rate <= end - oneAgentRate))) { finishExecution(); } // if 'interval' has passed and the currentLoad is still above 'end', // decrease the current load if (secondsLeft.getValue() == 0 && end != -1 && currentLoad > end) { - secondsLeft.setValue(interval); currentLoad -= rate; - LOG.debug("Current load: " + currentLoad); if (currentLoad < end) { currentLoad = end; } + + secondsLeft.setValue(interval / 1000); } } @@ -399,12 +500,13 @@ private void runRound() throws InterruptedException { testWorkers.add(worker); } } - - //TODO use this - Thread.sleep(1000); } } + public long getInitialDelay() { + return this.initialDelay; + } + @Override public ExecutorService getExecutorService() { return executorService; diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/Normal.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/Normal.java index 1cb0995..60878fd 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/Normal.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/Normal.java @@ -18,6 +18,11 @@ import com.adobe.qe.toughday.api.annotations.ConfigArgSet; import com.adobe.qe.toughday.internal.core.config.GlobalArgs; import com.adobe.qe.toughday.internal.core.engine.*; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.NormalRunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.NormalRunModeSplitter; + +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +30,7 @@ import java.util.concurrent.*; @Description(desc = "Runs tests normally.") -public class Normal implements RunMode { +public class Normal implements RunMode, Cloneable { private static final Logger LOG = LoggerFactory.getLogger(Normal.class); private static final int EPS = 1; @@ -40,8 +45,8 @@ public class Normal implements RunMode { private static final long DEFAULT_INTERVAL = 1000; private ExecutorService testsExecutorService; - private ScheduledExecutorService addWorkerScheduler = Executors.newSingleThreadScheduledExecutor(); - private ScheduledExecutorService removeWorkerScheduler = Executors.newSingleThreadScheduledExecutor(); + private ScheduledExecutorService rampingScheduler = Executors.newScheduledThreadPool(1); + private ScheduledFuture scheduledFuture = null; private final List testWorkers = Collections.synchronizedList(new LinkedList<>()); private final List runMaps = new ArrayList<>(); @@ -54,10 +59,23 @@ public class Normal implements RunMode { private long waitTime = DEFAULT_WAIT_TIME; private long interval = DEFAULT_INTERVAL; private int activeThreads = 0; + private long initialDelay = 0; private RunContext context = null; + private RunModeSplitter runModeSplitter = new NormalRunModeSplitter(); + private final RunModeBalancer normalRunModeBalancer = new NormalRunModeBalancer(); + private Engine engine; - @ConfigArgGet + + public Normal() { + /* this is required when running TD distributed because scheduled task might be cancelled and + * rescheduled when rebalancing the work between the agents. + */ + ScheduledThreadPoolExecutor scheduledPoolExecutor = (ScheduledThreadPoolExecutor) rampingScheduler; + scheduledPoolExecutor.setRemoveOnCancelPolicy(true); + } + + @ConfigArgGet(redistribute = true) public int getConcurrency() { return concurrency; } @@ -69,6 +87,11 @@ public void setConcurrency(String concurrencyString) { this.concurrency = Integer.parseInt(concurrencyString); } + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } + @ConfigArgGet public long getWaitTime() { return waitTime; @@ -95,12 +118,13 @@ public void setStart(String start) { this.start = Integer.valueOf(start); } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getRate() { return rate; } - @ConfigArgSet(required = false, desc = "The number of threads added per time unit. When it equals -1, it means it is not set.", defaultValue = "-1") + @ConfigArgSet(required = false, desc = "The number of threads added per time unit. When it equals -1, it means it is not set.", + defaultValue = "-1") public void setRate(String rate) { if (!rate.equals("-1")) { checkNotNegative(Long.parseLong(rate), "rate"); @@ -108,17 +132,22 @@ public void setRate(String rate) { this.rate = Integer.valueOf(rate); } - @ConfigArgGet - public long getInterval() { - return interval; + @ConfigArgGet(redistribute = true) + public String getInterval() { + return String.valueOf(this.interval / 1000) + 's'; + } + + public void setInitialDelay(long initialDelay) { + this.initialDelay = initialDelay; } - @ConfigArgSet(required = false, desc = "Used with rate to specify the time interval to add threads.", defaultValue = DEFAULT_INTERVAL_STRING) + @ConfigArgSet(required = false, desc = "Used with rate to specify the time interval to add threads.", + defaultValue = DEFAULT_INTERVAL_STRING) public void setInterval(String interval) { this.interval = GlobalArgs.parseDurationToSeconds(interval) * 1000; } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getEnd() { return end; } @@ -135,6 +164,43 @@ public int getActiveThreads() { return activeThreads; } + /** + * Method used for cancelling the execution of the task responsible for updating the number of worker threads. + * @return + */ + public boolean cancelPeriodicTask() { + if (this.scheduledFuture != null) { + return this.scheduledFuture.cancel(true); + } + + return true; + } + + /** + * Method used for scheduling the task responsible for updating the number of worker threads to execute + * periodically, every 'interval' milliseconds. + */ + public void schedulePeriodicTask() { + // added for testing purposes + if (this.engine == null) { + return; + } + + if (this.start < this.end) { + // execute 'rate' workers every 'interval' + this.scheduledFuture = this.rampingScheduler.scheduleAtFixedRate(this.getRampUpRunnable(this.engine, + this.engine.getCurrentPhase().getTestSuite()), this.initialDelay, this.interval, TimeUnit.MILLISECONDS); + } else if (this.end < this.start) { + // interrupt 'rate' workers every 'interval' + this.scheduledFuture = this.rampingScheduler.scheduleAtFixedRate(this.getRampDownRunnable(), this.initialDelay, + this.interval, TimeUnit.MILLISECONDS); + } + } + + public Engine getEngine() { + return this.engine; + } + private void checkNotNegative(long param, String property) { if (param < 0) { throw new IllegalArgumentException("Property " + property + " incorrectly configured as negative."); @@ -151,13 +217,14 @@ private void checkInvalidArgs() { } } - private boolean isVariableConcurrency() { + public boolean isVariableConcurrency() { return start != -1 && end != -1; } @Override public void runTests(Engine engine) { checkInvalidArgs(); + this.engine = engine; this.phase = engine.getCurrentPhase(); TestSuite testSuite = phase.getTestSuite(); @@ -168,7 +235,7 @@ public void runTests(Engine engine) { // namely every 'interval' milliseconds if (isVariableConcurrency()) { // if start and end were provided if (rate == -1) { - interval = (long)Math.floor(1000.0 * (phase.getDuration() - EPS) + interval = (long)Math.floor(1000.0 * (GlobalArgs.parseDurationToSeconds(phase.getDuration()) - EPS) / (start < end? end - start : start - end)); rate = 1; } @@ -184,16 +251,16 @@ public void runTests(Engine engine) { createAndExecuteWorker(engine, testSuite); } - // execute 'rate' workers every 'interval' - rampUp(engine, testSuite); - - // interrupt 'rate' workers every 'interval' - rampDown(); + schedulePeriodicTask(); } + public void finishAndDeleteWorker(AsyncTestWorker worker) { + worker.finishExecution(); + this.getRunContext().getTestWorkers().remove(worker); + activeThreads--; + } - - private void createAndExecuteWorker(Engine engine, TestSuite testSuite) { + public void createAndExecuteWorker(Engine engine, TestSuite testSuite) { AsyncTestWorkerImpl testWorker = new AsyncTestWorkerImpl(engine, phase, testSuite, phase.getPublishMode().getRunMap().newInstance()); synchronized (testWorkers) { testWorkers.add(testWorker); @@ -203,6 +270,8 @@ private void createAndExecuteWorker(Engine engine, TestSuite testSuite) { runMaps.add(testWorker.getLocalRunMap()); } + LOG.info("created test worker "); + try { testsExecutorService.execute(testWorker); } catch (OutOfMemoryError e) { @@ -210,64 +279,52 @@ private void createAndExecuteWorker(Engine engine, TestSuite testSuite) { } } - private void rampUp(Engine engine, TestSuite testSuite) { - // every 'interval' milliseconds, we'll create 'rate' workers - if (start < end) { - addWorkerScheduler.scheduleAtFixedRate(() -> { - for (int i = 0; i < rate; ++i) { - // if all the workers have been created - if (activeThreads >= end) { - addWorkerScheduler.shutdownNow(); + private Runnable getRampUpRunnable(Engine engine, TestSuite testSuite) { + return () -> { + for (int i = 0; i < rate; ++i) { + // if all the workers have been created + if (activeThreads >= end) { + rampingScheduler.shutdownNow(); - // mark workers as finished - for(AsyncTestWorker testWorker : testWorkers) { - testWorker.finishExecution(); - } - break; - } else { - createAndExecuteWorker(engine, testSuite); - } + // mark workers as finished + testWorkers.forEach(AsyncEngineWorker::finishExecution); + + break; + } else { + createAndExecuteWorker(engine, testSuite); } - }, 0, interval, TimeUnit.MILLISECONDS); - } + } + }; } - private void rampDown() { - // every 'interval' milliseconds, we'll stop 'rate' workers - if (end < start) { - ThreadPoolExecutor executor = (ThreadPoolExecutor)testsExecutorService; + private Runnable getRampDownRunnable() { + return () -> { + Iterator testWorkerIterator = testWorkers.iterator(); + int toRemove = rate; - removeWorkerScheduler.scheduleAtFixedRate(() -> { - Iterator testWorkerIterator = testWorkers.iterator(); - int toRemove = rate; + while (testWorkerIterator.hasNext()) { + // if all the workers have been removed + if (activeThreads <= end) { + rampingScheduler.shutdownNow(); - while (testWorkerIterator.hasNext()) { - // if all the workers have been removed - if (activeThreads <= end) { - removeWorkerScheduler.shutdownNow(); + // mark all the workers as finished, so the timeout checker will stop the execution + testWorkers.forEach(AsyncEngineWorker::finishExecution); - // mark all the workers as finished, so the timeout checker will stop the execution - for(AsyncTestWorker testWorker : testWorkers) { - testWorker.finishExecution(); - } + break; + } else { + AsyncTestWorker testWorker = testWorkerIterator.next(); + testWorker.finishExecution(); + + // remove the stopped worker + finishAndDeleteWorker(testWorker); + + // if rate users have been removed + if (toRemove == 0) { break; - } else { - AsyncTestWorker testWorker = testWorkerIterator.next(); - testWorker.finishExecution(); - - // remove the stopped worker - testWorkerIterator.remove(); - --toRemove; - --activeThreads; - - // if rate users have been removed - if (toRemove == 0) { - break; - } } } - }, 0, interval, TimeUnit.MILLISECONDS); - } + } + }; } public RunContext getRunContext() { @@ -298,13 +355,19 @@ public boolean isRunFinished() { } @Override - public void finishExecutionAndAwait() { - if (!addWorkerScheduler.isShutdown()) { - addWorkerScheduler.shutdownNow(); - } + public RunModeBalancer getRunModeBalancer() { + return this.normalRunModeBalancer; + } - if(!removeWorkerScheduler.isShutdown()) { - removeWorkerScheduler.shutdownNow(); + @Override + public RunModeSplitter getRunModeSplitter() { + return this.runModeSplitter; + } + + @Override + public void finishExecutionAndAwait() { + if (!rampingScheduler.isShutdown()) { + rampingScheduler.shutdownNow(); } synchronized (testWorkers) { @@ -426,5 +489,9 @@ public boolean hasExited() { } } + public long getInitialDelay() { + return this.initialDelay; + } + } diff --git a/toughday/src/main/java/com/adobe/qe/toughday/metrics/Percentile.java b/toughday/src/main/java/com/adobe/qe/toughday/metrics/Percentile.java index 7b703be..eebc5b4 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/metrics/Percentile.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/metrics/Percentile.java @@ -11,6 +11,7 @@ */ package com.adobe.qe.toughday.metrics; +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; import com.adobe.qe.toughday.api.annotations.Description; import com.adobe.qe.toughday.api.annotations.ConfigArgSet; import com.adobe.qe.toughday.api.core.RunMap; @@ -23,11 +24,16 @@ public class Percentile extends Metric { public Percentile setValue(String value) { this.value = Double.valueOf(value.substring(0,value.length() - 1)); if (this.name.equals(getClass().getSimpleName())) { - this.name = value + "p"; + this.name = value; } return this; } + @ConfigArgGet + public String getValue() { + return String.valueOf(this.value) + 'p'; + } + @Override public Object getValue(RunMap.TestStatistics testStatistics) { return testStatistics.getValueAtPercentile(value); diff --git a/toughday/src/test/java/com/adobe/qe/toughday/TestConfiguration.java b/toughday/src/test/java/com/adobe/qe/toughday/TestConfiguration.java index 05d02e1..75acc7c 100644 --- a/toughday/src/test/java/com/adobe/qe/toughday/TestConfiguration.java +++ b/toughday/src/test/java/com/adobe/qe/toughday/TestConfiguration.java @@ -14,8 +14,8 @@ import com.adobe.qe.toughday.internal.core.ReflectionsContainer; import com.adobe.qe.toughday.internal.core.TestSuite; import com.adobe.qe.toughday.internal.core.Timestamp; -import com.adobe.qe.toughday.internal.core.config.ConfigParams; import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; import com.adobe.qe.toughday.internal.core.config.PhaseParams; import com.adobe.qe.toughday.internal.core.engine.Phase; import com.adobe.qe.toughday.internal.core.engine.publishmodes.Intervals; @@ -278,7 +278,7 @@ public void testPhaseProperties() throws Exception { Assert.assertEquals(phase.getName(), "phase"); Assert.assertEquals(phase.getMeasurable(), false); - long duration = phase.getDuration(); + long duration = GlobalArgs.parseDurationToSeconds(phase.getDuration()); Assert.assertEquals(duration, 60); } @@ -385,7 +385,7 @@ public void testPhaseWithoutDuration() throws Exception { Assert.assertEquals(configuration.getPhasesWithoutDuration().size(), 1); - long duration = configuration.getPhases().get(1).getDuration(); + long duration = GlobalArgs.parseDurationToSeconds(configuration.getPhases().get(1).getDuration()); Assert.assertEquals(duration, 5L); } @@ -396,10 +396,10 @@ public void testTwoPhasesWithoutDuration() throws Exception { Assert.assertEquals(configuration.getPhasesWithoutDuration().size(), 2); - long duration = configuration.getPhases().get(1).getDuration(); + long duration = GlobalArgs.parseDurationToSeconds(configuration.getPhases().get(1).getDuration()); Assert.assertEquals(duration, 5L); - duration = configuration.getPhases().get(2).getDuration(); + duration = GlobalArgs.parseDurationToSeconds(configuration.getPhases().get(2).getDuration()); Assert.assertEquals(duration, 5L); } diff --git a/toughday/src/test/java/com/adobe/qe/toughday/TestConstantLoadMode.java b/toughday/src/test/java/com/adobe/qe/toughday/TestConstantLoadMode.java index bf9ee5a..c792afa 100644 --- a/toughday/src/test/java/com/adobe/qe/toughday/TestConstantLoadMode.java +++ b/toughday/src/test/java/com/adobe/qe/toughday/TestConstantLoadMode.java @@ -137,7 +137,7 @@ public void testCtLoadStartEndRateInterval() throws Exception { Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getStart(), 10); Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getEnd(), 100); Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getRate(), 5); - Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getInterval(), 60); + Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getInterval(), "60s"); } @After diff --git a/toughday/src/test/java/com/adobe/qe/toughday/TestNormalMode.java b/toughday/src/test/java/com/adobe/qe/toughday/TestNormalMode.java index b78442a..91377ec 100644 --- a/toughday/src/test/java/com/adobe/qe/toughday/TestNormalMode.java +++ b/toughday/src/test/java/com/adobe/qe/toughday/TestNormalMode.java @@ -138,7 +138,7 @@ public void testNormalStartEndRateInterval() throws Exception { Assert.assertEquals(runMode.getStart(), 10); Assert.assertEquals(runMode.getEnd(), 100); Assert.assertEquals(runMode.getRate(), 5); - Assert.assertEquals(runMode.getInterval(), 60000); + Assert.assertEquals(runMode.getInterval(), "60s"); } @After diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/AbstractRunModeBalancerTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/AbstractRunModeBalancerTest.java new file mode 100644 index 0000000..68e330d --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/AbstractRunModeBalancerTest.java @@ -0,0 +1,59 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class AbstractRunModeBalancerTest { + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullRunModeThrowsException() { + RunMode dummyRunMode = new DummyRunMode(); + dummyRunMode.getRunModeBalancer().getRunModePropertiesToRedistribute(DummyRunMode.class, null); + } + + @Test + public void testAllPropertiesAreCollected() { + DummyRunMode dummyRunMode = new DummyRunMode(); + Map actual = + dummyRunMode.getRunModeBalancer().getRunModePropertiesToRedistribute(DummyRunMode.class, dummyRunMode); + + Map expected = new HashMap<>(); + expected.put("property1", "prop1"); + expected.put("property2", "prop2"); + + Assert.assertEquals(expected, actual); + } + + @Test + public void testNewValuesAreAssignedForAllRunModeProperties() { + DummyRunMode dummyRunMode = new DummyRunMode(); + RedistributionInstructions redistributionInstructions = new RedistributionInstructions(); + + Map runModeChanges = new HashMap<>(); + runModeChanges.put("property1", "prop1_new"); + runModeChanges.put("property2", "prop2_new"); + + redistributionInstructions.setRunModeProperties(runModeChanges); + dummyRunMode.getRunModeBalancer().processRunModeInstructions(redistributionInstructions, dummyRunMode); + + Assert.assertEquals("prop1_new", dummyRunMode.getProperty1()); + Assert.assertEquals("prop2_new", dummyRunMode.getProperty2()); + Assert.assertEquals("prop3", dummyRunMode.getProperty3()); + + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/ConstantLoadRunModeSplitterTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/ConstantLoadRunModeSplitterTest.java new file mode 100644 index 0000000..808398d --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/ConstantLoadRunModeSplitterTest.java @@ -0,0 +1,98 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.engine.runmodes.ConstantLoad; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; + +public class ConstantLoadRunModeSplitterTest { + private ArrayList cmdLineArgs = new ArrayList<>(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test + public void testLoadDistribution() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=constantload", "load=182")); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + Assert.assertEquals(62, ((ConstantLoad) runModes.get("Agent1")).getLoad()); + Assert.assertEquals(60, ((ConstantLoad) runModes.get("Agent2")).getLoad()); + Assert.assertEquals(60, ((ConstantLoad) runModes.get("Agent3")).getLoad()); + } + + @Test + public void testVariableLoadDistribution() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=constantload", "start=3", "end=12", "rate=3")); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + + ConstantLoad firstAgentRunMode = (ConstantLoad) runModes.get("Agent1"); + Assert.assertEquals(2, firstAgentRunMode.getStart()); + Assert.assertEquals(6, firstAgentRunMode.getEnd()); + Assert.assertEquals(2, firstAgentRunMode.getRate()); + + ConstantLoad secondAgentRunMode = (ConstantLoad) runModes.get("Agent2"); + Assert.assertEquals(1, secondAgentRunMode.getStart()); + Assert.assertEquals(6, secondAgentRunMode.getEnd()); + Assert.assertEquals(1, secondAgentRunMode.getRate()); + } + + @Test + public void testVariableLoadWithRateLowerThanTheNumberOfAgents() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=constantload", "start=10", "end=20", "rate=2", "interval=2s")); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + ConstantLoad initialRunMode = (ConstantLoad) configuration.getPhases().get(0).getRunMode(); + + runModes.forEach((key, value) -> { + ConstantLoad taskRunMode = (ConstantLoad) value; + Assert.assertEquals(initialRunMode.getEnd(), taskRunMode.getEnd()); + Assert.assertEquals(initialRunMode.getRate(), taskRunMode.getOneAgentRate()); + Assert.assertEquals(6, taskRunMode.getRate()); + Assert.assertEquals("6s", taskRunMode.getInterval()); + + }); + + /* the values for rate property are checked for the first complete cycle + (nr_agents * interval period of time) */ + ConstantLoad firstAgentRunMode = (ConstantLoad) runModes.get("Agent1"); + Assert.assertEquals(10, firstAgentRunMode.getStart()); + Assert.assertEquals(0, firstAgentRunMode.getInitialDelay()); + + ConstantLoad secondAgentRunMode = (ConstantLoad) runModes.get("Agent2"); + Assert.assertEquals(12, secondAgentRunMode.getStart()); + Assert.assertEquals(2000, secondAgentRunMode.getInitialDelay()); + + ConstantLoad thirdAgentRunMode = (ConstantLoad) runModes.get("Agent3"); + Assert.assertEquals(14, thirdAgentRunMode.getStart()); + Assert.assertEquals(4000, thirdAgentRunMode.getInitialDelay()); + + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitorTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitorTest.java new file mode 100644 index 0000000..d0507dd --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitorTest.java @@ -0,0 +1,119 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; + +public class DistributedPhaseMonitorTest { + + private List cmdLineArgs; + private final DistributedPhaseMonitor distributedPhaseMonitor = new DistributedPhaseMonitor(); + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + + reflections.getTestClasses().put("MockTest", MockTest.class); + } + + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test + public void testIsPhaseExecuting() throws Exception { + // test when phase is null + Assert.assertFalse(this.distributedPhaseMonitor.isPhaseExecuting()); + + // test when no agent is executing TD + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + this.distributedPhaseMonitor.setPhase(configuration.getPhases().get(0)); + + Assert.assertFalse(this.distributedPhaseMonitor.isPhaseExecuting()); + + // test when phase is executed by agents + this.distributedPhaseMonitor.registerAgentRunningTD("Agent1"); + Assert.assertTrue(this.distributedPhaseMonitor.isPhaseExecuting()); + + this.distributedPhaseMonitor.addAgentWhichCompletedTheCurrentPhase("Agent1"); + } + + @Test + public void testUpdateCountPerTest() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest", "count=400")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + this.distributedPhaseMonitor.setPhase(configuration.getPhases().get(0)); + + // simulate executions per test + Map> executionsPerTest = this.distributedPhaseMonitor.getExecutions(); + Map executionsPerAgent = new HashMap() {{ + put(mockAgents.get(0), 100L); + put(mockAgents.get(1), 100L); + }}; + + executionsPerTest.put("MockTest", executionsPerAgent); + Assert.assertEquals(200L, (long) this.distributedPhaseMonitor.getExecutionsPerTest().get("MockTest")); + } + + @Test + public void testResetExecutions() { + this.distributedPhaseMonitor.getExecutions().clear(); + Map executionsPerAgent = new HashMap() {{ + put("Agent1", 100L); + }}; + + this.distributedPhaseMonitor.getExecutions().put("MockTest", executionsPerAgent); + this.distributedPhaseMonitor.resetExecutions(); + + Assert.assertEquals(0L, + (long) this.distributedPhaseMonitor.getExecutionsPerTest().get("MockTest")); + + this.distributedPhaseMonitor.getExecutions().clear(); + } + + + @Test + public void testGetExecutionsPerTest() { + this.distributedPhaseMonitor.getExecutions().clear(); + + Map executionsPerAgent = new HashMap() {{ + put("Agent1", 100L); + put("Agent2", 60L); + put("Agent3", 34L); + }}; + + this.distributedPhaseMonitor.getExecutions().put("MockTest", executionsPerAgent); + Assert.assertEquals(194L, (long) this.distributedPhaseMonitor.getExecutionsPerTest().get("MockTest")); + } + + @Test + public void testRemoveAgentFromActiveTDRunners() { + this.distributedPhaseMonitor.getExecutions().clear(); + + this.distributedPhaseMonitor.registerAgentRunningTD("Agent1"); + this.distributedPhaseMonitor.registerAgentRunningTD("Agent2"); + + this.distributedPhaseMonitor.getExecutions() + .forEach((key, value) -> Assert.assertTrue(value.containsKey("Agent2"))); + + this.distributedPhaseMonitor.removeAgentFromActiveTDRunners("Agent2"); + + this.distributedPhaseMonitor.getExecutions() + .forEach((key, value) -> Assert.assertFalse(value.containsKey("Agent2"))); + } + + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunMode.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunMode.java new file mode 100644 index 0000000..4619cec --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunMode.java @@ -0,0 +1,77 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; +import com.adobe.qe.toughday.api.annotations.ConfigArgSet; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; + +import java.util.concurrent.ExecutorService; + +public class DummyRunMode implements RunMode { + private RunModeBalancer dummyRunModeBalancer = new DummyRunModeBalancer(); + private String property1 = "prop1"; + private String property2 = "prop2"; + private String property3 = "prop3"; + + @ConfigArgSet(required = false, defaultValue = "p1") + public void setProperty1(String property1) { + this.property1 = property1; + } + + @ConfigArgSet(required = false, defaultValue = "p2") + public void setProperty2(String property2) { + this.property2 = property2; + } + + @ConfigArgSet(required = false, defaultValue = "p3") + public void setProperty3(String property3) { + this.property3 = property3; + } + + @ConfigArgGet(redistribute = true) + public String getProperty1() { + return this.property1; + } + + @ConfigArgGet(redistribute = true) + public String getProperty2() { + return this.property2; + } + + @ConfigArgGet + public String getProperty3() { + return this.property3; + } + + @Override + public void runTests(Engine engine) throws Exception { + + } + + @Override + public void finishExecutionAndAwait() { + + } + + @Override + public ExecutorService getExecutorService() { + return null; + } + + @Override + public RunContext getRunContext() { + return null; + } + + @Override + public RunModeSplitter getRunModeSplitter() { + return null; + } + + @Override + public RunModeBalancer getRunModeBalancer() { + return this.dummyRunModeBalancer; + } +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunModeBalancer.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunModeBalancer.java new file mode 100644 index 0000000..c744a07 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunModeBalancer.java @@ -0,0 +1,16 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.AbstractRunModeBalancer; + +public class DummyRunModeBalancer extends AbstractRunModeBalancer { + @Override + public void before(RedistributionInstructions redistributionInstructions, DummyRunMode runMode) { + + } + + @Override + public void after(RedistributionInstructions redistributionInstructions, DummyRunMode runMode) { + + } +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/MasterElectionTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/MasterElectionTest.java new file mode 100644 index 0000000..ff12249 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/MasterElectionTest.java @@ -0,0 +1,122 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.MasterElection; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.*; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import java.util.LinkedList; +import java.util.Queue; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class MasterElectionTest { + private MasterElection masterElection = MasterElection.getInstance(3); + + @Mock Driver mockDriver; + @Mock DriverState driverStateMock; + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + + } + + @Before + public void setup() { + this.masterElection.resetInvalidCandidates(); + } + + @Test + public void testAllCandidatesAreConsideredValidAfterTheInstanceIsCreated() { + Assert.assertTrue( MasterElection.getInstance(3).getInvalidCandidates().isEmpty()); + } + + @Test + public void testMarkCandidateAsInvalid() { + this.masterElection.markCandidateAsInvalid(0); + Queue expectedInvalidCandidates = new LinkedList() {{ + add(0); + }}; + Assert.assertEquals(expectedInvalidCandidates, masterElection.getInvalidCandidates()); + } + + @Test + public void testIsCandidateInvalid() { + this.masterElection.markCandidateAsInvalid(0); + + Assert.assertTrue(this.masterElection.isCandidateInvalid(0)); + Assert.assertFalse(this.masterElection.isCandidateInvalid(1)); + Assert.assertFalse(this.masterElection.isCandidateInvalid(2)); + } + + @Test + public void testResetInvalidCandidates() { + this.masterElection.markCandidateAsInvalid(0); + Assert.assertEquals(1, this.masterElection.getInvalidCandidates().size()); + + this.masterElection.resetInvalidCandidates(); + Assert.assertTrue(this.masterElection.getInvalidCandidates().isEmpty()); + } + + @Test + public void checkListOfInvalidCandidatesIsClearedWhenNoOptionAvailable() { + int currentDriverId = 2; + + Mockito.when(mockDriver.getDriverState()).thenReturn(driverStateMock); + Mockito.when(driverStateMock.getId()).thenReturn(currentDriverId); + Mockito.when(driverStateMock.getMasterIdLock()).thenReturn(new ReentrantReadWriteLock()); + + masterElection.markCandidateAsInvalid(0); + masterElection.markCandidateAsInvalid(1); + masterElection.markCandidateAsInvalid(2); + + // before starting the election process all candidates should be considered invalid + Assert.assertEquals(3, masterElection.getInvalidCandidates().size()); + + masterElection.electMaster(mockDriver); + + // after the election process, all candidates should be considered eligible to become the new master + Assert.assertTrue(masterElection.getInvalidCandidates().isEmpty()); + } + + @Test + public void testElectMasterFromCandidatePerspective() { + int expectedMaster = 1; + int currentDriverId = 2; + + Mockito.when(mockDriver.getDriverState()).thenReturn(driverStateMock); + Mockito.when(driverStateMock.getId()).thenReturn(currentDriverId); + Mockito.when(driverStateMock.getMasterIdLock()).thenReturn(new ReentrantReadWriteLock()); + + masterElection.markCandidateAsInvalid(0); + masterElection.electMaster(mockDriver); + + // check that the list of invalid candidates was not modified + Queue expectedInvalidCandidates = new LinkedList() {{ + add(0); + }}; + Assert.assertEquals(expectedInvalidCandidates, masterElection.getInvalidCandidates()); + + // check that current master (with id 2) is running as candidate + Mockito.verify(driverStateMock, Mockito.times(1)) + .setCurrentState(DriverState.Role.CANDIDATE); + + // check that driver-1 was elected as master + Mockito.verify(driverStateMock, Mockito.times(1)).setMasterId(expectedMaster); + + // check that the master election heartbeat task was scheduled + Mockito.verify(mockDriver, Mockito.times(1)).scheduleMasterHeartbeatTask(); + } + + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/NormalRunModeSplitterTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/NormalRunModeSplitterTest.java new file mode 100644 index 0000000..e723bab --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/NormalRunModeSplitterTest.java @@ -0,0 +1,111 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.engine.runmodes.Normal; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; + +public class NormalRunModeSplitterTest { + private ArrayList cmdLineArgs = new ArrayList<>(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test + public void testConcurrencyDistribution() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=normal", "concurrency=320")); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + + Assert.assertEquals(108, ((Normal) runModes.get("Agent1")).getConcurrency()); + Assert.assertEquals(106, ((Normal) runModes.get("Agent2")).getConcurrency()); + Assert.assertEquals(106, ((Normal) runModes.get("Agent3")).getConcurrency()); + } + + @Test + public void testVariableConcurrencyDistribution() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=normal", "start=3", "end=12", "rate=3")); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + Normal initialRunMode = (Normal) configuration.getRunMode(); + + Normal firstAgentRunMode = (Normal) runModes.get("Agent1"); + Assert.assertEquals(2, firstAgentRunMode.getStart()); + Assert.assertEquals(6, firstAgentRunMode.getEnd()); + Assert.assertEquals(2, firstAgentRunMode.getRate()); + // check that the interval property does not change + Assert.assertEquals(initialRunMode.getInterval(), firstAgentRunMode.getInterval()); + + Normal secondAgentRunMode = (Normal) runModes.get("Agent2"); + Assert.assertEquals(1, secondAgentRunMode.getStart()); + Assert.assertEquals(6, secondAgentRunMode.getEnd()); + Assert.assertEquals(1, secondAgentRunMode.getRate()); + // check that the interval property does not change + Assert.assertEquals(initialRunMode.getInterval(), secondAgentRunMode.getInterval()); + } + + + @Test + public void testVariableConcurrencyWithRateLowerThanNumberOfAgents() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=normal", "start=3", "end=9", "interval=1s", "rate=1")); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode initialRunMode = configuration.getRunMode(); + Map runModes = initialRunMode.getRunModeSplitter().distributeRunMode(initialRunMode, mockAgents); + + Normal firstAgentRunMode = (Normal) runModes.get("Agent1"); + Assert.assertEquals(3, firstAgentRunMode.getStart()); + Assert.assertEquals(6, firstAgentRunMode.getEnd()); + Assert.assertEquals("2s", firstAgentRunMode.getInterval()); + Assert.assertEquals(0, firstAgentRunMode.getInitialDelay()); + + Normal secondAgentRunMode = (Normal) runModes.get("Agent2"); + Assert.assertEquals(0, secondAgentRunMode.getStart()); + Assert.assertEquals(3, secondAgentRunMode.getEnd()); + Assert.assertEquals("2s", secondAgentRunMode.getInterval()); + Assert.assertEquals(1000, secondAgentRunMode.getInitialDelay()); + + // check that the initial rate is not modified + Assert.assertEquals(((Normal)initialRunMode).getRate(), firstAgentRunMode.getRate()); + Assert.assertEquals(((Normal)initialRunMode).getRate(), secondAgentRunMode.getRate()); + } + + @Test + public void testWaitTimeIsNotModified() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=normal", "waittime=1000")); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode initialRunMode = configuration.getPhases().get(0).getRunMode(); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + + Map runModes = initialRunMode.getRunModeSplitter().distributeRunMode(initialRunMode, mockAgents); + runModes.forEach((key, value) -> + Assert.assertEquals(((Normal)initialRunMode).getWaitTime(), + ((Normal) value).getWaitTime())); + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/PhaseSplitterTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/PhaseSplitterTest.java new file mode 100644 index 0000000..0331a51 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/PhaseSplitterTest.java @@ -0,0 +1,103 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.api.core.AbstractTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.PhaseSplitter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.*; + +import java.util.*; +import java.util.stream.Collectors; + +public class PhaseSplitterTest { + private final PhaseSplitter phaseSplitter = new PhaseSplitter(); + private List cmdLineArgs; + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + + reflections.getTestClasses().put("MockTest", MockTest.class); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullPhaseThrowsException() throws CloneNotSupportedException { + phaseSplitter.splitPhase(null, new ArrayList<>()); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullListOfAgentsThrowsException() throws CloneNotSupportedException { + phaseSplitter.splitPhase(new Phase(), null); + } + + @Test(expected = IllegalStateException.class) + public void testNoAvailableAgentsThrowsException() throws CloneNotSupportedException { + phaseSplitter.splitPhase(new Phase(), new ArrayList<>()); + } + + @Test + public void testNumberOfTasksIsEqualToTheNumberOfAgents() throws Exception { + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + configuration.getPhases().forEach(phase -> { + try { + Map taskMap = phaseSplitter.splitPhase(phase, mockAgents); + Assert.assertEquals(mockAgents.size(), taskMap.keySet().size()); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + } + }); + } + + @Test + public void testCountIsDistributedForTestsInTestSuite() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest", "name=Test1", "count=201")); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + + Phase phase = new Configuration(cmdLineArgs.toArray(new String[0])).getPhases().get(0); + Map taskMap = phaseSplitter.splitPhase(phase, mockAgents); + + taskMap.get("Agent1").getTestSuite().getTests().forEach(test -> { + Assert.assertEquals(101, test.getCount()); + }); + + taskMap.get("Agent2").getTestSuite().getTests().forEach(test -> { + Assert.assertEquals(100, test.getCount()); + }); + } + + @Test + public void testEachAgentRunsTheCompleteTestSuite() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest", "name=Test1", "--add", "MockTest", "name=Test2")); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + + Phase phase = new Configuration(cmdLineArgs.toArray(new String[0])).getPhases().get(0); + Set testNames = phase.getTestSuite().getTests().stream() + .map(AbstractTest::getName) + .collect(Collectors.toSet()); + + Map taskMap = phaseSplitter.splitPhase(phase, mockAgents); + taskMap.forEach((key, value) -> { + Set namesDiff = value.getTestSuite().getTests().stream() + .map(AbstractTest::getName) + .collect(Collectors.toSet()); + namesDiff.removeAll(testNames); + + Assert.assertEquals(0, namesDiff.size()); + }); + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/RedistributionInstructionsProcessorTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/RedistributionInstructionsProcessorTest.java new file mode 100644 index 0000000..4e7f97b --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/RedistributionInstructionsProcessorTest.java @@ -0,0 +1,86 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructionsProcessor; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.engine.runmodes.Normal; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.*; + +public class RedistributionInstructionsProcessorTest { + private List cmdLineArgs; + private final RedistributionInstructionsProcessor instructionsProcessor = new RedistributionInstructionsProcessor(); + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + + reflections.getTestClasses().put("MockTest", MockTest.class); + } + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullPhaseThrowsExceptions() throws IOException { + this.instructionsProcessor.processInstructions("", null); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullInstructionsThrowsException() throws Exception{ + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + this.instructionsProcessor.processInstructions(null, configuration.getPhases().get(0)); + } + + @Test + public void testProcessInstructions() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest", "--runmode", "type=normal", "rate=20", "interval=2s", + "end=20", "start=15")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + Phase phase = configuration.getPhases().get(0); + RedistributionInstructions instructions = new RedistributionInstructions(); + + Map runModeProperties = new HashMap() {{ + put("rate", "10"); + put("interval","5s"); + put("end", "200"); + }}; + Map counts = new HashMap() {{ + put("MockTest", 1200L); + }}; + + instructions.setRunModeProperties(runModeProperties); + instructions.setCounts(counts); + + ObjectMapper mapper = new ObjectMapper(); + String jsonInstructions = mapper.writeValueAsString(instructions); + + this.instructionsProcessor.processInstructions(jsonInstructions, phase); + + // test run mode properties were changed + Normal runmode = (Normal)phase.getRunMode(); + Assert.assertEquals(10, runmode.getRate()); + Assert.assertEquals("5s", runmode.getInterval()); + Assert.assertEquals(200, runmode.getEnd()); + + // test count properties were changed + Assert.assertEquals(1200, phase.getTestSuite().getTest("MockTest").getCount()); + + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgentsTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgentsTest.java new file mode 100644 index 0000000..fe46b05 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgentsTest.java @@ -0,0 +1,152 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.DummyRunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.mocks.MockMetric; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.Assert; +import org.junit.rules.ExpectedException; + +import java.util.*; + +public class YamlDumpConfigurationAsTaskForTDAgentsTest { + private List cmdLineArgs; + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + + reflections.getTestClasses().put("MockTest", MockTest.class); + reflections.getMetricClasses().put("MockMetric", MockMetric.class); + reflections.getRunModeClasses().put("DummyRunMode", DummyRunMode.class); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test + public void testNullCofigurationThrowsException() { + expectedEx.expect(IllegalArgumentException.class); + expectedEx.expectMessage("Configuration must not be null."); + + new YamlDumpConfigurationAsTaskForTDAgents(null); + } + + @Test() + public void testCollectConfigurablePropertiesWithIncompatibleTypeThrowsException() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + expectedEx.expect(IllegalArgumentException.class); + expectedEx.expectMessage("The object must have the specified type."); + + yamlDumpConfiguration.collectConfigurableProperties(DummyRunMode.class, configuration.getTestSuite().getTest("MockTest")); + } + + @Test + public void testCollectConfigurablePropertiesContainsOnlyModifiedProperties() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=dummyrunmode", "property1=value1", "property2=value2")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + + Map actualCollectedPropsForDummyRunMode = + yamlDumpConfiguration.collectConfigurableProperties(DummyRunMode.class, configuration.getRunMode()); + + Map expectedCollectedPropsForMockTest = + new HashMap() {{ + put("property1", "value1"); + put("property2", "value2"); + }}; + + Assert.assertEquals(expectedCollectedPropsForMockTest, actualCollectedPropsForDummyRunMode); + } + + @Test + public void testDumpedConfiguration() throws Exception{ + cmdLineArgs.addAll(Arrays.asList("--duration=10s", "--phase", "name=phase123", "--add", "MockTest", "name=Test1", "count=201", + "--add" , "MockMetric", "name=Metric123", "--runmode", "type=dummyrunmode", "--add", "CSVPublisher")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + String expectedYaml = + "globals:\n" + + " duration: 10s\n" + + " host: localhost\n" + + "phases:\n" + + "- duration: 10s\n" + + " metrics:\n" + + " - add: MockMetric\n" + + " properties:\n" + + " name: Metric123\n" + + " name: phase123\n" + + " publishers:\n" + + " - add: CSVPublisher\n" + + " properties:\n" + + " name: CSVPublisher\n" + + " aggregatedpublish: false\n" + + " publishmode:\n" + + " type: simple\n" + + " runmode:\n" + + " type: dummyrunmode\n" + + " tests:\n" + + " - add: MockTest\n" + + " properties:\n" + + " name: Test1\n" + + " count: 201\n"; + + Assert.assertEquals(expectedYaml, yamlDumpConfiguration.generateConfigurationObject()); + } + + @Test + public void testGetGlobals() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--duration=10s", "--port=1000", "--user=testUser", "--timeout=10")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + Map actualGlobals = yamlDumpConfiguration.getGlobals(); + + Map expectedGlobals = new HashMap() {{ + put("host", "localhost"); + put("duration", "10s"); + put("port", 1000); + put("user", "testUser"); + put("timeout", 10L); + }}; + + Assert.assertEquals(expectedGlobals, actualGlobals); + } + + @Test + public void testGetDistributedConfig() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--distributedconfig", "driverip=10.0.0.1", "heartbeatinterval=10s")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + Map actualDistributedConfig = yamlDumpConfiguration.getDistributedConfig(); + Map expectedDistributedConfig = new HashMap() {{ + put("driverip", "10.0.0.1"); + put("heartbeatinterval", "10s"); + }}; + + Assert.assertEquals(expectedDistributedConfig, actualDistributedConfig); + } + + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/CandidateRequestProcessorTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/CandidateRequestProcessorTest.java new file mode 100644 index 0000000..9512944 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/CandidateRequestProcessorTest.java @@ -0,0 +1,146 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.requests; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.PhaseParams; +import com.adobe.qe.toughday.internal.core.distributedtd.DistributedPhaseMonitor; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverUpdateInfo; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.MasterElection; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.RequestProcessor; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.RequestProcessorDispatcher; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.*; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.mockito.stubbing.Answer; +import spark.Request; + +import java.util.*; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class CandidateRequestProcessorTest { + @Mock Request request; + @Mock Driver mockDriver; + @Mock DriverState driverStateMock; + private DistributedPhaseMonitor distributedPhaseMonitor= new DistributedPhaseMonitor(); + private final MasterElection masterElection = MasterElection.getInstance(3); + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + + reflections.getTestClasses().put("MockTest", MockTest.class); + } + + @Before + public void setup() { + Mockito.when(mockDriver.getDriverState()).thenReturn(driverStateMock); + Mockito.when(driverStateMock.getCurrentState()).thenReturn(DriverState.Role.CANDIDATE); + Mockito.when(mockDriver.getMasterElection()).thenReturn(this.masterElection); + Mockito.when(driverStateMock.getId()).thenReturn(1); + + masterElection.resetInvalidCandidates(); + } + + @Test + public void testProcessUpdatesRequests() throws JsonProcessingException { + Queue registeredAgents = new LinkedList<>(Collections.singleton("10.0.0.1")); + + masterElection.markCandidateAsInvalid(0); + masterElection.markCandidateAsInvalid(1); + Mockito.when(driverStateMock.getRegisteredAgents()).thenReturn(registeredAgents); + + DriverUpdateInfo updates = new DriverUpdateInfo(1, DriverState.Role.CANDIDATE, + masterElection.getInvalidCandidates(), registeredAgents, "", ""); + ObjectMapper objectMapper = new ObjectMapper(); + String expectedYaml = objectMapper.writeValueAsString(updates); + + RequestProcessor processor = RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver); + + Assert.assertEquals(expectedYaml, processor.processUpdatesRequest(request, mockDriver)); + } + + @Test + public void testProcessMasterElectionRequest() { + Mockito.when(request.body()).then((Answer) answer -> "1"); + Mockito.when(driverStateMock.getMasterIdLock()).thenReturn(new ReentrantReadWriteLock()); + RequestProcessor processor = RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver); + + masterElection.markCandidateAsInvalid(0); + processor.processMasterElectionRequest(request, mockDriver); + + // test that driver-1 was added as invalid candidate + Queue expectedInvalidCandidates = new LinkedList<>(Arrays.asList(0, 1)); + Assert.assertEquals(expectedInvalidCandidates, masterElection.getInvalidCandidates()); + + // test that driver-2 was elected as master + Mockito.verify(driverStateMock).setMasterId(2); + + // test that driver-1 is running as a candidate after this process + Mockito.verify(driverStateMock).setCurrentState(DriverState.Role.CANDIDATE); + } + + @Test + public void testProcessPhaseCompletionAnnouncement() { + Mockito.when(request.body()).thenReturn("10.0.0.1"); + Mockito.when(request.queryParams("forward")).thenReturn("false"); + Mockito.when(mockDriver.getDistributedPhaseMonitor()).thenReturn(distributedPhaseMonitor); + + RequestProcessor processor = RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver); + processor.processPhaseCompletionAnnouncement(request); + + // test that the agent was added to the list of agents which finished executing the current phase + Assert.assertEquals(Collections.singletonList("10.0.0.1"), mockDriver.getDistributedPhaseMonitor().getAgentsWhichCompletedCurrentPhase()); + } + + @Test + public void processExecutionRequest() throws Exception { + List cmdLineArgs = new ArrayList() {{ + add("--phase"); + add("name=testPhase"); + add("--host=localhost"); + add("--add"); + add("MockTest"); + add("count=400"); + }}; + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + PhaseParams.namedPhases.clear(); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration + = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + String yamlConfig = yamlDumpConfiguration.generateConfigurationObject(); + + Mockito.when(request.body()).thenReturn(yamlConfig); + Mockito.when(request.queryParams("forward")).thenReturn("false"); + Mockito.when(mockDriver.getDistributedPhaseMonitor()).thenReturn(distributedPhaseMonitor); + Mockito.when(mockDriver.getConfiguration()).thenReturn(configuration); + + RequestProcessor processor = RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver); + processor.processExecutionRequest(request, mockDriver); + + // test that the configuration was set + Mockito.verify(mockDriver, Mockito.times(1)) + .setConfiguration(Mockito.any(Configuration.class)); + + // test that the first phase of the configuration to be executed is being monitored + Assert.assertEquals("testPhase", mockDriver.getDistributedPhaseMonitor().getPhase().getName()); + + // test that the sample content package should not be installed by the candidates + Assert.assertFalse(mockDriver.getConfiguration().getGlobalArgs().getInstallSampleContent()); + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/MasterRequestProcessorTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/MasterRequestProcessorTest.java new file mode 100644 index 0000000..952b90f --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/MasterRequestProcessorTest.java @@ -0,0 +1,135 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.requests; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.distributedtd.DistributedPhaseMonitor; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.RequestProcessor; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.RequestProcessorDispatcher; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.TaskBalancer; +import com.google.gson.Gson; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.*; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import spark.Request; +import spark.Response; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; + + +public class MasterRequestProcessorTest { + @Mock Request requestMock; + @Mock Response responseMock; + @Mock Driver mockDriver; + @Mock DistributedPhaseMonitor distributedPhaseMonitorMock; + @Mock DriverState driverStateMock; + @Mock TaskBalancer taskBalancerMock; + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + + reflections.getTestClasses().put("MockTest", MockTest.class); + } + + @Before + public void setup() { + Mockito.when(mockDriver.getDriverState()).thenReturn(driverStateMock); + Mockito.when(driverStateMock.getCurrentState()).thenReturn(DriverState.Role.MASTER); + } + + @Test + public void testSampleContentInstallationFailureForcesExecutionToStop() { + Mockito.when(requestMock.body()).thenReturn("false"); + RequestProcessor requestProcessor = RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver); + + requestProcessor.acknowledgeSampleContentSuccessfulInstallation(requestMock, mockDriver, responseMock); + + /* check that the driver finishes the distributed execution if TD sample content package was not successfully + /* installed + */ + Mockito.verify(mockDriver).finishDistributedExecution(); + } + + @Test + public void testProcessHeartbeatRequest() { + Map> executions = new HashMap<>(); + Map testsPerAgent = new HashMap() {{ + put("Agent1", 200L); + put("Agent2", 231L); + }}; + executions.put("MockTest", testsPerAgent); + + Mockito.when(mockDriver.getDistributedPhaseMonitor()).thenReturn(distributedPhaseMonitorMock); + Mockito.when(distributedPhaseMonitorMock.getExecutions()).thenReturn(executions); + + Gson gson = new Gson(); + String expected = gson.toJson(executions); + String actual = RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver) + .processHeartbeatRequest(requestMock, mockDriver); + + Assert.assertEquals(expected, actual); + } + + @Test(expected = IllegalStateException.class) + public void testProcessAgentFailureAnnouncementThrowsExceptionForMaster() { + /* the master is the one sending heartbeat messages to the agents running in the cluster and should therefore + * never receive this type of request from another driver. + */ + RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver) + .processAgentFailureAnnouncement(requestMock, mockDriver); + } + + private void mockitoSetupForRegisterRequest(String newAgentIP) { + Mockito.when(requestMock.body()).thenReturn(newAgentIP); + Mockito.when(requestMock.queryParams("forward")).thenReturn("false"); + Mockito.when(mockDriver.getDistributedPhaseMonitor()).thenReturn(distributedPhaseMonitorMock); + Mockito.when(mockDriver.getTaskBalancer()).thenReturn(taskBalancerMock); + } + + @Test + public void testProcessRegisterRequestWhenExecutionDidNotStarted() { + String newAgentIP = "10.0.0.1"; + + mockitoSetupForRegisterRequest(newAgentIP); + Mockito.when(distributedPhaseMonitorMock.isPhaseExecuting()).thenReturn(false); + Mockito.when(driverStateMock.getRegisteredAgents()).thenReturn(new LinkedList<>()); + + RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver) + .processRegisterRequest(requestMock, mockDriver); + + // check that the new agent was added to the list of active agents + Mockito.verify(driverStateMock, Mockito.times(1)).registerAgent(newAgentIP); + + // check that the work redistribution process is not scheduled when the execution did not started + Mockito.verify(taskBalancerMock, Mockito.times(0)). + scheduleWorkRedistributionProcess(mockDriver, newAgentIP, true); + } + + @Test + public void testRegisterRequestTriggersWorkRedistributionWhenTDIsExecuted() { + String newAgentIP = "10.0.0.1"; + + mockitoSetupForRegisterRequest(newAgentIP); + Mockito.when(distributedPhaseMonitorMock.isPhaseExecuting()).thenReturn(true); + + RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver) + .processRegisterRequest(requestMock, mockDriver); + Mockito.verify(taskBalancerMock, Mockito.times(1)) + .scheduleWorkRedistributionProcess(mockDriver, newAgentIP, true); + + } +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/RequestProcessorDispatcherTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/RequestProcessorDispatcherTest.java new file mode 100644 index 0000000..885ce9e --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/RequestProcessorDispatcherTest.java @@ -0,0 +1,56 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.requests; + +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.MasterRequestProcessor; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.RequestProcessorDispatcher; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.CandidateRequestProcessor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.*; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.mockito.stubbing.Answer; + +public class RequestProcessorDispatcherTest { + @Mock Driver mockDriver; + @Mock DriverState driverStateMock; + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + + @Test + public void testGetInstance() { + RequestProcessorDispatcher requestProcessorDispatcher = RequestProcessorDispatcher.getInstance(); + + Assert.assertNotNull(requestProcessorDispatcher); + } + + @Test + public void testMasterRequestProcessorIsReturnedForMaster() { + Mockito.when(mockDriver.getDriverState()).then((Answer) invocationOnMocl -> driverStateMock); + Mockito.when(driverStateMock.getCurrentState()).then((Answer) invocationOnMock -> DriverState.Role.MASTER); + + RequestProcessorDispatcher requestProcessorDispatcher = RequestProcessorDispatcher.getInstance(); + Assert.assertTrue(requestProcessorDispatcher.getRequestProcessor(mockDriver) instanceof MasterRequestProcessor); + } + + @Test + public void testCandidateRequestProcessorIsReturnedForCandidate() { + Mockito.when(mockDriver.getDriverState()).then((Answer) invocationOnMocl -> driverStateMock); + Mockito.when(driverStateMock.getCurrentState()).then((Answer) invocationOnMock -> DriverState.Role.CANDIDATE); + + RequestProcessorDispatcher requestProcessorDispatcher = RequestProcessorDispatcher.getInstance(); + Assert.assertTrue(requestProcessorDispatcher.getRequestProcessor(mockDriver) instanceof CandidateRequestProcessor); + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/mocks/MockMetric.java b/toughday/src/test/java/com/adobe/qe/toughday/mocks/MockMetric.java new file mode 100644 index 0000000..8b5f9ae --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/mocks/MockMetric.java @@ -0,0 +1,21 @@ +package com.adobe.qe.toughday.mocks; + +import com.adobe.qe.toughday.api.core.RunMap; +import com.adobe.qe.toughday.metrics.Metric; + +public class MockMetric extends Metric { + @Override + public Object getValue(RunMap.TestStatistics testStatistics) { + return null; + } + + @Override + public String getFormat() { + return null; + } + + @Override + public String getUnitOfMeasure() { + return null; + } +} diff --git a/toughday2-api/src/main/java/com/adobe/qe/toughday/api/annotations/ConfigArgGet.java b/toughday2-api/src/main/java/com/adobe/qe/toughday/api/annotations/ConfigArgGet.java index 45b22ec..f379cd8 100644 --- a/toughday2-api/src/main/java/com/adobe/qe/toughday/api/annotations/ConfigArgGet.java +++ b/toughday2-api/src/main/java/com/adobe/qe/toughday/api/annotations/ConfigArgGet.java @@ -18,7 +18,7 @@ /** * Use this annotation on a getter to expose it as a configuration property. - * These properties will be automatically picked up, shown in logging or in "dry" runmode + * These properties will be automatically picked up, shown in logging or in "dry" runmodes * Supported classes: subtypes of AbstractTest, subtypes of Publisher and * GlobalArgs. */ @@ -26,4 +26,14 @@ @Target(value = ElementType.METHOD) public @interface ConfigArgGet { String name() default ""; + + /** + * Use this field when you want to specify that a certain property must be taken into + * consideration when redistributing the work between the agents running in the cluster. + * The property will be automatically collected by the AbstractRunModeBalancer. + * Currently, this annotations is supported only for classes implementing the RunMode + * interface. + * @return + */ + boolean redistribute() default false; } From 7159962d5dea681af81a1676a0bed8c868284bbc Mon Sep 17 00:00:00 2001 From: meirosucristina <38101404+meirosucristina@users.noreply.github.com> Date: Tue, 26 Nov 2019 12:09:45 +0200 Subject: [PATCH 2/2] Update and rename driver_deployment.yaml to driver_statefulset.yaml --- driver_deployment.yaml | 58 ------------------------------------ driver_statefulset.yaml | 66 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 58 deletions(-) delete mode 100644 driver_deployment.yaml create mode 100644 driver_statefulset.yaml diff --git a/driver_deployment.yaml b/driver_deployment.yaml deleted file mode 100644 index cccf5a9..0000000 --- a/driver_deployment.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: driver -spec: - selector: - matchLabels: - app: driver - template: - metadata: - labels: - app: driver - spec: - containers: - - image: - name: driver-container - ports: - - containerPort: 4567 - - containerPort: 80 - livenessProbe: - httpGet: - path: /health - port: 4567 - initialDelaySeconds: 20 - periodSeconds: 3 - ---- - -apiVersion: v1 -kind: Service -metadata: - name: driver -spec: - ports: - - port: 80 - protocol: TCP - targetPort: 4567 - name: http - selector: - app: driver - ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: driver - annotations: - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/ssl-redirect: "false" - nginx.ingress.kubernetes.io/force-ssl-redirect: "false" -spec: - rules: - - http: - paths: - - backend: - serviceName: driver - servicePort: 80 - path: /config \ No newline at end of file diff --git a/driver_statefulset.yaml b/driver_statefulset.yaml new file mode 100644 index 0000000..4fc897e --- /dev/null +++ b/driver_statefulset.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: driver +spec: + selector: + matchLabels: + app: driver + serviceName: driver + replicas: 2 + template: + metadata: + labels: + app: driver + spec: + containers: + - image: + env: + - name: NR_DRIVERS + value: "2" # set this value to the value of spec.replicas field + name: driver-container + ports: + - containerPort: 4567 + - containerPort: 80 + livenessProbe: + httpGet: + path: /health + port: 4567 + initialDelaySeconds: 20 + periodSeconds: 3 + +--- + +# headless service required by StatefulSet resource +apiVersion: v1 +kind: Service +metadata: + name: driver + labels: + app: driver +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 4567 + name: http + selector: + app: driver + +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: driver + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" +spec: + rules: + - http: + paths: + - backend: + serviceName: driver + servicePort: 80 + path: /config