From 40f461e8cdfb9b28f0fe7115356076ceaa26483b Mon Sep 17 00:00:00 2001 From: Ioana Meirosu Date: Thu, 28 Feb 2019 18:27:37 +0200 Subject: [PATCH] Added the functionality to run TD distributed on Kubernetes. --- agent-job.yaml | 22 ++ driver_deployment.yaml | 58 ++++ toughday/pom.xml | 6 + .../main/java/com/adobe/qe/toughday/Main.java | 27 +- .../qe/toughday/internal/core/TestSuite.java | 20 +- .../internal/core/config/ConfigParams.java | 8 + .../internal/core/config/Configuration.java | 110 +++++-- .../internal/core/config/GlobalArgs.java | 7 +- .../core/config/parsers/cli/CliParser.java | 70 ++-- .../yaml/GenerateYamlConfiguration.java | 87 +---- .../core/config/parsers/yaml/YamlBuilder.java | 98 ++++++ .../parsers/yaml/YamlConfiguration.java | 3 + .../core/config/parsers/yaml/YamlParser.java | 36 ++- .../DistributedPhaseMonitor.java | 156 +++++++++ .../core/distributedtd/ExecutionTrigger.java | 51 +++ .../core/distributedtd/HttpUtils.java | 91 ++++++ ...amlDumpConfigurationAsTaskForTDAgents.java | 175 ++++++++++ .../core/distributedtd/cluster/Agent.java | 299 ++++++++++++++++++ .../cluster/DistributedConfig.java | 85 +++++ .../core/distributedtd/cluster/Driver.java | 287 +++++++++++++++++ .../RedistributionInstructions.java | 45 +++ .../RedistributionInstructionsProcessor.java | 75 +++++ .../redistribution/TaskBalancer.java | 287 +++++++++++++++++ .../runmodes/AbstractRunModeBalancer.java | 71 +++++ .../runmodes/ConstantLoadRunModeBalancer.java | 76 +++++ .../runmodes/NormalRunModeBalancer.java | 88 ++++++ .../runmodes/RunModeBalancer.java | 47 +++ .../splitters/PhaseSplitter.java | 108 +++++++ .../runmodes/ConstantLoadRunModeSplitter.java | 106 +++++++ .../runmodes/NormalRunModeSplitter.java | 113 +++++++ .../splitters/runmodes/RunModeSplitter.java | 31 ++ .../distributedtd/tasks/HeartbeatTask.java | 94 ++++++ .../core/engine/AsyncTimeoutChecker.java | 2 +- .../toughday/internal/core/engine/Engine.java | 14 +- .../toughday/internal/core/engine/Phase.java | 15 +- .../internal/core/engine/RunMode.java | 15 + .../core/engine/runmodes/ConstantLoad.java | 228 +++++++++---- .../internal/core/engine/runmodes/Normal.java | 214 ++++++++----- .../adobe/qe/toughday/metrics/Percentile.java | 8 +- .../adobe/qe/toughday/TestConfiguration.java | 10 +- .../qe/toughday/TestConstantLoadMode.java | 2 +- .../com/adobe/qe/toughday/TestNormalMode.java | 2 +- .../AbstractRunModeBalancerTest.java | 59 ++++ .../ConstantLoadRunModeSplitterTest.java | 98 ++++++ .../DistributedPhaseMonitorTest.java | 119 +++++++ .../core/distributedtd/DummyRunMode.java | 77 +++++ .../distributedtd/DummyRunModeBalancer.java | 16 + .../NormalRunModeSplitterTest.java | 111 +++++++ .../core/distributedtd/PhaseSplitterTest.java | 103 ++++++ ...distributionInstructionsProcessorTest.java | 86 +++++ ...umpConfigurationAsTaskForTDAgentsTest.java | 152 +++++++++ .../adobe/qe/toughday/mocks/MockMetric.java | 21 ++ .../api/annotations/ConfigArgGet.java | 12 +- 53 files changed, 3893 insertions(+), 308 deletions(-) create mode 100644 agent-job.yaml create mode 100644 driver_deployment.yaml create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlBuilder.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitor.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/ExecutionTrigger.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/HttpUtils.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgents.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Agent.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/DistributedConfig.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Driver.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructions.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructionsProcessor.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/TaskBalancer.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/AbstractRunModeBalancer.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/ConstantLoadRunModeBalancer.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/NormalRunModeBalancer.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/RunModeBalancer.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/PhaseSplitter.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/ConstantLoadRunModeSplitter.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/NormalRunModeSplitter.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/RunModeSplitter.java create mode 100644 toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/HeartbeatTask.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/AbstractRunModeBalancerTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/ConstantLoadRunModeSplitterTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitorTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunMode.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunModeBalancer.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/NormalRunModeSplitterTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/PhaseSplitterTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/RedistributionInstructionsProcessorTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgentsTest.java create mode 100644 toughday/src/test/java/com/adobe/qe/toughday/mocks/MockMetric.java diff --git a/agent-job.yaml b/agent-job.yaml new file mode 100644 index 0000000..78ac618 --- /dev/null +++ b/agent-job.yaml @@ -0,0 +1,22 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: agent +spec: + parallelism: 2 + ttlSecondsAfterFinished: 0 + template: + metadata: + labels: + app: agent + spec: + containers: + - name: agent-container + image: + livenessProbe: + httpGet: + path: /health + port: 4567 + initialDelaySeconds: 20 + periodSeconds: 3 + restartPolicy: Never diff --git a/driver_deployment.yaml b/driver_deployment.yaml new file mode 100644 index 0000000..cccf5a9 --- /dev/null +++ b/driver_deployment.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: driver +spec: + selector: + matchLabels: + app: driver + template: + metadata: + labels: + app: driver + spec: + containers: + - image: + name: driver-container + ports: + - containerPort: 4567 + - containerPort: 80 + livenessProbe: + httpGet: + path: /health + port: 4567 + initialDelaySeconds: 20 + periodSeconds: 3 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: driver +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 4567 + name: http + selector: + app: driver + +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: driver + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" +spec: + rules: + - http: + paths: + - backend: + serviceName: driver + servicePort: 80 + path: /config \ No newline at end of file diff --git a/toughday/pom.xml b/toughday/pom.xml index 4137f5e..f9182fb 100644 --- a/toughday/pom.xml +++ b/toughday/pom.xml @@ -271,5 +271,11 @@ test + + com.sparkjava + spark-core + 2.7.2 + + diff --git a/toughday/src/main/java/com/adobe/qe/toughday/Main.java b/toughday/src/main/java/com/adobe/qe/toughday/Main.java index 7b57ebb..21a5879 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/Main.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/Main.java @@ -11,14 +11,15 @@ */ package com.adobe.qe.toughday; - import com.adobe.qe.toughday.internal.core.engine.Engine; import com.adobe.qe.toughday.internal.core.config.parsers.cli.CliParser; import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.ExecutionTrigger; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Driver; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - /** * Main class. Creates a Configuration and an engine and runs the tests. */ @@ -31,8 +32,6 @@ public class Main { } public static void main(String[] args) { - - CliParser cliParser = new CliParser(); System.out.println(); @@ -51,15 +50,27 @@ public static void main(String[] args) { System.exit(1); } - Engine engine = new Engine(configuration); - engine.runTests(); + /* check if we should trigger an execution query in the cluster. */ + if (configuration.executeInDitributedMode()) { + new ExecutionTrigger(configuration).triggerDistributedExecution(); + System.exit(0); + } else if (configuration.getDistributedConfig().getAgent()) { + Agent agent = new Agent(); + agent.start(); + } else if (configuration.getDistributedConfig().getDriver()) { + Driver driver = new Driver(configuration); + driver.run(); + } else { + Engine engine = new Engine(configuration); + engine.runTests(); - System.exit(0); + System.exit(0); + } } catch (Throwable t) { LOG.error("Error encountered: " + (t.getMessage() != null ? t.getMessage() : "Please check toughday.log for more information.")); LogManager.getLogger(Engine.class).error("Error encountered", t); + System.exit(-1); } - System.exit(0); } } diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/TestSuite.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/TestSuite.java index 80ab294..88b7063 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/TestSuite.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/TestSuite.java @@ -21,7 +21,7 @@ /** * Test suite class. */ -public class TestSuite { +public class TestSuite implements Cloneable { private List setupStep; private String description = ""; private List tags = new ArrayList<>(); @@ -39,6 +39,24 @@ public TestSuite() { totalWeight = 0; } + /** + * Creates a copy of the current test suite. All tests contained by the test suite are cloned. + * @throws CloneNotSupportedException if the object to be cloned does not implement the Cloneable interface. + */ + public TestSuite clone() throws CloneNotSupportedException { + TestSuite newInstance = (TestSuite) super.clone(); + + /* clone all the tests in the TestSuite */ + newInstance.orderedTests = new ArrayList<>(); + newInstance.nameMap = new HashMap<>(); + + for (AbstractTest test : this.getTests()) { + newInstance.add(test.clone()); + } + + return newInstance; + } + /** * Method for adding a test. * @param test diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/ConfigParams.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/ConfigParams.java index e47fd3f..b59b1af 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/ConfigParams.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/ConfigParams.java @@ -26,6 +26,7 @@ public class ConfigParams implements Serializable { private List phasesParams = new ArrayList<>(); private Map globalParams = new HashMap<>(); + private Map distributedConfigParams = new HashMap<>(); private Map publishModeParams = new HashMap<>(); private Map runModeParams = new HashMap<>(); private List> items = new ArrayList<>(); @@ -94,6 +95,10 @@ public void setGlobalParams(Map globalParams) { this.globalParams = globalParams; } + public void setDistributedConfigParams(Map distributedConfigParams) { + this.distributedConfigParams = distributedConfigParams; + } + public void setPhasesParams(List phasesParams) { this.phasesParams = phasesParams; } @@ -163,12 +168,15 @@ public Map getGlobalParams(){ return globalParams; } + public Map getDistributedConfigParams() { return distributedConfigParams; } + public Map getPublishModeParams() { return publishModeParams; } public Map getRunModeParams() { return runModeParams; } public void merge(ConfigParams other) { globalParams.putAll(other.getGlobalParams()); + distributedConfigParams.putAll(other.distributedConfigParams); items.addAll(other.items); phasesParams.addAll(other.phasesParams); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/Configuration.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/Configuration.java index 4523850..285a6a9 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/Configuration.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/Configuration.java @@ -24,6 +24,7 @@ import com.adobe.qe.toughday.internal.core.engine.Phase; import com.adobe.qe.toughday.internal.core.engine.PublishMode; import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.DistributedConfig; import com.adobe.qe.toughday.metrics.Metric; import com.adobe.qe.toughday.publishers.CSVPublisher; import com.adobe.qe.toughday.publishers.ConsolePublisher; @@ -36,9 +37,7 @@ import org.apache.logging.log4j.core.config.LoggerConfig; import org.reflections.Reflections; -import java.io.File; -import java.io.IOException; -import java.lang.ref.WeakReference; +import java.io.*; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -63,6 +62,7 @@ public class Configuration { private PredefinedSuites predefinedSuites = new PredefinedSuites(); private GlobalArgs globalArgs; + private DistributedConfig distributedConfig; private RunMode runMode; private PublishMode publishMode; private TestSuite globalSuite; @@ -70,6 +70,7 @@ public class Configuration { private Map globalMetrics = new LinkedHashMap<>(); private List phases = new ArrayList<>(); private Set phasesWithoutDuration = new HashSet<>(); + private ConfigParams configParams; private boolean defaultSuiteAddedFromConfigExclude = false; private boolean anyMetricAdded = false; private boolean anyPublisherAdded = false; @@ -77,6 +78,37 @@ public class Configuration { private Map feeders = new LinkedHashMap<>(); private Map objects = new HashMap<>(); + public Configuration(String yamlConfig) + throws InvocationTargetException, NoSuchMethodException, InstantiationException, IOException, IllegalAccessException { + ConfigParams configParams = new YamlParser().parse(yamlConfig); + buildConfiguration(configParams); + } + + public Configuration(String[] cmdLineArgs) + throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException, IOException { + ConfigParams configParams = collectConfigurations(cmdLineArgs); + buildConfiguration(configParams); + + } + + /** + * Method for getting the property from a setter method + * + * @param methodName + * @return + */ + public static String propertyFromMethod(String methodName) { + return methodName.startsWith("set") || methodName.startsWith("get") ? StringUtils.lowerCase(methodName.substring(3)) : StringUtils.lowerCase(methodName); + } + + public static Map> getRequiredFieldsForClassAdded() { + return requiredFieldsForClassAdded; + } + + public ConfigParams getConfigParams() { + return this.configParams; + } + private void handleExtensions(ConfigParams configParams) { List extensionList = new ArrayList<>(); @@ -121,7 +153,8 @@ private void handleExtensions(ConfigParams configParams) { } /** - * Creates a jar file for each extension file that should be loaded. + * Creates a jar file for each extension file that should be loaded. + * * @param extensionList A list of names representing the jar files that should be loaded. */ private List createJarFiles(List extensionList) { @@ -139,7 +172,8 @@ private List createJarFiles(List extensionList) { } /** - * Creates an URL for each jar file, using its filename. + * Creates an URL for each jar file, using its filename. + * * @param extensionsFileNames * @return */ @@ -157,7 +191,6 @@ private URL[] formJarURLs(List extensionsFileNames) { } // loads all classes from the extension jar files using a new class loader. - private ClassLoader processJarFiles(List jarFiles, URL[] urls) throws MalformedURLException { ToughdayExtensionClassLoader classLoader = new ToughdayExtensionClassLoader(urls, Thread.currentThread().getContextClassLoader()); Map newClasses = new HashMap<>(); @@ -194,10 +227,14 @@ private ClassLoader processJarFiles(List jarFiles, URL[] urls) throws M return classLoader; } + public boolean executeInDitributedMode() { + return !configParams.getDistributedConfigParams().isEmpty() && + !this.getDistributedConfig().getAgent() && + !this.getDistributedConfig().getDriver(); + } - public Configuration(String[] cmdLineArgs) - throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException, IOException { - ConfigParams configParams = collectConfigurations(cmdLineArgs); + private void buildConfiguration(ConfigParams configParams) throws NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException, IOException { + this.configParams = ConfigParams.deepClone(configParams); ConfigParams copyOfConfigParams = ConfigParams.deepClone(configParams); Map items = new HashMap<>(); @@ -206,11 +243,12 @@ public Configuration(String[] cmdLineArgs) Map globalArgsMeta = configParams.getGlobalParams(); for (String helpOption : CliParser.availableHelpOptions) { - if (globalArgsMeta.containsKey(helpOption)) { - return; - } + if (globalArgsMeta.containsKey(helpOption)) { + return; + } } + this.distributedConfig = createObject(DistributedConfig.class, configParams.getDistributedConfigParams()); this.globalArgs = createObject(GlobalArgs.class, globalArgsMeta); configureLogPath(globalArgs.getLogPath()); @@ -232,6 +270,7 @@ public Configuration(String[] cmdLineArgs) GenerateYamlConfiguration generateYaml = new GenerateYamlConfiguration(copyOfConfigParams, items); generateYaml.createYamlConfigurationFile(); } + objects = null; } @@ -315,8 +354,8 @@ private Phase createPhase(ConfigParams configParams, PhaseParams phaseParams, Te phase.setPublishMode(publishMode); // compute the minimum timeout of the phase - phase.getTestSuite().setMinTimeout(globalArgs.getTimeout()); - for(AbstractTest test : phase.getTestSuite().getTests()) { + phase.getTestSuite().setMinTimeout(globalArgs.getTimeoutInSeconds()); + for (AbstractTest test : phase.getTestSuite().getTests()) { // set the count (the number of executions since the beginnin of the run) of each test to 0 phase.getCounts().put(test, new AtomicLong(0)); @@ -325,7 +364,7 @@ private Phase createPhase(ConfigParams configParams, PhaseParams phaseParams, Te items.put(test.getName(), test.getClass()); - if(test.getTimeout() < 0) { + if (test.getTimeout() < 0) { continue; } @@ -373,7 +412,7 @@ private void getConfigurationFromAnotherPhase(PhaseParams phaseParams) { // merge the current phase with the one whose name is the value of 'useconfig' phaseParams.merge(PhaseParams.namedPhases.get(useconfig), - new HashSet<>(Arrays.asList(name, useconfig))); + new HashSet<>(Arrays.asList(name, useconfig))); } } @@ -390,7 +429,7 @@ private void convertActionItems(List configItem((ConfigParams.NamedMetaObject) item.getValue(), items, testSuite, publishers, metrics); break; case EXCLUDE: - excludeItem(((ConfigParams.NamedMetaObject)item.getValue()).getName(), testSuite, publishers, metrics); + excludeItem(((ConfigParams.NamedMetaObject) item.getValue()).getName(), testSuite, publishers, metrics); break; } } @@ -405,7 +444,7 @@ private void configureDurationForPhases() { if (phase.getDuration() == null) { phasesWithoutDuration.add(phase); } else { - durationLeft -= phase.getDuration(); + durationLeft -= GlobalArgs.parseDurationToSeconds(phase.getDuration()); } } @@ -613,16 +652,6 @@ private void configureLogPath(String logPath) throws IOException { } } - /** - * Method for getting the property from a setter method - * - * @param methodName - * @return - */ - public static String propertyFromMethod(String methodName) { - return methodName.startsWith("set") || methodName.startsWith("get") ? StringUtils.lowerCase(methodName.substring(3)) : StringUtils.lowerCase(methodName); - } - /** * Method for setting an object properties annotated with ConfigArgSet using reflection * @@ -635,9 +664,9 @@ public static String propertyFromMethod(String methodName) { * @throws IllegalAccessException caused by reflection */ //TODO figure out if we can make this public static again - public T setObjectProperties(T object, Map args, boolean applyDefaults, Map feedersContext) throws InvocationTargetException, IllegalAccessException { + public T setObjectProperties(T object, Map args, boolean applyDefaults, Map feedersContext) throws InvocationTargetException, IllegalAccessException { Class classObject = object.getClass(); - LOGGER.info("Configuring object of class: " + classObject.getSimpleName()+" ["+classObject.getName()+"]"); + LOGGER.info("Configuring object of class: " + classObject.getSimpleName() + " [" + classObject.getName() + "]"); for (Method method : classObject.getMethods()) { callConfigArgSet(method, object, args, applyDefaults); FeederInjector.injectFeeder(method, object, args, applyDefaults, feedersContext, objects); @@ -683,7 +712,8 @@ private static void callConfigArgSet(Method method, Object object, Map T createObject(Class classObject, Map args) + + public T createObject(Class classObject, Map args) throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { return createObject(classObject, args, null); } @@ -700,7 +730,7 @@ public T createObject(Class classObject, Map a * @throws InstantiationException * @throws NoSuchMethodException */ - public T createObject(Class classObject, Map args, Map feederContext) + public T createObject(Class classObject, Map args, Map feederContext) throws IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException { Constructor constructor = null; @@ -837,6 +867,14 @@ public GlobalArgs getGlobalArgs() { return globalArgs; } + /** + * Getter for kubernetes config args + * @return + */ + public DistributedConfig getDistributedConfig() { + return this.distributedConfig; + } + /** * Getter for the run mode * @@ -870,8 +908,8 @@ public List getPhases() { return phases; } - public static Map> getRequiredFieldsForClassAdded() { - return requiredFieldsForClassAdded; + public void setPhases(List phases) { + this.phases = phases; } public TestSuite getTestSuite() { @@ -882,5 +920,7 @@ public Set getPhasesWithoutDuration() { return phasesWithoutDuration; } - public Collection getFeeders() { return feeders.values(); } + public Collection getFeeders() { + return feeders.values(); + } } diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/GlobalArgs.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/GlobalArgs.java index a58879e..4b8d618 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/GlobalArgs.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/GlobalArgs.java @@ -106,7 +106,6 @@ public static long parseDurationToSeconds(String duration) { long finalDuration = 0l; long intermDuration = 0l; - //if time unit is not specified, consider it seconds by default. if (duration.matches("^[0-9]+$")) { throw new IllegalArgumentException("Time unit is not specified"); } @@ -226,7 +225,11 @@ public Collection getPublishers() { @ConfigArgGet public long getTimeout() { - return timeout; + return timeout / 1000; + } + + public long getTimeoutInSeconds() { + return this.timeout; } @ConfigArgSet(required = false, desc = "How long a test will run before it will be interrupted and marked as failed. Expressed in seconds", diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/cli/CliParser.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/cli/CliParser.java index 498d5c3..f772b2d 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/cli/CliParser.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/cli/CliParser.java @@ -29,6 +29,7 @@ import com.adobe.qe.toughday.internal.core.engine.Engine; import com.adobe.qe.toughday.internal.core.engine.PublishMode; import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.DistributedConfig; import com.adobe.qe.toughday.metrics.Metric; import com.google.common.base.Joiner; import net.jodah.typetools.TypeResolver; @@ -53,9 +54,9 @@ public class CliParser implements ConfigurationParser { private static final String TEST_CLASS_HELP_HEADER = String.format(HELP_HEADER_FORMAT_WITH_TAGS, "Class", "Fully qualified domain name", "Tags", "Description"); private static final String PUBLISH_CLASS_HELP_HEADER = String.format(HELP_HEADER_FORMAT_NO_TAGS, "Class", "Fully qualified domain name", "Description"); private static final String METRIC_CLASS_HELP_HEADER = PUBLISH_CLASS_HELP_HEADER; - private static Method[] globalArgMethods = GlobalArgs.class.getMethods(); private static final String SUITE_HELP_HEADER = String.format(" %-40s %-40s %s", "Suite", "Tags", "Description"); private static Map> availableGlobalArgs = new HashMap<>(); + private static Map> availableDistributedConfigArgs = new HashMap<>(); private static List parserArgHelps = new ArrayList<>(); @@ -73,18 +74,24 @@ public class CliParser implements ConfigurationParser { add("tag"); }}); + private static void collectAvailableConfigurationOptions(Class type, Map> availableArgs) { + Arrays.stream(type.getMethods()) + .filter(method -> method.isAnnotationPresent(ConfigArgSet.class)) + .forEach(method -> { + ConfigArgSet annotation = method.getAnnotation(ConfigArgSet.class); + int order = annotation.order(); + if (!availableArgs.containsKey(order)) { + availableArgs.put(order, new HashMap<>()); + } + + availableArgs.get(order) + .put(Configuration.propertyFromMethod(method.getName()), annotation); + }); + } + static { - for (Method method : globalArgMethods) { - if (method.getAnnotation(ConfigArgSet.class) != null) { - ConfigArgSet annotation = method.getAnnotation(ConfigArgSet.class); - int order = annotation.order(); - if (null == availableGlobalArgs.get(order)) { - availableGlobalArgs.put(order, new HashMap()); - } - Map globalArgMap = availableGlobalArgs.get(order); - globalArgMap.put(Configuration.propertyFromMethod(method.getName()), annotation); - } - } + collectAvailableConfigurationOptions(GlobalArgs.class, availableGlobalArgs); + collectAvailableConfigurationOptions(DistributedConfig.class, availableDistributedConfigArgs); for (Class parserClass : ReflectionsContainer.getSubTypesOf(ConfigurationParser.class)) { for (Field field : parserClass.getDeclaredFields()) { @@ -230,6 +237,10 @@ private int parseObjectProperties(int startIndex, String[] cmdLineArgs, HashMap< return j - startIndex; } + private boolean isGlobalArg(String paramName) { + return availableGlobalArgs.values().stream().anyMatch(map -> map.containsKey(paramName)); + } + /** * Implementation of parser interface * @param cmdLineArgs command line arguments @@ -261,6 +272,9 @@ public ConfigParams parse(String[] cmdLineArgs) { } else if (arg.equals("runmode")) { skip = parseObjectProperties(i + 1, cmdLineArgs, args); configParams.setRunModeParams(args); + } else if (arg.equals("distributedconfig")) { + skip = parseObjectProperties(i + 1, cmdLineArgs, args); + configParams.setDistributedConfigParams(args); } else if (arg.equals("help")) { skip = 1; globalArgs.put("host", "N/A"); //TODO remove ugly hack @@ -268,14 +282,9 @@ public ConfigParams parse(String[] cmdLineArgs) { String[] res = parseProperty(arg); String key = res[0]; Object val = getObjectFromString(res[1]); - // if global param does not exist - boolean found = false; - for (Map argz : availableGlobalArgs.values()) { - if (argz.containsKey(key)) { - found = true; - break; - } - } + // if global param or distributed td config param does not exist + boolean found = isGlobalArg(key); + if (!found && !parserArgs.contains(key) && !availableHelpOptions.contains(key) && !helpOptionsParameters.contains(key) && !key.equals("suite") && !key.equals("suitesetup")) { throw new IllegalArgumentException("Unrecognized argument --" + key); @@ -468,24 +477,27 @@ public void printShortHelp(boolean printSuitesTests) { System.out.println("\t java -jar toughday.jar --host=localhost --add extension.jar --add com.adobe.qe.toughday.tests.extensionTest"); System.out.println("\t java -jar toughday.jar --suite=toughday --add BASICMetrics --add Average decimals=3 --exclude Failed"); - System.out.println("\r\nGlobal arguments:"); + System.out.println("\r\nExamples for running TD distributed: \r\n"); + System.out.println("\t java -jar toughday.jar --host=localhost --distributedconfig driverip=1.1.1.1"); + System.out.println("\t java -jar toughday.jar --host=localhost --distributedconfig driverip=1.1.1.1 heartbeatinterval=10s --suite=toughday"); - for (Integer order : availableGlobalArgs.keySet()) { - Map paramGroup = availableGlobalArgs.get(order); - for (String param : paramGroup.keySet()) { - System.out.printf("\t--%-32s\t Default: %s - %s\r\n", - param + "=val", paramGroup.get(param).defaultValue(), paramGroup.get(param).desc()); - } - } + System.out.println("\r\nGlobal arguments:"); + availableGlobalArgs.forEach((order, paramGroup) -> + paramGroup.forEach((key, value) -> System.out.printf("\t--%-32s\t Default: %s - %s\r\n", + key + "=val", value.defaultValue(), value.desc()))); for (ParserArgHelp parserArgHelp : parserArgHelps) { System.out.printf("\t--%-32s\t Default: %s - %s\r\n", parserArgHelp.name() + "=val", parserArgHelp.defaultValue(), parserArgHelp.description()); } - //System.out.printf("\t%-32s\t %s\r\n", "--suitesetup=val", getSuiteSetupDescription()); System.out.printf("\t%-32s\t %s\r\n", "--suite=val", "Default: toughday - Where \"val\" can be one or a list (separated by commas) of the predefined suites"); + System.out.println("\r\n Distributed run arguments (--distributedconfig):"); + availableDistributedConfigArgs.forEach((order, paramGroup) -> + paramGroup.forEach((key, value) -> System.out.printf("\t%-32s\t Default: %s - %s\r\n", + key + "=val", value.defaultValue(), value.desc()))); + System.out.println("\r\nAvailable run modes (--runmode):"); for(Map.Entry> runMode : ReflectionsContainer.getInstance().getRunModeClasses().entrySet()) { Description description = runMode.getValue().getAnnotation(Description.class); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/GenerateYamlConfiguration.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/GenerateYamlConfiguration.java index dec7b78..1216be3 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/GenerateYamlConfiguration.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/GenerateYamlConfiguration.java @@ -17,17 +17,11 @@ import com.adobe.qe.toughday.internal.core.Timestamp; import com.adobe.qe.toughday.internal.core.config.ConfigParams; import com.adobe.qe.toughday.internal.core.config.PhaseParams; -import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml; -import org.yaml.snakeyaml.introspector.Property; -import org.yaml.snakeyaml.nodes.NodeTuple; -import org.yaml.snakeyaml.nodes.Tag; -import org.yaml.snakeyaml.representer.Representer; import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; -import java.lang.reflect.Method; import java.util.*; public class GenerateYamlConfiguration { @@ -43,6 +37,7 @@ public class GenerateYamlConfiguration { private static final String DEFAULT_YAML_CONFIGURATION_FILENAME = "toughday_"; private static final String DEFAULT_YAML_EXTENSION = ".yaml"; private static final String TIMESTAMP = Timestamp.START_TIME; + public static final String yamlConfigFilename = DEFAULT_YAML_CONFIGURATION_FILENAME + TIMESTAMP + DEFAULT_YAML_EXTENSION; public GenerateYamlConfiguration(ConfigParams configParams, Map items) { this.configParams = configParams; @@ -61,6 +56,13 @@ public Map getGlobals() { return globals; } + public Map getDistributedConfig() { + Map distributedConfigParms = configParams.getDistributedConfigParams(); + // remove this so that the driver won't try to trigger the execution + distributedConfigParms.remove("driverip"); + return distributedConfigParms; + } + public Map getPublishmode() { return configParams.getPublishModeParams(); } @@ -185,85 +187,28 @@ private void excludeAction(String item, int index) { } } - // Configure yaml representer to exclude class tags when dumping an object. - private void configureYamlRepresenterToExcludeClassTags(Representer representer) { - // Tag.MAP is by default ignored when dumping an object - representer.addClassTag(GenerateYamlConfiguration.class, Tag.MAP); - for (Class klass : ReflectionsContainer.getSubTypesOf(YamlDumpAction.class)) { - representer.addClassTag(klass, Tag.MAP); - } + public String createYamlStringRepresentation() { + Yaml yaml = YamlBuilder.getYamlInstance(); + return yaml.dump(this); } /** * Creates a YAML configuration file. */ public void createYamlConfigurationFile() { + String yamlStringRepresentation = createYamlStringRepresentation(); - final String filename = DEFAULT_YAML_CONFIGURATION_FILENAME + TIMESTAMP + DEFAULT_YAML_EXTENSION; - - FileWriter fileWriter = null; try { - fileWriter = new FileWriter(filename); - } catch (IOException e) { - e.printStackTrace(); - } - - BufferedWriter bufferedWriter = new BufferedWriter(fileWriter); - org.yaml.snakeyaml.constructor.Constructor constructor = new org.yaml.snakeyaml.constructor.Constructor - (GenerateYamlConfiguration.class); - - DumperOptions dumperOptions = new DumperOptions(); - dumperOptions.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - dumperOptions.setAllowReadOnlyProperties(true); - - // Configure the representer to ignore empty fields when dumping the object. By default, each empty filed is represented as {}. - - Representer representer = new Representer() { - @Override - protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) { - - Method method = null; - try { - if (propertyValue == null) { - propertyValue = ""; - } - method = propertyValue.getClass().getMethod("isEmpty"); - } catch (NoSuchMethodException e) { } - - if (method == null) { - return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); - } else { - - try { - if (Boolean.valueOf(method.invoke(propertyValue).toString())) { - return null; - } - } catch (Throwable t) { - t.printStackTrace(); - } - - return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); - } - } - }; + FileWriter fileWriter = new FileWriter(yamlConfigFilename); + BufferedWriter bufferedWriter = new BufferedWriter(fileWriter); - configureYamlRepresenterToExcludeClassTags(representer); - - // dump configuration - Yaml yaml = new Yaml(constructor, representer, dumperOptions); - String yamlObjectRepresentation = yaml.dump(this); - - try { - bufferedWriter.write(yamlObjectRepresentation); + bufferedWriter.write(yamlStringRepresentation); bufferedWriter.flush(); - } catch (IOException e) { - e.printStackTrace(); - } - try { fileWriter.close(); bufferedWriter.close(); + } catch (IOException e) { e.printStackTrace(); } diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlBuilder.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlBuilder.java new file mode 100644 index 0000000..c846679 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlBuilder.java @@ -0,0 +1,98 @@ +package com.adobe.qe.toughday.internal.core.config.parsers.yaml; + +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.introspector.Property; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Representer; + +import java.lang.reflect.Method; + +/** + * Class responsible for building and configuring a Yaml instance that knows how to dump a configuration + * in the desired format(without class tags, empty objects etc.) + */ +public class YamlBuilder { + + private static Yaml instance = null; + + // Configure yaml representer to exclude class tags when dumping an object. + private static void configureYamlRepresenterToExcludeClassTags(Representer representer) { + // Tag.MAP is by default ignored when dumping an object + representer.addClassTag(GenerateYamlConfiguration.class, Tag.MAP); + representer.addClassTag(YamlDumpConfigurationAsTaskForTDAgents.class, Tag.MAP); + for (Class type : ReflectionsContainer.getSubTypesOf(YamlDumpAction.class)) { + representer.addClassTag(type, Tag.MAP); + } + + } + + private static void buildInstance() { + DumperOptions dumperOptions = getDumperOptions(); + + // Configure the representer to ignore empty fields when dumping the object. By default, each empty filed is represented as {}. + Representer representer = getRepresenter(); + configureYamlRepresenterToExcludeClassTags(representer); + + Constructor constructor = new Constructor(GenerateYamlConfiguration.class); + instance = new Yaml(constructor, representer, dumperOptions); + } + + /** + * Getter for the Yaml instance. + */ + public static Yaml getYamlInstance() { + if (instance == null) { + buildInstance(); + } + + return instance; + } + + + private static Representer getRepresenter() { + return new Representer() { + @Override + protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) { + + Method method = null; + try { + if (propertyValue == null) { + propertyValue = ""; + } + method = propertyValue.getClass().getMethod("isEmpty"); + } catch (NoSuchMethodException ignored) { } + + if (method == null) { + return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); + } else { + + try { + if (Boolean.valueOf(method.invoke(propertyValue).toString())) { + return null; + } + } catch (Throwable t) { + /* no action required here. The only implication is that the empty collection will be included + * in the Yaml representation when dumping an object. + */ + } + + return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); + } + } + }; + } + + private static DumperOptions getDumperOptions() { + DumperOptions dumperOptions = new DumperOptions(); + + dumperOptions.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + dumperOptions.setAllowReadOnlyProperties(true); + + return dumperOptions; + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlConfiguration.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlConfiguration.java index 46f3202..4d40077 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlConfiguration.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlConfiguration.java @@ -21,6 +21,9 @@ public class YamlConfiguration { private ConfigParams configParams = new ConfigParams(); + private void setDistributedConfig(Map distributedConfig) { + this.configParams.setDistributedConfigParams(distributedConfig); + } public void setGlobals(Map globals) { this.configParams.setGlobalParams(globals); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlParser.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlParser.java index b511938..a5cef99 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlParser.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlParser.java @@ -43,6 +43,31 @@ public String description() { } }; + private Constructor getYamlConstructor() { + Constructor constructor = new Constructor(YamlConfiguration.class); + TypeDescription yamlParserDesc = new TypeDescription(YamlConfiguration.class); + + yamlParserDesc.putListPropertyType("tests", YamlParseAction.class); + yamlParserDesc.putListPropertyType("phases", YamlParsePhase.class); + yamlParserDesc.putListPropertyType("publishers", YamlParseAction.class); + yamlParserDesc.putListPropertyType("metrics", YamlParseAction.class); + yamlParserDesc.putListPropertyType("extensions", YamlParseAction.class); + yamlParserDesc.putListPropertyType("feeders", YamlParseAction.class); + + constructor.addTypeDescription(yamlParserDesc); + + return constructor; + } + + public ConfigParams parse(String stringYaml) { + Constructor constructor = getYamlConstructor(); + + Yaml yaml = new Yaml(constructor); + YamlConfiguration yamlConfig = (YamlConfiguration) yaml.load(stringYaml); + + return yamlConfig.getConfigParams(); + } + @Override public ConfigParams parse(String[] cmdLineArgs) { String configFilePath = null; @@ -55,16 +80,7 @@ public ConfigParams parse(String[] cmdLineArgs) { if(configFilePath != null) { try { - Constructor constructor = new Constructor(YamlConfiguration.class); - TypeDescription yamlParserDesc = new TypeDescription(YamlConfiguration.class); - yamlParserDesc.putListPropertyType("tests", YamlParseAction.class); - yamlParserDesc.putListPropertyType("phases", YamlParsePhase.class); - yamlParserDesc.putListPropertyType("publishers", YamlParseAction.class); - yamlParserDesc.putListPropertyType("metrics", YamlParseAction.class); - yamlParserDesc.putListPropertyType("extensions", YamlParseAction.class); - yamlParserDesc.putListPropertyType("feeders", YamlParseAction.class); - - constructor.addTypeDescription(yamlParserDesc); + Constructor constructor = getYamlConstructor(); Yaml yaml = new Yaml(constructor); YamlConfiguration yamlConfig = (YamlConfiguration) yaml.load(new FileInputStream(configFilePath)); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitor.java new file mode 100644 index 0000000..c14bfe3 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitor.java @@ -0,0 +1,156 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; + +/** + * Class responsible for monitoring the execution of a distributed phase. + */ +public class DistributedPhaseMonitor { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private final ExecutorService executorService = Executors.newFixedThreadPool(1); + private final List agentsRunningTD = new ArrayList<>(); + // key = name of the test; value = map(key = name of the agent, value = nr of tests executed) + private Map> executions = new HashMap<>(); + private Phase phase; + private long phaseStartTime = 0; + + /** + * Returns true is the exections of the phase has started. + */ + public boolean isPhaseExecuting() { + return this.phase != null && !this.agentsRunningTD.isEmpty(); + } + + /** + * Setter for the execution start time of the phase. + */ + public void setPhaseStartTime(long phaseStartTime) { + this.phaseStartTime = phaseStartTime; + } + + /** + * Getter for the execution start time of the phase + */ + public long getPhaseStartTime() { + return this.phaseStartTime; + } + + /** + * Setter for the phase being monitored + */ + public void setPhase(Phase phase) { + this.phase = phase; + this.phase.getTestSuite().getTests().forEach(test -> executions.put(test.getName(), new HashMap<>())); + } + + /** + * Adds the agent received as a parameter to the list of agents which are currently executing the + * phase being monitored. + * @param agentIdentifier : ip address that identifies the agent inside the cluster + */ + public void registerAgentRunningTD(String agentIdentifier) { + this.agentsRunningTD.add(agentIdentifier); + + } + + /** + * Removes tha agent received as a parameter from the list of agents which are currently executing + * the phase being monitored. + * @param agentIdentifier : ip address that identifies the agent inside the cluster + */ + public void removeAgentFromActiveTDRunners(String agentIdentifier) { + this.agentsRunningTD.remove(agentIdentifier); + // remove agent from executions map + this.executions.forEach((test, executionsPerAgent) -> this.executions.get(test).remove(agentIdentifier)); + } + + /** + * Method for waiting until all agents finish executing the phase being monitored. + */ + public boolean waitForPhaseCompletion(int retries) { + while (retries > 0) { + Future waitPhaseCompletion = executorService.submit(() -> { + while (!this.agentsRunningTD.isEmpty()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // this will not cause further problems => it can be ignored + } + } + }); + + try { + waitPhaseCompletion.get(); + return true; + } catch (InterruptedException | ExecutionException e) { + retries--; + } + } + + LOG.warn("Exception occurred while waiting for the completion of the current phase. The rest " + + "of the phases will no longer be executed"); + return false; + + } + + /** + * Updates the number of executions/test that must be executed by the agents running in the cluster. This method is + * called during the work redistribution process. + */ + public void updateCountPerTest() { + phase.getTestSuite().getTests().forEach(test -> { + long remained = test.getCount() - this.getExecutionsPerTest().get(test.getName()); + if (remained < 0) { + // set this to 0 so that the agents will know to delete the test from the test suite + test.setCount("0"); + } else { + test.setCount(String.valueOf(remained)); + } + }); + } + + /** + * Returns the number of executions/test/agent. + */ + public Map> getExecutions() { + return this.executions; + } + + /** + * Resets the number of execution/test for each agent executing the phase being monitored. + */ + public void resetExecutions() { + this.executions.forEach((testName, executionsPerAgent) -> + executionsPerAgent.keySet().forEach(agentName -> executionsPerAgent.put(agentName, 0L))); + } + + /** + * Returns the number of executions/test executed by all the agents running in the cluster. + */ + public Map getExecutionsPerTest() { + Map executionsPerTest = new HashMap<>(); + + this.executions.forEach((testName, executionsPerAgent) -> + executionsPerTest.put(testName, executionsPerAgent.values().stream().mapToLong(x -> x).sum())); + + return executionsPerTest; + } + + /** + * Getter for the phase being monitored by this class. + */ + public Phase getPhase() { + return this.phase; + } + +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/ExecutionTrigger.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/ExecutionTrigger.java new file mode 100644 index 0000000..5299344 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/ExecutionTrigger.java @@ -0,0 +1,51 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.GenerateYamlConfiguration; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Driver; +import org.apache.http.HttpResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashMap; + + +/** + * Class responsible for sending a request to the driver component running in the cluster. This + * request will trigger the distributed execution of TD. + * */ +public class ExecutionTrigger { + + protected static final Logger LOG = LogManager.getLogger(ExecutionTrigger.class); + private static final String DEFAULT_CLUSTER_PORT = "80"; + + private final Configuration configuration; + private final String executionPath; + + public ExecutionTrigger(Configuration configuration) { + this.configuration = configuration; + // sanity check + if (configuration.getDistributedConfig().getDriverIp() == null || configuration.getDistributedConfig().getDriverIp().isEmpty()) { + throw new IllegalStateException("The public ip address at which the driver's service is accessible " + + " is required when running TD in distributed mode."); + } + + this.executionPath = HttpUtils.URL_PREFIX + configuration.getDistributedConfig().getDriverIp() + ":" + DEFAULT_CLUSTER_PORT + + Driver.EXECUTION_PATH; + } + + /** + * Method used for triggering the execution of TD in distributed mode. + */ + public void triggerDistributedExecution() { + GenerateYamlConfiguration generateYaml = new GenerateYamlConfiguration(this.configuration.getConfigParams(), new HashMap<>()); + String yamlConfig = generateYaml.createYamlStringRepresentation(); + HttpUtils httpUtils = new HttpUtils(); + + HttpResponse response = httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, yamlConfig, executionPath, 3); + if (response == null) { + LOG.warn("TD execution request could not be sent to driver. Make sure that driver is up" + + " and ready to process requests."); + } + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/HttpUtils.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/HttpUtils.java new file mode 100644 index 0000000..b583c61 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/HttpUtils.java @@ -0,0 +1,91 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.api.annotations.labels.Nullable; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.io.IOException; + +/** + * Class responsible for handling the communication between the agents and the drivers running in the cluster. + */ +public class HttpUtils { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + public static final String POST_METHOD = "POST"; + public static final String GET_METHOD = "GET"; + public static final String URL_PREFIX = "http://"; + public static final int HTTP_REQUEST_RETRIES = 3; + + private HttpResponse sendGetRequest(String URI) { + HttpClient httpClient = HttpClientBuilder.create().build(); + HttpGet request = new HttpGet(URI); + + try { + return httpClient.execute(request); + } catch (IOException e) { + LOG.warn("Http request could not be sent to " + URI + ". Received error " + e.getMessage()); + } + + return null; + } + + private HttpResponse sendPostRequest(String requestContent, String URI) { + HttpClient httpClient = HttpClientBuilder.create().build(); + HttpPost request = new HttpPost(URI); + + try { + StringEntity params = new StringEntity(requestContent); + request.setEntity(params); + request.setHeader("Content-type", "text/plain"); + + // submit request and wait for ack from agent + return httpClient.execute(request); + } catch (IOException e) { + LOG.warn("Http request could not be sent to " + URI + ". Received error " + e.getMessage()); + } + + return null; + } + + private boolean checkSuccessfulRequest(HttpResponse response) { + return response != null && response.getStatusLine().getStatusCode() >= 200 && + response.getStatusLine().getStatusCode() < 300; + } + + /** + * Method used for sending http requests. + * @param requestContent content of the request + * @param URI the path that uniquely identifies the component in the cluster and the http endpoint to be used + * when sending the request + * @param retries how many times to retry sending the request in case of failure + * @return true is the the request is successfully sent, false otherwise. + */ + @Nullable + public HttpResponse sendHttpRequest(String requestType, String requestContent, String URI, int retries) { + HttpResponse response = null; + + while (retries > 0) { + if (requestType.equals(POST_METHOD)) { + response = sendPostRequest(requestContent, URI); + } else if (requestType.equals(GET_METHOD)) { + response = sendGetRequest(URI); + } + + if (checkSuccessfulRequest(response)) { + return response; + } + + retries--; + } + + return null; + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgents.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgents.java new file mode 100644 index 0000000..9c3ba5f --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgents.java @@ -0,0 +1,175 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; +import com.adobe.qe.toughday.api.annotations.ConfigArgSet; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.*; + +import com.adobe.qe.toughday.internal.core.config.parsers.cli.CliParser; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.YamlBuilder; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.YamlDumpAddAction; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.YamlDumpPhase; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.DistributedConfig; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.yaml.snakeyaml.Yaml; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.*; + +/** + * Knows how to dump a Configuration object in yaml format in order to send it to the agents running in the cluster. + * This class assumes that it is not necessary to dump the run mode and the publish mode fields of the Configuration + * class since each Phase contained by this configuration already defines them. + * + * It is also assumed that the only action to be taken into consideration when dumping the configuration is Actions.ADD + * since all the other actions were already processed before sending the execution query to the driver. + */ +public class YamlDumpConfigurationAsTaskForTDAgents { + private static final Logger LOG= LogManager.getLogger(CliParser.class); + + private Configuration configuration; + private List phases = new ArrayList<>(); + + public List getPhases() { + return this.phases; + } + + /** + * This method is used for collecting all configurable properties that were assigned a new value(different than the + * default one) from the command line or the yaml configuration file used for running ToughDay. + * @param type class of the object whose properties are collected + * @param object instance + */ + protected Map collectConfigurableProperties(Class type, Object object) { + if (!type.isAssignableFrom(object.getClass())) { + throw new IllegalArgumentException("The object must have the specified type."); + } + + Map configurableArgs = new HashMap<>(); + + /* add all inherited configurable properties */ + if (type.getSuperclass() != Object.class) { + configurableArgs.putAll(collectConfigurableProperties(type.getSuperclass(), object)); + } + + Arrays.stream(type.getDeclaredMethods()) + .filter(method -> method.isAnnotationPresent(ConfigArgGet.class)) + .forEach(method -> { + String property = Configuration.propertyFromMethod(method.getName()); + try { + Object value = method.invoke(object); + + Class[] parametersType = {String.class}; + Method m = type.getMethod(method.getName().replace("get", "set"), parametersType); + String defaultValue = m.getAnnotation(ConfigArgSet.class).defaultValue(); + + // skip all default values + if (!String.valueOf(value).equals(defaultValue) && value != null) { + if (value instanceof Level) { + configurableArgs.put(property, ((Level) value).name()); + } else { + configurableArgs.put(property, value); + } + } + } catch (IllegalAccessException | InvocationTargetException e) { + LOG.warn("Configurable property " + property + " could not be collected when dumping the " + + "configuration. Please check that public methods with @ConfigArgSet and @ConfigArgGet " + + "were defined for this property."); + } catch ( NoSuchMethodException e) { + // skip for now + LOG.warn("No @ConfigArgGet method was defined for property " + property + ". The configuration " + + "will be dumped without this property."); + } + }); + + return configurableArgs; + } + + private void addAction(ConfigParams.ClassMetaObject item, YamlDumpPhase yamlDumpPhase) { + YamlDumpAddAction addAction = new YamlDumpAddAction(item.getClassName(), item.getParameters()); + if (ReflectionsContainer.getInstance().isTestClass(item.getClassName())) { + yamlDumpPhase.getTests().add(addAction); + } else if (ReflectionsContainer.getInstance().isMetricClass(item.getClassName())) { + yamlDumpPhase.getMetrics().add(addAction); + } else if (ReflectionsContainer.getInstance().isPublisherClass(item.getClassName())) { + yamlDumpPhase.getPublishers().add(addAction); + } + } + + private void createItem(Object object, List> items ) { + Map parameters = collectConfigurableProperties(object.getClass(), object); + ConfigParams.MetaObject metaObject = new ConfigParams.ClassMetaObject(object.getClass().getSimpleName(), parameters); + items.add(new AbstractMap.SimpleEntry<>(Actions.ADD, metaObject)); + } + + private void buildYamlDumpPhases() { + configuration.getPhases().forEach(phase -> { + // collect all configurable properties + Map properties = collectConfigurableProperties(Phase.class, phase); + Map runMode = collectConfigurableProperties(phase.getRunMode().getClass(), phase.getRunMode()); + Map publishMode = collectConfigurableProperties(phase.getPublishMode().getClass(), phase.getPublishMode()); + + // add required type parameter for run mode and publish mode + runMode.put("type", phase.getRunMode().getClass().getSimpleName().toLowerCase()); + publishMode.put("type", phase.getPublishMode().getClass().getSimpleName().toLowerCase()); + + List> items = new ArrayList<>(); + + // create tests + phase.getTestSuite().getTests().forEach(test -> createItem(test, items)); + // create metrics + phase.getMetrics().forEach(metric -> createItem(metric, items)); + // create publishers + phase.getPublishers().forEach(publisher -> createItem(publisher, items)); + + YamlDumpPhase yamlDumpPhase = new YamlDumpPhase(properties, runMode, publishMode); + items.forEach(entry -> addAction((ConfigParams.ClassMetaObject)entry.getValue(), yamlDumpPhase)); + + this.phases.add(yamlDumpPhase); + }); + } + + public YamlDumpConfigurationAsTaskForTDAgents(Configuration configuration) { + if (configuration == null) { + throw new IllegalArgumentException("Configuration must not be null."); + } + + this.configuration = configuration; + } + + /** + * Getter for global params. + */ + public Map getGlobals() { + Map globals = + collectConfigurableProperties(GlobalArgs.class, configuration.getGlobalArgs()); + /* this is required because duration is internally converted from string(which includes the unit of measure) + to long value(duration in seconds) so we need to manually add the unit of measure when dumping it. */ + globals.put("duration", String.valueOf(globals.get("duration")) + 's'); + + return globals; + } + + + /** + * Getter for the distributed configuration. + */ + public Map getDistributedConfig() { + return collectConfigurableProperties(DistributedConfig.class, configuration.getDistributedConfig()); + } + + /** + * Dumps the configuration received in the constructor. + */ + public String generateConfigurationObject() { + buildYamlDumpPhases(); + Yaml yaml = YamlBuilder.getYamlInstance(); + + return yaml.dump(this); + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Agent.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Agent.java new file mode 100644 index 0000000..85e9749 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Agent.java @@ -0,0 +1,299 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster; + +import com.adobe.qe.toughday.api.core.AbstractTest; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.config.PhaseParams; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructionsProcessor; +import com.google.gson.Gson; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.net.*; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.URL_PREFIX; +import static com.adobe.qe.toughday.internal.core.engine.Engine.installToughdayContentPackage; +import static spark.Spark.*; + +/** + * Agent component for running TD distributed. + */ +public class Agent { + private static final String PORT = "4567"; + protected static final Logger LOG = LogManager.getLogger(Engine.class); + private final ExecutorService tdExecutorService = Executors.newFixedThreadPool(1); + private final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(1); + + public enum Status { + IDLE, /* Agent is waiting to receive tasks from the driver */ + BUILDING_CONFIG, /* Agent has received one task from the driver and it is building the TD configuration */ + RUNNING /* Agent is running TD tests */ + } + + // available routes + private static final String INSTALL_SAMPLE_CONTENT_PATH = "/sampleContent"; + private static final String SUBMIT_TASK_PATH = "/submitTask"; + private static final String FINISH_PATH = "/finish"; + private static final String HEARTBEAT_PATH = "/heartbeat"; + private static final String REBALANCE_PATH = "/rebalance"; + private static final String HEALTH_PATH = "/health"; + private static final String GET_STATUS_PATH = "/status"; + + private HttpUtils httpUtils = new HttpUtils(); + private Status status = Status.IDLE; + private final Lock statusLock = new ReentrantLock(); + + /** + * Returns the http URL that the driver should use for finished the execution of the agent. + * @param agentItAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getFinishPath(String agentItAddress) { + return URL_PREFIX + agentItAddress + ":" + PORT + FINISH_PATH; + } + + /** + * Returns the http URL that the driver should use for sending heartbeat messages to the agent. + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getHeartbeatPath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + PORT + HEARTBEAT_PATH; + } + + /** + * Returns the http URL that the driver should use for sending a task to the agent. + * @param agentIpAdress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getSubmissionTaskPath(String agentIpAdress) { + return URL_PREFIX + agentIpAdress + ":" + PORT + SUBMIT_TASK_PATH; + } + + /** + * Returns the http URL that the driver should use for sending redistribution instructions to the agent. + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getRebalancePath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + PORT + REBALANCE_PATH; + } + + /** + * Returns the http URL that the driver should use for requiring the agent to install the TD sample content package. + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getInstallSampleContentPath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + PORT + INSTALL_SAMPLE_CONTENT_PATH; + } + + /** + * Returns the http URL that the driver should use for getting the current status of the agent + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getGetStatusPath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + PORT + GET_STATUS_PATH; + } + + private Engine engine; + private static String ipAddress; + + static { + try { + ipAddress = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + System.exit(-1); + } + } + + private final RedistributionInstructionsProcessor redistributionInstructionsProcessor = new RedistributionInstructionsProcessor(); + + public static boolean announcePhaseCompletion() { + /* the current master might be dead so we should retry this for a certain amount of time before shutting + * down the execution. + */ + HttpUtils httpUtils = new HttpUtils(); + HttpResponse response = null; + long duration = GlobalArgs.parseDurationToSeconds("60s"); + + while (duration > 0 && response == null) { + response = httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, ipAddress, + Driver.getPhaseFinishedByAgentPath(), HTTP_REQUEST_RETRIES); + + try { + Thread.sleep(10 * 1000L); // try again in 10 seconds + } catch (InterruptedException e) { + // skip this since this thread is not interrupted by anyone + } finally { + duration -= GlobalArgs.parseDurationToSeconds("10s"); + } + } + + return response != null; + } + + public void start() { + register(); + + /* expose http endpoint to allow the driver to ask for ToughDay sample content package to be installed */ + post(INSTALL_SAMPLE_CONTENT_PATH, ((request, response) -> { + String yamlConfig = request.body(); + Configuration configuration = new Configuration(yamlConfig); + + this.tdExecutorService.submit(() -> { + boolean installed = true; + + try { + installToughdayContentPackage(configuration.getGlobalArgs()); + } catch (Exception e) { + installed = false; + LOG.error("Error encountered when installing TD sample content", e); + } + + HttpResponse driverResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, String.valueOf(installed), + Driver.getSampleContentAckPath(), HTTP_REQUEST_RETRIES); + if (driverResponse == null) { + LOG.error("Agent " + ipAddress + " could not announce the driver that Toughday sample content" + + " package was installed."); + System.exit(-1); + } + + }); + + // clear all phases + PhaseParams.namedPhases.clear(); + return ""; + })); + + /* Expose http endpoint for receiving ToughDay execution request from the driver */ + post(SUBMIT_TASK_PATH, ((request, response) -> { + try { + this.statusLock.lock(); + if (this.status != Status.IDLE) { + LOG.info("Agent is currently executing a different task. Request will be ignored."); + + // set status code to Conflict + response.status(HttpStatus.SC_CONFLICT); + return "Agent is already running a task received from the Driver."; + } + + // update current status + this.status = Status.BUILDING_CONFIG; + } finally { + this.statusLock.unlock(); + } + + String yamlTask = request.body(); + LOG.info("[Agent] Received task:\n" + yamlTask); + + Configuration configuration = new Configuration(yamlTask); + this.engine = new Engine(configuration); + configuration.getDistributedConfig().setAgent("true"); + + tdExecutorService.submit(() -> { + this.statusLock.lock(); + this.status = Status.RUNNING; + this.statusLock.unlock(); + + this.engine.runTests(); + + if (!announcePhaseCompletion()) { + LOG.error("Agent " + ipAddress + " could not inform driver that phase was executed."); + System.exit(-1); + } + + this.statusLock.lock(); + this.status = Status.IDLE; + this.statusLock.unlock(); + }); + + return ""; + })); + + /* Expose http endpoint to be used by the driver for heartbeat messages */ + get(HEARTBEAT_PATH, ((request, response) -> + { + // send to driver the total number of executions/test + Gson gson = new Gson(); + Map currentCounts = new HashMap<>(); + + // check if execution has started + if (this.status != Status.RUNNING) { + return gson.toJson(currentCounts); + } + + Map phaseCounts = engine.getCurrentPhase().getCounts(); + phaseCounts.forEach((test, count) -> currentCounts.put(test.getName(), count.get())); + + return gson.toJson(currentCounts); + })); + + + + /* expose http endpoint for receiving redistribution requests from the driver */ + post(REBALANCE_PATH, (((request, response) -> { + try { + statusLock.lock(); + if (this.status == Status.IDLE) { + LOG.warn("Rebalance was requested before submitting a task. Request will be ignored."); + return ""; + } + } finally { + statusLock.unlock(); + } + + String instructionsMessage = request.body(); + LOG.info("[Agent] Received " + instructionsMessage + " from driver"); + this.redistributionInstructionsProcessor.processInstructions(instructionsMessage, this.engine.getConfiguration().getPhases().get(0)); + + return ""; + }))); + + + /* expose http endpoint for health checks */ + get(HEALTH_PATH, ((request, response) -> "Healthy")); + + /* expose http endpoint for getting the current status of the agent */ + get(GET_STATUS_PATH, ((request, response) -> { + try { + this.statusLock.lock(); + return this.status; + } finally { + this.statusLock.unlock(); + } + })); + + /* expose http endpoint for finishing the execution of the agent */ + post(FINISH_PATH, ((request, response) -> { + scheduledExecutorService.schedule(() -> { + this.tdExecutorService.shutdown(); + this.scheduledExecutorService.shutdown(); + + System.exit(0); + }, GlobalArgs.parseDurationToSeconds("3s"), TimeUnit.SECONDS); + + return ""; + })); + } + + /* Method responsible for registering the current agent to the driver. It should be the + * first method executed. + */ + private void register() { + HttpResponse response = + this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, ipAddress, + Driver.getAgentRegisterPath(), HTTP_REQUEST_RETRIES); + if (response == null) { + System.exit(-1); + } + + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/DistributedConfig.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/DistributedConfig.java new file mode 100644 index 0000000..9e127bb --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/DistributedConfig.java @@ -0,0 +1,85 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster; + +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; +import com.adobe.qe.toughday.api.annotations.ConfigArgSet; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; + +/** + * Contains all the configurable arguments when running TD distributed. + */ +public class DistributedConfig { + private static final String DEFAULT_HEARTBEAT_INTERVAL = "5s"; + private static final String DEFAULT_REDISTRIBUTION_WAIT_TIME = "3s"; + + private boolean agent = false; + private boolean driver = false; + private String driverIp = null; + private String heartbeatInterval = DEFAULT_HEARTBEAT_INTERVAL; + private String redistributionWaitTime = DEFAULT_REDISTRIBUTION_WAIT_TIME; + + @ConfigArgSet(required = false, desc = "The public ip address of the cluster. The driver" + + " service must be accessible at this address. This property is required when running in distributed mode.") + public void setDriverIp(String driverIp) { + this.driverIp = driverIp; + } + + @ConfigArgGet + public String getDriverIp() { + return this.driverIp; + } + + @ConfigArgGet + public boolean getAgent() { return this.agent; } + + @ConfigArgSet(required = false, defaultValue = "false", desc = "If true, TD runs as a cluster agent, waiting to receive" + + " a task from the driver.") + public void setAgent(String agent) { + this.agent = Boolean.parseBoolean(agent); + } + + @ConfigArgGet + public boolean getDriver() { + return this.driver; + } + + @ConfigArgSet(required = false, defaultValue = "false", desc = "If true, TD runs as a driver in the cluster," + + " distributing the work between the agents.") + public void setDriver(String driver) { + this.driver = Boolean.parseBoolean(driver); + } + + @ConfigArgSet(required = false, defaultValue = DEFAULT_HEARTBEAT_INTERVAL, desc = "Period of time for sending " + + "heartbeat messages to the agents in the cluster.") + public void setHeartbeatInterval(String heartbeatInterval) { + this.heartbeatInterval = heartbeatInterval; + } + + @ConfigArgGet + public String getHeartbeatInterval() { + return this.heartbeatInterval; + } + + @ConfigArgSet(required = false, defaultValue = DEFAULT_REDISTRIBUTION_WAIT_TIME, desc = "The minimum amount of time " + + " to wait before scheduling work redistribution if required.") + public void setRedistributionWaitTime(String redistributionWaitTime) { + this.redistributionWaitTime = redistributionWaitTime; + } + + @ConfigArgGet + public String getRedistributionWaitTime() { + return this.redistributionWaitTime; + } + + public long getRedistributionWaitTimeInSeconds() { + return GlobalArgs.parseDurationToSeconds(this.redistributionWaitTime); + } + + public long getHeartbeatIntervalInSeconds() { + return GlobalArgs.parseDurationToSeconds(this.heartbeatInterval); + } + + public void merge(DistributedConfig other) { + this.setHeartbeatInterval(other.getHeartbeatInterval()); + this.setRedistributionWaitTime(other.redistributionWaitTime); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Driver.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Driver.java new file mode 100644 index 0000000..6452d70 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Driver.java @@ -0,0 +1,287 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.distributedtd.DistributedPhaseMonitor; +import com.adobe.qe.toughday.internal.core.distributedtd.tasks.HeartbeatTask; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.TaskBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.PhaseSplitter; +import org.apache.http.HttpResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.*; +import java.util.concurrent.*; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.URL_PREFIX; +import static com.adobe.qe.toughday.internal.core.engine.Engine.logGlobal; +import static spark.Spark.*; + +/** + * Driver component for the cluster. + */ +public class Driver { + // routes + public static final String EXECUTION_PATH = "/config"; + private static final String REGISTER_PATH = "/registerAgent"; + private static final String PHASE_FINISHED_BY_AGENT_PATH = "/phaseFinished"; + private static final String HEALTH_PATH = "/health"; + private static final String SAMPLE_CONTENT_ACK_PATH = "/contentAck"; + + private static final String HOSTNAME = "driver"; + private static final String PORT = "80"; + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private final ScheduledExecutorService heartbeatScheduler = Executors.newSingleThreadScheduledExecutor(); + private final ExecutorService executorService = Executors.newFixedThreadPool(1); + private final HttpUtils httpUtils = new HttpUtils(); + private final Queue agents = new ConcurrentLinkedQueue<>(); + private final TaskBalancer taskBalancer = TaskBalancer.getInstance(); + private DistributedPhaseMonitor distributedPhaseMonitor = new DistributedPhaseMonitor(); + private Configuration configuration; + private Configuration driverConfiguration; + private final Object object = new Object(); + + public Driver(Configuration configuration) { + this.driverConfiguration = configuration; + } + + /** + * Returns the http URL that should be used by the agents in order to register themselves. + */ + public static String getAgentRegisterPath() { + return URL_PREFIX + HOSTNAME + ":" + PORT + REGISTER_PATH; + } + + /** + * Returns the http URL that should be used by the agents whenever they finished executing the task received from + * the driver. + */ + public static String getPhaseFinishedByAgentPath() { + return URL_PREFIX + HOSTNAME + ":" + PORT + PHASE_FINISHED_BY_AGENT_PATH; + } + + /** + * Returns the http URL that should be used by the agent chosen to install the TD sample content package for + * informing the driver that the installation was completed successfully. + */ + public static String getSampleContentAckPath() { + return URL_PREFIX + HOSTNAME + ":" + PORT + SAMPLE_CONTENT_ACK_PATH; + } + + private void waitForSampleContentToBeInstalled() { + synchronized (object) { + try { + object.wait(); + } catch (InterruptedException e) { + LOG.error("Failed to install ToughDay sample content package. Execution will be stopped."); + finishDistributedExecution(); + System.exit(-1); + } + } + } + + private void installToughdayContentPackage(Configuration configuration) { + logGlobal("Installing ToughDay 2 Content Package..."); + GlobalArgs globalArgs = configuration.getGlobalArgs(); + + if (globalArgs.getDryRun() || !globalArgs.getInstallSampleContent()) { + return; + } + + HttpResponse agentResponse = null; + List agentsCopy = new ArrayList<>(this.agents); + + while (agentResponse == null && agentsCopy.size() > 0) { + // pick one agent to install the sample content + String agentIpAddress = agentsCopy.remove(0); + String URI = Agent.getInstallSampleContentPath(agentIpAddress); + LOG.info("Installing sample content request was sent to agent " + agentIpAddress); + + YamlDumpConfigurationAsTaskForTDAgents dumpConfig = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + String yamlTask = dumpConfig.generateConfigurationObject(); + + agentResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, yamlTask, URI, HTTP_REQUEST_RETRIES); + } + + // we should wait until the we receive confirmation that the sample content was successfully installed + waitForSampleContentToBeInstalled(); + + logGlobal("Finished installing ToughDay 2 Content Package."); + globalArgs.setInstallSampleContent("false"); + + } + + private void mergeDistributedConfigParams(Configuration configuration) { + if (this.driverConfiguration.getDistributedConfig().getHeartbeatIntervalInSeconds() == + this.configuration.getDistributedConfig().getHeartbeatIntervalInSeconds()) { + + this.driverConfiguration.getDistributedConfig().merge(configuration.getDistributedConfig()); + return; + } + + // cancel heartbeat task and reschedule it with the new period + this.heartbeatScheduler.shutdownNow(); + this.driverConfiguration.getDistributedConfig().merge(configuration.getDistributedConfig()); + scheduleHeartbeatTask(); + } + + private void finishAgents() { + agents.forEach(agentIp -> { + LOG.info("[Driver] Finishing agent " + agentIp); + HttpResponse response = + httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, "", Agent.getFinishPath(agentIp), HTTP_REQUEST_RETRIES); + if (response == null) { + // the assumption is that the agent will be killed when he fails to respond to heartbeat request + LOG.warn("Driver could not finish the execution on agent " + agentIp + "."); + } + }); + + agents.clear(); + } + + private void finishDistributedExecution() { + this.executorService.shutdownNow(); + // finish tasks + this.heartbeatScheduler.shutdownNow(); + + finishAgents(); + } + + private void handleExecutionRequest(Configuration configuration) { + installToughdayContentPackage(configuration); + mergeDistributedConfigParams(configuration); + + PhaseSplitter phaseSplitter = new PhaseSplitter(); + + for (Phase phase : configuration.getPhases()) { + try { + Map tasks = phaseSplitter.splitPhase(phase, new ArrayList<>(agents)); + this.distributedPhaseMonitor.setPhase(phase); + + for (String agentIp : agents) { + configuration.setPhases(Collections.singletonList(tasks.get(agentIp))); + + // convert configuration to yaml representation + YamlDumpConfigurationAsTaskForTDAgents dumpConfig = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + String yamlTask = dumpConfig.generateConfigurationObject(); + + /* send query to agent and register running task */ + String URI = Agent.getSubmissionTaskPath(agentIp); + HttpResponse response = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, yamlTask, URI, HTTP_REQUEST_RETRIES); + + if (response != null) { + this.distributedPhaseMonitor.registerAgentRunningTD(agentIp); + LOG.info("Task was submitted to agent " + agentIp); + } else { + /* the assumption is that the agent is no longer active in the cluster and he will fail to respond + * to the heartbeat request sent by the driver. This will automatically trigger process of + * redistributing the work + * */ + LOG.info("Task\n" + yamlTask + " could not be submitted to agent " + agentIp + + ". Work will be rebalanced once the agent fails to respond to heartbeat request."); + } + } + + // al execution queries were sent => set phase execution start time + this.distributedPhaseMonitor.setPhaseStartTime(System.currentTimeMillis()); + + // we should wait until all agents complete the current tasks in order to execute phases sequentially + if (!this.distributedPhaseMonitor.waitForPhaseCompletion(3)) { + break; + } + + LOG.info("Phase " + phase.getName() + " finished execution successfully."); + + } catch (CloneNotSupportedException e) { + LOG.error("Phase " + phase.getName() + " could not de divided into tasks to be sent to the agents.", e); + + LOG.info("Finishing agents"); + finishAgents(); + + System.exit(-1); + } + } + + finishDistributedExecution(); + } + + private void scheduleHeartbeatTask() { + // we should periodically send heartbeat messages from driver to all the agents + heartbeatScheduler.scheduleAtFixedRate(new HeartbeatTask(this.agents, this.distributedPhaseMonitor, + this.configuration, this.driverConfiguration), + 0, this.driverConfiguration.getDistributedConfig().getHeartbeatIntervalInSeconds(), TimeUnit.SECONDS); + } + + /** + * Starts the execution of the driver. + */ + public void run() { + /* expose http endpoint for running TD with the given configuration */ + post(EXECUTION_PATH, ((request, response) -> { + String yamlConfiguration = request.body(); + Configuration configuration = new Configuration(yamlConfiguration); + this.configuration = configuration; + + // handle execution in a different thread to be able to quickly respond to this request + this.executorService.submit(() -> handleExecutionRequest(configuration)); + + return ""; + })); + + /* health check http endpoint */ + get(HEALTH_PATH, ((request, response) -> "Healthy")); + + /* http endpoint used by the agent installing the sample content to announce if the installation was + * successful or not. */ + post(SAMPLE_CONTENT_ACK_PATH, ((request, response) -> { + boolean installed = Boolean.parseBoolean(request.body()); + + if (!installed) { + LOG.error("Failed to install the ToughDay sample content package. Execution will be stopped."); + finishDistributedExecution(); + System.exit(-1); + } + + synchronized (object) { + object.notify(); + } + + return ""; + })); + + /* expose http endpoint to allow agents to announce when they finished executing the current phase */ + post(PHASE_FINISHED_BY_AGENT_PATH, ((request, response) -> { + String agentIp = request.body(); + + LOG.info("Agent " + agentIp + " finished executing the current phase."); + this.distributedPhaseMonitor.removeAgentFromActiveTDRunners(agentIp); + + return ""; + })); + + /* expose http endpoint for registering new agents in the cluster */ + post(REGISTER_PATH, (request, response) -> { + String agentIp = request.body(); + + LOG.info("[driver] Registered agent with ip " + agentIp); + if (!this.distributedPhaseMonitor.isPhaseExecuting()) { + agents.add(agentIp); + LOG.info("[driver] active agents " + agents.toString()); + return ""; + } + + this.taskBalancer.scheduleWorkRedistributionProcess(distributedPhaseMonitor, agents, configuration, + driverConfiguration.getDistributedConfig(), agentIp, true); + + return ""; + }); + + scheduleHeartbeatTask(); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructions.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructions.java new file mode 100644 index 0000000..383ad67 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructions.java @@ -0,0 +1,45 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution; + +import java.util.Map; + +/** Contains all the information needed by the agents for updating their configuration when the work needs to be + * rebalanced. + */ +public class RedistributionInstructions { + private Map counts; + private Map runModeProperties; + + // dummy constructor, required for dumping the class + public RedistributionInstructions() { } + + public RedistributionInstructions(Map counts, Map runModeProperties) { + this.counts = counts; + this.runModeProperties = runModeProperties; + } + + /** + * Getter for counts. + */ + public Map getCounts() { + return this.counts; + } + + /** + * Setter for counts. + */ + public void setCounts(Map counts) { + this.counts = counts; + } + + /** + * Getter for run mode properties that must be redistributed. + */ + public Map getRunModeProperties() { return this.runModeProperties; } + + /** + * Setter for run mode properties that must be redistributed. + */ + public void setRunModeProperties(Map runModeProperties) { + this.runModeProperties = runModeProperties; + } +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructionsProcessor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructionsProcessor.java new file mode 100644 index 0000000..7147eff --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructionsProcessor.java @@ -0,0 +1,75 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution; + +import com.adobe.qe.toughday.internal.core.TestSuite; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.ConstantLoadRunModeBalancer; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Class responsible for processing the rebalancing request received by an agent from the driver when the + * work must be rebalanced. + */ +public class RedistributionInstructionsProcessor { + protected static final Logger LOG = LogManager.getLogger(ConstantLoadRunModeBalancer.class); + + private void processTestSuiteChanges(Map counts, Phase phase) { + TestSuite testSuite = phase.getTestSuite(); + long nr = testSuite.getTests().stream() + .filter(test -> !counts.containsKey(test.getName())) + .count(); + if (nr > 0) { + throw new IllegalStateException("Instructions were not received for each test in the test suite."); + } + + testSuite.getTests() + .stream() + .filter(test -> counts.containsKey(test.getName())) + .forEach(test -> { + // remove tests for which the count property was achieved + if (counts.get(test.getName()) == 0) { + phase.getCounts().remove(test); + } else { + // reset number of tests executed so far + phase.getCounts().put(test, new AtomicLong(0)); + // update number of executions left for this test + test.setCount(String.valueOf(counts.get(test.getName()))); + } + }); + } + + private void processRunModeChanges(RedistributionInstructions redistributionInstructions, RunMode runMode) { + runMode.getRunModeBalancer().before(redistributionInstructions, runMode); + + runMode.getRunModeBalancer().processRunModeInstructions(redistributionInstructions, runMode); + + runMode.getRunModeBalancer().after(redistributionInstructions, runMode); + } + + /** + * Method used for processing the rebalance request. + * @param jsonInstructions : redistribution instructions received from the driver + * @param phase : the current phase being executed by the agents. + * @throws IOException : if the instructions don't have the appropriate format. + */ + public void processInstructions(String jsonInstructions, Phase phase) throws IOException { + if (phase == null || jsonInstructions == null) { + throw new IllegalArgumentException("Phase and redistribution instructions must not be null during work " + + "redistribution process."); + } + + ObjectMapper objectMapper = new ObjectMapper(); + RedistributionInstructions redistributionInstructions = + objectMapper.readValue(jsonInstructions, RedistributionInstructions.class); + + // update values for each modified property + processRunModeChanges(redistributionInstructions, phase.getRunMode()); + processTestSuiteChanges(redistributionInstructions.getCounts(), phase); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/TaskBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/TaskBalancer.java new file mode 100644 index 0000000..121b85c --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/TaskBalancer.java @@ -0,0 +1,287 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution; + +import com.adobe.qe.toughday.internal.core.TestSuite; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.DistributedPhaseMonitor; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.DistributedConfig; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.PhaseSplitter; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpResponse; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.*; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; + +/** + * Class responsible for balancing the work between the agents running the cluster + * whenever the number of agents is changing. + */ +public class TaskBalancer { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + private static TaskBalancer instance = null; + + private RedistributionStatus status = RedistributionStatus.UNNECESSARY; + private final PhaseSplitter phaseSplitter = new PhaseSplitter(); + private final HttpUtils httpUtils = new HttpUtils(); + private final ConcurrentLinkedQueue inactiveAgents = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue recentlyAddedAgents = new ConcurrentLinkedQueue<>(); + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + + public enum RedistributionStatus { + UNNECESSARY, // no need to redistribute the work + SCHEDULED, // redistribution is scheduled + RESCHEDULE_REQUIRED, /* nr of agents running in the cluster has changed while the work was redistributed so + * we must reschedule the process */ + EXECUTING // work redistribution process is executing + } + + /** + * Returns a list with all the agents that failed to respond to the heartbeat request sent + * by the driver. + */ + public ConcurrentLinkedQueue getInactiveAgents() { + return this.inactiveAgents; + } + + private TaskBalancer() {} + + /** + * Returns a singleton instance of this class. + */ + public static TaskBalancer getInstance() { + if (instance == null) { + instance = new TaskBalancer(); + } + + return instance; + } + + private Map getTestSuitePropertiesToRedistribute(TestSuite taskTestSuite) { + HashMap map = new HashMap<>(); + taskTestSuite.getTests().forEach(test -> map.put(test.getName(), test.getCount())); + + return map; + } + + private void addNewAgent(String agentIdentifier) { + this.recentlyAddedAgents.add(agentIdentifier); + } + + private void addInactiveAgent(String agentIdentifier) { + this.inactiveAgents.add(agentIdentifier); + } + + private void waitUntilAllAgentsAreReadyForRebalancing(Queue agents) { + boolean ready = false; + + while(!ready) { + ready = true; + + // get status of all new agents + for (String agentIpAddress : agents) { + HttpResponse agentResponse = this.httpUtils.sendHttpRequest(HttpUtils.GET_METHOD, "", + Agent.getGetStatusPath(agentIpAddress), HTTP_REQUEST_RETRIES); + + if (agentResponse != null) { + try { + String response = EntityUtils.toString(agentResponse.getEntity()); + if (!response.equals(Agent.Status.RUNNING.toString())) { + ready = false; + break; + } + } catch (IOException e) { + LOG.warn("Could not confirm that agent " + agentIpAddress + " is ready for processing " + + "redistribution instructions") ; + } + } + } + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // skip this and continue waiting for all agents to be ready for redistribution + } + } + } + + private void sendInstructionsToExistingAgents(Map phases, Queue activeAgents) { + ObjectMapper mapper = new ObjectMapper(); + + // send instructions to agents that are already running TD + activeAgents.stream() + .filter(ipAddress -> !this.inactiveAgents.contains(ipAddress)) // filter agents that become inactive + .forEach(ipAddress -> { + String agentURI = Agent.getRebalancePath(ipAddress); + TestSuite testSuite = phases.get(ipAddress).getTestSuite(); + RunMode runMode = phases.get(ipAddress).getRunMode(); + + // build redistribution instructions + Map testSuiteProperties = getTestSuitePropertiesToRedistribute(testSuite); + Map runModeProperties = runMode.getRunModeBalancer() + .getRunModePropertiesToRedistribute(runMode.getClass(), runMode); + RedistributionInstructions redistributionInstructions = + new RedistributionInstructions(testSuiteProperties, runModeProperties); + try { + String message = mapper.writeValueAsString(redistributionInstructions); + LOG.info("[Redistribution] Sending " + message + " to agent " + ipAddress); + + HttpResponse response = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, message, + agentURI, HTTP_REQUEST_RETRIES); + if (response == null) { + /* redistribution will be automatically triggered as soon as this agent fails to respond to + * heartbeat request + */ + LOG.warn("Redistribution instructions could not be sent to agent " + ipAddress + "."); + } + + } catch (JsonProcessingException e) { + LOG.error("Unexpected exception while sending redistribution instructions", e); + LOG.warn("Agent " + ipAddress + " will continue to run with the configuration it had before " + + "the process of redistribution was triggered."); + } + + }); + } + + private void sendExecutionRequestsToNewAgents(List recentlyAddedAgents, Map phases, + Configuration configuration) { + // for the recently added agents, send execution queries + recentlyAddedAgents.forEach(newAgentIpAddress -> { + configuration.setPhases(Collections.singletonList(phases.get(newAgentIpAddress))); + YamlDumpConfigurationAsTaskForTDAgents dumpConfig = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + String yamlTask = dumpConfig.generateConfigurationObject(); + + LOG.info("Sending execution request + " + yamlTask + " to new agent " + newAgentIpAddress); + HttpResponse response = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, yamlTask, + Agent.getSubmissionTaskPath(newAgentIpAddress), 3); + + if (response == null) { + /* the assumption is that the agent will fail to respond to heartbeat request => work will be + * automatically redistributed when this happens. + */ + LOG.info("Failed to send task to new agent " + newAgentIpAddress + "."); + } + + }); + } + + private void updateStateOfAgents(DistributedPhaseMonitor distributedPhaseMonitor, + Queue activeAgents, List recentlyAddedAgents, + Configuration configuration) throws CloneNotSupportedException { + if (!distributedPhaseMonitor.isPhaseExecuting()) { + return; + } + + Phase phase = distributedPhaseMonitor.getPhase(); + // update number of tests that are left to be executed by the agents + distributedPhaseMonitor.updateCountPerTest(); + Map phases = this.phaseSplitter.splitPhaseForRebalancingWork(phase, new ArrayList<>(activeAgents), + new ArrayList<>(recentlyAddedAgents), distributedPhaseMonitor.getPhaseStartTime()); + // wait until all agents are able to process redistribution instructions + waitUntilAllAgentsAreReadyForRebalancing(activeAgents); + sendInstructionsToExistingAgents(phases, activeAgents); + sendExecutionRequestsToNewAgents(recentlyAddedAgents, phases, configuration); + + } + + private void excludeInactiveAgents(List agentsToBeExcluded, Queue activeAgents, + DistributedPhaseMonitor distributedPhaseMonitor) { + agentsToBeExcluded.forEach(activeAgents::remove); + agentsToBeExcluded.forEach(this.inactiveAgents::remove); + // we should not wait for task completion since the agent running it left the cluster + agentsToBeExcluded.forEach(distributedPhaseMonitor::removeAgentFromActiveTDRunners); + } + + private void after(DistributedPhaseMonitor distributedPhaseMonitor, Queue activeAgents, + List newAgents, Configuration configuration, DistributedConfig distributedConfig) { + distributedPhaseMonitor.resetExecutions(); + // mark recently added agents as active agents executing tasks + activeAgents.addAll(newAgents); + newAgents.forEach(distributedPhaseMonitor::registerAgentRunningTD); + newAgents.forEach(this.recentlyAddedAgents::remove); + LOG.info("[Redistribution] Finished redistributing the work"); + + if (this.status == RedistributionStatus.RESCHEDULE_REQUIRED) { + this.status = RedistributionStatus.SCHEDULED; + LOG.info("[driver] Scheduling delayed redistribution process for agents " + + this.recentlyAddedAgents.toString()); + this.scheduler.schedule(() -> { + LOG.info("[Redistribution] starting delayed work redistribution process"); + rebalanceWork(distributedPhaseMonitor, activeAgents, configuration, distributedConfig); + }, distributedConfig.getRedistributionWaitTimeInSeconds(), TimeUnit.SECONDS); + } else { + this.status = RedistributionStatus.UNNECESSARY; + } + + + } + + private void rebalanceWork(DistributedPhaseMonitor distributedPhaseMonitor, Queue activeAgents, + Configuration configuration, DistributedConfig distributedConfig) { + this.status = RedistributionStatus.EXECUTING; + + List newAgents = new ArrayList<>(recentlyAddedAgents); + List inactiveAgents = new ArrayList<>(this.inactiveAgents); + + // remove all agents who failed answering the heartbeat request in the past + excludeInactiveAgents(inactiveAgents, activeAgents, distributedPhaseMonitor); + LOG.info("[Redistribution] Starting the process...."); + + try { + updateStateOfAgents(distributedPhaseMonitor, activeAgents, newAgents, configuration); + } catch (CloneNotSupportedException e) { + LOG.warn(""); + } + + after(distributedPhaseMonitor, activeAgents, newAgents, configuration, distributedConfig); + } + + /** + * Method used for scheduling the process of redistributing the work between the agents running in the cluster. This + * method is called whenever the number of active agents is changing. + * @param distributedPhaseMonitor : instance monitoring the current phase executed by the agents running TD + * @param activeAgents : agents that are considered to be up and running in the cluster + * @param configuration : TD configuration + * @param distributedConfig : configuration parameters for running TD distributed + * @param agentIdentifier : the identifier of the agent that generated this call + * @param activeAgent : true when the agent represented by agentIdentifier has joined the cluster; false if the + * agent has become inactive(he failed to respond to heartbeat request) + */ + public void scheduleWorkRedistributionProcess(DistributedPhaseMonitor distributedPhaseMonitor, + Queue activeAgents, Configuration configuration, + DistributedConfig distributedConfig, String agentIdentifier, + boolean activeAgent) { + + if (activeAgent) { + this.addNewAgent(agentIdentifier); + } else { + this.addInactiveAgent(agentIdentifier); + } + + if (this.status == RedistributionStatus.UNNECESSARY) { + this.status = RedistributionStatus.SCHEDULED; + LOG.info("Scheduling redistribution process to start in " + + distributedConfig.getRedistributionWaitTimeInSeconds() + " seconds."); + + // schedule work redistribution process + this.scheduler.schedule(() -> rebalanceWork(distributedPhaseMonitor, activeAgents,configuration,distributedConfig), + distributedConfig.getRedistributionWaitTimeInSeconds(), TimeUnit.SECONDS); + } else if (this.status == RedistributionStatus.EXECUTING) { + LOG.info("Work redistribution process must be rescheduled."); + this.status = RedistributionStatus.RESCHEDULE_REQUIRED; + } + + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/AbstractRunModeBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/AbstractRunModeBalancer.java new file mode 100644 index 0000000..b6dd2a0 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/AbstractRunModeBalancer.java @@ -0,0 +1,71 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes; + +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; +import com.adobe.qe.toughday.api.annotations.ConfigArgSet; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public abstract class AbstractRunModeBalancer implements RunModeBalancer { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + @Override + public Map getRunModePropertiesToRedistribute(Class type, T object) { + final Map properties = new HashMap<>(); + + if (object == null) { + throw new IllegalArgumentException("Run mode object must not be null."); + } + + Arrays.stream(type.getDeclaredMethods()) + .filter(method -> method.isAnnotationPresent(ConfigArgGet.class)) + .filter(method -> method.getAnnotation(ConfigArgGet.class).redistribute()) + .forEach(method -> { + String propertyName = Configuration.propertyFromMethod(method.getName()); + try { + Object value = method.invoke(object); + properties.put(propertyName, String.valueOf(value)); + } catch (IllegalAccessException | InvocationTargetException e) { + LOG.warn("Property " + propertyName + " could not be collected for redistribution instructions." + + " Received error " + e.getMessage()); + } + }); + + return properties; + } + + @Override + public void processRunModeInstructions(RedistributionInstructions redistributionInstructions, T runMode) { + if (redistributionInstructions == null || runMode == null) { + throw new IllegalArgumentException("Rebalance instructions and run mode must not be null."); + } + + Map runModeProperties = redistributionInstructions.getRunModeProperties(); + if (runModeProperties == null || runModeProperties.isEmpty()) { + return; + } + + Arrays.stream(runMode.getClass().getDeclaredMethods()) + .filter(method -> runModeProperties.containsKey(Configuration.propertyFromMethod(method.getName()))) + .filter(method -> method.isAnnotationPresent(ConfigArgSet.class)) + .forEach(method -> { + String property = Configuration.propertyFromMethod(method.getName()); + LOG.info("[Agent] Setting property " + property + " to " + runModeProperties.get(property)); + + try { + method.invoke(runMode, runModeProperties.get(property)); + } catch (IllegalAccessException | InvocationTargetException e) { + LOG.warn("[Agent] Property " + property + " could not be set to " + runModeProperties.get(property)); + } + + }); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/ConstantLoadRunModeBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/ConstantLoadRunModeBalancer.java new file mode 100644 index 0000000..48b2a6d --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/ConstantLoadRunModeBalancer.java @@ -0,0 +1,76 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes; + +import com.adobe.qe.toughday.internal.core.engine.runmodes.ConstantLoad; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; + + +public class ConstantLoadRunModeBalancer extends AbstractRunModeBalancer { + protected static final Logger LOG = LogManager.getLogger(ConstantLoadRunModeBalancer.class); + private static final String CURRENT_LOAD_PROPERTY = "currentload"; + + @Override + public Map getRunModePropertiesToRedistribute(Class type, ConstantLoad runMode) { + Map runModeProps = super.getRunModePropertiesToRedistribute(type, runMode); + + if (runMode.isVariableLoad()) { + runModeProps.put(CURRENT_LOAD_PROPERTY, String.valueOf(runMode.getCurrentLoad())); + } + + return runModeProps; + } + + @Override + public void before(RedistributionInstructions redistributionInstructions, ConstantLoad runMode) { + if (runMode.isVariableLoad()) { + // We must cancel the scheduled task and reschedule it with the new values for 'period' and initial delay. + boolean cancelled = runMode.cancelPeriodicTask(); + if (!cancelled) { + LOG.warn("[Driver] Periodic task for updating the load could not be cancelled. Run mode properties " + + "might not be respected."); + } else { + LOG.info("[Driver] Periodic task for updating the load was cancelled."); + } + } + + Map runModeProperties = redistributionInstructions.getRunModeProperties(); + runModeProperties.forEach((property, propValue) -> + beforeChangingValueOfProperty(property, propValue, runMode)); + } + + private void beforeChangingValueOfProperty(String property, String newValue, ConstantLoad runMode) { + if (property.equals("load") && !runMode.isVariableLoad()) { + long newLoad = Long.parseLong(newValue); + long difference = runMode.getLoad() - newLoad; + + if (difference > 0 ) { + runMode.removeRunMaps(difference); + } else { + runMode.addRunMaps(Math.abs(difference)); + } + } + } + + @Override + public void processRunModeInstructions(RedistributionInstructions redistributionInstructions, ConstantLoad runMode) { + super.processRunModeInstructions(redistributionInstructions, runMode); + + Map runModeProps = redistributionInstructions.getRunModeProperties(); + if (runModeProps.containsKey(CURRENT_LOAD_PROPERTY)) { + LOG.info("[Driver[ Setting current load to " + runModeProps.get(CURRENT_LOAD_PROPERTY)); + runMode.setCurrentLoad(Integer.parseInt(runModeProps.get(CURRENT_LOAD_PROPERTY))); + } + } + + @Override + public void after(RedistributionInstructions redistributionInstructions, ConstantLoad runMode) { + if (runMode.isVariableLoad()) { + runMode.schedulePeriodicTask(); + LOG.info("Periodic task for updating the load was rescheduled with interval " + runMode.getInterval() + + " and initial delay " + runMode.getInitialDelay()); + } + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/NormalRunModeBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/NormalRunModeBalancer.java new file mode 100644 index 0000000..d2584a9 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/NormalRunModeBalancer.java @@ -0,0 +1,88 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes; + +import com.adobe.qe.toughday.internal.core.engine.AsyncEngineWorker; +import com.adobe.qe.toughday.internal.core.engine.AsyncTestWorker; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.runmodes.Normal; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class NormalRunModeBalancer extends AbstractRunModeBalancer { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + @Override + public void before(RedistributionInstructions redistributionInstructions, Normal runMode) { + if (runMode.isVariableConcurrency()) { + /* We must cancel the scheduled task for updating concurrency and reschedule it with the new values + * for interval and initial delay. + */ + boolean cancelled = runMode.cancelPeriodicTask(); + if (!cancelled) { + LOG.warn("Periodic task for updating the number of threads could not be cancelled. Normal run mode" + + " properties might not be respected."); + } + } + + Map runModeProperties = redistributionInstructions.getRunModeProperties(); + runModeProperties.forEach((propertyName, propValue) -> this.beforeChangingValueOfProperty(propertyName, propValue, runMode)); + } + + private void beforeChangingValueOfProperty(String property, String newValue, Normal runMode) { + if (property.equals("concurrency")) { + long currentConcurrency = runMode.isVariableConcurrency() ? runMode.getActiveThreads() : + runMode.getConcurrency(); + long newConcurrency = Long.parseLong(newValue); + long difference = currentConcurrency - newConcurrency; + + if (difference > 0) { + reduceConcurrency(difference, runMode); + } else { + increaseConcurrency(Math.abs(difference), runMode); + } + + concurrencySanityChecks(runMode, newConcurrency); + } + } + + private void concurrencySanityChecks(Normal runMode, long newConcurrency) { + // check that we have the exact number of active test workers + if (runMode.getRunContext().getTestWorkers().size() != newConcurrency) { + throw new IllegalStateException("[redistribution] TestWorkers size is " + + runMode.getRunContext().getTestWorkers().size() + " but" + " new value for concurrency is " + newConcurrency); + } + + // check that all test workers are active + if (runMode.getRunContext().getTestWorkers().stream().anyMatch(AsyncEngineWorker::isFinished)) { + throw new IllegalStateException("[redistribution] " + + "There are finished test workers in the list of active workers."); + } + } + + private void reduceConcurrency(long reduce, Normal runMode) { + List workerList = new ArrayList<>(runMode.getRunContext().getTestWorkers()); + + for (int i = 0; i < reduce; i++) { + LOG.info("[Agent - redistribution] Finished test worker " + workerList.get(i).getWorkerThread().getId()); + runMode.finishAndDeleteWorker(workerList.get(i)); + } + } + + private void increaseConcurrency(long increase, Normal runMode) { + for (int i = 0; i < increase; i++) { + runMode.createAndExecuteWorker(runMode.getEngine(), runMode.getEngine().getCurrentPhase().getTestSuite()); + } + } + + @Override + public void after(RedistributionInstructions redistributionInstructions, Normal runMode) { + if (runMode.isVariableConcurrency()) { + runMode.schedulePeriodicTask(); + LOG.info("Task responsible for updating the number threads used to run tests was rescheduled"); + } + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/RunModeBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/RunModeBalancer.java new file mode 100644 index 0000000..1a79a49 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/RunModeBalancer.java @@ -0,0 +1,47 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes; + +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; + +import java.util.Map; + +/** + * Contains all the methods that should be implemented in order to describe how a certain + * run mode must be redistributed when the number of agents running in the cluster modifies. + * @param : type of the run mode + */ +public interface RunModeBalancer { + /** + * Collects all the properties that must be taken into consideration when redistributing + * the work between the agents. + * @param type Class of the run mode + * @param runMode + * @return + */ + Map getRunModePropertiesToRedistribute(Class type, T runMode); + + /** + * This method is called before changing the actual value of the properties to be + * modified. It should be used for creating the appropriate environment for running + * with the run mode configuration. + * @param redistributionInstructions instructions for redistributing the work + * @param runMode + */ + void before(RedistributionInstructions redistributionInstructions, T runMode); + + /** + * This method is responsible for changing the values of the properties that must be + * redistributed according to the redistribution instructions received as a parameter. + * @param redistributionInstructions instructions for redistributing the work + * @param runMode + */ + void processRunModeInstructions(RedistributionInstructions redistributionInstructions, T runMode); + + /** + * The last method called when the process of redistributing the work is + * executing. + * @param redistributionInstructions instructions for redistributing the work + * @param runMode + */ + void after(RedistributionInstructions redistributionInstructions, T runMode); +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/PhaseSplitter.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/PhaseSplitter.java new file mode 100644 index 0000000..1bb1ed7 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/PhaseSplitter.java @@ -0,0 +1,108 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.splitters; + +import com.adobe.qe.toughday.internal.core.TestSuite; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.engine.RunMode; + +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Class responsible for partitioning phases into multiple phases to be distributed between the agents running in + * the cluster. + */ +public class PhaseSplitter { + + private Map distributeTestSuite(TestSuite initialTestSuite, List agents) + throws CloneNotSupportedException { + Map taskTestSuites = new HashMap<>(); + + for (int i = 0; i < agents.size(); i++) { + TestSuite clone = initialTestSuite.clone(); + + clone.getTests().forEach(test -> { + if (taskTestSuites.isEmpty()) { + test.setCount(String.valueOf(test.getCount() / agents.size() + test.getCount() % agents.size())); + } else { + test.setCount(String.valueOf(test.getCount() / agents.size())); + } + }); + + taskTestSuites.put(agents.get(i), clone); + } + + return taskTestSuites; + } + + private void sanityChecks(Phase phase, List agents) { + if (phase == null || agents == null) { + throw new IllegalArgumentException("Phase/List of agents must not be null"); + } + + if (agents.isEmpty()) { + throw new IllegalStateException("At least one agent must be running in the cluster."); + } + + } + + private Map mapPhaseToAgent(Phase phase,Map partitionRunModes, + Map partitionTestSuites, List agents) + throws CloneNotSupportedException { + Map phasePerAgent = new HashMap<>(); + + for (String agent : agents) { + Phase taskPhase = (Phase) phase.clone(); + TestSuite taskTestSuite = partitionTestSuites.get(agent); + + // set the count (the number of executions since the beginning of the run) of each test to 0 + taskTestSuite.getTests().forEach(test -> taskPhase.getCounts().put(test, new AtomicLong(0))); + + taskPhase.setTestSuite(taskTestSuite); + taskPhase.setRunMode(partitionRunModes.get(agent)); + + phasePerAgent.put(agent, taskPhase); + } + + return phasePerAgent; + } + + /** + * Knows how to divide a phase into a number of phases equal to the number of agents running in the cluster. + * @param phase the phase to be partitioned. + * @param agents list with all the agents able to receive a task and to execute it. + */ + public Map splitPhase(Phase phase, List agents) throws CloneNotSupportedException { + sanityChecks(phase, agents); + + Map partitionRunModes = phase.getRunMode().getRunModeSplitter().distributeRunMode(phase.getRunMode(), agents); + Map partitionTestSuites = distributeTestSuite(phase.getTestSuite(), agents); + + return mapPhaseToAgent(phase, partitionRunModes, partitionTestSuites, agents); + } + + /** + * Knows how to divide a phase into a number of phases equal to the number of agents running in the cluster taking + * into consideration the agents that recently joined the cluster and triggered the work redistribution process. + * @param phase : the phase to be partitioned + * @param existingAgents : agents that were running TD before redistributing the work + * @param newAgents : agents that recently joined the cluster + * @param phaseStartTime : execution start time of the phase + * @return : a map having as key the IP address of an Agent running in the cluster and the Phase to be executed by + * this agent as value + */ + public Map splitPhaseForRebalancingWork(Phase phase, List existingAgents, + List newAgents, long phaseStartTime) + throws CloneNotSupportedException { + sanityChecks(phase, existingAgents); + + List allAgents = new ArrayList<>(existingAgents); + allAgents.addAll(newAgents); + + Map partitionRunModes = + phase.getRunMode().getRunModeSplitter().distributeRunModeForRebalancingWork(phase.getRunMode(), existingAgents, + newAgents, phaseStartTime); + Map partitionTestSuites = distributeTestSuite(phase.getTestSuite(), allAgents); + + return mapPhaseToAgent(phase, partitionRunModes, partitionTestSuites, allAgents); + } +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/ConstantLoadRunModeSplitter.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/ConstantLoadRunModeSplitter.java new file mode 100644 index 0000000..ba95c53 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/ConstantLoadRunModeSplitter.java @@ -0,0 +1,106 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes; + +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.runmodes.ConstantLoad; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Knows how to split the constant load run mode into multiple run modes to be distributed to the agents + * running in the cluster. + */ +public class ConstantLoadRunModeSplitter implements RunModeSplitter { + + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private ConstantLoad setParamsForDistributedRunMode(ConstantLoad runMode, int nrAgents, int rateRemainder, + int startRemainder, int endRemainder, + int loadRemainder, int agentId) throws CloneNotSupportedException { + ConstantLoad clone = (ConstantLoad) runMode.clone(); + + + if (runMode.isVariableLoad()) { + if (runMode.getRate() >= nrAgents) { + clone.setStart(String.valueOf(runMode.getStart()/ nrAgents + startRemainder)); + clone.setEnd(String.valueOf(runMode.getEnd()/ nrAgents + endRemainder)); + clone.setRate(String.valueOf(runMode.getRate()/ nrAgents + rateRemainder)); + } else { + clone.setInitialDelay(agentId * GlobalArgs.parseDurationToSeconds(runMode.getInterval()) * 1000); + clone.setOneAgentRate(runMode.getRate()); + clone.setStart(String.valueOf(runMode.getStart()+ agentId * runMode.getRate())); + clone.setRate(String.valueOf(nrAgents * runMode.getRate())); + long interval = GlobalArgs.parseDurationToSeconds(runMode.getInterval()); + clone.setInterval(String.valueOf(interval * nrAgents) + 's'); + } + + return clone; + } + + /* we must distribute the load between the agents */ + clone.setLoad(String.valueOf(runMode.getLoad() / nrAgents + loadRemainder)); + + return clone; + } + + @Override + public Map distributeRunMode(ConstantLoad runMode, List agents) + throws CloneNotSupportedException { + int nrAgents = agents.size(); + Map runModes = new HashMap<>(); + + ConstantLoad firstTask = setParamsForDistributedRunMode(runMode, nrAgents, runMode.getRate() % nrAgents, + runMode.getStart() % nrAgents, runMode.getEnd() % nrAgents, runMode.getLoad() % nrAgents, 0); + runModes.put(agents.get(0), firstTask); + + for (int i = 1; i < nrAgents; i++) { + ConstantLoad task = setParamsForDistributedRunMode(runMode, nrAgents, 0, 0, 0, 0, i); + runModes.put(agents.get(i), task); + } + + return runModes; + } + + @Override + public Map distributeRunModeForRebalancingWork(ConstantLoad runMode, List existingAgents, + List newAgents, long phaseStartTime) throws CloneNotSupportedException { + List agents = new ArrayList<>(existingAgents); + agents.addAll(newAgents); + + Map taskRunModes = distributeRunMode(runMode, agents); + newAgents.forEach(agentName -> taskRunModes.get(agentName).setStart("0")); + + if (!runMode.isVariableLoad()) { + return taskRunModes; + } + + // compute the current load to determine the new values for start/current load + long endTime = System.currentTimeMillis(); + long diff = (endTime - phaseStartTime) / 1000; // convert to seconds + int estimatedCurrentLoad = ((int)(diff / GlobalArgs.parseDurationToSeconds(runMode.getInterval()))) + * runMode.getRate() + runMode.getStart(); + + LOG.info("Phase was executed for " + diff + " seconds"); + LOG.info("Estimated current load " + estimatedCurrentLoad); + + // set start property for new agents + newAgents.forEach(agentName -> taskRunModes.get(agentName) + .setStart(String.valueOf(estimatedCurrentLoad / agents.size()))); + + // set current load for old agents + taskRunModes.get(existingAgents.get(0)).setCurrentLoad(estimatedCurrentLoad / agents.size() + + estimatedCurrentLoad % agents.size()); + for (int i = 1; i < existingAgents.size(); i++) { + taskRunModes.get(existingAgents.get(i)).setCurrentLoad(estimatedCurrentLoad / agents.size()); + } + + return taskRunModes; + + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/NormalRunModeSplitter.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/NormalRunModeSplitter.java new file mode 100644 index 0000000..67eac1e --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/NormalRunModeSplitter.java @@ -0,0 +1,113 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes; + +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.runmodes.Normal; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Knows how to split the normal run mode into multiple normal run modes to be distributed to the agents + * running in the cluster. + */ +public class NormalRunModeSplitter implements RunModeSplitter { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private Normal setParamsForDistributedRunMode(Normal runMode, int nrAgents, int rateRemainder, int endRemainder, + int startRemainder, int concurrencyRemainder, int agentId) throws CloneNotSupportedException { + Normal clone = (Normal) runMode.clone(); + + + if (runMode.isVariableConcurrency()) { + if (runMode.getRate() >= nrAgents) { + clone.setRate(String.valueOf(runMode.getRate() / nrAgents + rateRemainder)); + clone.setStart(String.valueOf(runMode.getStart() / nrAgents + startRemainder)); + clone.setEnd(String.valueOf(runMode.getEnd() / nrAgents + endRemainder)); + } else { + clone.setInitialDelay(agentId * GlobalArgs.parseDurationToSeconds(runMode.getInterval()) * 1000); + long interval = GlobalArgs.parseDurationToSeconds(runMode.getInterval()); + clone.setInterval(String.valueOf(interval * nrAgents) + 's'); + + if (agentId > 0) { + clone.setStart("0"); + clone.setEnd(String.valueOf((runMode.getEnd() - runMode.getStart()) / nrAgents)); + } else { + long diff = runMode.getEnd() - runMode.getStart(); + clone.setEnd(String.valueOf(diff / nrAgents + diff % nrAgents + runMode.getStart())); + } + } + } else { + clone.setConcurrency(String.valueOf(runMode.getConcurrency()/ nrAgents + concurrencyRemainder)); + } + + return clone; + } + + + @Override + public Map distributeRunMode(Normal runMode, List agents) throws CloneNotSupportedException { + if (runMode == null) { + throw new IllegalArgumentException("Run mode must not be null."); + } + + Map taskRunModes = new HashMap<>(); + int nrAgents = agents.size(); + + Normal firstTask = setParamsForDistributedRunMode(runMode, nrAgents, runMode.getRate() % nrAgents, runMode.getEnd() % nrAgents, + runMode.getStart() % nrAgents, runMode.getConcurrency() % nrAgents, 0); + taskRunModes.put(agents.get(0), firstTask); + + for (int i = 1; i < nrAgents; i++) { + Normal task = setParamsForDistributedRunMode(runMode, nrAgents, 0, 0, 0, 0, i); + taskRunModes.put(agents.get(i), task); + } + + return taskRunModes; + } + + @Override + public Map distributeRunModeForRebalancingWork(Normal runMode, List existingAgents, + List newAgents, long phaseStartTime) throws CloneNotSupportedException { + if (runMode == null) { + throw new IllegalArgumentException("Run mode must not be null."); + } + + List agents = new ArrayList<>(existingAgents); + agents.addAll(newAgents); + Map taskRunModes = distributeRunMode(runMode, agents); + + // set start property to 'rate' for each new agent + if (!runMode.isVariableConcurrency()) { + return taskRunModes; + } + + // compute the expected current concurrency + long endTime = System.currentTimeMillis(); + long diff = (endTime - phaseStartTime) / 1000; + int estimatedConcurrency = ((int)(diff / GlobalArgs.parseDurationToSeconds(runMode.getInterval()))) + * runMode.getRate() + runMode.getStart(); + + LOG.info("Phase was executed for " + diff + " seconds"); + LOG.info("Estimated concurrency " + estimatedConcurrency); + + // set start property for new agents + newAgents.forEach(agentName -> + taskRunModes.get(agentName).setStart(String.valueOf(estimatedConcurrency / agents.size()))); + + // set desired active threads for old agents + taskRunModes.get(existingAgents.get(0)).setConcurrency(String.valueOf(estimatedConcurrency / agents.size() + + estimatedConcurrency % agents.size())); + for (int i = 1; i < existingAgents.size(); i++) { + taskRunModes.get(existingAgents.get(i)).setConcurrency(String.valueOf(estimatedConcurrency / agents.size())); + } + + return taskRunModes; + + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/RunModeSplitter.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/RunModeSplitter.java new file mode 100644 index 0000000..95015af --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/RunModeSplitter.java @@ -0,0 +1,31 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes; + +import com.adobe.qe.toughday.internal.core.engine.RunMode; + +import java.util.List; +import java.util.Map; + +/** + * Common interface for all run mode splitters. Implement this interface to specify how a certain type of run mode + * must be partitioned into multiple run modes to be distributed to the agents running in the cluster. + * @param Type of the run mode to be partitioned + */ +public interface RunModeSplitter { + /** + * Used for distributing the run mode when before sending TD execution requests to the agents + * running in the cluster. + * @param runMode object to be partitioned + * @param agents names of the agents waiting to receive TD execution requests from the driver. + */ + Map distributeRunMode(T runMode, List agents) throws CloneNotSupportedException; + + /** + * Used for redistributing the run mode when the process of rebalancing the work is triggered. + * @param runMode object to be partitioned + * @param existingAgents list of agents that were running TD before starting the work redistribution process. + * @param newAgents list of agents which recently joined the cluster + * @param phaseStartTime : time when the phase execution started + */ + Map distributeRunModeForRebalancingWork(T runMode, List existingAgents, List newAgents, + long phaseStartTime) throws CloneNotSupportedException; +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/HeartbeatTask.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/HeartbeatTask.java new file mode 100644 index 0000000..c931145 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/HeartbeatTask.java @@ -0,0 +1,94 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.tasks; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.DistributedPhaseMonitor; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.TaskBalancer; +import com.google.gson.Gson; +import org.apache.http.HttpResponse; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.*; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; + +/** + * Class used for implementing the heartbeat protocol for monitoring the state of the agents running in the cluster. + */ +public class HeartbeatTask implements Runnable { + protected static final Logger LOG = LogManager.getLogger(Driver.class); + + private final Queue agents; + private DistributedPhaseMonitor distributedPhaseMonitor; + private Configuration configuration; + private Configuration driverConfiguration; + private final HttpUtils httpUtils = new HttpUtils(); + private final TaskBalancer taskBalancer = TaskBalancer.getInstance(); + + public HeartbeatTask(Queue agents, DistributedPhaseMonitor distributedPhaseMonitor, + Configuration configuration, Configuration driverConfiguration) { + this.agents = agents; + this.distributedPhaseMonitor = distributedPhaseMonitor; + this.configuration = configuration; + this.driverConfiguration = driverConfiguration; + } + + private void processHeartbeatResponse(String agentIp, HttpResponse agentResponse) throws IOException { + // the agent has sent his statistic for executions/test => aggregate counts + Gson gson = new Gson(); + String yamlCounts = EntityUtils.toString(agentResponse.getEntity()); + + // gson treats numbers as double values by default + Map doubleAgentCounts = gson.fromJson(yamlCounts, Map.class); + LOG.info("[Heartbeat] Received execution state from agent " + agentIp + ": " + doubleAgentCounts.toString()); + + // recently added agents might not execute tests yet + if (doubleAgentCounts.isEmpty()) { + return; + } + + this.distributedPhaseMonitor.getExecutions().forEach((testName, executionsPerAgent) -> + executionsPerAgent.put(agentIp, doubleAgentCounts.get(testName).longValue())); + } + + /** + * Method used for sending heartbeat messages to all the agents running in the cluster. This method should be + * called periodically during the entire distributed execution of ToughDay. + */ + @Override + public void run() { + List activeAgents = new ArrayList<>(agents); + // remove agents which previously failed to respond to heartbeat request + this.taskBalancer.getInactiveAgents().forEach(activeAgents::remove); + + for (String agentIp : activeAgents) { + String URI = Agent.getHeartbeatPath(agentIp); + HttpResponse agentResponse = httpUtils.sendHttpRequest(HttpUtils.GET_METHOD, "", URI, HTTP_REQUEST_RETRIES); + if (agentResponse != null) { + try { + processHeartbeatResponse(agentIp, agentResponse); + } catch (IOException e) { + // the system may reach the limit of available IO resources + LOG.warn("Could not process heartbeat information received form agent " + agentIp); + } + continue; + } + + if (!this.distributedPhaseMonitor.isPhaseExecuting()) { + agents.remove(agentIp); + continue; + } + + this.taskBalancer.scheduleWorkRedistributionProcess(this.distributedPhaseMonitor, this.agents, + this.configuration, this.driverConfiguration.getDistributedConfig(), + agentIp, false); + } + + LOG.info("Number of executions per test: " + this.distributedPhaseMonitor.getExecutionsPerTest()); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/AsyncTimeoutChecker.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/AsyncTimeoutChecker.java index bda8e69..0b867a2 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/AsyncTimeoutChecker.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/AsyncTimeoutChecker.java @@ -44,7 +44,7 @@ private void interruptWorkerIfTimeout(AsyncTestWorker worker) { return; Long testTimeout = currentTest.getTimeout(); - long timeout = testTimeout >= 0 ? testTimeout : engine.getGlobalArgs().getTimeout(); + long timeout = testTimeout >= 0 ? testTimeout : engine.getGlobalArgs().getTimeoutInSeconds(); if (!worker.getMutex().tryLock()) { /* nothing to interrupt. if the test was running diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Engine.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Engine.java index bcdda9b..d5084f3 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Engine.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Engine.java @@ -23,6 +23,7 @@ import com.adobe.qe.toughday.api.annotations.ConfigArgGet; import com.adobe.qe.toughday.internal.core.config.Configuration; import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; import com.adobe.qe.toughday.metrics.Metric; import com.adobe.qe.toughday.tests.sequential.AEMTestBase; import com.adobe.qe.toughday.tests.utils.PackageManagerClient; @@ -370,6 +371,7 @@ public void run() { } finally { currentPhaseLock.readLock().unlock(); } + Engine.logGlobal("Test execution finished at: " + Engine.getCurrentDateTime()); LogManager.shutdown(); } @@ -411,7 +413,7 @@ public void run() { } RunMode currentRunmode = phase.getRunMode(); - Long currentDuration = phase.getDuration(); + Long currentDuration = GlobalArgs.parseDurationToSeconds(phase.getDuration()); currentRunmode.runTests(this); long start = System.currentTimeMillis(); @@ -423,6 +425,10 @@ public void run() { LOG.info("Phase interrupted."); long elapsed = System.currentTimeMillis() - start; + if (configuration.getDistributedConfig().getAgent()) { + Agent.announcePhaseCompletion(); + } + // if the phase finishes sooner than its duration, // the remainder is split equally between the remaining phases // whose duration was not given as input @@ -532,7 +538,11 @@ protected void shutdownAndAwaitTermination(ExecutorService pool) { } public Phase getCurrentPhase() { - return currentPhase; + currentPhaseLock.readLock().lock(); + Phase phase = currentPhase; + currentPhaseLock.readLock().unlock(); + + return phase; } public ReadWriteLock getCurrentPhaseLock() { diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Phase.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Phase.java index 53f8385..a0d9a1c 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Phase.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Phase.java @@ -24,7 +24,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicLong; -public class Phase { +public class Phase implements Cloneable { private static final String DEFAULT_MEASURABILITY = "true"; private String name; @@ -39,6 +39,15 @@ public class Phase { private Map metrics = new LinkedHashMap<>(); private Map counts = new HashMap<>(); + /** + * Creates shallow copy. + * @throws CloneNotSupportedException if the object to be cloned does not implement the Cloneable interface. + */ + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } + @ConfigArgGet public String getName() { return name; @@ -71,8 +80,8 @@ public void setUseconfig(String useconfig) { } @ConfigArgGet - public Long getDuration() { - return duration; + public String getDuration() { + return duration == null ? null : String.valueOf(duration) + 's'; } @ConfigArgSet(required = false, desc = "The duration of the current phase.") diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/RunMode.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/RunMode.java index d29f263..3450a99 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/RunMode.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/RunMode.java @@ -12,6 +12,8 @@ package com.adobe.qe.toughday.internal.core.engine; import com.adobe.qe.toughday.api.core.RunMap; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; import java.util.Collection; import java.util.concurrent.ExecutorService; @@ -22,6 +24,19 @@ public interface RunMode { ExecutorService getExecutorService(); RunContext getRunContext(); + /** + * Returns the RunModeSplitter instance which knows how to split the run mode for running ToughDay distributed. + * @param type of the run mode. + */ + RunModeSplitter getRunModeSplitter(); + + /** + * Returns the RunModeBalancer instance which knows how to process the redistribution instructions received from + * the driver when the number of agents running in the cluster modifies and the work must be redistributed. + * @param type of the run node. + */ + RunModeBalancer getRunModeBalancer(); + interface RunContext { Collection getTestWorkers(); Collection getRunMaps(); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/ConstantLoad.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/ConstantLoad.java index 36318d5..f2df205 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/ConstantLoad.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/ConstantLoad.java @@ -18,21 +18,26 @@ import com.adobe.qe.toughday.api.annotations.ConfigArgGet; import com.adobe.qe.toughday.api.annotations.ConfigArgSet; import com.adobe.qe.toughday.internal.core.TestSuite; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.ConstantLoadRunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; import com.adobe.qe.toughday.internal.core.engine.*; import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.ConstantLoadRunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.ConstantLoadRunModeSplitter; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; import org.apache.commons.lang3.mutable.MutableLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; @Description(desc = "Generates a constant load of test executions, regardless of their execution time.") -public class ConstantLoad implements RunMode { +public class ConstantLoad implements RunMode, Cloneable { private static final Logger LOG = LoggerFactory.getLogger(ConstantLoad.class); private static final String DEFAULT_LOAD_STRING = "50"; @@ -52,22 +57,32 @@ public class ConstantLoad implements RunMode { private long interval = DEFAULT_INTERVAL; private int rate; private int currentLoad; + //private int currentLoad; + private long initialDelay = 0; + /* field used for checking the finish condition when running TD distributed with the rate + * smaller than number of agents in the cluster. */ + private int oneAgentRate = 0; + private ScheduledExecutorService runRoundScheduler = Executors.newScheduledThreadPool(1); private TestCache testCache; private Phase phase; + private RunModeSplitter runModeSplitter = new ConstantLoadRunModeSplitter(); + private ScheduledFuture scheduledFuture = null; + private final ConstantLoadRunModeBalancer runModeBalancer = new ConstantLoadRunModeBalancer(); private Boolean measurable = true; - @ConfigArgSet(required = false, defaultValue = DEFAULT_LOAD_STRING, desc = "Set the load, in requests per second for the \"constantload\" runmode.") + @ConfigArgSet(required = false, defaultValue = DEFAULT_LOAD_STRING, + desc = "Set the load, in requests per second for the \"constantload\" runmodes.") public void setLoad(String load) { checkNotNegative(Long.parseLong(load), "load"); this.load = Integer.parseInt(load); } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getLoad() { return this.load; } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getStart() { return start; } @@ -81,12 +96,13 @@ public void setStart(String start) { this.start = Integer.valueOf(start); } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getRate() { return rate; } - @ConfigArgSet(required = false, desc = "The increase in load per time unit. When it equals -1, it means it is not set.", defaultValue = "-1") + @ConfigArgSet(required = false, desc = "The increase in load per time unit. When it equals -1, it means it is not set.", + defaultValue = "-1") public void setRate(String rate) { if (!rate.equals("-1")) { checkNotNegative(Long.parseLong(rate), "rate"); @@ -94,17 +110,18 @@ public void setRate(String rate) { this.rate = Integer.valueOf(rate); } - @ConfigArgGet - public long getInterval() { - return interval; + @ConfigArgGet(redistribute = true) + public String getInterval() { + return String.valueOf(interval / 1000) + 's'; } - @ConfigArgSet(required = false, desc = "Used with rate to specify the time interval to add increase the load.", defaultValue = DEFAULT_INTERVAL_STRING) + @ConfigArgSet(required = false, desc = "Used with rate to specify the time interval to add increase the load.", + defaultValue = DEFAULT_INTERVAL_STRING) public void setInterval(String interval) { - this.interval = GlobalArgs.parseDurationToSeconds(interval); + this.interval = GlobalArgs.parseDurationToSeconds(interval) * 1000; } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getEnd() { return end; } @@ -117,6 +134,31 @@ public void setEnd(String end) { this.end = Integer.valueOf(end); } + public ConstantLoad() { + /* this is required when running TD distributed because scheduled task might be cancelled and + * rescheduled when rebalancing the work between the agents. + */ + ScheduledThreadPoolExecutor scheduledPoolExecutor = (ScheduledThreadPoolExecutor) runRoundScheduler; + scheduledPoolExecutor.setRemoveOnCancelPolicy(true); + } + + public int getOneAgentRate() { + return this.oneAgentRate; + } + + public void setCurrentLoad(int currentLoad) { + this.currentLoad = currentLoad; + } + + public int getCurrentLoad() { + return this.currentLoad; + } + + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } + private static class TestCache { public Map> cache = new HashMap<>(); @@ -135,7 +177,7 @@ public void add(@NotNull AbstractTest test) { } } - private boolean isVariableLoad() { + public boolean isVariableLoad() { return start != -1 && end != -1; } @@ -155,6 +197,22 @@ private void checkInvalidArgs() { } } + public void addRunMaps(long nr) { + for (long i = 0; i < nr; i++) { + synchronized (runMaps) { + runMaps.add(phase.getPublishMode().getRunMap().newInstance()); + } + } + } + + public void removeRunMaps(long nr) { + for (long i = 0; i < nr; i++) { + synchronized (runMaps) { + this.runMaps.remove(0); + } + } + } + @Override public void runTests(Engine engine) { checkInvalidArgs(); @@ -169,11 +227,7 @@ public void runTests(Engine engine) { load = Math.max(start, end); } - for(int i = 0; i < load; i++) { - synchronized (runMaps) { - runMaps.add(phase.getPublishMode().getRunMap().newInstance()); - } - } + addRunMaps(load); this.scheduler = new AsyncTestWorkerScheduler(engine); executorService.execute(scheduler); @@ -193,11 +247,29 @@ public Collection getRunMaps() { @Override public boolean isRunFinished() { - return scheduler.isFinished(); + return scheduler != null && scheduler.isFinished(); } }; } + @Override + public RunModeBalancer getRunModeBalancer() { + return this.runModeBalancer; + } + + public void setInitialDelay(long initialDelay) { + this.initialDelay = initialDelay; + } + + public void setOneAgentRate(int oneAgentRate) { + this.oneAgentRate = oneAgentRate; + } + + @Override + public RunModeSplitter getRunModeSplitter() { + return this.runModeSplitter; + } + @Override public void finishExecutionAndAwait() { scheduler.finishExecution(); @@ -231,6 +303,26 @@ public void finishExecutionAndAwait() { } + /** + * Method used for cancelling the task responsible for updating the current load. + * @return true if the task is successfully cancelled; false otherwise + */ + public boolean cancelPeriodicTask() { + return this.scheduledFuture.cancel(true); + } + + + /** + * Method used for scheduling the task responsible for updating the current load to be executed every 'interval' + * milliseconds. + */ + public void schedulePeriodicTask() { + MutableLong secondsLeft = new MutableLong((initialDelay + interval) / 1000); + + this.runRoundScheduler.scheduleAtFixedRate(this.scheduler.getRunnableToSchedule(secondsLeft), 0, + GlobalArgs.parseDurationToSeconds("1s"), TimeUnit.SECONDS); + } + private class AsyncTestWorkerImpl extends AsyncTestWorker { private AbstractTest test; private RunMap runMap; @@ -269,98 +361,107 @@ public boolean hasExited() { } } - private class AsyncTestWorkerScheduler extends AsyncEngineWorker { private Engine engine; + public AsyncTestWorkerScheduler(Engine engine) { this.engine = engine; } - private void configureRateAndInterval(MutableLong secondsLeft) { + private void configureRateAndInterval() { //the difference from the beginning load to the end one int loadDifference = Math.abs(end - start); // suppose load will increase by second - secondsLeft.setValue(1); - rate = (int)Math.floor(1.0 * secondsLeft.getValue() * loadDifference - / phase.getDuration()); + long newInterval = 1; + rate = (int)Math.floor(1.0 * newInterval* loadDifference + / GlobalArgs.parseDurationToSeconds(phase.getDuration())); // if the rate becomes too small, increase the interval at which the load is increased while (rate < 1) { - secondsLeft.increment(); - rate = (int)Math.floor(1.0 * secondsLeft.getValue() * loadDifference - / phase.getDuration()); + newInterval += 1; + rate = (int)Math.floor(1.0 * newInterval * loadDifference + / GlobalArgs.parseDurationToSeconds(phase.getDuration())); } - interval = secondsLeft.getValue(); + interval = newInterval * 1000; // set interval in milliseconds } - @Override - public void run() { - try { - currentLoad = load; - MutableLong secondsLeft = new MutableLong(interval); + private Runnable getRunnableToSchedule(MutableLong secondsLeft) { + return () -> { + if (!isFinished()) { + try { + // run the current run with the current load + runRound(); + secondsLeft.decrement(); - // if the rate was not specified and start and end were - if (isVariableLoad()) { - if (rate == -1) { - configureRateAndInterval(secondsLeft); - } + rampUp(secondsLeft); - currentLoad = start; - } + rampDown(secondsLeft); - while (!isFinished()) { - // run the current run with the current load - runRound(); + } catch (InterruptedException e) { + finishExecution(); + LOG.warn("Constant load scheduler thread was interrupted."); - secondsLeft.decrement(); + // gracefully shut down scheduler + runRoundScheduler.shutdownNow(); + } + } + }; + } - // ramp up the load if 'start' was specified - rampUp(secondsLeft); + @Override + public void run() { + currentLoad = load; - // ramp down the load if 'end' was specified - rampDown(secondsLeft); + // if the rate was not specified and start and end were + if (isVariableLoad()) { + if (rate == -1) { + configureRateAndInterval(); } - } catch (InterruptedException e) { - finishExecution(); - LOG.warn("Constant load scheduler thread was interrupted."); + + currentLoad = start; } + + MutableLong secondsLeft = new MutableLong((interval + initialDelay) / 1000); + scheduledFuture = runRoundScheduler.scheduleAtFixedRate(getRunnableToSchedule(secondsLeft), + 0, GlobalArgs.parseDurationToSeconds("1s"), TimeUnit.SECONDS); + } private void rampUp(MutableLong secondsLeft) { - if (currentLoad == end) { + if (currentLoad == end || ((oneAgentRate > 0) && (currentLoad + rate >= end + oneAgentRate))) { finishExecution(); } // if 'interval' has passed and the current load is still below 'end', // increase the current load if (secondsLeft.getValue() == 0 && end != -1 && currentLoad < end) { - secondsLeft.setValue(interval); currentLoad += rate; - LOG.debug("Current load: " + currentLoad); if (currentLoad > end) { currentLoad = end; } + + secondsLeft.setValue(interval / 1000); } } private void rampDown(MutableLong secondsLeft) { - if (currentLoad == end) { + if (currentLoad == end || ((oneAgentRate > 0) && (currentLoad - rate <= end - oneAgentRate))) { finishExecution(); } // if 'interval' has passed and the currentLoad is still above 'end', // decrease the current load if (secondsLeft.getValue() == 0 && end != -1 && currentLoad > end) { - secondsLeft.setValue(interval); currentLoad -= rate; - LOG.debug("Current load: " + currentLoad); if (currentLoad < end) { currentLoad = end; } + + secondsLeft.setValue(interval / 1000); } } @@ -399,12 +500,13 @@ private void runRound() throws InterruptedException { testWorkers.add(worker); } } - - //TODO use this - Thread.sleep(1000); } } + public long getInitialDelay() { + return this.initialDelay; + } + @Override public ExecutorService getExecutorService() { return executorService; diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/Normal.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/Normal.java index 1cb0995..991b937 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/Normal.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/Normal.java @@ -18,6 +18,11 @@ import com.adobe.qe.toughday.api.annotations.ConfigArgSet; import com.adobe.qe.toughday.internal.core.config.GlobalArgs; import com.adobe.qe.toughday.internal.core.engine.*; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.NormalRunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.NormalRunModeSplitter; + +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +30,7 @@ import java.util.concurrent.*; @Description(desc = "Runs tests normally.") -public class Normal implements RunMode { +public class Normal implements RunMode, Cloneable { private static final Logger LOG = LoggerFactory.getLogger(Normal.class); private static final int EPS = 1; @@ -40,8 +45,8 @@ public class Normal implements RunMode { private static final long DEFAULT_INTERVAL = 1000; private ExecutorService testsExecutorService; - private ScheduledExecutorService addWorkerScheduler = Executors.newSingleThreadScheduledExecutor(); - private ScheduledExecutorService removeWorkerScheduler = Executors.newSingleThreadScheduledExecutor(); + private ScheduledExecutorService rampingScheduler = Executors.newScheduledThreadPool(1); + private ScheduledFuture scheduledFuture = null; private final List testWorkers = Collections.synchronizedList(new LinkedList<>()); private final List runMaps = new ArrayList<>(); @@ -54,10 +59,23 @@ public class Normal implements RunMode { private long waitTime = DEFAULT_WAIT_TIME; private long interval = DEFAULT_INTERVAL; private int activeThreads = 0; + private long initialDelay = 0; private RunContext context = null; + private RunModeSplitter runModeSplitter = new NormalRunModeSplitter(); + private final RunModeBalancer normalRunModeBalancer = new NormalRunModeBalancer(); + private Engine engine; - @ConfigArgGet + + public Normal() { + /* this is required when running TD distributed because scheduled task might be cancelled and + * rescheduled when rebalancing the work between the agents. + */ + ScheduledThreadPoolExecutor scheduledPoolExecutor = (ScheduledThreadPoolExecutor) rampingScheduler; + scheduledPoolExecutor.setRemoveOnCancelPolicy(true); + } + + @ConfigArgGet(redistribute = true) public int getConcurrency() { return concurrency; } @@ -69,6 +87,11 @@ public void setConcurrency(String concurrencyString) { this.concurrency = Integer.parseInt(concurrencyString); } + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } + @ConfigArgGet public long getWaitTime() { return waitTime; @@ -95,12 +118,13 @@ public void setStart(String start) { this.start = Integer.valueOf(start); } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getRate() { return rate; } - @ConfigArgSet(required = false, desc = "The number of threads added per time unit. When it equals -1, it means it is not set.", defaultValue = "-1") + @ConfigArgSet(required = false, desc = "The number of threads added per time unit. When it equals -1, it means it is not set.", + defaultValue = "-1") public void setRate(String rate) { if (!rate.equals("-1")) { checkNotNegative(Long.parseLong(rate), "rate"); @@ -108,17 +132,22 @@ public void setRate(String rate) { this.rate = Integer.valueOf(rate); } - @ConfigArgGet - public long getInterval() { - return interval; + @ConfigArgGet(redistribute = true) + public String getInterval() { + return String.valueOf(this.interval / 1000) + 's'; + } + + public void setInitialDelay(long initialDelay) { + this.initialDelay = initialDelay; } - @ConfigArgSet(required = false, desc = "Used with rate to specify the time interval to add threads.", defaultValue = DEFAULT_INTERVAL_STRING) + @ConfigArgSet(required = false, desc = "Used with rate to specify the time interval to add threads.", + defaultValue = DEFAULT_INTERVAL_STRING) public void setInterval(String interval) { this.interval = GlobalArgs.parseDurationToSeconds(interval) * 1000; } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getEnd() { return end; } @@ -135,6 +164,42 @@ public int getActiveThreads() { return activeThreads; } + /** + * Method used for cancelling the execution of the task responsible for updating the number of worker threads. + * @return + */ + public boolean cancelPeriodicTask() { + if (this.scheduledFuture != null) { + return this.scheduledFuture.cancel(true); + } + + return true; + } + + /** + * Method used for scheduling the task responsible for updating the number of worker threads to execute + * periodically, every 'interval' milliseconds. + */ + public void schedulePeriodicTask() { + if (this.scheduledFuture == null) { + return; + } + + if (this.start < this.end) { + // execute 'rate' workers every 'interval' + this.scheduledFuture = this.rampingScheduler.scheduleAtFixedRate(this.getRampUpRunnable(this.engine, + this.engine.getCurrentPhase().getTestSuite()), this.initialDelay, this.interval, TimeUnit.MILLISECONDS); + } else if (this.end < this.start) { + // interrupt 'rate' workers every 'interval' + this.scheduledFuture = this.rampingScheduler.scheduleAtFixedRate(this.getRampDownRunnable(), this.initialDelay, + this.interval, TimeUnit.MILLISECONDS); + } + } + + public Engine getEngine() { + return this.engine; + } + private void checkNotNegative(long param, String property) { if (param < 0) { throw new IllegalArgumentException("Property " + property + " incorrectly configured as negative."); @@ -151,13 +216,14 @@ private void checkInvalidArgs() { } } - private boolean isVariableConcurrency() { + public boolean isVariableConcurrency() { return start != -1 && end != -1; } @Override public void runTests(Engine engine) { checkInvalidArgs(); + this.engine = engine; this.phase = engine.getCurrentPhase(); TestSuite testSuite = phase.getTestSuite(); @@ -168,7 +234,7 @@ public void runTests(Engine engine) { // namely every 'interval' milliseconds if (isVariableConcurrency()) { // if start and end were provided if (rate == -1) { - interval = (long)Math.floor(1000.0 * (phase.getDuration() - EPS) + interval = (long)Math.floor(1000.0 * (GlobalArgs.parseDurationToSeconds(phase.getDuration()) - EPS) / (start < end? end - start : start - end)); rate = 1; } @@ -184,16 +250,16 @@ public void runTests(Engine engine) { createAndExecuteWorker(engine, testSuite); } - // execute 'rate' workers every 'interval' - rampUp(engine, testSuite); - - // interrupt 'rate' workers every 'interval' - rampDown(); + schedulePeriodicTask(); } + public void finishAndDeleteWorker(AsyncTestWorker worker) { + worker.finishExecution(); + this.getRunContext().getTestWorkers().remove(worker); + activeThreads--; + } - - private void createAndExecuteWorker(Engine engine, TestSuite testSuite) { + public void createAndExecuteWorker(Engine engine, TestSuite testSuite) { AsyncTestWorkerImpl testWorker = new AsyncTestWorkerImpl(engine, phase, testSuite, phase.getPublishMode().getRunMap().newInstance()); synchronized (testWorkers) { testWorkers.add(testWorker); @@ -203,6 +269,8 @@ private void createAndExecuteWorker(Engine engine, TestSuite testSuite) { runMaps.add(testWorker.getLocalRunMap()); } + LOG.info("created test worker "); + try { testsExecutorService.execute(testWorker); } catch (OutOfMemoryError e) { @@ -210,64 +278,52 @@ private void createAndExecuteWorker(Engine engine, TestSuite testSuite) { } } - private void rampUp(Engine engine, TestSuite testSuite) { - // every 'interval' milliseconds, we'll create 'rate' workers - if (start < end) { - addWorkerScheduler.scheduleAtFixedRate(() -> { - for (int i = 0; i < rate; ++i) { - // if all the workers have been created - if (activeThreads >= end) { - addWorkerScheduler.shutdownNow(); + private Runnable getRampUpRunnable(Engine engine, TestSuite testSuite) { + return () -> { + for (int i = 0; i < rate; ++i) { + // if all the workers have been created + if (activeThreads >= end) { + rampingScheduler.shutdownNow(); - // mark workers as finished - for(AsyncTestWorker testWorker : testWorkers) { - testWorker.finishExecution(); - } - break; - } else { - createAndExecuteWorker(engine, testSuite); - } + // mark workers as finished + testWorkers.forEach(AsyncEngineWorker::finishExecution); + + break; + } else { + createAndExecuteWorker(engine, testSuite); } - }, 0, interval, TimeUnit.MILLISECONDS); - } + } + }; } - private void rampDown() { - // every 'interval' milliseconds, we'll stop 'rate' workers - if (end < start) { - ThreadPoolExecutor executor = (ThreadPoolExecutor)testsExecutorService; + private Runnable getRampDownRunnable() { + return () -> { + Iterator testWorkerIterator = testWorkers.iterator(); + int toRemove = rate; - removeWorkerScheduler.scheduleAtFixedRate(() -> { - Iterator testWorkerIterator = testWorkers.iterator(); - int toRemove = rate; + while (testWorkerIterator.hasNext()) { + // if all the workers have been removed + if (activeThreads <= end) { + rampingScheduler.shutdownNow(); - while (testWorkerIterator.hasNext()) { - // if all the workers have been removed - if (activeThreads <= end) { - removeWorkerScheduler.shutdownNow(); + // mark all the workers as finished, so the timeout checker will stop the execution + testWorkers.forEach(AsyncEngineWorker::finishExecution); - // mark all the workers as finished, so the timeout checker will stop the execution - for(AsyncTestWorker testWorker : testWorkers) { - testWorker.finishExecution(); - } + break; + } else { + AsyncTestWorker testWorker = testWorkerIterator.next(); + testWorker.finishExecution(); + + // remove the stopped worker + finishAndDeleteWorker(testWorker); + + // if rate users have been removed + if (toRemove == 0) { break; - } else { - AsyncTestWorker testWorker = testWorkerIterator.next(); - testWorker.finishExecution(); - - // remove the stopped worker - testWorkerIterator.remove(); - --toRemove; - --activeThreads; - - // if rate users have been removed - if (toRemove == 0) { - break; - } } } - }, 0, interval, TimeUnit.MILLISECONDS); - } + } + }; } public RunContext getRunContext() { @@ -298,13 +354,19 @@ public boolean isRunFinished() { } @Override - public void finishExecutionAndAwait() { - if (!addWorkerScheduler.isShutdown()) { - addWorkerScheduler.shutdownNow(); - } + public RunModeBalancer getRunModeBalancer() { + return this.normalRunModeBalancer; + } - if(!removeWorkerScheduler.isShutdown()) { - removeWorkerScheduler.shutdownNow(); + @Override + public RunModeSplitter getRunModeSplitter() { + return this.runModeSplitter; + } + + @Override + public void finishExecutionAndAwait() { + if (!rampingScheduler.isShutdown()) { + rampingScheduler.shutdownNow(); } synchronized (testWorkers) { @@ -426,5 +488,9 @@ public boolean hasExited() { } } + public long getInitialDelay() { + return this.initialDelay; + } + } diff --git a/toughday/src/main/java/com/adobe/qe/toughday/metrics/Percentile.java b/toughday/src/main/java/com/adobe/qe/toughday/metrics/Percentile.java index 7b703be..eebc5b4 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/metrics/Percentile.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/metrics/Percentile.java @@ -11,6 +11,7 @@ */ package com.adobe.qe.toughday.metrics; +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; import com.adobe.qe.toughday.api.annotations.Description; import com.adobe.qe.toughday.api.annotations.ConfigArgSet; import com.adobe.qe.toughday.api.core.RunMap; @@ -23,11 +24,16 @@ public class Percentile extends Metric { public Percentile setValue(String value) { this.value = Double.valueOf(value.substring(0,value.length() - 1)); if (this.name.equals(getClass().getSimpleName())) { - this.name = value + "p"; + this.name = value; } return this; } + @ConfigArgGet + public String getValue() { + return String.valueOf(this.value) + 'p'; + } + @Override public Object getValue(RunMap.TestStatistics testStatistics) { return testStatistics.getValueAtPercentile(value); diff --git a/toughday/src/test/java/com/adobe/qe/toughday/TestConfiguration.java b/toughday/src/test/java/com/adobe/qe/toughday/TestConfiguration.java index 05d02e1..75acc7c 100644 --- a/toughday/src/test/java/com/adobe/qe/toughday/TestConfiguration.java +++ b/toughday/src/test/java/com/adobe/qe/toughday/TestConfiguration.java @@ -14,8 +14,8 @@ import com.adobe.qe.toughday.internal.core.ReflectionsContainer; import com.adobe.qe.toughday.internal.core.TestSuite; import com.adobe.qe.toughday.internal.core.Timestamp; -import com.adobe.qe.toughday.internal.core.config.ConfigParams; import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; import com.adobe.qe.toughday.internal.core.config.PhaseParams; import com.adobe.qe.toughday.internal.core.engine.Phase; import com.adobe.qe.toughday.internal.core.engine.publishmodes.Intervals; @@ -278,7 +278,7 @@ public void testPhaseProperties() throws Exception { Assert.assertEquals(phase.getName(), "phase"); Assert.assertEquals(phase.getMeasurable(), false); - long duration = phase.getDuration(); + long duration = GlobalArgs.parseDurationToSeconds(phase.getDuration()); Assert.assertEquals(duration, 60); } @@ -385,7 +385,7 @@ public void testPhaseWithoutDuration() throws Exception { Assert.assertEquals(configuration.getPhasesWithoutDuration().size(), 1); - long duration = configuration.getPhases().get(1).getDuration(); + long duration = GlobalArgs.parseDurationToSeconds(configuration.getPhases().get(1).getDuration()); Assert.assertEquals(duration, 5L); } @@ -396,10 +396,10 @@ public void testTwoPhasesWithoutDuration() throws Exception { Assert.assertEquals(configuration.getPhasesWithoutDuration().size(), 2); - long duration = configuration.getPhases().get(1).getDuration(); + long duration = GlobalArgs.parseDurationToSeconds(configuration.getPhases().get(1).getDuration()); Assert.assertEquals(duration, 5L); - duration = configuration.getPhases().get(2).getDuration(); + duration = GlobalArgs.parseDurationToSeconds(configuration.getPhases().get(2).getDuration()); Assert.assertEquals(duration, 5L); } diff --git a/toughday/src/test/java/com/adobe/qe/toughday/TestConstantLoadMode.java b/toughday/src/test/java/com/adobe/qe/toughday/TestConstantLoadMode.java index bf9ee5a..c792afa 100644 --- a/toughday/src/test/java/com/adobe/qe/toughday/TestConstantLoadMode.java +++ b/toughday/src/test/java/com/adobe/qe/toughday/TestConstantLoadMode.java @@ -137,7 +137,7 @@ public void testCtLoadStartEndRateInterval() throws Exception { Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getStart(), 10); Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getEnd(), 100); Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getRate(), 5); - Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getInterval(), 60); + Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getInterval(), "60s"); } @After diff --git a/toughday/src/test/java/com/adobe/qe/toughday/TestNormalMode.java b/toughday/src/test/java/com/adobe/qe/toughday/TestNormalMode.java index b78442a..91377ec 100644 --- a/toughday/src/test/java/com/adobe/qe/toughday/TestNormalMode.java +++ b/toughday/src/test/java/com/adobe/qe/toughday/TestNormalMode.java @@ -138,7 +138,7 @@ public void testNormalStartEndRateInterval() throws Exception { Assert.assertEquals(runMode.getStart(), 10); Assert.assertEquals(runMode.getEnd(), 100); Assert.assertEquals(runMode.getRate(), 5); - Assert.assertEquals(runMode.getInterval(), 60000); + Assert.assertEquals(runMode.getInterval(), "60s"); } @After diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/AbstractRunModeBalancerTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/AbstractRunModeBalancerTest.java new file mode 100644 index 0000000..68e330d --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/AbstractRunModeBalancerTest.java @@ -0,0 +1,59 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class AbstractRunModeBalancerTest { + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullRunModeThrowsException() { + RunMode dummyRunMode = new DummyRunMode(); + dummyRunMode.getRunModeBalancer().getRunModePropertiesToRedistribute(DummyRunMode.class, null); + } + + @Test + public void testAllPropertiesAreCollected() { + DummyRunMode dummyRunMode = new DummyRunMode(); + Map actual = + dummyRunMode.getRunModeBalancer().getRunModePropertiesToRedistribute(DummyRunMode.class, dummyRunMode); + + Map expected = new HashMap<>(); + expected.put("property1", "prop1"); + expected.put("property2", "prop2"); + + Assert.assertEquals(expected, actual); + } + + @Test + public void testNewValuesAreAssignedForAllRunModeProperties() { + DummyRunMode dummyRunMode = new DummyRunMode(); + RedistributionInstructions redistributionInstructions = new RedistributionInstructions(); + + Map runModeChanges = new HashMap<>(); + runModeChanges.put("property1", "prop1_new"); + runModeChanges.put("property2", "prop2_new"); + + redistributionInstructions.setRunModeProperties(runModeChanges); + dummyRunMode.getRunModeBalancer().processRunModeInstructions(redistributionInstructions, dummyRunMode); + + Assert.assertEquals("prop1_new", dummyRunMode.getProperty1()); + Assert.assertEquals("prop2_new", dummyRunMode.getProperty2()); + Assert.assertEquals("prop3", dummyRunMode.getProperty3()); + + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/ConstantLoadRunModeSplitterTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/ConstantLoadRunModeSplitterTest.java new file mode 100644 index 0000000..808398d --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/ConstantLoadRunModeSplitterTest.java @@ -0,0 +1,98 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.engine.runmodes.ConstantLoad; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; + +public class ConstantLoadRunModeSplitterTest { + private ArrayList cmdLineArgs = new ArrayList<>(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test + public void testLoadDistribution() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=constantload", "load=182")); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + Assert.assertEquals(62, ((ConstantLoad) runModes.get("Agent1")).getLoad()); + Assert.assertEquals(60, ((ConstantLoad) runModes.get("Agent2")).getLoad()); + Assert.assertEquals(60, ((ConstantLoad) runModes.get("Agent3")).getLoad()); + } + + @Test + public void testVariableLoadDistribution() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=constantload", "start=3", "end=12", "rate=3")); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + + ConstantLoad firstAgentRunMode = (ConstantLoad) runModes.get("Agent1"); + Assert.assertEquals(2, firstAgentRunMode.getStart()); + Assert.assertEquals(6, firstAgentRunMode.getEnd()); + Assert.assertEquals(2, firstAgentRunMode.getRate()); + + ConstantLoad secondAgentRunMode = (ConstantLoad) runModes.get("Agent2"); + Assert.assertEquals(1, secondAgentRunMode.getStart()); + Assert.assertEquals(6, secondAgentRunMode.getEnd()); + Assert.assertEquals(1, secondAgentRunMode.getRate()); + } + + @Test + public void testVariableLoadWithRateLowerThanTheNumberOfAgents() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=constantload", "start=10", "end=20", "rate=2", "interval=2s")); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + ConstantLoad initialRunMode = (ConstantLoad) configuration.getPhases().get(0).getRunMode(); + + runModes.forEach((key, value) -> { + ConstantLoad taskRunMode = (ConstantLoad) value; + Assert.assertEquals(initialRunMode.getEnd(), taskRunMode.getEnd()); + Assert.assertEquals(initialRunMode.getRate(), taskRunMode.getOneAgentRate()); + Assert.assertEquals(6, taskRunMode.getRate()); + Assert.assertEquals("6s", taskRunMode.getInterval()); + + }); + + /* the values for rate property are checked for the first complete cycle + (nr_agents * interval period of time) */ + ConstantLoad firstAgentRunMode = (ConstantLoad) runModes.get("Agent1"); + Assert.assertEquals(10, firstAgentRunMode.getStart()); + Assert.assertEquals(0, firstAgentRunMode.getInitialDelay()); + + ConstantLoad secondAgentRunMode = (ConstantLoad) runModes.get("Agent2"); + Assert.assertEquals(12, secondAgentRunMode.getStart()); + Assert.assertEquals(2000, secondAgentRunMode.getInitialDelay()); + + ConstantLoad thirdAgentRunMode = (ConstantLoad) runModes.get("Agent3"); + Assert.assertEquals(14, thirdAgentRunMode.getStart()); + Assert.assertEquals(4000, thirdAgentRunMode.getInitialDelay()); + + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitorTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitorTest.java new file mode 100644 index 0000000..8587294 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitorTest.java @@ -0,0 +1,119 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; + +public class DistributedPhaseMonitorTest { + + private List cmdLineArgs; + private final DistributedPhaseMonitor distributedPhaseMonitor = new DistributedPhaseMonitor(); + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + + reflections.getTestClasses().put("MockTest", MockTest.class); + } + + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test + public void testIsPhaseExecuting() throws Exception { + // test when phase is null + Assert.assertFalse(this.distributedPhaseMonitor.isPhaseExecuting()); + + // test when no agent is executing TD + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + this.distributedPhaseMonitor.setPhase(configuration.getPhases().get(0)); + + Assert.assertFalse(this.distributedPhaseMonitor.isPhaseExecuting()); + + // test when phase is executed by agents + this.distributedPhaseMonitor.registerAgentRunningTD("Agent1"); + Assert.assertTrue(this.distributedPhaseMonitor.isPhaseExecuting()); + + this.distributedPhaseMonitor.removeAgentFromActiveTDRunners("Agent1"); + } + + @Test + public void testUpdateCountPerTest() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest", "count=400")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + this.distributedPhaseMonitor.setPhase(configuration.getPhases().get(0)); + + // simulate executions per test + Map> executionsPerTest = this.distributedPhaseMonitor.getExecutions(); + Map executionsPerAgent = new HashMap() {{ + put(mockAgents.get(0), 100L); + put(mockAgents.get(1), 100L); + }}; + + executionsPerTest.put("MockTest", executionsPerAgent); + Assert.assertEquals(200L, (long) this.distributedPhaseMonitor.getExecutionsPerTest().get("MockTest")); + } + + @Test + public void testResetExecutions() { + this.distributedPhaseMonitor.getExecutions().clear(); + Map executionsPerAgent = new HashMap() {{ + put("Agent1", 100L); + }}; + + this.distributedPhaseMonitor.getExecutions().put("MockTest", executionsPerAgent); + this.distributedPhaseMonitor.resetExecutions(); + + Assert.assertEquals(0L, + (long) this.distributedPhaseMonitor.getExecutionsPerTest().get("MockTest")); + + this.distributedPhaseMonitor.getExecutions().clear(); + } + + + @Test + public void testGetExecutionsPerTest() { + this.distributedPhaseMonitor.getExecutions().clear(); + + Map executionsPerAgent = new HashMap() {{ + put("Agent1", 100L); + put("Agent2", 60L); + put("Agent3", 34L); + }}; + + this.distributedPhaseMonitor.getExecutions().put("MockTest", executionsPerAgent); + Assert.assertEquals(194L, (long) this.distributedPhaseMonitor.getExecutionsPerTest().get("MockTest")); + } + + @Test + public void testRemoveAgentFromActiveTDRunners() { + this.distributedPhaseMonitor.getExecutions().clear(); + + this.distributedPhaseMonitor.registerAgentRunningTD("Agent1"); + this.distributedPhaseMonitor.registerAgentRunningTD("Agent2"); + + this.distributedPhaseMonitor.getExecutions() + .forEach((key, value) -> Assert.assertTrue(value.containsKey("Agent2"))); + + this.distributedPhaseMonitor.removeAgentFromActiveTDRunners("Agent2"); + + this.distributedPhaseMonitor.getExecutions() + .forEach((key, value) -> Assert.assertFalse(value.containsKey("Agent2"))); + } + + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunMode.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunMode.java new file mode 100644 index 0000000..4619cec --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunMode.java @@ -0,0 +1,77 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; +import com.adobe.qe.toughday.api.annotations.ConfigArgSet; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; + +import java.util.concurrent.ExecutorService; + +public class DummyRunMode implements RunMode { + private RunModeBalancer dummyRunModeBalancer = new DummyRunModeBalancer(); + private String property1 = "prop1"; + private String property2 = "prop2"; + private String property3 = "prop3"; + + @ConfigArgSet(required = false, defaultValue = "p1") + public void setProperty1(String property1) { + this.property1 = property1; + } + + @ConfigArgSet(required = false, defaultValue = "p2") + public void setProperty2(String property2) { + this.property2 = property2; + } + + @ConfigArgSet(required = false, defaultValue = "p3") + public void setProperty3(String property3) { + this.property3 = property3; + } + + @ConfigArgGet(redistribute = true) + public String getProperty1() { + return this.property1; + } + + @ConfigArgGet(redistribute = true) + public String getProperty2() { + return this.property2; + } + + @ConfigArgGet + public String getProperty3() { + return this.property3; + } + + @Override + public void runTests(Engine engine) throws Exception { + + } + + @Override + public void finishExecutionAndAwait() { + + } + + @Override + public ExecutorService getExecutorService() { + return null; + } + + @Override + public RunContext getRunContext() { + return null; + } + + @Override + public RunModeSplitter getRunModeSplitter() { + return null; + } + + @Override + public RunModeBalancer getRunModeBalancer() { + return this.dummyRunModeBalancer; + } +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunModeBalancer.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunModeBalancer.java new file mode 100644 index 0000000..c744a07 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunModeBalancer.java @@ -0,0 +1,16 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.AbstractRunModeBalancer; + +public class DummyRunModeBalancer extends AbstractRunModeBalancer { + @Override + public void before(RedistributionInstructions redistributionInstructions, DummyRunMode runMode) { + + } + + @Override + public void after(RedistributionInstructions redistributionInstructions, DummyRunMode runMode) { + + } +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/NormalRunModeSplitterTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/NormalRunModeSplitterTest.java new file mode 100644 index 0000000..e723bab --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/NormalRunModeSplitterTest.java @@ -0,0 +1,111 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.engine.runmodes.Normal; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; + +public class NormalRunModeSplitterTest { + private ArrayList cmdLineArgs = new ArrayList<>(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test + public void testConcurrencyDistribution() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=normal", "concurrency=320")); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + + Assert.assertEquals(108, ((Normal) runModes.get("Agent1")).getConcurrency()); + Assert.assertEquals(106, ((Normal) runModes.get("Agent2")).getConcurrency()); + Assert.assertEquals(106, ((Normal) runModes.get("Agent3")).getConcurrency()); + } + + @Test + public void testVariableConcurrencyDistribution() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=normal", "start=3", "end=12", "rate=3")); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + Normal initialRunMode = (Normal) configuration.getRunMode(); + + Normal firstAgentRunMode = (Normal) runModes.get("Agent1"); + Assert.assertEquals(2, firstAgentRunMode.getStart()); + Assert.assertEquals(6, firstAgentRunMode.getEnd()); + Assert.assertEquals(2, firstAgentRunMode.getRate()); + // check that the interval property does not change + Assert.assertEquals(initialRunMode.getInterval(), firstAgentRunMode.getInterval()); + + Normal secondAgentRunMode = (Normal) runModes.get("Agent2"); + Assert.assertEquals(1, secondAgentRunMode.getStart()); + Assert.assertEquals(6, secondAgentRunMode.getEnd()); + Assert.assertEquals(1, secondAgentRunMode.getRate()); + // check that the interval property does not change + Assert.assertEquals(initialRunMode.getInterval(), secondAgentRunMode.getInterval()); + } + + + @Test + public void testVariableConcurrencyWithRateLowerThanNumberOfAgents() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=normal", "start=3", "end=9", "interval=1s", "rate=1")); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode initialRunMode = configuration.getRunMode(); + Map runModes = initialRunMode.getRunModeSplitter().distributeRunMode(initialRunMode, mockAgents); + + Normal firstAgentRunMode = (Normal) runModes.get("Agent1"); + Assert.assertEquals(3, firstAgentRunMode.getStart()); + Assert.assertEquals(6, firstAgentRunMode.getEnd()); + Assert.assertEquals("2s", firstAgentRunMode.getInterval()); + Assert.assertEquals(0, firstAgentRunMode.getInitialDelay()); + + Normal secondAgentRunMode = (Normal) runModes.get("Agent2"); + Assert.assertEquals(0, secondAgentRunMode.getStart()); + Assert.assertEquals(3, secondAgentRunMode.getEnd()); + Assert.assertEquals("2s", secondAgentRunMode.getInterval()); + Assert.assertEquals(1000, secondAgentRunMode.getInitialDelay()); + + // check that the initial rate is not modified + Assert.assertEquals(((Normal)initialRunMode).getRate(), firstAgentRunMode.getRate()); + Assert.assertEquals(((Normal)initialRunMode).getRate(), secondAgentRunMode.getRate()); + } + + @Test + public void testWaitTimeIsNotModified() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=normal", "waittime=1000")); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode initialRunMode = configuration.getPhases().get(0).getRunMode(); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + + Map runModes = initialRunMode.getRunModeSplitter().distributeRunMode(initialRunMode, mockAgents); + runModes.forEach((key, value) -> + Assert.assertEquals(((Normal)initialRunMode).getWaitTime(), + ((Normal) value).getWaitTime())); + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/PhaseSplitterTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/PhaseSplitterTest.java new file mode 100644 index 0000000..0331a51 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/PhaseSplitterTest.java @@ -0,0 +1,103 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.api.core.AbstractTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.PhaseSplitter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.*; + +import java.util.*; +import java.util.stream.Collectors; + +public class PhaseSplitterTest { + private final PhaseSplitter phaseSplitter = new PhaseSplitter(); + private List cmdLineArgs; + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + + reflections.getTestClasses().put("MockTest", MockTest.class); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullPhaseThrowsException() throws CloneNotSupportedException { + phaseSplitter.splitPhase(null, new ArrayList<>()); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullListOfAgentsThrowsException() throws CloneNotSupportedException { + phaseSplitter.splitPhase(new Phase(), null); + } + + @Test(expected = IllegalStateException.class) + public void testNoAvailableAgentsThrowsException() throws CloneNotSupportedException { + phaseSplitter.splitPhase(new Phase(), new ArrayList<>()); + } + + @Test + public void testNumberOfTasksIsEqualToTheNumberOfAgents() throws Exception { + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + configuration.getPhases().forEach(phase -> { + try { + Map taskMap = phaseSplitter.splitPhase(phase, mockAgents); + Assert.assertEquals(mockAgents.size(), taskMap.keySet().size()); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + } + }); + } + + @Test + public void testCountIsDistributedForTestsInTestSuite() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest", "name=Test1", "count=201")); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + + Phase phase = new Configuration(cmdLineArgs.toArray(new String[0])).getPhases().get(0); + Map taskMap = phaseSplitter.splitPhase(phase, mockAgents); + + taskMap.get("Agent1").getTestSuite().getTests().forEach(test -> { + Assert.assertEquals(101, test.getCount()); + }); + + taskMap.get("Agent2").getTestSuite().getTests().forEach(test -> { + Assert.assertEquals(100, test.getCount()); + }); + } + + @Test + public void testEachAgentRunsTheCompleteTestSuite() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest", "name=Test1", "--add", "MockTest", "name=Test2")); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + + Phase phase = new Configuration(cmdLineArgs.toArray(new String[0])).getPhases().get(0); + Set testNames = phase.getTestSuite().getTests().stream() + .map(AbstractTest::getName) + .collect(Collectors.toSet()); + + Map taskMap = phaseSplitter.splitPhase(phase, mockAgents); + taskMap.forEach((key, value) -> { + Set namesDiff = value.getTestSuite().getTests().stream() + .map(AbstractTest::getName) + .collect(Collectors.toSet()); + namesDiff.removeAll(testNames); + + Assert.assertEquals(0, namesDiff.size()); + }); + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/RedistributionInstructionsProcessorTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/RedistributionInstructionsProcessorTest.java new file mode 100644 index 0000000..4e7f97b --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/RedistributionInstructionsProcessorTest.java @@ -0,0 +1,86 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructionsProcessor; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.engine.runmodes.Normal; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.*; + +public class RedistributionInstructionsProcessorTest { + private List cmdLineArgs; + private final RedistributionInstructionsProcessor instructionsProcessor = new RedistributionInstructionsProcessor(); + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + + reflections.getTestClasses().put("MockTest", MockTest.class); + } + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullPhaseThrowsExceptions() throws IOException { + this.instructionsProcessor.processInstructions("", null); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullInstructionsThrowsException() throws Exception{ + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + this.instructionsProcessor.processInstructions(null, configuration.getPhases().get(0)); + } + + @Test + public void testProcessInstructions() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest", "--runmode", "type=normal", "rate=20", "interval=2s", + "end=20", "start=15")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + Phase phase = configuration.getPhases().get(0); + RedistributionInstructions instructions = new RedistributionInstructions(); + + Map runModeProperties = new HashMap() {{ + put("rate", "10"); + put("interval","5s"); + put("end", "200"); + }}; + Map counts = new HashMap() {{ + put("MockTest", 1200L); + }}; + + instructions.setRunModeProperties(runModeProperties); + instructions.setCounts(counts); + + ObjectMapper mapper = new ObjectMapper(); + String jsonInstructions = mapper.writeValueAsString(instructions); + + this.instructionsProcessor.processInstructions(jsonInstructions, phase); + + // test run mode properties were changed + Normal runmode = (Normal)phase.getRunMode(); + Assert.assertEquals(10, runmode.getRate()); + Assert.assertEquals("5s", runmode.getInterval()); + Assert.assertEquals(200, runmode.getEnd()); + + // test count properties were changed + Assert.assertEquals(1200, phase.getTestSuite().getTest("MockTest").getCount()); + + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgentsTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgentsTest.java new file mode 100644 index 0000000..fe46b05 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgentsTest.java @@ -0,0 +1,152 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.DummyRunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.mocks.MockMetric; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.Assert; +import org.junit.rules.ExpectedException; + +import java.util.*; + +public class YamlDumpConfigurationAsTaskForTDAgentsTest { + private List cmdLineArgs; + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + + reflections.getTestClasses().put("MockTest", MockTest.class); + reflections.getMetricClasses().put("MockMetric", MockMetric.class); + reflections.getRunModeClasses().put("DummyRunMode", DummyRunMode.class); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test + public void testNullCofigurationThrowsException() { + expectedEx.expect(IllegalArgumentException.class); + expectedEx.expectMessage("Configuration must not be null."); + + new YamlDumpConfigurationAsTaskForTDAgents(null); + } + + @Test() + public void testCollectConfigurablePropertiesWithIncompatibleTypeThrowsException() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + expectedEx.expect(IllegalArgumentException.class); + expectedEx.expectMessage("The object must have the specified type."); + + yamlDumpConfiguration.collectConfigurableProperties(DummyRunMode.class, configuration.getTestSuite().getTest("MockTest")); + } + + @Test + public void testCollectConfigurablePropertiesContainsOnlyModifiedProperties() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=dummyrunmode", "property1=value1", "property2=value2")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + + Map actualCollectedPropsForDummyRunMode = + yamlDumpConfiguration.collectConfigurableProperties(DummyRunMode.class, configuration.getRunMode()); + + Map expectedCollectedPropsForMockTest = + new HashMap() {{ + put("property1", "value1"); + put("property2", "value2"); + }}; + + Assert.assertEquals(expectedCollectedPropsForMockTest, actualCollectedPropsForDummyRunMode); + } + + @Test + public void testDumpedConfiguration() throws Exception{ + cmdLineArgs.addAll(Arrays.asList("--duration=10s", "--phase", "name=phase123", "--add", "MockTest", "name=Test1", "count=201", + "--add" , "MockMetric", "name=Metric123", "--runmode", "type=dummyrunmode", "--add", "CSVPublisher")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + String expectedYaml = + "globals:\n" + + " duration: 10s\n" + + " host: localhost\n" + + "phases:\n" + + "- duration: 10s\n" + + " metrics:\n" + + " - add: MockMetric\n" + + " properties:\n" + + " name: Metric123\n" + + " name: phase123\n" + + " publishers:\n" + + " - add: CSVPublisher\n" + + " properties:\n" + + " name: CSVPublisher\n" + + " aggregatedpublish: false\n" + + " publishmode:\n" + + " type: simple\n" + + " runmode:\n" + + " type: dummyrunmode\n" + + " tests:\n" + + " - add: MockTest\n" + + " properties:\n" + + " name: Test1\n" + + " count: 201\n"; + + Assert.assertEquals(expectedYaml, yamlDumpConfiguration.generateConfigurationObject()); + } + + @Test + public void testGetGlobals() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--duration=10s", "--port=1000", "--user=testUser", "--timeout=10")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + Map actualGlobals = yamlDumpConfiguration.getGlobals(); + + Map expectedGlobals = new HashMap() {{ + put("host", "localhost"); + put("duration", "10s"); + put("port", 1000); + put("user", "testUser"); + put("timeout", 10L); + }}; + + Assert.assertEquals(expectedGlobals, actualGlobals); + } + + @Test + public void testGetDistributedConfig() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--distributedconfig", "driverip=10.0.0.1", "heartbeatinterval=10s")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + Map actualDistributedConfig = yamlDumpConfiguration.getDistributedConfig(); + Map expectedDistributedConfig = new HashMap() {{ + put("driverip", "10.0.0.1"); + put("heartbeatinterval", "10s"); + }}; + + Assert.assertEquals(expectedDistributedConfig, actualDistributedConfig); + } + + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/mocks/MockMetric.java b/toughday/src/test/java/com/adobe/qe/toughday/mocks/MockMetric.java new file mode 100644 index 0000000..8b5f9ae --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/mocks/MockMetric.java @@ -0,0 +1,21 @@ +package com.adobe.qe.toughday.mocks; + +import com.adobe.qe.toughday.api.core.RunMap; +import com.adobe.qe.toughday.metrics.Metric; + +public class MockMetric extends Metric { + @Override + public Object getValue(RunMap.TestStatistics testStatistics) { + return null; + } + + @Override + public String getFormat() { + return null; + } + + @Override + public String getUnitOfMeasure() { + return null; + } +} diff --git a/toughday2-api/src/main/java/com/adobe/qe/toughday/api/annotations/ConfigArgGet.java b/toughday2-api/src/main/java/com/adobe/qe/toughday/api/annotations/ConfigArgGet.java index 45b22ec..f379cd8 100644 --- a/toughday2-api/src/main/java/com/adobe/qe/toughday/api/annotations/ConfigArgGet.java +++ b/toughday2-api/src/main/java/com/adobe/qe/toughday/api/annotations/ConfigArgGet.java @@ -18,7 +18,7 @@ /** * Use this annotation on a getter to expose it as a configuration property. - * These properties will be automatically picked up, shown in logging or in "dry" runmode + * These properties will be automatically picked up, shown in logging or in "dry" runmodes * Supported classes: subtypes of AbstractTest, subtypes of Publisher and * GlobalArgs. */ @@ -26,4 +26,14 @@ @Target(value = ElementType.METHOD) public @interface ConfigArgGet { String name() default ""; + + /** + * Use this field when you want to specify that a certain property must be taken into + * consideration when redistributing the work between the agents running in the cluster. + * The property will be automatically collected by the AbstractRunModeBalancer. + * Currently, this annotations is supported only for classes implementing the RunMode + * interface. + * @return + */ + boolean redistribute() default false; }