diff --git a/agent-job.yaml b/agent-job.yaml new file mode 100644 index 0000000..78ac618 --- /dev/null +++ b/agent-job.yaml @@ -0,0 +1,22 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: agent +spec: + parallelism: 2 + ttlSecondsAfterFinished: 0 + template: + metadata: + labels: + app: agent + spec: + containers: + - name: agent-container + image: + livenessProbe: + httpGet: + path: /health + port: 4567 + initialDelaySeconds: 20 + periodSeconds: 3 + restartPolicy: Never diff --git a/driver_statefulset.yaml b/driver_statefulset.yaml new file mode 100644 index 0000000..4fc897e --- /dev/null +++ b/driver_statefulset.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: driver +spec: + selector: + matchLabels: + app: driver + serviceName: driver + replicas: 2 + template: + metadata: + labels: + app: driver + spec: + containers: + - image: + env: + - name: NR_DRIVERS + value: "2" # set this value to the value of spec.replicas field + name: driver-container + ports: + - containerPort: 4567 + - containerPort: 80 + livenessProbe: + httpGet: + path: /health + port: 4567 + initialDelaySeconds: 20 + periodSeconds: 3 + +--- + +# headless service required by StatefulSet resource +apiVersion: v1 +kind: Service +metadata: + name: driver + labels: + app: driver +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 4567 + name: http + selector: + app: driver + +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: driver + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" +spec: + rules: + - http: + paths: + - backend: + serviceName: driver + servicePort: 80 + path: /config diff --git a/toughday/pom.xml b/toughday/pom.xml index 4137f5e..f9182fb 100644 --- a/toughday/pom.xml +++ b/toughday/pom.xml @@ -271,5 +271,11 @@ test + + com.sparkjava + spark-core + 2.7.2 + + diff --git a/toughday/src/main/java/com/adobe/qe/toughday/Main.java b/toughday/src/main/java/com/adobe/qe/toughday/Main.java index 7b57ebb..e2243b4 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/Main.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/Main.java @@ -11,14 +11,15 @@ */ package com.adobe.qe.toughday; - import com.adobe.qe.toughday.internal.core.engine.Engine; import com.adobe.qe.toughday.internal.core.config.parsers.cli.CliParser; import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.ExecutionTrigger; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - /** * Main class. Creates a Configuration and an engine and runs the tests. */ @@ -31,8 +32,6 @@ public class Main { } public static void main(String[] args) { - - CliParser cliParser = new CliParser(); System.out.println(); @@ -51,15 +50,27 @@ public static void main(String[] args) { System.exit(1); } - Engine engine = new Engine(configuration); - engine.runTests(); + /* check if we should trigger an execution query in the cluster. */ + if (configuration.executeInDitributedMode()) { + new ExecutionTrigger(configuration).triggerDistributedExecution(); + System.exit(0); + } else if (configuration.getDistributedConfig().getAgent()) { + Agent agent = new Agent(); + agent.start(); + } else if (configuration.getDistributedConfig().getDriver()) { + Driver driver = new Driver(configuration); + driver.run(); + } else { + Engine engine = new Engine(configuration); + engine.runTests(); - System.exit(0); + System.exit(0); + } } catch (Throwable t) { LOG.error("Error encountered: " + (t.getMessage() != null ? t.getMessage() : "Please check toughday.log for more information.")); LogManager.getLogger(Engine.class).error("Error encountered", t); + System.exit(-1); } - System.exit(0); } } diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/TestSuite.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/TestSuite.java index 80ab294..88b7063 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/TestSuite.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/TestSuite.java @@ -21,7 +21,7 @@ /** * Test suite class. */ -public class TestSuite { +public class TestSuite implements Cloneable { private List setupStep; private String description = ""; private List tags = new ArrayList<>(); @@ -39,6 +39,24 @@ public TestSuite() { totalWeight = 0; } + /** + * Creates a copy of the current test suite. All tests contained by the test suite are cloned. + * @throws CloneNotSupportedException if the object to be cloned does not implement the Cloneable interface. + */ + public TestSuite clone() throws CloneNotSupportedException { + TestSuite newInstance = (TestSuite) super.clone(); + + /* clone all the tests in the TestSuite */ + newInstance.orderedTests = new ArrayList<>(); + newInstance.nameMap = new HashMap<>(); + + for (AbstractTest test : this.getTests()) { + newInstance.add(test.clone()); + } + + return newInstance; + } + /** * Method for adding a test. * @param test diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/ConfigParams.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/ConfigParams.java index e47fd3f..b59b1af 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/ConfigParams.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/ConfigParams.java @@ -26,6 +26,7 @@ public class ConfigParams implements Serializable { private List phasesParams = new ArrayList<>(); private Map globalParams = new HashMap<>(); + private Map distributedConfigParams = new HashMap<>(); private Map publishModeParams = new HashMap<>(); private Map runModeParams = new HashMap<>(); private List> items = new ArrayList<>(); @@ -94,6 +95,10 @@ public void setGlobalParams(Map globalParams) { this.globalParams = globalParams; } + public void setDistributedConfigParams(Map distributedConfigParams) { + this.distributedConfigParams = distributedConfigParams; + } + public void setPhasesParams(List phasesParams) { this.phasesParams = phasesParams; } @@ -163,12 +168,15 @@ public Map getGlobalParams(){ return globalParams; } + public Map getDistributedConfigParams() { return distributedConfigParams; } + public Map getPublishModeParams() { return publishModeParams; } public Map getRunModeParams() { return runModeParams; } public void merge(ConfigParams other) { globalParams.putAll(other.getGlobalParams()); + distributedConfigParams.putAll(other.distributedConfigParams); items.addAll(other.items); phasesParams.addAll(other.phasesParams); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/Configuration.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/Configuration.java index 4523850..285a6a9 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/Configuration.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/Configuration.java @@ -24,6 +24,7 @@ import com.adobe.qe.toughday.internal.core.engine.Phase; import com.adobe.qe.toughday.internal.core.engine.PublishMode; import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.DistributedConfig; import com.adobe.qe.toughday.metrics.Metric; import com.adobe.qe.toughday.publishers.CSVPublisher; import com.adobe.qe.toughday.publishers.ConsolePublisher; @@ -36,9 +37,7 @@ import org.apache.logging.log4j.core.config.LoggerConfig; import org.reflections.Reflections; -import java.io.File; -import java.io.IOException; -import java.lang.ref.WeakReference; +import java.io.*; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -63,6 +62,7 @@ public class Configuration { private PredefinedSuites predefinedSuites = new PredefinedSuites(); private GlobalArgs globalArgs; + private DistributedConfig distributedConfig; private RunMode runMode; private PublishMode publishMode; private TestSuite globalSuite; @@ -70,6 +70,7 @@ public class Configuration { private Map globalMetrics = new LinkedHashMap<>(); private List phases = new ArrayList<>(); private Set phasesWithoutDuration = new HashSet<>(); + private ConfigParams configParams; private boolean defaultSuiteAddedFromConfigExclude = false; private boolean anyMetricAdded = false; private boolean anyPublisherAdded = false; @@ -77,6 +78,37 @@ public class Configuration { private Map feeders = new LinkedHashMap<>(); private Map objects = new HashMap<>(); + public Configuration(String yamlConfig) + throws InvocationTargetException, NoSuchMethodException, InstantiationException, IOException, IllegalAccessException { + ConfigParams configParams = new YamlParser().parse(yamlConfig); + buildConfiguration(configParams); + } + + public Configuration(String[] cmdLineArgs) + throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException, IOException { + ConfigParams configParams = collectConfigurations(cmdLineArgs); + buildConfiguration(configParams); + + } + + /** + * Method for getting the property from a setter method + * + * @param methodName + * @return + */ + public static String propertyFromMethod(String methodName) { + return methodName.startsWith("set") || methodName.startsWith("get") ? StringUtils.lowerCase(methodName.substring(3)) : StringUtils.lowerCase(methodName); + } + + public static Map> getRequiredFieldsForClassAdded() { + return requiredFieldsForClassAdded; + } + + public ConfigParams getConfigParams() { + return this.configParams; + } + private void handleExtensions(ConfigParams configParams) { List extensionList = new ArrayList<>(); @@ -121,7 +153,8 @@ private void handleExtensions(ConfigParams configParams) { } /** - * Creates a jar file for each extension file that should be loaded. + * Creates a jar file for each extension file that should be loaded. + * * @param extensionList A list of names representing the jar files that should be loaded. */ private List createJarFiles(List extensionList) { @@ -139,7 +172,8 @@ private List createJarFiles(List extensionList) { } /** - * Creates an URL for each jar file, using its filename. + * Creates an URL for each jar file, using its filename. + * * @param extensionsFileNames * @return */ @@ -157,7 +191,6 @@ private URL[] formJarURLs(List extensionsFileNames) { } // loads all classes from the extension jar files using a new class loader. - private ClassLoader processJarFiles(List jarFiles, URL[] urls) throws MalformedURLException { ToughdayExtensionClassLoader classLoader = new ToughdayExtensionClassLoader(urls, Thread.currentThread().getContextClassLoader()); Map newClasses = new HashMap<>(); @@ -194,10 +227,14 @@ private ClassLoader processJarFiles(List jarFiles, URL[] urls) throws M return classLoader; } + public boolean executeInDitributedMode() { + return !configParams.getDistributedConfigParams().isEmpty() && + !this.getDistributedConfig().getAgent() && + !this.getDistributedConfig().getDriver(); + } - public Configuration(String[] cmdLineArgs) - throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException, IOException { - ConfigParams configParams = collectConfigurations(cmdLineArgs); + private void buildConfiguration(ConfigParams configParams) throws NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException, IOException { + this.configParams = ConfigParams.deepClone(configParams); ConfigParams copyOfConfigParams = ConfigParams.deepClone(configParams); Map items = new HashMap<>(); @@ -206,11 +243,12 @@ public Configuration(String[] cmdLineArgs) Map globalArgsMeta = configParams.getGlobalParams(); for (String helpOption : CliParser.availableHelpOptions) { - if (globalArgsMeta.containsKey(helpOption)) { - return; - } + if (globalArgsMeta.containsKey(helpOption)) { + return; + } } + this.distributedConfig = createObject(DistributedConfig.class, configParams.getDistributedConfigParams()); this.globalArgs = createObject(GlobalArgs.class, globalArgsMeta); configureLogPath(globalArgs.getLogPath()); @@ -232,6 +270,7 @@ public Configuration(String[] cmdLineArgs) GenerateYamlConfiguration generateYaml = new GenerateYamlConfiguration(copyOfConfigParams, items); generateYaml.createYamlConfigurationFile(); } + objects = null; } @@ -315,8 +354,8 @@ private Phase createPhase(ConfigParams configParams, PhaseParams phaseParams, Te phase.setPublishMode(publishMode); // compute the minimum timeout of the phase - phase.getTestSuite().setMinTimeout(globalArgs.getTimeout()); - for(AbstractTest test : phase.getTestSuite().getTests()) { + phase.getTestSuite().setMinTimeout(globalArgs.getTimeoutInSeconds()); + for (AbstractTest test : phase.getTestSuite().getTests()) { // set the count (the number of executions since the beginnin of the run) of each test to 0 phase.getCounts().put(test, new AtomicLong(0)); @@ -325,7 +364,7 @@ private Phase createPhase(ConfigParams configParams, PhaseParams phaseParams, Te items.put(test.getName(), test.getClass()); - if(test.getTimeout() < 0) { + if (test.getTimeout() < 0) { continue; } @@ -373,7 +412,7 @@ private void getConfigurationFromAnotherPhase(PhaseParams phaseParams) { // merge the current phase with the one whose name is the value of 'useconfig' phaseParams.merge(PhaseParams.namedPhases.get(useconfig), - new HashSet<>(Arrays.asList(name, useconfig))); + new HashSet<>(Arrays.asList(name, useconfig))); } } @@ -390,7 +429,7 @@ private void convertActionItems(List configItem((ConfigParams.NamedMetaObject) item.getValue(), items, testSuite, publishers, metrics); break; case EXCLUDE: - excludeItem(((ConfigParams.NamedMetaObject)item.getValue()).getName(), testSuite, publishers, metrics); + excludeItem(((ConfigParams.NamedMetaObject) item.getValue()).getName(), testSuite, publishers, metrics); break; } } @@ -405,7 +444,7 @@ private void configureDurationForPhases() { if (phase.getDuration() == null) { phasesWithoutDuration.add(phase); } else { - durationLeft -= phase.getDuration(); + durationLeft -= GlobalArgs.parseDurationToSeconds(phase.getDuration()); } } @@ -613,16 +652,6 @@ private void configureLogPath(String logPath) throws IOException { } } - /** - * Method for getting the property from a setter method - * - * @param methodName - * @return - */ - public static String propertyFromMethod(String methodName) { - return methodName.startsWith("set") || methodName.startsWith("get") ? StringUtils.lowerCase(methodName.substring(3)) : StringUtils.lowerCase(methodName); - } - /** * Method for setting an object properties annotated with ConfigArgSet using reflection * @@ -635,9 +664,9 @@ public static String propertyFromMethod(String methodName) { * @throws IllegalAccessException caused by reflection */ //TODO figure out if we can make this public static again - public T setObjectProperties(T object, Map args, boolean applyDefaults, Map feedersContext) throws InvocationTargetException, IllegalAccessException { + public T setObjectProperties(T object, Map args, boolean applyDefaults, Map feedersContext) throws InvocationTargetException, IllegalAccessException { Class classObject = object.getClass(); - LOGGER.info("Configuring object of class: " + classObject.getSimpleName()+" ["+classObject.getName()+"]"); + LOGGER.info("Configuring object of class: " + classObject.getSimpleName() + " [" + classObject.getName() + "]"); for (Method method : classObject.getMethods()) { callConfigArgSet(method, object, args, applyDefaults); FeederInjector.injectFeeder(method, object, args, applyDefaults, feedersContext, objects); @@ -683,7 +712,8 @@ private static void callConfigArgSet(Method method, Object object, Map T createObject(Class classObject, Map args) + + public T createObject(Class classObject, Map args) throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { return createObject(classObject, args, null); } @@ -700,7 +730,7 @@ public T createObject(Class classObject, Map a * @throws InstantiationException * @throws NoSuchMethodException */ - public T createObject(Class classObject, Map args, Map feederContext) + public T createObject(Class classObject, Map args, Map feederContext) throws IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException { Constructor constructor = null; @@ -837,6 +867,14 @@ public GlobalArgs getGlobalArgs() { return globalArgs; } + /** + * Getter for kubernetes config args + * @return + */ + public DistributedConfig getDistributedConfig() { + return this.distributedConfig; + } + /** * Getter for the run mode * @@ -870,8 +908,8 @@ public List getPhases() { return phases; } - public static Map> getRequiredFieldsForClassAdded() { - return requiredFieldsForClassAdded; + public void setPhases(List phases) { + this.phases = phases; } public TestSuite getTestSuite() { @@ -882,5 +920,7 @@ public Set getPhasesWithoutDuration() { return phasesWithoutDuration; } - public Collection getFeeders() { return feeders.values(); } + public Collection getFeeders() { + return feeders.values(); + } } diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/GlobalArgs.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/GlobalArgs.java index a58879e..dc3d7e8 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/GlobalArgs.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/GlobalArgs.java @@ -59,7 +59,11 @@ public class GlobalArgs implements com.adobe.qe.toughday.api.core.config.GlobalA private boolean saveConfig = Boolean.parseBoolean(DEFAULT_SAVE_CONFIG); private boolean showSteps = false; private boolean hostValidationEnabled = true; + private boolean k8sRun = false; + private boolean k8sAgent = false; + private boolean k8sdriver = false; private String logPath; + private String driverIp = null; /** * Constructor @@ -106,7 +110,6 @@ public static long parseDurationToSeconds(String duration) { long finalDuration = 0l; long intermDuration = 0l; - //if time unit is not specified, consider it seconds by default. if (duration.matches("^[0-9]+$")) { throw new IllegalArgumentException("Time unit is not specified"); } @@ -226,7 +229,11 @@ public Collection getPublishers() { @ConfigArgGet public long getTimeout() { - return timeout; + return timeout / 1000; + } + + public long getTimeoutInSeconds() { + return this.timeout; } @ConfigArgSet(required = false, desc = "How long a test will run before it will be interrupted and marked as failed. Expressed in seconds", diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/cli/CliParser.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/cli/CliParser.java index 498d5c3..f772b2d 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/cli/CliParser.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/cli/CliParser.java @@ -29,6 +29,7 @@ import com.adobe.qe.toughday.internal.core.engine.Engine; import com.adobe.qe.toughday.internal.core.engine.PublishMode; import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.DistributedConfig; import com.adobe.qe.toughday.metrics.Metric; import com.google.common.base.Joiner; import net.jodah.typetools.TypeResolver; @@ -53,9 +54,9 @@ public class CliParser implements ConfigurationParser { private static final String TEST_CLASS_HELP_HEADER = String.format(HELP_HEADER_FORMAT_WITH_TAGS, "Class", "Fully qualified domain name", "Tags", "Description"); private static final String PUBLISH_CLASS_HELP_HEADER = String.format(HELP_HEADER_FORMAT_NO_TAGS, "Class", "Fully qualified domain name", "Description"); private static final String METRIC_CLASS_HELP_HEADER = PUBLISH_CLASS_HELP_HEADER; - private static Method[] globalArgMethods = GlobalArgs.class.getMethods(); private static final String SUITE_HELP_HEADER = String.format(" %-40s %-40s %s", "Suite", "Tags", "Description"); private static Map> availableGlobalArgs = new HashMap<>(); + private static Map> availableDistributedConfigArgs = new HashMap<>(); private static List parserArgHelps = new ArrayList<>(); @@ -73,18 +74,24 @@ public class CliParser implements ConfigurationParser { add("tag"); }}); + private static void collectAvailableConfigurationOptions(Class type, Map> availableArgs) { + Arrays.stream(type.getMethods()) + .filter(method -> method.isAnnotationPresent(ConfigArgSet.class)) + .forEach(method -> { + ConfigArgSet annotation = method.getAnnotation(ConfigArgSet.class); + int order = annotation.order(); + if (!availableArgs.containsKey(order)) { + availableArgs.put(order, new HashMap<>()); + } + + availableArgs.get(order) + .put(Configuration.propertyFromMethod(method.getName()), annotation); + }); + } + static { - for (Method method : globalArgMethods) { - if (method.getAnnotation(ConfigArgSet.class) != null) { - ConfigArgSet annotation = method.getAnnotation(ConfigArgSet.class); - int order = annotation.order(); - if (null == availableGlobalArgs.get(order)) { - availableGlobalArgs.put(order, new HashMap()); - } - Map globalArgMap = availableGlobalArgs.get(order); - globalArgMap.put(Configuration.propertyFromMethod(method.getName()), annotation); - } - } + collectAvailableConfigurationOptions(GlobalArgs.class, availableGlobalArgs); + collectAvailableConfigurationOptions(DistributedConfig.class, availableDistributedConfigArgs); for (Class parserClass : ReflectionsContainer.getSubTypesOf(ConfigurationParser.class)) { for (Field field : parserClass.getDeclaredFields()) { @@ -230,6 +237,10 @@ private int parseObjectProperties(int startIndex, String[] cmdLineArgs, HashMap< return j - startIndex; } + private boolean isGlobalArg(String paramName) { + return availableGlobalArgs.values().stream().anyMatch(map -> map.containsKey(paramName)); + } + /** * Implementation of parser interface * @param cmdLineArgs command line arguments @@ -261,6 +272,9 @@ public ConfigParams parse(String[] cmdLineArgs) { } else if (arg.equals("runmode")) { skip = parseObjectProperties(i + 1, cmdLineArgs, args); configParams.setRunModeParams(args); + } else if (arg.equals("distributedconfig")) { + skip = parseObjectProperties(i + 1, cmdLineArgs, args); + configParams.setDistributedConfigParams(args); } else if (arg.equals("help")) { skip = 1; globalArgs.put("host", "N/A"); //TODO remove ugly hack @@ -268,14 +282,9 @@ public ConfigParams parse(String[] cmdLineArgs) { String[] res = parseProperty(arg); String key = res[0]; Object val = getObjectFromString(res[1]); - // if global param does not exist - boolean found = false; - for (Map argz : availableGlobalArgs.values()) { - if (argz.containsKey(key)) { - found = true; - break; - } - } + // if global param or distributed td config param does not exist + boolean found = isGlobalArg(key); + if (!found && !parserArgs.contains(key) && !availableHelpOptions.contains(key) && !helpOptionsParameters.contains(key) && !key.equals("suite") && !key.equals("suitesetup")) { throw new IllegalArgumentException("Unrecognized argument --" + key); @@ -468,24 +477,27 @@ public void printShortHelp(boolean printSuitesTests) { System.out.println("\t java -jar toughday.jar --host=localhost --add extension.jar --add com.adobe.qe.toughday.tests.extensionTest"); System.out.println("\t java -jar toughday.jar --suite=toughday --add BASICMetrics --add Average decimals=3 --exclude Failed"); - System.out.println("\r\nGlobal arguments:"); + System.out.println("\r\nExamples for running TD distributed: \r\n"); + System.out.println("\t java -jar toughday.jar --host=localhost --distributedconfig driverip=1.1.1.1"); + System.out.println("\t java -jar toughday.jar --host=localhost --distributedconfig driverip=1.1.1.1 heartbeatinterval=10s --suite=toughday"); - for (Integer order : availableGlobalArgs.keySet()) { - Map paramGroup = availableGlobalArgs.get(order); - for (String param : paramGroup.keySet()) { - System.out.printf("\t--%-32s\t Default: %s - %s\r\n", - param + "=val", paramGroup.get(param).defaultValue(), paramGroup.get(param).desc()); - } - } + System.out.println("\r\nGlobal arguments:"); + availableGlobalArgs.forEach((order, paramGroup) -> + paramGroup.forEach((key, value) -> System.out.printf("\t--%-32s\t Default: %s - %s\r\n", + key + "=val", value.defaultValue(), value.desc()))); for (ParserArgHelp parserArgHelp : parserArgHelps) { System.out.printf("\t--%-32s\t Default: %s - %s\r\n", parserArgHelp.name() + "=val", parserArgHelp.defaultValue(), parserArgHelp.description()); } - //System.out.printf("\t%-32s\t %s\r\n", "--suitesetup=val", getSuiteSetupDescription()); System.out.printf("\t%-32s\t %s\r\n", "--suite=val", "Default: toughday - Where \"val\" can be one or a list (separated by commas) of the predefined suites"); + System.out.println("\r\n Distributed run arguments (--distributedconfig):"); + availableDistributedConfigArgs.forEach((order, paramGroup) -> + paramGroup.forEach((key, value) -> System.out.printf("\t%-32s\t Default: %s - %s\r\n", + key + "=val", value.defaultValue(), value.desc()))); + System.out.println("\r\nAvailable run modes (--runmode):"); for(Map.Entry> runMode : ReflectionsContainer.getInstance().getRunModeClasses().entrySet()) { Description description = runMode.getValue().getAnnotation(Description.class); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/GenerateYamlConfiguration.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/GenerateYamlConfiguration.java index dec7b78..1216be3 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/GenerateYamlConfiguration.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/GenerateYamlConfiguration.java @@ -17,17 +17,11 @@ import com.adobe.qe.toughday.internal.core.Timestamp; import com.adobe.qe.toughday.internal.core.config.ConfigParams; import com.adobe.qe.toughday.internal.core.config.PhaseParams; -import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml; -import org.yaml.snakeyaml.introspector.Property; -import org.yaml.snakeyaml.nodes.NodeTuple; -import org.yaml.snakeyaml.nodes.Tag; -import org.yaml.snakeyaml.representer.Representer; import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; -import java.lang.reflect.Method; import java.util.*; public class GenerateYamlConfiguration { @@ -43,6 +37,7 @@ public class GenerateYamlConfiguration { private static final String DEFAULT_YAML_CONFIGURATION_FILENAME = "toughday_"; private static final String DEFAULT_YAML_EXTENSION = ".yaml"; private static final String TIMESTAMP = Timestamp.START_TIME; + public static final String yamlConfigFilename = DEFAULT_YAML_CONFIGURATION_FILENAME + TIMESTAMP + DEFAULT_YAML_EXTENSION; public GenerateYamlConfiguration(ConfigParams configParams, Map items) { this.configParams = configParams; @@ -61,6 +56,13 @@ public Map getGlobals() { return globals; } + public Map getDistributedConfig() { + Map distributedConfigParms = configParams.getDistributedConfigParams(); + // remove this so that the driver won't try to trigger the execution + distributedConfigParms.remove("driverip"); + return distributedConfigParms; + } + public Map getPublishmode() { return configParams.getPublishModeParams(); } @@ -185,85 +187,28 @@ private void excludeAction(String item, int index) { } } - // Configure yaml representer to exclude class tags when dumping an object. - private void configureYamlRepresenterToExcludeClassTags(Representer representer) { - // Tag.MAP is by default ignored when dumping an object - representer.addClassTag(GenerateYamlConfiguration.class, Tag.MAP); - for (Class klass : ReflectionsContainer.getSubTypesOf(YamlDumpAction.class)) { - representer.addClassTag(klass, Tag.MAP); - } + public String createYamlStringRepresentation() { + Yaml yaml = YamlBuilder.getYamlInstance(); + return yaml.dump(this); } /** * Creates a YAML configuration file. */ public void createYamlConfigurationFile() { + String yamlStringRepresentation = createYamlStringRepresentation(); - final String filename = DEFAULT_YAML_CONFIGURATION_FILENAME + TIMESTAMP + DEFAULT_YAML_EXTENSION; - - FileWriter fileWriter = null; try { - fileWriter = new FileWriter(filename); - } catch (IOException e) { - e.printStackTrace(); - } - - BufferedWriter bufferedWriter = new BufferedWriter(fileWriter); - org.yaml.snakeyaml.constructor.Constructor constructor = new org.yaml.snakeyaml.constructor.Constructor - (GenerateYamlConfiguration.class); - - DumperOptions dumperOptions = new DumperOptions(); - dumperOptions.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - dumperOptions.setAllowReadOnlyProperties(true); - - // Configure the representer to ignore empty fields when dumping the object. By default, each empty filed is represented as {}. - - Representer representer = new Representer() { - @Override - protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) { - - Method method = null; - try { - if (propertyValue == null) { - propertyValue = ""; - } - method = propertyValue.getClass().getMethod("isEmpty"); - } catch (NoSuchMethodException e) { } - - if (method == null) { - return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); - } else { - - try { - if (Boolean.valueOf(method.invoke(propertyValue).toString())) { - return null; - } - } catch (Throwable t) { - t.printStackTrace(); - } - - return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); - } - } - }; + FileWriter fileWriter = new FileWriter(yamlConfigFilename); + BufferedWriter bufferedWriter = new BufferedWriter(fileWriter); - configureYamlRepresenterToExcludeClassTags(representer); - - // dump configuration - Yaml yaml = new Yaml(constructor, representer, dumperOptions); - String yamlObjectRepresentation = yaml.dump(this); - - try { - bufferedWriter.write(yamlObjectRepresentation); + bufferedWriter.write(yamlStringRepresentation); bufferedWriter.flush(); - } catch (IOException e) { - e.printStackTrace(); - } - try { fileWriter.close(); bufferedWriter.close(); + } catch (IOException e) { e.printStackTrace(); } diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlBuilder.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlBuilder.java new file mode 100644 index 0000000..93345db --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlBuilder.java @@ -0,0 +1,99 @@ +package com.adobe.qe.toughday.internal.core.config.parsers.yaml; + +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; + +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.introspector.Property; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Representer; + +import java.lang.reflect.Method; + +/** + * Class responsible for building and configuring a Yaml instance that knows how to dump a configuration + * in the desired format(without class tags, empty objects etc.) + */ +public class YamlBuilder { + + private static Yaml instance = null; + + // Configure yaml representer to exclude class tags when dumping an object. + private static void configureYamlRepresenterToExcludeClassTags(Representer representer) { + // Tag.MAP is by default ignored when dumping an object + representer.addClassTag(GenerateYamlConfiguration.class, Tag.MAP); + representer.addClassTag(YamlDumpConfigurationAsTaskForTDAgents.class, Tag.MAP); + for (Class type : ReflectionsContainer.getSubTypesOf(YamlDumpAction.class)) { + representer.addClassTag(type, Tag.MAP); + } + + } + + private static void buildInstance() { + DumperOptions dumperOptions = getDumperOptions(); + + // Configure the representer to ignore empty fields when dumping the object. By default, each empty filed is represented as {}. + Representer representer = getRepresenter(); + configureYamlRepresenterToExcludeClassTags(representer); + + Constructor constructor = new Constructor(GenerateYamlConfiguration.class); + instance = new Yaml(constructor, representer, dumperOptions); + } + + /** + * Getter for the Yaml instance. + */ + public static Yaml getYamlInstance() { + if (instance == null) { + buildInstance(); + } + + return instance; + } + + + private static Representer getRepresenter() { + return new Representer() { + @Override + protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) { + + Method method = null; + try { + if (propertyValue == null) { + propertyValue = ""; + } + method = propertyValue.getClass().getMethod("isEmpty"); + } catch (NoSuchMethodException ignored) { } + + if (method == null) { + return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); + } else { + + try { + if (Boolean.valueOf(method.invoke(propertyValue).toString())) { + return null; + } + } catch (Throwable t) { + /* no action required here. The only implication is that the empty collection will be included + * in the Yaml representation when dumping an object. + */ + } + + return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); + } + } + }; + } + + private static DumperOptions getDumperOptions() { + DumperOptions dumperOptions = new DumperOptions(); + + dumperOptions.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + dumperOptions.setAllowReadOnlyProperties(true); + + return dumperOptions; + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlConfiguration.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlConfiguration.java index 46f3202..4d40077 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlConfiguration.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlConfiguration.java @@ -21,6 +21,9 @@ public class YamlConfiguration { private ConfigParams configParams = new ConfigParams(); + private void setDistributedConfig(Map distributedConfig) { + this.configParams.setDistributedConfigParams(distributedConfig); + } public void setGlobals(Map globals) { this.configParams.setGlobalParams(globals); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlParser.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlParser.java index b511938..a5cef99 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlParser.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/config/parsers/yaml/YamlParser.java @@ -43,6 +43,31 @@ public String description() { } }; + private Constructor getYamlConstructor() { + Constructor constructor = new Constructor(YamlConfiguration.class); + TypeDescription yamlParserDesc = new TypeDescription(YamlConfiguration.class); + + yamlParserDesc.putListPropertyType("tests", YamlParseAction.class); + yamlParserDesc.putListPropertyType("phases", YamlParsePhase.class); + yamlParserDesc.putListPropertyType("publishers", YamlParseAction.class); + yamlParserDesc.putListPropertyType("metrics", YamlParseAction.class); + yamlParserDesc.putListPropertyType("extensions", YamlParseAction.class); + yamlParserDesc.putListPropertyType("feeders", YamlParseAction.class); + + constructor.addTypeDescription(yamlParserDesc); + + return constructor; + } + + public ConfigParams parse(String stringYaml) { + Constructor constructor = getYamlConstructor(); + + Yaml yaml = new Yaml(constructor); + YamlConfiguration yamlConfig = (YamlConfiguration) yaml.load(stringYaml); + + return yamlConfig.getConfigParams(); + } + @Override public ConfigParams parse(String[] cmdLineArgs) { String configFilePath = null; @@ -55,16 +80,7 @@ public ConfigParams parse(String[] cmdLineArgs) { if(configFilePath != null) { try { - Constructor constructor = new Constructor(YamlConfiguration.class); - TypeDescription yamlParserDesc = new TypeDescription(YamlConfiguration.class); - yamlParserDesc.putListPropertyType("tests", YamlParseAction.class); - yamlParserDesc.putListPropertyType("phases", YamlParsePhase.class); - yamlParserDesc.putListPropertyType("publishers", YamlParseAction.class); - yamlParserDesc.putListPropertyType("metrics", YamlParseAction.class); - yamlParserDesc.putListPropertyType("extensions", YamlParseAction.class); - yamlParserDesc.putListPropertyType("feeders", YamlParseAction.class); - - constructor.addTypeDescription(yamlParserDesc); + Constructor constructor = getYamlConstructor(); Yaml yaml = new Yaml(constructor); YamlConfiguration yamlConfig = (YamlConfiguration) yaml.load(new FileInputStream(configFilePath)); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitor.java new file mode 100644 index 0000000..5e29cd9 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitor.java @@ -0,0 +1,186 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; + +/** + * Class responsible for monitoring the execution of a distributed phase. + */ +public class DistributedPhaseMonitor { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private final ExecutorService executorService = Executors.newFixedThreadPool(1); + private final List agentsRunningTD = new ArrayList<>(); + private final List agentsWhichCompletedCurrentPhase = new ArrayList<>(); + // key = name of the test; value = map(key = name of the agent, value = nr of tests executed) + private Map> executions = new HashMap<>(); + private Phase phase; + private long phaseStartTime = 0; + + /** + * Returns true is the exections of the phase has started. + */ + public boolean isPhaseExecuting() { + return this.phase != null && !this.agentsRunningTD.isEmpty(); + } + + public List getAgentsRunningTD() { + return this.agentsRunningTD; + } + + public List getAgentsWhichCompletedCurrentPhase() { + return this.agentsWhichCompletedCurrentPhase; + } + + public void setExecutions(Map> executions) { + this.executions = executions; + } + + /** + * Setter for the execution start time of the phase. + */ + public void setPhaseStartTime(long phaseStartTime) { + this.phaseStartTime = phaseStartTime; + } + + /** + * Getter for the execution start time of the phase + */ + public long getPhaseStartTime() { + return this.phaseStartTime; + } + + /** + * Setter for the phase being monitored + */ + public void setPhase(Phase phase) { + this.phase = phase; + this.phase.getTestSuite().getTests().forEach(test -> executions.put(test.getName(), new HashMap<>())); + + // reset agents which completed the current phase + this.agentsWhichCompletedCurrentPhase.clear(); + } + + /** + * Adds the agent received as a parameter to the list of agents which are currently executing the + * phase being monitored. + * @param agentIdentifier : ip address that identifies the agent inside the cluster + */ + public void registerAgentRunningTD(String agentIdentifier) { + if (!this.agentsRunningTD.contains(agentIdentifier)) { + this.agentsRunningTD.add(agentIdentifier); + } + + } + + public void addAgentWhichCompletedTheCurrentPhase(String agentIdentifier) { + if (!this.agentsWhichCompletedCurrentPhase.contains(agentIdentifier)) { + this.agentsWhichCompletedCurrentPhase.add(agentIdentifier); + } + } + + /** + * Removes tha agent received as a parameter from the list of agents which are currently executing + * the phase being monitored. + * @param agentIdentifier : ip address that identifies the agent inside the cluster + */ + public void removeAgentFromActiveTDRunners(String agentIdentifier) { + this.agentsRunningTD.remove(agentIdentifier); + + if (agentsWhichCompletedCurrentPhase.contains(agentIdentifier)) { + this.agentsWhichCompletedCurrentPhase.remove(agentIdentifier); + } + + // remove agent from executions map + this.executions.forEach((test, executionsPerAgent) -> this.executions.get(test).remove(agentIdentifier)); + } + + /** + * Method for waiting until all agents finish executing the phase being monitored. + */ + public boolean waitForPhaseCompletion(int retries) { + while (retries > 0) { + Future waitPhaseCompletion = executorService.submit(() -> { + + while (this.agentsRunningTD.size() != this.agentsWhichCompletedCurrentPhase.size()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // this will not cause further problems => it can be ignored + } + } + }); + + try { + waitPhaseCompletion.get(); + return true; + } catch (InterruptedException | ExecutionException e) { + retries--; + } + } + + LOG.warn("Exception occurred while waiting for the completion of the current phase. The rest " + + "of the phases will no longer be executed"); + return false; + + } + + /** + * Updates the number of executions/test that must be executed by the agents running in the cluster. This method is + * called during the work redistribution process. + */ + public void updateCountPerTest() { + phase.getTestSuite().getTests().forEach(test -> { + long remained = test.getCount() - this.getExecutionsPerTest().get(test.getName()); + if (remained < 0) { + // set this to 0 so that the agents will know to delete the test from the test suite + test.setCount("0"); + } else { + test.setCount(String.valueOf(remained)); + } + }); + } + + /** + * Returns the number of executions/test/agent. + */ + public Map> getExecutions() { + return this.executions; + } + + /** + * Resets the number of execution/test for each agent executing the phase being monitored. + */ + public void resetExecutions() { + this.executions.forEach((testName, executionsPerAgent) -> + executionsPerAgent.keySet().forEach(agentName -> executionsPerAgent.put(agentName, 0L))); + } + + /** + * Returns the number of executions/test executed by all the agents running in the cluster. + */ + public Map getExecutionsPerTest() { + Map executionsPerTest = new HashMap<>(); + + this.executions.forEach((testName, executionsPerAgent) -> + executionsPerTest.put(testName, executionsPerAgent.values().stream().mapToLong(x -> x).sum())); + + return executionsPerTest; + } + + /** + * Getter for the phase being monitored by this class. + */ + public Phase getPhase() { + return this.phase; + } + +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/ExecutionTrigger.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/ExecutionTrigger.java new file mode 100644 index 0000000..064a415 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/ExecutionTrigger.java @@ -0,0 +1,47 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.GenerateYamlConfiguration; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import org.apache.http.HttpResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashMap; + + +/** + * Class responsible for sending a request to the driver component running in the cluster. This + * request will trigger the distributed execution of TD. + * */ +public class ExecutionTrigger { + + protected static final Logger LOG = LogManager.getLogger(ExecutionTrigger.class); + private final Configuration configuration; + + + public ExecutionTrigger(Configuration configuration) { + this.configuration = configuration; + // sanity check + if (configuration.getDistributedConfig().getDriverIp() == null || configuration.getDistributedConfig().getDriverIp().isEmpty()) { + throw new IllegalStateException("The public ip address at which the driver's service is accessible " + + " is required when running TD in distributed mode."); + } + } + + /** + * Method used for triggering the execution of TD in distributed mode. + */ + public void triggerDistributedExecution() { + GenerateYamlConfiguration generateYaml = new GenerateYamlConfiguration(this.configuration.getConfigParams(), new HashMap<>()); + String yamlConfig = generateYaml.createYamlStringRepresentation(); + HttpUtils httpUtils = new HttpUtils(); + + HttpResponse response = httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, yamlConfig, + Driver.getExecutionPath(this.configuration.getDistributedConfig().getDriverIp(), HttpUtils.SVC_PORT, true), 3); + if (response == null) { + LOG.warn("TD execution request could not be sent to driver. Make sure that driver is up" + + " and ready to process requests."); + } + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/HttpUtils.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/HttpUtils.java new file mode 100644 index 0000000..91917b2 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/HttpUtils.java @@ -0,0 +1,100 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.api.annotations.labels.Nullable; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.io.IOException; + +/** + * Class responsible for handling the communication between the agents and the drivers running in the cluster. + */ +public class HttpUtils { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + public static final String POST_METHOD = "POST"; + public static final String GET_METHOD = "GET"; + public static final String URL_PREFIX = "http://"; + public static final int HTTP_REQUEST_RETRIES = 3; + public static final String SVC_PORT = "80"; + public static final String SPARK_PORT = "4567"; + public static final String FORWARD_QUERY_PARAM = "?forward="; + + + + private HttpResponse sendGetRequest(String URI) { + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(10 * 1000) // 5 seconds + .build(); + HttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(config).build(); + HttpGet request = new HttpGet(URI); + + try { + return httpClient.execute(request); + } catch (IOException e) { + LOG.warn("Http request could not be sent to " + URI + ". Received error " + e.getMessage()); + } + + return null; + } + + private HttpResponse sendPostRequest(String requestContent, String URI) { + HttpClient httpClient = HttpClientBuilder.create().build(); + HttpPost request = new HttpPost(URI); + + try { + StringEntity params = new StringEntity(requestContent); + request.setEntity(params); + request.setHeader("Content-type", "text/plain"); + + // submit request and wait for ack from agent + return httpClient.execute(request); + } catch (IOException e) { + LOG.warn("Http request could not be sent to " + URI + ". Received error " + e.getMessage()); + } + + return null; + } + + private boolean checkSuccessfulRequest(HttpResponse response) { + return response != null && response.getStatusLine().getStatusCode() >= 200 && + response.getStatusLine().getStatusCode() < 300; + } + + /** + * Method used for sending http requests. + * @param requestContent content of the request + * @param URI the path that uniquely identifies the component in the cluster and the http endpoint to be used + * when sending the request + * @param retries how many times to retry sending the request in case of failure + * @return true is the the request is successfully sent, false otherwise. + */ + @Nullable + public HttpResponse sendHttpRequest(String requestType, String requestContent, String URI, int retries) { + HttpResponse response = null; + + while (retries > 0) { + if (requestType.equals(POST_METHOD)) { + response = sendPostRequest(requestContent, URI); + } else if (requestType.equals(GET_METHOD)) { + response = sendGetRequest(URI); + } + + if (checkSuccessfulRequest(response)) { + return response; + } + + retries--; + } + + return null; + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgents.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgents.java new file mode 100644 index 0000000..9c3ba5f --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgents.java @@ -0,0 +1,175 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; +import com.adobe.qe.toughday.api.annotations.ConfigArgSet; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.*; + +import com.adobe.qe.toughday.internal.core.config.parsers.cli.CliParser; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.YamlBuilder; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.YamlDumpAddAction; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.YamlDumpPhase; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.DistributedConfig; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.yaml.snakeyaml.Yaml; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.*; + +/** + * Knows how to dump a Configuration object in yaml format in order to send it to the agents running in the cluster. + * This class assumes that it is not necessary to dump the run mode and the publish mode fields of the Configuration + * class since each Phase contained by this configuration already defines them. + * + * It is also assumed that the only action to be taken into consideration when dumping the configuration is Actions.ADD + * since all the other actions were already processed before sending the execution query to the driver. + */ +public class YamlDumpConfigurationAsTaskForTDAgents { + private static final Logger LOG= LogManager.getLogger(CliParser.class); + + private Configuration configuration; + private List phases = new ArrayList<>(); + + public List getPhases() { + return this.phases; + } + + /** + * This method is used for collecting all configurable properties that were assigned a new value(different than the + * default one) from the command line or the yaml configuration file used for running ToughDay. + * @param type class of the object whose properties are collected + * @param object instance + */ + protected Map collectConfigurableProperties(Class type, Object object) { + if (!type.isAssignableFrom(object.getClass())) { + throw new IllegalArgumentException("The object must have the specified type."); + } + + Map configurableArgs = new HashMap<>(); + + /* add all inherited configurable properties */ + if (type.getSuperclass() != Object.class) { + configurableArgs.putAll(collectConfigurableProperties(type.getSuperclass(), object)); + } + + Arrays.stream(type.getDeclaredMethods()) + .filter(method -> method.isAnnotationPresent(ConfigArgGet.class)) + .forEach(method -> { + String property = Configuration.propertyFromMethod(method.getName()); + try { + Object value = method.invoke(object); + + Class[] parametersType = {String.class}; + Method m = type.getMethod(method.getName().replace("get", "set"), parametersType); + String defaultValue = m.getAnnotation(ConfigArgSet.class).defaultValue(); + + // skip all default values + if (!String.valueOf(value).equals(defaultValue) && value != null) { + if (value instanceof Level) { + configurableArgs.put(property, ((Level) value).name()); + } else { + configurableArgs.put(property, value); + } + } + } catch (IllegalAccessException | InvocationTargetException e) { + LOG.warn("Configurable property " + property + " could not be collected when dumping the " + + "configuration. Please check that public methods with @ConfigArgSet and @ConfigArgGet " + + "were defined for this property."); + } catch ( NoSuchMethodException e) { + // skip for now + LOG.warn("No @ConfigArgGet method was defined for property " + property + ". The configuration " + + "will be dumped without this property."); + } + }); + + return configurableArgs; + } + + private void addAction(ConfigParams.ClassMetaObject item, YamlDumpPhase yamlDumpPhase) { + YamlDumpAddAction addAction = new YamlDumpAddAction(item.getClassName(), item.getParameters()); + if (ReflectionsContainer.getInstance().isTestClass(item.getClassName())) { + yamlDumpPhase.getTests().add(addAction); + } else if (ReflectionsContainer.getInstance().isMetricClass(item.getClassName())) { + yamlDumpPhase.getMetrics().add(addAction); + } else if (ReflectionsContainer.getInstance().isPublisherClass(item.getClassName())) { + yamlDumpPhase.getPublishers().add(addAction); + } + } + + private void createItem(Object object, List> items ) { + Map parameters = collectConfigurableProperties(object.getClass(), object); + ConfigParams.MetaObject metaObject = new ConfigParams.ClassMetaObject(object.getClass().getSimpleName(), parameters); + items.add(new AbstractMap.SimpleEntry<>(Actions.ADD, metaObject)); + } + + private void buildYamlDumpPhases() { + configuration.getPhases().forEach(phase -> { + // collect all configurable properties + Map properties = collectConfigurableProperties(Phase.class, phase); + Map runMode = collectConfigurableProperties(phase.getRunMode().getClass(), phase.getRunMode()); + Map publishMode = collectConfigurableProperties(phase.getPublishMode().getClass(), phase.getPublishMode()); + + // add required type parameter for run mode and publish mode + runMode.put("type", phase.getRunMode().getClass().getSimpleName().toLowerCase()); + publishMode.put("type", phase.getPublishMode().getClass().getSimpleName().toLowerCase()); + + List> items = new ArrayList<>(); + + // create tests + phase.getTestSuite().getTests().forEach(test -> createItem(test, items)); + // create metrics + phase.getMetrics().forEach(metric -> createItem(metric, items)); + // create publishers + phase.getPublishers().forEach(publisher -> createItem(publisher, items)); + + YamlDumpPhase yamlDumpPhase = new YamlDumpPhase(properties, runMode, publishMode); + items.forEach(entry -> addAction((ConfigParams.ClassMetaObject)entry.getValue(), yamlDumpPhase)); + + this.phases.add(yamlDumpPhase); + }); + } + + public YamlDumpConfigurationAsTaskForTDAgents(Configuration configuration) { + if (configuration == null) { + throw new IllegalArgumentException("Configuration must not be null."); + } + + this.configuration = configuration; + } + + /** + * Getter for global params. + */ + public Map getGlobals() { + Map globals = + collectConfigurableProperties(GlobalArgs.class, configuration.getGlobalArgs()); + /* this is required because duration is internally converted from string(which includes the unit of measure) + to long value(duration in seconds) so we need to manually add the unit of measure when dumping it. */ + globals.put("duration", String.valueOf(globals.get("duration")) + 's'); + + return globals; + } + + + /** + * Getter for the distributed configuration. + */ + public Map getDistributedConfig() { + return collectConfigurableProperties(DistributedConfig.class, configuration.getDistributedConfig()); + } + + /** + * Dumps the configuration received in the constructor. + */ + public String generateConfigurationObject() { + buildYamlDumpPhases(); + Yaml yaml = YamlBuilder.getYamlInstance(); + + return yaml.dump(this); + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Agent.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Agent.java new file mode 100644 index 0000000..edd6070 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/Agent.java @@ -0,0 +1,303 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster; + +import com.adobe.qe.toughday.api.core.AbstractTest; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.config.PhaseParams; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructionsProcessor; +import com.google.gson.Gson; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.net.*; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.URL_PREFIX; +import static com.adobe.qe.toughday.internal.core.engine.Engine.installToughdayContentPackage; +import static spark.Spark.*; + +/** + * Agent component for running TD distributed. + */ +public class Agent { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + private final ExecutorService tdExecutorService = Executors.newFixedThreadPool(1); + private final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(1); + + public enum Status { + IDLE, /* Agent is waiting to receive tasks from the driver */ + BUILDING_CONFIG, /* Agent has received one task from the driver and it is building the TD configuration */ + RUNNING, /* Agent is running TD tests */ + PHASE_COMPLETED /* Agent finished executing the current phase */ + } + + // available routes + private static final String INSTALL_SAMPLE_CONTENT_PATH = "/sampleContent"; + private static final String SUBMIT_TASK_PATH = "/submitTask"; + private static final String FINISH_PATH = "/finish"; + private static final String HEARTBEAT_PATH = "/heartbeat"; + private static final String REBALANCE_PATH = "/rebalance"; + private static final String HEALTH_PATH = "/health"; + private static final String GET_STATUS_PATH = "/status"; + + private HttpUtils httpUtils = new HttpUtils(); + private Status status = Status.IDLE; + private final Lock statusLock = new ReentrantLock(); + + + /** + * Returns the http URL that the driver should use for finished the execution of the agent. + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getFinishPath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + HttpUtils.SPARK_PORT + FINISH_PATH; + } + + /** + * Returns the http URL that the driver should use for sending heartbeat messages to the agent. + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getHeartbeatPath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + HttpUtils.SPARK_PORT + HEARTBEAT_PATH; + } + + /** + * Returns the http URL that the driver should use for sending a task to the agent. + * @param agentIpAdress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getSubmissionTaskPath(String agentIpAdress) { + return URL_PREFIX + agentIpAdress + ":" + HttpUtils.SPARK_PORT + SUBMIT_TASK_PATH; + } + + /** + * Returns the http URL that the driver should use for sending redistribution instructions to the agent. + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getRebalancePath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + HttpUtils.SPARK_PORT + REBALANCE_PATH; + } + + /** + * Returns the http URL that the driver should use for requiring the agent to install the TD sample content package. + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getInstallSampleContentPath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + HttpUtils.SPARK_PORT + INSTALL_SAMPLE_CONTENT_PATH; + } + + /** + * Returns the http URL that the driver should use for getting the current status of the agent + * @param agentIpAddress : the ip address that uniquely identifies the agent in the cluster + */ + public static String getGetStatusPath(String agentIpAddress) { + return URL_PREFIX + agentIpAddress + ":" + HttpUtils.SPARK_PORT + GET_STATUS_PATH; + } + + private Engine engine; + private static String ipAddress; + + static { + try { + ipAddress = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + System.exit(-1); + } + } + + private final RedistributionInstructionsProcessor redistributionInstructionsProcessor = new RedistributionInstructionsProcessor(); + + public boolean announcePhaseCompletion() { + /* the current master might be dead so we should retry this for a certain amount of time before shutting + * down the execution. + */ + + HttpUtils httpUtils = new HttpUtils(); + HttpResponse response = null; + long duration = GlobalArgs.parseDurationToSeconds("60s"); + + while (duration > 0 && response == null) { + response = httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, ipAddress, + Driver.getPhaseFinishedByAgentPath("driver", HttpUtils.SVC_PORT, true), HTTP_REQUEST_RETRIES); + + try { + Thread.sleep(10 * 1000L); // try again in 10 seconds + } catch (InterruptedException e) { + // skip this since this thread is not interrupted by anyone + } finally { + duration -= GlobalArgs.parseDurationToSeconds("10s"); + } + } + + return response != null; + } + + public void start() { + register(); + + /* expose http endpoint to allow the driver to ask for ToughDay sample content package to be installed */ + post(INSTALL_SAMPLE_CONTENT_PATH, ((request, response) -> { + String yamlConfig = request.body(); + Configuration configuration = new Configuration(yamlConfig); + + this.tdExecutorService.submit(() -> { + boolean installed = true; + + try { + installToughdayContentPackage(configuration.getGlobalArgs()); + } catch (Exception e) { + installed = false; + LOG.error("Error encountered when installing TD sample content", e); + } + + HttpResponse driverResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, String.valueOf(installed), + Driver.getSampleContentAckPath("driver", HttpUtils.SVC_PORT), HTTP_REQUEST_RETRIES); + if (driverResponse == null) { + LOG.error("Agent " + ipAddress + " could not announce the driver that Toughday sample content" + + " package was installed."); + System.exit(-1); + } + + }); + + // clear all phases + PhaseParams.namedPhases.clear(); + return ""; + })); + + /* Expose http endpoint for receiving ToughDay execution request from the driver */ + post(SUBMIT_TASK_PATH, ((request, response) -> { + try { + this.statusLock.lock(); + if (this.status != Status.IDLE) { + LOG.info("Agent is currently executing a different task. Request will be ignored."); + + // set status code to Conflict + response.status(HttpStatus.SC_CONFLICT); + return "Agent is already running a task received from the Driver."; + } + + // update current status + this.status = Status.BUILDING_CONFIG; + } finally { + this.statusLock.unlock(); + } + + String yamlTask = request.body(); + LOG.info("[Agent] Received task:\n" + yamlTask); + + Configuration configuration = new Configuration(yamlTask); + this.engine = new Engine(configuration); + this.engine.setAgent(this); + + tdExecutorService.submit(() -> { + this.statusLock.lock(); + this.status = Status.RUNNING; + this.statusLock.unlock(); + + this.engine.runTests(); + + if (!announcePhaseCompletion()) { + LOG.error("Agent " + ipAddress + " could not inform driver that phase was executed."); + System.exit(-1); + } + + this.statusLock.lock(); + this.status = Status.PHASE_COMPLETED; + this.statusLock.unlock(); + + }); + + return ""; + })); + + /* Expose http endpoint to be used by the driver for heartbeat messages */ + get(HEARTBEAT_PATH, ((request, response) -> + { + // send to driver the total number of executions/test + Gson gson = new Gson(); + Map currentCounts = new HashMap<>(); + + // check if execution has started + if (this.status != Status.RUNNING) { + return gson.toJson(currentCounts); + } + + Map phaseCounts = engine.getCurrentPhase().getCounts(); + phaseCounts.forEach((test, count) -> currentCounts.put(test.getName(), count.get())); + + return gson.toJson(currentCounts); + })); + + + + /* expose http endpoint for receiving redistribution requests from the driver */ + post(REBALANCE_PATH, (((request, response) -> { + try { + statusLock.lock(); + if (this.status == Status.IDLE) { + LOG.warn("Rebalance was requested before submitting a task. Request will be ignored."); + return ""; + } + } finally { + statusLock.unlock(); + } + + String instructionsMessage = request.body(); + LOG.info("[Agent] Received " + instructionsMessage + " from driver"); + this.redistributionInstructionsProcessor.processInstructions(instructionsMessage, this.engine.getConfiguration().getPhases().get(0)); + + return ""; + }))); + + + /* expose http endpoint for health checks */ + get(HEALTH_PATH, ((request, response) -> "Healthy")); + + /* expose http endpoint for getting the current status of the agent */ + get(GET_STATUS_PATH, ((request, response) -> { + try { + this.statusLock.lock(); + return this.status; + } finally { + this.statusLock.unlock(); + } + })); + + /* expose http endpoint for finishing the execution of the agent */ + post(FINISH_PATH, ((request, response) -> { + scheduledExecutorService.schedule(() -> { + this.tdExecutorService.shutdown(); + this.scheduledExecutorService.shutdown(); + + System.exit(0); + }, GlobalArgs.parseDurationToSeconds("3s"), TimeUnit.SECONDS); + + return ""; + })); + } + + /* Method responsible for registering the current agent to the driver. It should be the + * first method executed. + */ + private void register() { + LOG.info("Registering..."); + HttpResponse response = + this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, ipAddress, + Driver.getAgentRegisterPath("driver", HttpUtils.SVC_PORT, true), HTTP_REQUEST_RETRIES); + if (response == null) { + System.exit(-1); + } + + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/DistributedConfig.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/DistributedConfig.java new file mode 100644 index 0000000..62ecf1a --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/DistributedConfig.java @@ -0,0 +1,98 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster; + +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; +import com.adobe.qe.toughday.api.annotations.ConfigArgSet; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; + +/** + * Contains all the configurable arguments when running TD distributed. + */ +public class DistributedConfig { + private static final String DEFAULT_HEARTBEAT_INTERVAL = "5s"; + private static final String DEFAULT_REDISTRIBUTION_WAIT_TIME = "3s"; + private static final String DEFAULT_CLUSTER_NAMESPACE = "default"; + + private boolean agent = false; + private boolean driver = false; + private String driverIp = null; + private String heartbeatInterval = DEFAULT_HEARTBEAT_INTERVAL; + private String redistributionWaitTime = DEFAULT_REDISTRIBUTION_WAIT_TIME; + private String clusterNamespace = DEFAULT_CLUSTER_NAMESPACE; + + @ConfigArgSet(required = false, desc = "The public ip address of the cluster. The driver" + + " service must be accessible at this address. This property is required when running in distributed mode.") + public void setDriverIp(String driverIp) { + this.driverIp = driverIp; + } + + @ConfigArgGet + public String getDriverIp() { + return this.driverIp; + } + + @ConfigArgGet + public boolean getAgent() { return this.agent; } + + @ConfigArgSet(required = false, defaultValue = "false", desc = "If true, TD runs as a cluster agent, waiting to receive" + + " a task from the driver.") + public void setAgent(String agent) { + this.agent = Boolean.parseBoolean(agent); + } + + @ConfigArgGet + public boolean getDriver() { + return this.driver; + } + + @ConfigArgSet(required = false, defaultValue = "false", desc = "If true, TD runs as a driver in the cluster," + + " distributing the work between the agents.") + public void setDriver(String driver) { + this.driver = Boolean.parseBoolean(driver); + } + + @ConfigArgSet(required = false, defaultValue = DEFAULT_HEARTBEAT_INTERVAL, desc = "Period of time for sending " + + "heartbeat messages to the agents in the cluster.") + public void setHeartbeatInterval(String heartbeatInterval) { + this.heartbeatInterval = heartbeatInterval; + } + + @ConfigArgGet + public String getHeartbeatInterval() { + return this.heartbeatInterval; + } + + @ConfigArgSet(required = false, defaultValue = DEFAULT_REDISTRIBUTION_WAIT_TIME, desc = "The minimum amount of time " + + " to wait before scheduling work redistribution if required.") + public void setRedistributionWaitTime(String redistributionWaitTime) { + this.redistributionWaitTime = redistributionWaitTime; + } + + @ConfigArgGet + public String getRedistributionWaitTime() { + return this.redistributionWaitTime; + } + + public long getRedistributionWaitTimeInSeconds() { + return GlobalArgs.parseDurationToSeconds(this.redistributionWaitTime); + } + + public long getHeartbeatIntervalInSeconds() { + return GlobalArgs.parseDurationToSeconds(this.heartbeatInterval); + } + + @ConfigArgSet(required = false, defaultValue = DEFAULT_CLUSTER_NAMESPACE, desc = "The namespace in the cluster " + + " where the resources(agents and drivers) were deployed.") + public void setClusterNamespace(String clusterNamespace) { + this.clusterNamespace = clusterNamespace; + } + + @ConfigArgGet + public String getClusterNamespace() { + return this.clusterNamespace; + } + + public void merge(DistributedConfig other) { + this.setHeartbeatInterval(other.getHeartbeatInterval()); + this.setRedistributionWaitTime(other.redistributionWaitTime); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/Driver.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/Driver.java new file mode 100644 index 0000000..138bd10 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/Driver.java @@ -0,0 +1,401 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.RequestProcessorDispatcher; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.PhaseSplitter; +import com.adobe.qe.toughday.internal.core.distributedtd.tasks.MasterHeartbeatTask; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.distributedtd.DistributedPhaseMonitor; +import com.adobe.qe.toughday.internal.core.distributedtd.tasks.HeartbeatTask; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.TaskBalancer; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import org.apache.http.HttpResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.*; +import java.util.concurrent.*; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.*; +import static spark.Spark.*; + +/** + * Driver component for the cluster. + */ +public class Driver { + // routes + public static final String EXECUTION_PATH = "/config"; + private static final String REGISTER_PATH = "/registerAgent"; + private static final String PHASE_FINISHED_BY_AGENT_PATH = "/phaseFinished"; + private static final String HEALTH_PATH = "/health"; + private static final String SAMPLE_CONTENT_ACK_PATH = "/contentAck"; + private static final String MASTER_ELECTION_PATH = "/masterElection"; + private static final String ASK_FOR_UPDATES_PATH = "/driverUpdates"; + private static final String GET_NR_DRIVERS_PATH = "/getNrDrivers"; + private static final String HEARTBEAT_PATH = "/heartbeat"; + private static final String AGENT_FAILURE_PATH = "/agentFailure"; + + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private final ScheduledExecutorService heartbeatScheduler = Executors.newSingleThreadScheduledExecutor(); + private final ExecutorService executorService = Executors.newFixedThreadPool(1); + private final HttpUtils httpUtils = new HttpUtils(); + private final TaskBalancer taskBalancer = TaskBalancer.getInstance(); + private DistributedPhaseMonitor distributedPhaseMonitor = new DistributedPhaseMonitor(); + private Configuration configuration; + private DriverState driverState; + private MasterElection masterElection; + private ScheduledFuture scheduledFuture = null; + + public DistributedPhaseMonitor getDistributedPhaseMonitor() { + return this.distributedPhaseMonitor; + } + + /** + * Getter for the task balancer instance. + */ + public TaskBalancer getTaskBalancer() { + return this.taskBalancer; + } + + /** + * Getter for the TD configuration which must be executed in distributed mode. + */ + public Configuration getConfiguration() { + return this.configuration; + } + + /** + * Getter for the state of the driver. + */ + public DriverState getDriverState() { + return this.driverState; + } + + /** + * Getter for master election instance. + */ + public MasterElection getMasterElection() { + return this.masterElection; + } + + /** + * Constructor + * @param configuration : configuration given when the driver was deployed into the cluster. + */ + public Driver(Configuration configuration) { + try { + String hostname = InetAddress.getLocalHost().getHostName(); + this.driverState = new DriverState(hostname, configuration); + } catch (UnknownHostException e) { + System.exit(-1); + } + + this.masterElection = MasterElection.getInstance(this.driverState.getNrDrivers()); + this.masterElection.electMasterWhenDriverJoinsTheCluster(this); + + LOG.info("Driver " + this.driverState.getHostname() + " elected as master " + this.driverState.getMasterId()); + } + + /** + * Returns the http URL that should be used for triggering the distributed execution in the cluster. + * @param driverIdentifier : the identifier of the driver that will received this execution query. + * @param port : the port on which the query must be sent. + * @param forwardReq : true if the request must be forwarded to all the other drivers running in the cluster; false + * otherwise. + */ + public static String getExecutionPath(String driverIdentifier, String port, boolean forwardReq) { + return HttpUtils.URL_PREFIX + driverIdentifier + ":" + port + Driver.EXECUTION_PATH + + HttpUtils.FORWARD_QUERY_PARAM + forwardReq; + } + + /** + * Returns the http URL that should be used by the agents whenever they finished executing the task received from + * the driver. + */ + public static String getPhaseFinishedByAgentPath(String driverIdentifier, String port, boolean forwardReq) { + return URL_PREFIX + driverIdentifier + ":" + port + PHASE_FINISHED_BY_AGENT_PATH + + HttpUtils.FORWARD_QUERY_PARAM + forwardReq; + } + + /** + * Returns the http URL that should be used by the agent chosen to install the TD sample content package for + * informing the driver that the installation was completed successfully. + */ + public static String getSampleContentAckPath(String driverIdentifier, String port) { + return URL_PREFIX + driverIdentifier + ":" + port + SAMPLE_CONTENT_ACK_PATH; + } + + /** + * Returns the http URL that should be used by the driver which detected that the current Master died to inform all + * the other drivers running in the cluster that the master election process must be triggered. + * @param driverHostname : hostname of the driver exposing this http endpoint + */ + public static String getMasterElectionPath(String driverHostname, String port) { + return URL_PREFIX + driverHostname + ":" + port + MASTER_ELECTION_PATH; + } + + /** + * Returns the http URL that should be used by the agents to register themselves after joining the cluster. + * @param driverIdentifier : specifies what kind of pods should received this request. + * @param port : the port on which the driver is listening for this type of request. + * @param forwardReq : true if this request should be forwarded to all the drivers running in the cluster; false + * otherwise + */ + public static String getAgentRegisterPath(String driverIdentifier, String port, boolean forwardReq) { + return URL_PREFIX + driverIdentifier + ":" + port + REGISTER_PATH + HttpUtils.FORWARD_QUERY_PARAM + forwardReq; + } + + + /** + * Returns the http URL that should be used by the Candidates to periodically check if the current Master is running + * properly. + * @param driverHostName : hostname that uniquely identifies the driver which should receive the heartbeat message + */ + public static String getHeartbeatPath(String driverHostName) { + return URL_PREFIX + driverHostName + ":" + HttpUtils.SPARK_PORT + HEARTBEAT_PATH; + } + + /** + * Returns the http URL that should be used by the drivers to get information about the current state of the + * execution. + * @param driverHostName : hostname of the driver component exposing this http endpoint + */ + public static String getAskForUpdatesPath(String driverHostName) { + return URL_PREFIX + driverHostName + ":" + SPARK_PORT + ASK_FOR_UPDATES_PATH; + } + + /** + * Returns the http URL that should be used by the current master to announce that a certain agent failed to respond + * to heartbeat messages. + * @param driverHostName : hostname of the driver component that should receive the http request + */ + public static String getAgentFailurePath(String driverHostName) { + return URL_PREFIX + driverHostName + ":" + SPARK_PORT + AGENT_FAILURE_PATH; + } + + /** + * Getter for the executor service used to submit the task of executing ToughDay distributed. The execution request + * must be handled in a different thread in order to be able to quickly send a response for this request. + */ + public ExecutorService getExecutorService() { + return this.executorService; + } + + /** + * Setter for the TD configuration to be executed in distributed mode. + */ + public void setConfiguration(Configuration configuration) { + this.configuration = configuration; + } + + /** + * Method used for announcing the agents that the TD configuration was successfully executed and they should stop + * their execution. + */ + public void finishAgents() { + this.driverState.getRegisteredAgents().forEach(agentIp -> { + LOG.info("[Driver] Finishing agent " + agentIp); + HttpResponse response = + httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, "", Agent.getFinishPath(agentIp), HTTP_REQUEST_RETRIES); + if (response == null) { + // the assumption is that the agent will be killed when he fails to respond to heartbeat request + LOG.warn("Driver could not finish the execution on agent " + agentIp + "."); + } + }); + + this.driverState.getRegisteredAgents().clear(); + } + + /** + * Method used to finish the distributed execution. The agents will be informed that there are no more phases to be + * executed and they will stop running. + */ + public void finishDistributedExecution() { + finishAgents(); + + this.executorService.shutdownNow(); + // finish tasks + this.heartbeatScheduler.shutdownNow(); + } + + /** + * Method used for scheduling the periodic task of sending heartbeat messages to all the agents running in the + * cluster to be executed at every 'heartbeatInterval' seconds. + */ + public void scheduleHeartbeatTask() { + // we should periodically send heartbeat messages from driver to all the agents + heartbeatScheduler.scheduleAtFixedRate(new HeartbeatTask(this), + 0, this.driverState.getDriverConfig().getDistributedConfig().getHeartbeatIntervalInSeconds(), TimeUnit.SECONDS); + } + + /** + * Method used for scheduling the periodic task of sending heartbeat messages to the driver playing the role of the + * Master to be executed at every 10 seconds. + */ + public void scheduleMasterHeartbeatTask() { + // we should periodically send heartbeat messages from candidates to check id the master is still running + this.scheduledFuture = this.heartbeatScheduler.scheduleAtFixedRate(new MasterHeartbeatTask(this), 0, + GlobalArgs.parseDurationToSeconds("10s"), TimeUnit.SECONDS); + } + + /** + * Method used for cancelling the periodic task of sending heartbeat messages to the driver running as Master in the + * cluster. + */ + public void cancelMasterHeartBeatTask() { + if (this.scheduledFuture != null) { + if(!this.scheduledFuture.cancel(true) && !this.scheduledFuture.isDone()) { + LOG.warn("Could not cancel task used to periodically send heartbeat messages to the Master."); + } + } + } + + /** + * Method used for cancelling the periodic task of sending heartbeat messaged to the agents running in the cluster. + */ + public void cancelHeartbeatTask() { + this.heartbeatScheduler.shutdownNow(); + } + + /** + * Method used to resume the distributed execution of ToughDay whenever a new master is elected in the cluster. + */ + public void resumeExecution() { + if (this.configuration == null) { + return; + } + + LOG.info("Resuming execution..."); + + // wait until current phase is successfully finished + if (!distributedPhaseMonitor.waitForPhaseCompletion(3)) { + finishDistributedExecution(); + return; + } + + LOG.info("Phase " + distributedPhaseMonitor.getPhase().getName() + " finished execution successfully."); + List remainingPhases = new ArrayList<>(configuration.getPhases()); + remainingPhases.remove(remainingPhases.get(0)); + + LOG.info("Phases left to be executed: " + remainingPhases.toString()); + this.configuration.setPhases(remainingPhases); + + // continue executing all the other phases + executePhases(); + } + + + /** + * Method used for running the phases of the TD configuration received to be executed distributed. This method + * should only be called once, when the master receives the query to execute TD. Whenever a new master is elected, + * the {@link #resumeExecution()} method should be used. + */ + public void executePhases() { + PhaseSplitter phaseSplitter = new PhaseSplitter(); + + LOG.info("[Driver] Executing phases + " + this.configuration.getPhases().toString()); + for (Phase phase : configuration.getPhases()) { + try { + Map tasks = phaseSplitter.splitPhase(phase, new ArrayList<>(this.driverState.getRegisteredAgents())); + this.distributedPhaseMonitor.setPhase(phase); + + for (String agentIp : this.driverState.getRegisteredAgents()) { + configuration.setPhases(Collections.singletonList(tasks.get(agentIp))); + + // convert configuration to yaml representation + YamlDumpConfigurationAsTaskForTDAgents dumpConfig + = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + String yamlTask = dumpConfig.generateConfigurationObject(); + + /* send query to agent and register running task */ + String URI = Agent.getSubmissionTaskPath(agentIp); + HttpResponse response = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, yamlTask, URI, HTTP_REQUEST_RETRIES); + + if (response != null) { + this.distributedPhaseMonitor.registerAgentRunningTD(agentIp); + LOG.info("Task was submitted to agent " + agentIp); + } else { + /* the assumption is that the agent is no longer active in the cluster and he will fail to respond + * to the heartbeat request sent by the driver. This will automatically trigger process of + * redistributing the work + * */ + LOG.info("Task\n" + yamlTask + " could not be submitted to agent " + agentIp + + ". Work will be rebalanced once the agent fails to respond to heartbeat request."); + } + } + + // al execution queries were sent => set phase execution start time + this.distributedPhaseMonitor.setPhaseStartTime(System.currentTimeMillis()); + + // we should wait until all agents complete the current tasks in order to execute phases sequentially + if (!this.distributedPhaseMonitor.waitForPhaseCompletion(3)) { + break; + } + + LOG.info("Phase " + phase.getName() + " finished execution successfully."); + + } catch (CloneNotSupportedException e) { + LOG.error("Phase " + phase.getName() + " could not de divided into tasks to be sent to the agents.", e); + + LOG.info("Finishing agents"); + finishAgents(); + + System.exit(-1); + } + } + + finishDistributedExecution(); + } + + /** + * Starts the execution of the driver. + */ + public void run() { + RequestProcessorDispatcher dispatcher = RequestProcessorDispatcher.getInstance(); + /* expose http endpoint for running TD with the given configuration */ + post(EXECUTION_PATH, ((request, response) -> dispatcher.getRequestProcessor(this) + .processExecutionRequest(request, this) + )); + + /* health check http endpoint */ + get(HEALTH_PATH, ((request, response) -> "Healthy")); + + /* expose http endpoint to allow agents to get the number of drivers running in the cluster */ + get(GET_NR_DRIVERS_PATH, ((request, response) -> String.valueOf(this.getDriverState().getNrDrivers()))); + + /* http endpoint used by the agent installing the sample content to announce if the installation was + * successful or not. */ + post(SAMPLE_CONTENT_ACK_PATH, ((request, response) -> + dispatcher.getRequestProcessor(this) + .acknowledgeSampleContentSuccessfulInstallation(request, this, response))); + + /* expose http endpoint to allow agents to announce when they finished executing the current phase */ + post(PHASE_FINISHED_BY_AGENT_PATH, ((request, response) -> + dispatcher.getRequestProcessor(this).processPhaseCompletionAnnouncement(request))); + + /* expose http endpoint for triggering the master election process */ + post(MASTER_ELECTION_PATH, ((request, response) -> + dispatcher.getRequestProcessor(this).processMasterElectionRequest(request, this))); + + /* expose http endpoint for sending information about the current state of the distributed execution */ + get(ASK_FOR_UPDATES_PATH, ((request, response) -> dispatcher.getRequestProcessor(this) + .processUpdatesRequest(request, this))); + + get(HEARTBEAT_PATH, ((request, response) -> dispatcher.getRequestProcessor(this) + .processHeartbeatRequest(request, this))); + + post(AGENT_FAILURE_PATH, ((request, response) -> dispatcher.getRequestProcessor(this) + .processAgentFailureAnnouncement(request, this))); + + /* expose http endpoint for registering new agents in the cluster */ + post(REGISTER_PATH, (request, response) -> dispatcher.getRequestProcessor(this) + .processRegisterRequest(request, this)); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverState.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverState.java new file mode 100644 index 0000000..2a868ea --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverState.java @@ -0,0 +1,214 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.regex.Pattern; + +/** + * Class responsible for encapsulating all the information required for describing the state of a Driver component. + */ +public class DriverState { + // Documentation says that each service has the following DNS A record: my-svc.my-namespace.svc.cluster.local + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private final String hostname; + private static final String SVC_NAME = "driver"; + private final Queue agents = new ConcurrentLinkedQueue<>(); + private final int id; + private int masterId = -1; + private int nrDrivers; + private Configuration driverConfig; + private Role currentState; + private final ReadWriteLock masterIdLock = new ReentrantReadWriteLock(); + + public enum Role { + CANDIDATE, + MASTER + } + + /** + * Returns the URI to be used for identifying a driver component using its id. + * @param id : the unique identifier associated with the driver component. + */ + public String getPathForId(int id) { + return SVC_NAME + "-" + id + "." + SVC_NAME + "." + + this.driverConfig.getDistributedConfig().getClusterNamespace() + ".svc.cluster.local"; + } + + /** + * Returns the lock used for synchronised access to the id of the driver component considered to be the Master. + */ + public ReadWriteLock getMasterIdLock() { + return this.masterIdLock; + } + + /** + * Constructor. + * @param hostname : hostname used to access the driver service inside the cluster. + * @param driverConfig : TD configuration used for deploying the driver. + */ + public DriverState(String hostname, Configuration driverConfig) { + this.driverConfig = driverConfig; + this.hostname = hostname; + + String name_pattern = "driver-[0-9]+\\." + SVC_NAME + "\\." + this.driverConfig.getDistributedConfig().getClusterNamespace() + + "\\.svc\\.cluster\\.local"; + + if (!Pattern.matches(name_pattern, hostname)) { + throw new IllegalStateException("Driver's name should respect the following format: driver-...svc.cluster.local"); + } + + String podName = hostname.substring(0, hostname.indexOf("." + SVC_NAME + "." + + this.driverConfig.getDistributedConfig().getClusterNamespace())); + + this.id = Integer.parseInt(podName.substring(podName.lastIndexOf("-") + 1)); + this.nrDrivers = Integer.parseInt(System.getenv("NR_DRIVERS")); + this.currentState = Role.CANDIDATE; + } + + /** + * Getter for the hostname. + */ + public String getHostname() { + return this.hostname; + } + + /** + * Getter for the id associated with the driver. + */ + public int getId() { + return this.id; + } + + /** + * Getter for the id of the driver considered to be the current master. + */ + public int getMasterId() { + return this.masterId; + } + + /** + * Getter for the number of driver components deployed in the cluster. + */ + public int getNrDrivers() { + return this.nrDrivers; + } + + /** + * Returns the list of active agents running in the cluster. + */ + public Queue getRegisteredAgents() { + return this.agents; + } + + /** + * Adds the agent to the list of agents running in the cluster. + * @param agentIdentifier : ipAddress used to uniquely identify the agent inside the cluster. + */ + public void registerAgent(String agentIdentifier) { + if (!this.agents.contains(agentIdentifier)) { + this.agents.add(agentIdentifier); + } + } + + /** + * Removes an agent from the list of active agents running in the cluster. + * @param agentIdentifier : ipAddress that uniquely identifies the agent to be removed. + */ + public void removeAgent(String agentIdentifier) { + this.agents.remove(agentIdentifier); + } + + /** + * Getter for the TD configuration of the driver component. + */ + public Configuration getDriverConfig() { + return this.driverConfig; + } + + /** + * Getter for the current status of the driver component. (Master or Candidate) + */ + public Role getCurrentState() { + return this.currentState; + } + + /** + * Setter the current status of the driver component. + */ + public void setCurrentState(Role role) { + this.currentState = role; + } + + /** + * Setter for the id of the driver running as master in the cluster. + */ + public void setMasterId(int id) { + this.masterId = id; + } + + /** + * Method used for updating the state of the cluster when information is received from another drivers. This method + * is called usually when a new driver joins the cluster and needs to know the current state of the execution (for + * example how many active agents are registered in the cluster) + * @param updates : information received from other drivers running in the cluster + * @param driverInstance : the Driver instance which received the updates. + */ + public void updateDriverState(DriverUpdateInfo updates, Driver driverInstance) { + // excludes candidates which are not allowed to become master + updates.getInvalidCandidates().forEach(driverInstance.getMasterElection()::markCandidateAsInvalid); + + // add all agents + updates.getRegisteredAgents().forEach(driverInstance.getDriverState()::registerAgent); + LOG.info("After processing update instructions, registered agents is " + + driverInstance.getDriverState().getRegisteredAgents().toString()); + LOG.info("After processing update instructions, invalid candidates are: " + + updates.getInvalidCandidates().toString()); + + // build TD configuration to be executed distributed + if (!updates.getYamlConfig().isEmpty() && driverInstance.getConfiguration() == null) { + LOG.info("Building configuration received from the other drivers running in the cluster."); + try { + Configuration configuration = new Configuration(updates.getYamlConfig()); + driverInstance.setConfiguration(configuration); + } catch (InvocationTargetException | NoSuchMethodException | InstantiationException | IOException | + IllegalAccessException e) { + /* if driver can't build the configuration executed distributed it will not be able to become the master + * and to coordinate the entire execution + */ + LOG.warn("Exception occurred when building ToughDay configuration to be executed distributed. Driver" + + " will leave the cluster"); + System.exit(-1); + } + + // delete phases that were previously executed + List phases = driverInstance.getConfiguration().getPhases(); + List previouslyExecutedPhases = new ArrayList<>(); + for (Phase phase : phases) { + if (phase.getName().equals(updates.getCurrentPhaseName())) { + break; + } + + previouslyExecutedPhases.add(phase); + } + LOG.info("phases that will be removed " + previouslyExecutedPhases.toString()); + phases.removeAll(previouslyExecutedPhases); + + // set current phase to be monitored + driverInstance.getDistributedPhaseMonitor().setPhase(phases.get(0)); + LOG.info("Current phase being executed is " + driverInstance.getDistributedPhaseMonitor().getPhase().getName()); + } + } +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverUpdateInfo.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverUpdateInfo.java new file mode 100644 index 0000000..958d8bc --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/DriverUpdateInfo.java @@ -0,0 +1,135 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver; + +import com.adobe.qe.toughday.internal.core.engine.Engine; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.Queue; + +/** + * Contains all the information needed for updating the state of a new driver when joining the cluster. + */ +public class DriverUpdateInfo { + private int driverId; + private DriverState.Role role; + private Queue invalidCandidates; + private Queue registeredAgents; + private String yamlConfig; + private String currentPhaseName; + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + + // TODO: add registeredAgents currently running tasks + // TODO: add registeredAgents which finished running the current phase + + + // dummy constructor used to dump the class + public DriverUpdateInfo() {} + + /** + * Constructor. + * @param driverId : the id of the driver which sent this updates. + * @param role : the status of the driver which sent this updates (Master or Candidate). + * @param invalidCandidates : list of drivers that are not eligible to become the new master if the current one dies + * @param registeredAgents : list of active agents running in the cluster. + * @param yamlConfig : TD configuration to be executed in distributed mode. This string is empty if the + * configuration was not received yet. + * @param currentPhaseName : the name of the current phase being executed by the agents running in the cluster. This + * is empty if the execution did not started yet. + */ + public DriverUpdateInfo(int driverId, DriverState.Role role, Queue invalidCandidates, + Queue registeredAgents, String yamlConfig, String currentPhaseName) { + this.driverId = driverId; + this.role = role; + this.invalidCandidates = invalidCandidates; + this.registeredAgents = registeredAgents; + this.yamlConfig = yamlConfig; + this.currentPhaseName = currentPhaseName; + } + + /** + * Getter for the id of the driver sending the updates. + */ + public int getDriverId() { + return this.driverId; + } + + /** + * Setter for the driver sending the updates. + */ + public void setDriverId(int driverId) { + this.driverId = driverId; + } + + /** + * Setter for the status of the driver sending the updates. + */ + public void setRole(DriverState.Role driverState) { + this.role = driverState; + } + + /** + * Getter for the status of the driver sending the updates. + * @return + */ + public DriverState.Role getRole() { + return this.role; + } + + /** + * Setter for the list of candidates to be excluded when electing a new master. + */ + public void setInvalidCandidates(Queue invalidCandidates) { + this.invalidCandidates = invalidCandidates; + } + + /** + * Getter for the list of candidates to be excluded when electing a new master. + * @return + */ + public Queue getInvalidCandidates() { + return this.invalidCandidates; + } + + /** + * Setter for the list of agents which are currently running in the cluster. + */ + public void setRegisteredAgents(Queue registeredAgents) { + this.registeredAgents = registeredAgents; + } + + /** + * Getter for the list of agents which are currently running in the cluster. + */ + public Queue getRegisteredAgents() { + return this.registeredAgents; + } + + /** + * Setter for the YAML representation of the TD configuration to be executed in distributed mode. + */ + public void setYamlConfig(String yamlConfig) { + this.yamlConfig = yamlConfig; + } + + /** + * Getter for the YAML representation of the TD configuration to be executed in distributed mode. + */ + public String getYamlConfig() { + return this.yamlConfig; + } + + /** + * Getter for the name of the phase being executed by the agents. + */ + public String getCurrentPhaseName() { + return this.currentPhaseName; + } + + /** + * Setter for the name of the phase being executed by the agents. + */ + public void setCurrentPhaseName(String currentPhaseName) { + this.currentPhaseName = currentPhaseName; + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/MasterElection.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/MasterElection.java new file mode 100644 index 0000000..c1c2945 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/MasterElection.java @@ -0,0 +1,264 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver; + +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpResponse; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Class responsible for implementing the master election process to be executed whenever a new Master must be elected + * in the cluster. + */ +public class MasterElection { + private int nrDrivers; + private Queue candidates; + protected static final Logger LOG = LogManager.getLogger(Engine.class); + private static MasterElection instance = null; + + private MasterElection(int nrDrivers) { + this.nrDrivers = nrDrivers; + this.candidates = IntStream.rangeClosed(0, nrDrivers - 1).boxed() + .collect(Collectors.toCollection(ConcurrentLinkedQueue::new)); + } + + /** + * Returns a Singleton instance of this class. This method is not thread safe and it should not be called from + * multiple threads. + * @param nrDrivers : number of driver components deployed in the cluster. + */ + public static MasterElection getInstance(int nrDrivers) { + if (instance == null) { + instance = new MasterElection(nrDrivers); + } + + return instance; + } + + /** + * Mark the given candidate as unable to become the new master, in case the current one fails. + * @param candidateId : id used to identify the driver component which should no be taken into consideration when + * electing a new master. + */ + public void markCandidateAsInvalid(int candidateId) { + this.candidates.remove(candidateId); + } + + /** + * Return true if the given candidate should not be chosen as the new Master; false otherwise. + */ + public boolean isCandidateInvalid(int candidateId) { + return !this.candidates.contains(candidateId); + } + + /** + * Return a list with all the ids of the candidates that should not be taken into consideration when electing a new + * master in the cluster. + */ + public Queue getInvalidCandidates() { + Queue invalidCandidates = IntStream.rangeClosed(0, nrDrivers - 1).boxed() + .collect(Collectors.toCollection(LinkedList::new)); + invalidCandidates.removeAll(this.candidates); + + return invalidCandidates; + } + + private void before() { + // restore all valid candidates in case there is no option; + if (candidates.isEmpty()) { + resetInvalidCandidates(); + } + } + + /** + * Resets the list of candidates considered invalid. After this method is called, all the drivers will be considered + * eligible to be elected as the new master. + */ + public void resetInvalidCandidates() { + LOG.info("Resetting list of candidates to be considered for master election"); + this.candidates = IntStream.rangeClosed(0, nrDrivers - 1).boxed().collect(Collectors.toCollection(ConcurrentLinkedQueue::new)); + } + + /** + * Method used for electing a new master when a Driver component is joining the cluster. + * @param newDriver : driver instance which joined the cluster recently and which is currently not aware if there is + * already a driver which plays the role of the Master. + */ + public void electMasterWhenDriverJoinsTheCluster(Driver newDriver) { + // check if there is already a master running in the cluster + collectUpdatesFromAllDrivers(newDriver); + DriverState driverState = newDriver.getDriverState(); + + // if no master was detected, trigger the master election process + driverState.getMasterIdLock().readLock().lock(); + if (driverState.getMasterId() == -1) { + driverState.getMasterIdLock().readLock().unlock(); + electMaster(newDriver); + return; + } + + driverState.getMasterIdLock().readLock().unlock(); + after(newDriver); + } + + /** + * Method used for electing a new Master whenever the previous one fails to respond to heartbeat messages. + * @param driver : the driver instance electing a new master. + */ + public void electMaster(Driver driver) { + before(); + + // set new master + LOG.info("Electing new master " + candidates.stream().min(Integer::compareTo).get()); + + driver.getDriverState().getMasterIdLock().writeLock().lock(); + driver.getDriverState().setMasterId(candidates.stream().min(Integer::compareTo).get()); + driver.getDriverState().getMasterIdLock().writeLock().unlock(); + + after(driver); + } + + private List getIdleAgents(Driver driver) { + HttpUtils httpUtils = new HttpUtils(); + List idleAgents = new ArrayList<>(); + + driver.getDriverState().getRegisteredAgents() + .forEach(agentIp -> { + HttpResponse agentResponse = httpUtils.sendHttpRequest(HttpUtils.GET_METHOD, "", + Agent.getGetStatusPath(agentIp), HttpUtils.HTTP_REQUEST_RETRIES); + if (agentResponse != null) { + try { + String status = EntityUtils.toString(agentResponse.getEntity()); + if (status.equals(Agent.Status.IDLE.toString())) { + LOG.info("Agent " + agentIp + " is idle."); + idleAgents.add(agentIp); + } else if (status.equals(Agent.Status.PHASE_COMPLETED.toString())) { + LOG.info(("Agent " + agentIp + " finished executing the current phase.")); + driver.getDistributedPhaseMonitor().registerAgentRunningTD(agentIp); + driver.getDistributedPhaseMonitor().addAgentWhichCompletedTheCurrentPhase(agentIp); + } else { + LOG.info("Agent " + agentIp + " is running TD."); + driver.getDistributedPhaseMonitor().registerAgentRunningTD(agentIp); + } + } catch (IOException e) { + LOG.warn("Could not check status of agent. Current phase might not be executed with the " + + "desired configuration."); + } + } + }); + + return idleAgents; + } + + private void after(Driver driver) { + // cancel heartbeat task for the diver elected as the new master + driver.getDriverState().getMasterIdLock().readLock().lock(); + + // running as master + if (driver.getDriverState().getMasterId() == driver.getDriverState().getId()) { + LOG.info("Running as MASTER"); + + driver.getDriverState().setCurrentState(DriverState.Role.MASTER); + LOG.info("Stopping master heartbeat periodic task since this driver was elected as master."); + driver.cancelMasterHeartBeatTask(); + + /* check if work redistribution is required */ + List idleAgents = getIdleAgents(driver); + + /* the assumption is that all the registered agents are currently executing TD test or they've completed + * the execution of the current phase + */ + if (driver.getConfiguration() != null && !idleAgents.isEmpty()) { + LOG.info("New master must redistribute the work between the agents because of idle agents " + + idleAgents.toString()); + idleAgents.forEach(idleAgent -> + driver.getTaskBalancer().scheduleWorkRedistributionProcess(driver, idleAgent, true)); + } + + // schedule heartbeat task for periodically checking agents + LOG.info("Scheduling heartbeat task for monitoring agents..."); + driver.scheduleHeartbeatTask(); + + // resume execution + driver.getExecutorService().submit(driver::resumeExecution); + } else { + LOG.info("Running as CANDIDATE"); + + // running as candidate + driver.getDriverState().setCurrentState(DriverState.Role.CANDIDATE); + + // schedule heartbeat task to periodically monitor the agents + driver.scheduleMasterHeartbeatTask(); + } + + driver.getDriverState().getMasterIdLock().readLock().unlock(); + } + + private void processUpdatesFromDriver(Driver currentDriver, String yamlUpdates) { + LOG.info("Current driver " + currentDriver.getDriverState().getId() + " is processing updates:\n" + yamlUpdates); + ObjectMapper objectMapper = new ObjectMapper(); + DriverUpdateInfo updates = null; + + try { + updates = objectMapper.readValue(yamlUpdates, DriverUpdateInfo.class); + } catch (IOException e) { + LOG.info("Unable to process updates about the cluster state. Driver will restart now..."); + System.exit(-1); + } + + currentDriver.getDriverState().updateDriverState(updates, currentDriver) + ; + + // set master if updates were received from the current master running in the cluster + if (updates.getRole() == DriverState.Role.MASTER) { + LOG.info("Received instructions that " + currentDriver.getDriverState().getPathForId(updates.getDriverId()) + "is the " + + "current master running in the cluster."); + currentDriver.getDriverState().getMasterIdLock().writeLock().lock(); + currentDriver.getDriverState().setMasterId(updates.getDriverId()); + currentDriver.getDriverState().getMasterIdLock().writeLock().unlock(); + } + } + + private void collectUpdatesFromAllDrivers(Driver currentDriver) { + HttpUtils httpUtils = new HttpUtils(); + List ids = IntStream.rangeClosed(0, nrDrivers - 1).boxed().collect(Collectors.toList()); + + // send request to all drivers, except the current one + List paths = ids.stream() + .filter(id -> id != currentDriver.getDriverState().getId()) + .map(id -> currentDriver.getDriverState().getPathForId(id)) + .map(Driver::getAskForUpdatesPath) + .collect(Collectors.toList()); + + for (String URI : paths) { + HttpResponse driverResponse = httpUtils.sendHttpRequest(HttpUtils.GET_METHOD, "", URI, + HttpUtils.HTTP_REQUEST_RETRIES); + if (driverResponse != null) { + try { + String yamlUpdates = EntityUtils.toString(driverResponse.getEntity()); + LOG.info("Received updates: " + yamlUpdates); + + processUpdatesFromDriver(currentDriver, yamlUpdates); + + // if updates were received from the master -> finish process + if (currentDriver.getDriverState().getMasterId() != -1) { + break; + } + } catch (IOException e) { + LOG.info("Unable to process updates about the cluster state. Driver will restart now..."); + System.exit(-1); + } + } + } + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/AbstractRequestProcessor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/AbstractRequestProcessor.java new file mode 100644 index 0000000..b382bed --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/AbstractRequestProcessor.java @@ -0,0 +1,200 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.parsers.yaml.GenerateYamlConfiguration; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverUpdateInfo; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.MasterElection; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import spark.Request; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; + +/** + * Base implementation for the RequestProcessor interface. Implements the common actions that should be taken by both + * driver components (master and candidates) when receiving certain types of HTTP requests. + */ +public abstract class AbstractRequestProcessor implements RequestProcessor { + protected final HttpUtils httpUtils = new HttpUtils(); + protected static final Logger LOG = LogManager.getLogger(Engine.class); + protected Driver driverInstance; + + /** + * Constructor. + * @param driverInstance : the driver Instance which will receive the http requests to be processed. + */ + public AbstractRequestProcessor(Driver driverInstance) { + this.driverInstance = driverInstance; + } + + /** + * Returns a list containing the URIs that should be used for forwarding a request to all the other drivers running + * in the cluster. + * @param driverInstance : the driver instance that must forward the request. + */ + protected List getDriverPathsForRedirectingRequests(Driver driverInstance) { + return IntStream.rangeClosed(0, driverInstance.getDriverState().getNrDrivers() - 1).boxed() + .filter(id -> id != driverInstance.getDriverState().getId()) // exclude current driver + .map(id -> driverInstance.getDriverState().getPathForId(id)) + .collect(Collectors.toCollection(LinkedList::new)); + } + + @Override + public String processRegisterRequest(Request request, Driver driverInstance) { + String agentIp = request.body(); + DriverState driverState = this.driverInstance.getDriverState(); + + if (request.queryParams("forward").equals("true")) { + /* register new agents to all the drivers running in the cluster */ + for (int i = 0; i < driverState.getNrDrivers(); i++) { + /* skip current driver */ + if (i == driverState.getId()) { + continue; + } + + LOG.info(this.driverInstance.getDriverState().getHostname() + ": sending agent register request for agent " + agentIp + "" + + "to driver " + this.driverInstance.getDriverState().getPathForId(i)); + HttpResponse regResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, agentIp, + Driver.getAgentRegisterPath(driverState.getPathForId(i), HttpUtils.SPARK_PORT, false), HTTP_REQUEST_RETRIES); + if (regResponse == null) { + // the assumption is that the new driver will receive the full list of active agents after being restarted + LOG.info("Driver " + driverState.getHostname() + "failed to send register request for agent " + agentIp + + "to driver " + driverState.getPathForId(i)); + } + } + } + + return ""; + } + + @Override + public String processUpdatesRequest(Request request, Driver driverInstance) throws JsonProcessingException { + LOG.info("Driver has requested updates about the state of the cluster."); + String currentPhaseName = ""; + String yamlConfig = ""; + DriverState driverState = this.driverInstance.getDriverState(); + + /* send configuration received to be executed in distributed mode and the phase being executed at this moment, + * if applicable. + */ + if (driverInstance.getConfiguration() != null) { + GenerateYamlConfiguration generateYaml = + new GenerateYamlConfiguration(driverInstance.getConfiguration().getConfigParams(), new HashMap<>()); + yamlConfig = generateYaml.createYamlStringRepresentation(); + if (driverInstance.getDistributedPhaseMonitor().isPhaseExecuting()) { + currentPhaseName = this.driverInstance.getDistributedPhaseMonitor().getPhase().getName(); + } + } + + // build information to send to the driver that recently joined the cluster + DriverUpdateInfo driverUpdateInfo = new DriverUpdateInfo(driverState.getId(), + driverState.getCurrentState(), this.driverInstance.getMasterElection().getInvalidCandidates(), + driverState.getRegisteredAgents(), yamlConfig, currentPhaseName); + + ObjectMapper objectMapper = new ObjectMapper(); + String yamlUpdateInfo = objectMapper.writeValueAsString(driverUpdateInfo); + LOG.info("Create YAML update info: " + yamlUpdateInfo); + + // set response + return yamlUpdateInfo; + } + + @Override + public String processMasterElectionRequest(Request request, Driver driverInstance) { + int failedDriverId = Integer.parseInt(request.body()); + MasterElection masterElection = this.driverInstance.getMasterElection(); + // check if this news was already processed + if (masterElection.isCandidateInvalid(failedDriverId)) { + return ""; + } + + LOG.info("Driver was informed that the current master (id: " + failedDriverId + ") died"); + masterElection.markCandidateAsInvalid(failedDriverId); + + // pick a new leader + masterElection.electMaster(driverInstance); + LOG.info("New master was elected: " + this.driverInstance.getDriverState().getMasterId()); + + return ""; + } + + @Override + public String processPhaseCompletionAnnouncement(Request request) { + String agentIp = request.body(); + DriverState driverState = this.driverInstance.getDriverState(); + + LOG.info("Agent " + agentIp + " finished executing the current phase."); + this.driverInstance.getDistributedPhaseMonitor().addAgentWhichCompletedTheCurrentPhase(agentIp); + + /* if this is the first driver receiving this type of request, forward it to all the other drivers running in + * the cluster. + */ + if (request.queryParams("forward").equals("true")) { + for (int i = 0; i < driverState.getNrDrivers(); i++) { + /* skip current driver and inactive drivers */ + if (i == driverState.getId()) { + continue; + } + + LOG.info(driverState.getHostname() + ": sending agent announcement for phase completion " + + agentIp + "" + "to driver " + driverState.getPathForId(i)); + HttpResponse response = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, agentIp, + Driver.getPhaseFinishedByAgentPath(driverState.getPathForId(i), HttpUtils.SPARK_PORT, false), + HTTP_REQUEST_RETRIES); + + if (response == null) { + // the assumption is that the new driver will receive the full list of active agents after being restarted + LOG.info("Driver " + driverState.getHostname() + "failed to send announcement for phase " + + "of agent " + agentIp + "to driver " + driverState.getPathForId(i)); + } + + } + + } + + // TODO: update current phase for stand-by drivers if the phase was successfully finished + return ""; + } + + + @Override + public String processExecutionRequest(Request request, Driver driverInstance) throws Exception { + String yamlConfiguration = request.body(); + LOG.info("Received execution request for TD configuration:\n"); + LOG.info(yamlConfiguration); + + // save TD configuration which must be executed in distributed mode + driverInstance.setConfiguration(new Configuration(yamlConfiguration)); + + // send TD configuration to all the other drivers running in the cluster + if (request.queryParams("forward").equals("true")) { + List forwardPaths = this.getDriverPathsForRedirectingRequests(driverInstance); + forwardPaths.forEach(forwardPath -> { + LOG.info("Forwarding execution request to driver " + forwardPath); + HttpResponse driverResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, request.body(), + Driver.getExecutionPath(forwardPath, HttpUtils.SPARK_PORT, false), HttpUtils.HTTP_REQUEST_RETRIES); + if (driverResponse == null) { + /* the assumption is that the driver will fail to respond to heartbeat request and will receive this + * information after being restarted. + */ + LOG.warn("Unable to forward execution request to " + forwardPath); + } + }); + } + + return ""; + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/CandidateRequestProcessor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/CandidateRequestProcessor.java new file mode 100644 index 0000000..0d30611 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/CandidateRequestProcessor.java @@ -0,0 +1,94 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests; + +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import org.apache.http.HttpResponse; +import spark.Request; +import spark.Response; + +/** + * Specifies how a candidate will process the HTTP requests received from the other drivers or the agents running in the + * cluster. + */ +public class CandidateRequestProcessor extends AbstractRequestProcessor { + private static CandidateRequestProcessor instance = null; + + /** + * Returns an instance of this class. + * @param driver : the driver instance that will use this class for processing HTTP requests. + */ + public static CandidateRequestProcessor getInstance(Driver driver) { + if (instance == null || !instance.driverInstance.equals(driver)) { + instance = new CandidateRequestProcessor(driver); + } + + return instance; + } + + private CandidateRequestProcessor(Driver driverInstance) { + super(driverInstance); + } + + @Override + public String processRegisterRequest(Request request, Driver driverInstance) { + super.processRegisterRequest(request, driverInstance); + + String agentIp = request.body(); + this.driverInstance.getDriverState().registerAgent(agentIp); + LOG.info("[driver] Registered agent with ip " + agentIp); + LOG.info("[driver] active agents " + this.driverInstance.getDriverState().getRegisteredAgents().toString()); + + return ""; + } + + @Override + public String acknowledgeSampleContentSuccessfulInstallation(Request request, Driver driverInstance, Response response) { + DriverState driverState = driverInstance.getDriverState(); + + LOG.info("Candidate driver is redirecting sample content ack to the master..."); + // another driver should be chosen for this responsibility + if (driverState.getMasterId() == -1) { + response.status(503); + } + // request should be forwarded to the current master + HttpResponse masterResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, request.body(), + Driver.getSampleContentAckPath(driverState.getPathForId(driverState.getMasterId()), HttpUtils.SPARK_PORT), + HttpUtils.HTTP_REQUEST_RETRIES); + + if (masterResponse == null) { + response.status(503); + } + + return ""; + } + + @Override + public String processExecutionRequest(Request request, Driver driverInstance) throws Exception { + super.processExecutionRequest(request, driverInstance); + + // candidates will assume that the execution started successfully + driverInstance.getDistributedPhaseMonitor().setPhase(driverInstance.getConfiguration().getPhases().get(0)); + // the assumption is that the master is responsible for installing TD sample content + driverInstance.getConfiguration().getGlobalArgs().setInstallSampleContent("false"); + + return ""; + } + + @Override + public String processHeartbeatRequest(Request request, Driver driverInstance) { + throw new IllegalStateException("Candidates should never receive this type of requests."); + } + + @Override + public String processAgentFailureAnnouncement(Request request, Driver driverInstance) { + String agentIdentifier = request.body(); + + LOG.info("The driver was announced by the master that agent " + agentIdentifier + " failed to respond" + + " to heartbeat request."); + this.driverInstance.getDistributedPhaseMonitor().removeAgentFromActiveTDRunners(agentIdentifier); + this.driverInstance.getDriverState().removeAgent(agentIdentifier); + + return ""; + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/MasterRequestProcessor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/MasterRequestProcessor.java new file mode 100644 index 0000000..d88282c --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/MasterRequestProcessor.java @@ -0,0 +1,168 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.google.gson.Gson; +import org.apache.http.HttpResponse; +import spark.Request; +import spark.Response; + +import java.util.ArrayList; +import java.util.List; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; +import static com.adobe.qe.toughday.internal.core.engine.Engine.logGlobal; + +/** + * Specifies how the master will process the HTTP requests received from the other components running in the cluster. + */ +public class MasterRequestProcessor extends AbstractRequestProcessor { + private final Object object = new Object(); + private static MasterRequestProcessor instance = null; + + /** + * Returns an instance of this class. + * @param driver : the driver instance that will use this class for processing HTTP requests. + */ + public static MasterRequestProcessor getInstance(Driver driver) { + if (instance == null || !instance.driverInstance.equals(driver)) { + instance = new MasterRequestProcessor(driver); + } + + return instance; + } + + private MasterRequestProcessor(Driver driverInstance) { + super(driverInstance); + } + + private void waitForSampleContentToBeInstalled(Driver currentDriver) { + synchronized (object) { + try { + object.wait(); + } catch (InterruptedException e) { + LOG.error("Failed to install ToughDay sample content package. Execution will be stopped."); + currentDriver.finishDistributedExecution(); + System.exit(-1); + } + } + } + + private void installToughdayContentPackage(Configuration configuration, Driver currentDriver) { + logGlobal("Installing ToughDay 2 Content Package..."); + GlobalArgs globalArgs = configuration.getGlobalArgs(); + + if (globalArgs.getDryRun() || !globalArgs.getInstallSampleContent()) { + return; + } + + HttpResponse agentResponse = null; + List agentsCopy = new ArrayList<>(this.driverInstance.getDriverState().getRegisteredAgents()); + + while (agentResponse == null && agentsCopy.size() > 0) { + // pick one agent to install the sample content + String agentIpAddress = agentsCopy.remove(0); + String URI = Agent.getInstallSampleContentPath(agentIpAddress); + LOG.info("Installing sample content request was sent to agent " + agentIpAddress); + + YamlDumpConfigurationAsTaskForTDAgents dumpConfig + = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + String yamlTask = dumpConfig.generateConfigurationObject(); + + agentResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, yamlTask, URI, HTTP_REQUEST_RETRIES); + } + + // we should wait until the we receive confirmation that the sample content was successfully installed + waitForSampleContentToBeInstalled(currentDriver); + + logGlobal("Finished installing ToughDay 2 Content Package."); + globalArgs.setInstallSampleContent("false"); + + } + + private void mergeDistributedConfigParams(Configuration configuration, Driver currentDriver) { + DriverState driverState = currentDriver.getDriverState(); + + if (driverState.getDriverConfig().getDistributedConfig().getHeartbeatIntervalInSeconds() == + currentDriver.getConfiguration().getDistributedConfig().getHeartbeatIntervalInSeconds()) { + + driverState.getDriverConfig().getDistributedConfig().merge(configuration.getDistributedConfig()); + return; + } + + // cancel heartbeat task and reschedule it with the new period + currentDriver.cancelHeartbeatTask(); + driverState.getDriverConfig().getDistributedConfig().merge(configuration.getDistributedConfig()); + currentDriver.scheduleHeartbeatTask(); + } + + + private void handleExecutionRequest(Configuration configuration, Driver currentDriver) { + installToughdayContentPackage(configuration, currentDriver); + mergeDistributedConfigParams(configuration, currentDriver); + + currentDriver.executePhases(); + } + + @Override + public String acknowledgeSampleContentSuccessfulInstallation(Request request, Driver driverInstance, Response response) { + boolean installed = Boolean.parseBoolean(request.body()); + + if (!installed) { + LOG.error("Failed to install the ToughDay sample content package. Execution will be stopped."); + driverInstance.finishDistributedExecution(); + } + + synchronized (this.object) { + this.object.notify(); + } + + return ""; + } + + @Override + public String processExecutionRequest(Request request, Driver driverInstance) throws Exception { + super.processExecutionRequest(request, driverInstance); + + // handle execution in a different thread to be able to quickly respond to this request + driverInstance.getExecutorService().submit(() -> handleExecutionRequest(driverInstance.getConfiguration(), driverInstance)); + + return ""; + } + + @Override + public String processHeartbeatRequest(Request request, Driver driverInstance) { + // send executions/test to candidates to keep them updated + Gson gson = new Gson(); + return gson.toJson(driverInstance.getDistributedPhaseMonitor().getExecutions()); + } + + @Override + public String processAgentFailureAnnouncement(Request request, Driver driverInstance) { + throw new IllegalStateException("Master should never receive this type of request."); + } + + @Override + public String processRegisterRequest(Request request, Driver driverInstance) { + super.processRegisterRequest(request, driverInstance); + String agentIp = request.body(); + DriverState driverState = this.driverInstance.getDriverState(); + LOG.info("[driver] Registered agent with ip " + agentIp); + + if (!this.driverInstance.getDistributedPhaseMonitor().isPhaseExecuting()) { + driverState.registerAgent(agentIp); + LOG.info("[driver] active agents " + driverState.getRegisteredAgents().toString()); + return ""; + } + + // master must schedule the work redistribution process when a new agent is registering + this.driverInstance.getTaskBalancer().scheduleWorkRedistributionProcess(this.driverInstance, agentIp, true); + + return ""; + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessor.java new file mode 100644 index 0000000..63d8c88 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessor.java @@ -0,0 +1,67 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests; + +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.fasterxml.jackson.core.JsonProcessingException; +import spark.Request; +import spark.Response; + +/** + * Common interface for handling http requests received by the driver components running in the cluster. + */ +public interface RequestProcessor { + /** + * Method responsible for processing the request of registering a new Agent which joined the cluster recently. + * @param request : http request received from the Agent. + * @param driverInstance : the Driver instance processing this request + */ + String processRegisterRequest(Request request, Driver driverInstance); + + /** + * Method responsible for processing the request sent by the drivers running in the cluster to ask for updates + * regarding the current state of the cluster. This method is called whenever a new Driver component joins the + * cluster. + * @param request : http request received from a Driver + * @param driverInstance : The driver instance processing this request + * @return Yaml representation of the current state of the cluster (generated by dumping the DriverUpdateInfo class) + */ + String processUpdatesRequest(Request request, Driver driverInstance) throws JsonProcessingException; + + /** + * Method responsible for processing the request sent by a driver to announce that the master election process must + * be triggered. + * @param request : http request used for triggering the master election process. + * @param driverInstance : the Driver instance processing this request. + */ + String processMasterElectionRequest(Request request, Driver driverInstance); + + /** + * Method responsible for processing the request sent by an agent to announce that the Phase received as a task from + * the driver was successfully executed. + * @param request : http request sent by an agent which completed the execution of a phase. + */ + String processPhaseCompletionAnnouncement(Request request); + + /** + * Method responsible for processing the request sent by the agent responsible to install the TD sample content + * package to announce that the installation was successfully completed. + * @param request : http request sent by the agent. + * @param driverInstance : the Driver instance processing this request + * @param response + * @return + */ + String acknowledgeSampleContentSuccessfulInstallation(Request request, Driver driverInstance, Response response); + + /** + * Method responsible for processing the request used to trigger the distributed execution of ToughDay. + * @param request : http request + * @param driverInstance : the Driver instance processing this request + * @throws Exception : if the Configuration for running ToughDay cannot be successfully built and dumped as a YAML + * string. + */ + String processExecutionRequest(Request request, Driver driverInstance) throws Exception; + + String processHeartbeatRequest(Request request, Driver driverInstance); + + String processAgentFailureAnnouncement(Request request, Driver driverInstance); + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessorDispatcher.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessorDispatcher.java new file mode 100644 index 0000000..e03adb9 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/cluster/driver/requests/RequestProcessorDispatcher.java @@ -0,0 +1,38 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests; + +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; + +/** + * Class responsible for choosing the appropriate RequestProcessor implementation to be used for processing requests by + * the driver, depending on the role played by the Driver receiving the request (Master or Candidate). + */ +public class RequestProcessorDispatcher { + private static RequestProcessorDispatcher instance = null; + + private RequestProcessorDispatcher() { } + + /** + * Returns an instance of this class. + */ + public static RequestProcessorDispatcher getInstance() { + if (instance == null) { + instance = new RequestProcessorDispatcher(); + } + + return instance; + } + + /** + * Returns the appropriate RequestProcessor to be used for processing the requests received by the driver. + * @param driver : the Driver instance that must process the http requests. + */ + public RequestProcessor getRequestProcessor(Driver driver) { + if (driver.getDriverState().getCurrentState() == DriverState.Role.MASTER) { + return MasterRequestProcessor.getInstance(driver); + } else { + return CandidateRequestProcessor.getInstance(driver); + } + + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructions.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructions.java new file mode 100644 index 0000000..383ad67 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructions.java @@ -0,0 +1,45 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution; + +import java.util.Map; + +/** Contains all the information needed by the agents for updating their configuration when the work needs to be + * rebalanced. + */ +public class RedistributionInstructions { + private Map counts; + private Map runModeProperties; + + // dummy constructor, required for dumping the class + public RedistributionInstructions() { } + + public RedistributionInstructions(Map counts, Map runModeProperties) { + this.counts = counts; + this.runModeProperties = runModeProperties; + } + + /** + * Getter for counts. + */ + public Map getCounts() { + return this.counts; + } + + /** + * Setter for counts. + */ + public void setCounts(Map counts) { + this.counts = counts; + } + + /** + * Getter for run mode properties that must be redistributed. + */ + public Map getRunModeProperties() { return this.runModeProperties; } + + /** + * Setter for run mode properties that must be redistributed. + */ + public void setRunModeProperties(Map runModeProperties) { + this.runModeProperties = runModeProperties; + } +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructionsProcessor.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructionsProcessor.java new file mode 100644 index 0000000..7147eff --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/RedistributionInstructionsProcessor.java @@ -0,0 +1,75 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution; + +import com.adobe.qe.toughday.internal.core.TestSuite; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.ConstantLoadRunModeBalancer; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Class responsible for processing the rebalancing request received by an agent from the driver when the + * work must be rebalanced. + */ +public class RedistributionInstructionsProcessor { + protected static final Logger LOG = LogManager.getLogger(ConstantLoadRunModeBalancer.class); + + private void processTestSuiteChanges(Map counts, Phase phase) { + TestSuite testSuite = phase.getTestSuite(); + long nr = testSuite.getTests().stream() + .filter(test -> !counts.containsKey(test.getName())) + .count(); + if (nr > 0) { + throw new IllegalStateException("Instructions were not received for each test in the test suite."); + } + + testSuite.getTests() + .stream() + .filter(test -> counts.containsKey(test.getName())) + .forEach(test -> { + // remove tests for which the count property was achieved + if (counts.get(test.getName()) == 0) { + phase.getCounts().remove(test); + } else { + // reset number of tests executed so far + phase.getCounts().put(test, new AtomicLong(0)); + // update number of executions left for this test + test.setCount(String.valueOf(counts.get(test.getName()))); + } + }); + } + + private void processRunModeChanges(RedistributionInstructions redistributionInstructions, RunMode runMode) { + runMode.getRunModeBalancer().before(redistributionInstructions, runMode); + + runMode.getRunModeBalancer().processRunModeInstructions(redistributionInstructions, runMode); + + runMode.getRunModeBalancer().after(redistributionInstructions, runMode); + } + + /** + * Method used for processing the rebalance request. + * @param jsonInstructions : redistribution instructions received from the driver + * @param phase : the current phase being executed by the agents. + * @throws IOException : if the instructions don't have the appropriate format. + */ + public void processInstructions(String jsonInstructions, Phase phase) throws IOException { + if (phase == null || jsonInstructions == null) { + throw new IllegalArgumentException("Phase and redistribution instructions must not be null during work " + + "redistribution process."); + } + + ObjectMapper objectMapper = new ObjectMapper(); + RedistributionInstructions redistributionInstructions = + objectMapper.readValue(jsonInstructions, RedistributionInstructions.class); + + // update values for each modified property + processRunModeChanges(redistributionInstructions, phase.getRunMode()); + processTestSuiteChanges(redistributionInstructions.getCounts(), phase); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/TaskBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/TaskBalancer.java new file mode 100644 index 0000000..b7bd012 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/TaskBalancer.java @@ -0,0 +1,313 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution; + +import com.adobe.qe.toughday.internal.core.TestSuite; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.DistributedPhaseMonitor; +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.DistributedConfig; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.PhaseSplitter; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpResponse; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.*; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; + +/** + * Class responsible for balancing the work between the agents running the cluster + * whenever the number of agents is changing. + */ +public class TaskBalancer { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + private static TaskBalancer instance = null; + + private RedistributionStatus status = RedistributionStatus.UNNECESSARY; + private final PhaseSplitter phaseSplitter = new PhaseSplitter(); + private final HttpUtils httpUtils = new HttpUtils(); + private final ConcurrentLinkedQueue inactiveAgents = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue recentlyAddedAgents = new ConcurrentLinkedQueue<>(); + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + + public enum RedistributionStatus { + UNNECESSARY, // no need to redistribute the work + SCHEDULED, // redistribution is scheduled + RESCHEDULE_REQUIRED, /* nr of agents running in the cluster has changed while the work was redistributed so + * we must reschedule the process */ + EXECUTING // work redistribution process is executing + } + + /** + * Returns a list with all the agents that failed to respond to the heartbeat request sent + * by the driver. + */ + public ConcurrentLinkedQueue getInactiveAgents() { + return this.inactiveAgents; + } + + private TaskBalancer() {} + + /** + * Returns a singleton instance of this class. + */ + public static TaskBalancer getInstance() { + if (instance == null) { + instance = new TaskBalancer(); + } + + return instance; + } + + private Map getTestSuitePropertiesToRedistribute(TestSuite taskTestSuite) { + HashMap map = new HashMap<>(); + taskTestSuite.getTests().forEach(test -> map.put(test.getName(), test.getCount())); + + return map; + } + + private void addNewAgent(String agentIdentifier) { + this.recentlyAddedAgents.add(agentIdentifier); + } + + private void addInactiveAgent(String agentIdentifier) { + this.inactiveAgents.add(agentIdentifier); + } + + private void waitUntilAllAgentsAreReadyForRebalancing(List agents) { + boolean ready = false; + + while(!ready) { + ready = true; + + // get status of all new agents + for (String agentIpAddress : agents) { + HttpResponse agentResponse = this.httpUtils.sendHttpRequest(HttpUtils.GET_METHOD, "", + Agent.getGetStatusPath(agentIpAddress), HTTP_REQUEST_RETRIES); + + if (agentResponse != null) { + try { + String response = EntityUtils.toString(agentResponse.getEntity()); + if (!response.equals(Agent.Status.RUNNING.toString())) { + ready = false; + break; + } + } catch (IOException e) { + LOG.warn("Could not confirm that agent " + agentIpAddress + " is ready for processing " + + "redistribution instructions") ; + } + } + } + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // skip this and continue waiting for all agents to be ready for redistribution + } + } + } + + private void sendInstructionsToExistingAgents(Map phases, List activeAgents) { + ObjectMapper mapper = new ObjectMapper(); + + // send instructions to agents that are already running TD + activeAgents.stream() + .filter(ipAddress -> !this.inactiveAgents.contains(ipAddress)) // filter agents that become inactive + .forEach(ipAddress -> { + String agentURI = Agent.getRebalancePath(ipAddress); + TestSuite testSuite = phases.get(ipAddress).getTestSuite(); + RunMode runMode = phases.get(ipAddress).getRunMode(); + + // build redistribution instructions + Map testSuiteProperties = getTestSuitePropertiesToRedistribute(testSuite); + Map runModeProperties = runMode.getRunModeBalancer() + .getRunModePropertiesToRedistribute(runMode.getClass(), runMode); + RedistributionInstructions redistributionInstructions = + new RedistributionInstructions(testSuiteProperties, runModeProperties); + try { + String message = mapper.writeValueAsString(redistributionInstructions); + LOG.info("[Redistribution] Sending " + message + " to agent " + ipAddress); + + HttpResponse response = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, message, + agentURI, HTTP_REQUEST_RETRIES); + if (response == null) { + /* redistribution will be automatically triggered as soon as this agent fails to respond to + * heartbeat request + */ + LOG.warn("Redistribution instructions could not be sent to agent " + ipAddress + "."); + } + + } catch (JsonProcessingException e) { + LOG.error("Unexpected exception while sending redistribution instructions", e); + LOG.warn("Agent " + ipAddress + " will continue to run with the configuration it had before " + + "the process of redistribution was triggered."); + } + + }); + } + + private void sendExecutionRequestsToNewAgents(List recentlyAddedAgents, Map phases, + Configuration configuration, DriverState driverState) { + // for the recently added agents, send execution queries + recentlyAddedAgents.forEach(newAgentIpAddress -> { + configuration.setPhases(Collections.singletonList(phases.get(newAgentIpAddress))); + YamlDumpConfigurationAsTaskForTDAgents dumpConfig + = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + String yamlTask = dumpConfig.generateConfigurationObject(); + + + LOG.info("Sending execution request + " + yamlTask + " to new agent " + newAgentIpAddress); + HttpResponse response = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, yamlTask, + Agent.getSubmissionTaskPath(newAgentIpAddress), 3); + + if (response == null) { + /* the assumption is that the agent will fail to respond to heartbeat request => work will be + * automatically redistributed when this happens. + */ + LOG.info("Failed to send task to new agent " + newAgentIpAddress + "."); + } + }); + } + + private void updateStateOfAgents(Driver driverInstance, List recentlyAddedAgents) throws CloneNotSupportedException { + DistributedPhaseMonitor distributedPhaseMonitor = driverInstance.getDistributedPhaseMonitor(); + DriverState driverState = driverInstance.getDriverState(); + + if (!distributedPhaseMonitor.isPhaseExecuting()) { + return; + } + + Phase phase = distributedPhaseMonitor.getPhase(); + // update number of tests that are left to be executed by the agents + distributedPhaseMonitor.updateCountPerTest(); + Map phases = this.phaseSplitter.splitPhaseForRebalancingWork(phase, + new LinkedList<>(driverState.getRegisteredAgents()), + new ArrayList<>(recentlyAddedAgents), distributedPhaseMonitor.getPhaseStartTime()); + + // wait until all agents are able to process redistribution instructions + waitUntilAllAgentsAreReadyForRebalancing(distributedPhaseMonitor.getAgentsRunningTD()); + sendInstructionsToExistingAgents(phases, distributedPhaseMonitor.getAgentsRunningTD()); + sendExecutionRequestsToNewAgents(recentlyAddedAgents, phases, driverInstance.getConfiguration(), driverState); + + } + + private void after(Driver driverInstance, List newAgents, List inactiveAgents) { + DistributedPhaseMonitor distributedPhaseMonitor = driverInstance.getDistributedPhaseMonitor(); + DriverState driverState = driverInstance.getDriverState(); + + distributedPhaseMonitor.resetExecutions(); + + // mark recently added agents as active agents executing tasks + newAgents.forEach(driverState::registerAgent); + newAgents.forEach(distributedPhaseMonitor::registerAgentRunningTD); + newAgents.forEach(this.recentlyAddedAgents::remove); + + // inform all the other drivers about the agents which failed to respond to heartbeat request + inactiveAgents.forEach(agentIp -> { + for (int i = 0; i < driverInstance.getDriverState().getNrDrivers(); i++) { + if (i == driverInstance.getDriverState().getId()) { + continue; + } + + HttpResponse driverResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, agentIp, + Driver.getAgentFailurePath(driverInstance.getDriverState().getPathForId(i)), HTTP_REQUEST_RETRIES); + if (driverResponse == null) { + /* the assumption is that if this driver will become the master it will detect this change once the + * heartbeat task is scheduled so no further actions should be done at this moment + */ + LOG.warn("Failed to announce driver-" + i + " that agent " + agentIp + " died."); + } + + LOG.info("Successfully announced driver-" + i + " that the agent " + agentIp + " died"); + } + + }); + + LOG.info("[Redistribution] Finished redistributing the work"); + + if (this.status == RedistributionStatus.RESCHEDULE_REQUIRED) { + this.status = RedistributionStatus.SCHEDULED; + LOG.info("[driver] Scheduling delayed redistribution process for agents " + + this.recentlyAddedAgents.toString()); + this.scheduler.schedule(() -> { + LOG.info("[Redistribution] starting delayed work redistribution process"); + rebalanceWork(driverInstance); + }, driverState.getDriverConfig().getDistributedConfig().getRedistributionWaitTimeInSeconds(), TimeUnit.SECONDS); + } else { + this.status = RedistributionStatus.UNNECESSARY; + } + + + } + + private void excludeInactiveAgents(List agentsToBeExcluded, Driver driverInstance) { + agentsToBeExcluded.forEach(driverInstance.getDriverState()::removeAgent); + agentsToBeExcluded.forEach(this.inactiveAgents::remove); + + // we should not wait for task completion since the agent running it left the cluster + agentsToBeExcluded.forEach(driverInstance.getDistributedPhaseMonitor()::removeAgentFromActiveTDRunners); + } + + private void rebalanceWork(Driver driverInstance) { + this.status = RedistributionStatus.EXECUTING; + + List newAgents = new ArrayList<>(recentlyAddedAgents); + List inactiveAgents = new ArrayList<>(this.inactiveAgents); + + // remove all agents who failed answering the heartbeat request in the past + excludeInactiveAgents(inactiveAgents, driverInstance); + LOG.info("[Redistribution] Starting the process...."); + + try { + updateStateOfAgents(driverInstance, newAgents); + } catch (CloneNotSupportedException e) { + LOG.warn(""); + } + + after(driverInstance, newAgents, inactiveAgents); + } + + /** + * Method used for scheduling the process of redistributing the work between the agents running in the cluster. This + * method is called whenever the number of active agents is changing. + * @param driverInstance : the driver instance that will schedule the work redistribution process + * @param agentIdentifier : the identifier of the agent that generated this call + * @param activeAgent : true when the agent represented by agentIdentifier has joined the cluster; false if the + * agent has become inactive(he failed to respond to heartbeat request) + */ + public void scheduleWorkRedistributionProcess(Driver driverInstance, String agentIdentifier, boolean activeAgent) { + DriverState driverState = driverInstance.getDriverState(); + + if (activeAgent) { + this.addNewAgent(agentIdentifier); + } else { + this.addInactiveAgent(agentIdentifier); + } + + DistributedConfig distributedConfig = driverState.getDriverConfig().getDistributedConfig(); + if (this.status == RedistributionStatus.UNNECESSARY) { + this.status = RedistributionStatus.SCHEDULED; + LOG.info("Scheduling redistribution process to start in " + + distributedConfig.getRedistributionWaitTimeInSeconds() + " seconds."); + + // schedule work redistribution process + this.scheduler.schedule(() -> rebalanceWork(driverInstance), + distributedConfig.getRedistributionWaitTimeInSeconds(), TimeUnit.SECONDS); + } else if (this.status == RedistributionStatus.EXECUTING) { + LOG.info("Work redistribution process must be rescheduled."); + this.status = RedistributionStatus.RESCHEDULE_REQUIRED; + } + + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/AbstractRunModeBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/AbstractRunModeBalancer.java new file mode 100644 index 0000000..b6dd2a0 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/AbstractRunModeBalancer.java @@ -0,0 +1,71 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes; + +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; +import com.adobe.qe.toughday.api.annotations.ConfigArgSet; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public abstract class AbstractRunModeBalancer implements RunModeBalancer { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + @Override + public Map getRunModePropertiesToRedistribute(Class type, T object) { + final Map properties = new HashMap<>(); + + if (object == null) { + throw new IllegalArgumentException("Run mode object must not be null."); + } + + Arrays.stream(type.getDeclaredMethods()) + .filter(method -> method.isAnnotationPresent(ConfigArgGet.class)) + .filter(method -> method.getAnnotation(ConfigArgGet.class).redistribute()) + .forEach(method -> { + String propertyName = Configuration.propertyFromMethod(method.getName()); + try { + Object value = method.invoke(object); + properties.put(propertyName, String.valueOf(value)); + } catch (IllegalAccessException | InvocationTargetException e) { + LOG.warn("Property " + propertyName + " could not be collected for redistribution instructions." + + " Received error " + e.getMessage()); + } + }); + + return properties; + } + + @Override + public void processRunModeInstructions(RedistributionInstructions redistributionInstructions, T runMode) { + if (redistributionInstructions == null || runMode == null) { + throw new IllegalArgumentException("Rebalance instructions and run mode must not be null."); + } + + Map runModeProperties = redistributionInstructions.getRunModeProperties(); + if (runModeProperties == null || runModeProperties.isEmpty()) { + return; + } + + Arrays.stream(runMode.getClass().getDeclaredMethods()) + .filter(method -> runModeProperties.containsKey(Configuration.propertyFromMethod(method.getName()))) + .filter(method -> method.isAnnotationPresent(ConfigArgSet.class)) + .forEach(method -> { + String property = Configuration.propertyFromMethod(method.getName()); + LOG.info("[Agent] Setting property " + property + " to " + runModeProperties.get(property)); + + try { + method.invoke(runMode, runModeProperties.get(property)); + } catch (IllegalAccessException | InvocationTargetException e) { + LOG.warn("[Agent] Property " + property + " could not be set to " + runModeProperties.get(property)); + } + + }); + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/ConstantLoadRunModeBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/ConstantLoadRunModeBalancer.java new file mode 100644 index 0000000..48b2a6d --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/ConstantLoadRunModeBalancer.java @@ -0,0 +1,76 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes; + +import com.adobe.qe.toughday.internal.core.engine.runmodes.ConstantLoad; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; + + +public class ConstantLoadRunModeBalancer extends AbstractRunModeBalancer { + protected static final Logger LOG = LogManager.getLogger(ConstantLoadRunModeBalancer.class); + private static final String CURRENT_LOAD_PROPERTY = "currentload"; + + @Override + public Map getRunModePropertiesToRedistribute(Class type, ConstantLoad runMode) { + Map runModeProps = super.getRunModePropertiesToRedistribute(type, runMode); + + if (runMode.isVariableLoad()) { + runModeProps.put(CURRENT_LOAD_PROPERTY, String.valueOf(runMode.getCurrentLoad())); + } + + return runModeProps; + } + + @Override + public void before(RedistributionInstructions redistributionInstructions, ConstantLoad runMode) { + if (runMode.isVariableLoad()) { + // We must cancel the scheduled task and reschedule it with the new values for 'period' and initial delay. + boolean cancelled = runMode.cancelPeriodicTask(); + if (!cancelled) { + LOG.warn("[Driver] Periodic task for updating the load could not be cancelled. Run mode properties " + + "might not be respected."); + } else { + LOG.info("[Driver] Periodic task for updating the load was cancelled."); + } + } + + Map runModeProperties = redistributionInstructions.getRunModeProperties(); + runModeProperties.forEach((property, propValue) -> + beforeChangingValueOfProperty(property, propValue, runMode)); + } + + private void beforeChangingValueOfProperty(String property, String newValue, ConstantLoad runMode) { + if (property.equals("load") && !runMode.isVariableLoad()) { + long newLoad = Long.parseLong(newValue); + long difference = runMode.getLoad() - newLoad; + + if (difference > 0 ) { + runMode.removeRunMaps(difference); + } else { + runMode.addRunMaps(Math.abs(difference)); + } + } + } + + @Override + public void processRunModeInstructions(RedistributionInstructions redistributionInstructions, ConstantLoad runMode) { + super.processRunModeInstructions(redistributionInstructions, runMode); + + Map runModeProps = redistributionInstructions.getRunModeProperties(); + if (runModeProps.containsKey(CURRENT_LOAD_PROPERTY)) { + LOG.info("[Driver[ Setting current load to " + runModeProps.get(CURRENT_LOAD_PROPERTY)); + runMode.setCurrentLoad(Integer.parseInt(runModeProps.get(CURRENT_LOAD_PROPERTY))); + } + } + + @Override + public void after(RedistributionInstructions redistributionInstructions, ConstantLoad runMode) { + if (runMode.isVariableLoad()) { + runMode.schedulePeriodicTask(); + LOG.info("Periodic task for updating the load was rescheduled with interval " + runMode.getInterval() + + " and initial delay " + runMode.getInitialDelay()); + } + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/NormalRunModeBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/NormalRunModeBalancer.java new file mode 100644 index 0000000..d2584a9 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/NormalRunModeBalancer.java @@ -0,0 +1,88 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes; + +import com.adobe.qe.toughday.internal.core.engine.AsyncEngineWorker; +import com.adobe.qe.toughday.internal.core.engine.AsyncTestWorker; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.runmodes.Normal; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class NormalRunModeBalancer extends AbstractRunModeBalancer { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + @Override + public void before(RedistributionInstructions redistributionInstructions, Normal runMode) { + if (runMode.isVariableConcurrency()) { + /* We must cancel the scheduled task for updating concurrency and reschedule it with the new values + * for interval and initial delay. + */ + boolean cancelled = runMode.cancelPeriodicTask(); + if (!cancelled) { + LOG.warn("Periodic task for updating the number of threads could not be cancelled. Normal run mode" + + " properties might not be respected."); + } + } + + Map runModeProperties = redistributionInstructions.getRunModeProperties(); + runModeProperties.forEach((propertyName, propValue) -> this.beforeChangingValueOfProperty(propertyName, propValue, runMode)); + } + + private void beforeChangingValueOfProperty(String property, String newValue, Normal runMode) { + if (property.equals("concurrency")) { + long currentConcurrency = runMode.isVariableConcurrency() ? runMode.getActiveThreads() : + runMode.getConcurrency(); + long newConcurrency = Long.parseLong(newValue); + long difference = currentConcurrency - newConcurrency; + + if (difference > 0) { + reduceConcurrency(difference, runMode); + } else { + increaseConcurrency(Math.abs(difference), runMode); + } + + concurrencySanityChecks(runMode, newConcurrency); + } + } + + private void concurrencySanityChecks(Normal runMode, long newConcurrency) { + // check that we have the exact number of active test workers + if (runMode.getRunContext().getTestWorkers().size() != newConcurrency) { + throw new IllegalStateException("[redistribution] TestWorkers size is " + + runMode.getRunContext().getTestWorkers().size() + " but" + " new value for concurrency is " + newConcurrency); + } + + // check that all test workers are active + if (runMode.getRunContext().getTestWorkers().stream().anyMatch(AsyncEngineWorker::isFinished)) { + throw new IllegalStateException("[redistribution] " + + "There are finished test workers in the list of active workers."); + } + } + + private void reduceConcurrency(long reduce, Normal runMode) { + List workerList = new ArrayList<>(runMode.getRunContext().getTestWorkers()); + + for (int i = 0; i < reduce; i++) { + LOG.info("[Agent - redistribution] Finished test worker " + workerList.get(i).getWorkerThread().getId()); + runMode.finishAndDeleteWorker(workerList.get(i)); + } + } + + private void increaseConcurrency(long increase, Normal runMode) { + for (int i = 0; i < increase; i++) { + runMode.createAndExecuteWorker(runMode.getEngine(), runMode.getEngine().getCurrentPhase().getTestSuite()); + } + } + + @Override + public void after(RedistributionInstructions redistributionInstructions, Normal runMode) { + if (runMode.isVariableConcurrency()) { + runMode.schedulePeriodicTask(); + LOG.info("Task responsible for updating the number threads used to run tests was rescheduled"); + } + } +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/RunModeBalancer.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/RunModeBalancer.java new file mode 100644 index 0000000..1a79a49 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/redistribution/runmodes/RunModeBalancer.java @@ -0,0 +1,47 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes; + +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; + +import java.util.Map; + +/** + * Contains all the methods that should be implemented in order to describe how a certain + * run mode must be redistributed when the number of agents running in the cluster modifies. + * @param : type of the run mode + */ +public interface RunModeBalancer { + /** + * Collects all the properties that must be taken into consideration when redistributing + * the work between the agents. + * @param type Class of the run mode + * @param runMode + * @return + */ + Map getRunModePropertiesToRedistribute(Class type, T runMode); + + /** + * This method is called before changing the actual value of the properties to be + * modified. It should be used for creating the appropriate environment for running + * with the run mode configuration. + * @param redistributionInstructions instructions for redistributing the work + * @param runMode + */ + void before(RedistributionInstructions redistributionInstructions, T runMode); + + /** + * This method is responsible for changing the values of the properties that must be + * redistributed according to the redistribution instructions received as a parameter. + * @param redistributionInstructions instructions for redistributing the work + * @param runMode + */ + void processRunModeInstructions(RedistributionInstructions redistributionInstructions, T runMode); + + /** + * The last method called when the process of redistributing the work is + * executing. + * @param redistributionInstructions instructions for redistributing the work + * @param runMode + */ + void after(RedistributionInstructions redistributionInstructions, T runMode); +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/PhaseSplitter.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/PhaseSplitter.java new file mode 100644 index 0000000..1bb1ed7 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/PhaseSplitter.java @@ -0,0 +1,108 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.splitters; + +import com.adobe.qe.toughday.internal.core.TestSuite; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.engine.RunMode; + +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Class responsible for partitioning phases into multiple phases to be distributed between the agents running in + * the cluster. + */ +public class PhaseSplitter { + + private Map distributeTestSuite(TestSuite initialTestSuite, List agents) + throws CloneNotSupportedException { + Map taskTestSuites = new HashMap<>(); + + for (int i = 0; i < agents.size(); i++) { + TestSuite clone = initialTestSuite.clone(); + + clone.getTests().forEach(test -> { + if (taskTestSuites.isEmpty()) { + test.setCount(String.valueOf(test.getCount() / agents.size() + test.getCount() % agents.size())); + } else { + test.setCount(String.valueOf(test.getCount() / agents.size())); + } + }); + + taskTestSuites.put(agents.get(i), clone); + } + + return taskTestSuites; + } + + private void sanityChecks(Phase phase, List agents) { + if (phase == null || agents == null) { + throw new IllegalArgumentException("Phase/List of agents must not be null"); + } + + if (agents.isEmpty()) { + throw new IllegalStateException("At least one agent must be running in the cluster."); + } + + } + + private Map mapPhaseToAgent(Phase phase,Map partitionRunModes, + Map partitionTestSuites, List agents) + throws CloneNotSupportedException { + Map phasePerAgent = new HashMap<>(); + + for (String agent : agents) { + Phase taskPhase = (Phase) phase.clone(); + TestSuite taskTestSuite = partitionTestSuites.get(agent); + + // set the count (the number of executions since the beginning of the run) of each test to 0 + taskTestSuite.getTests().forEach(test -> taskPhase.getCounts().put(test, new AtomicLong(0))); + + taskPhase.setTestSuite(taskTestSuite); + taskPhase.setRunMode(partitionRunModes.get(agent)); + + phasePerAgent.put(agent, taskPhase); + } + + return phasePerAgent; + } + + /** + * Knows how to divide a phase into a number of phases equal to the number of agents running in the cluster. + * @param phase the phase to be partitioned. + * @param agents list with all the agents able to receive a task and to execute it. + */ + public Map splitPhase(Phase phase, List agents) throws CloneNotSupportedException { + sanityChecks(phase, agents); + + Map partitionRunModes = phase.getRunMode().getRunModeSplitter().distributeRunMode(phase.getRunMode(), agents); + Map partitionTestSuites = distributeTestSuite(phase.getTestSuite(), agents); + + return mapPhaseToAgent(phase, partitionRunModes, partitionTestSuites, agents); + } + + /** + * Knows how to divide a phase into a number of phases equal to the number of agents running in the cluster taking + * into consideration the agents that recently joined the cluster and triggered the work redistribution process. + * @param phase : the phase to be partitioned + * @param existingAgents : agents that were running TD before redistributing the work + * @param newAgents : agents that recently joined the cluster + * @param phaseStartTime : execution start time of the phase + * @return : a map having as key the IP address of an Agent running in the cluster and the Phase to be executed by + * this agent as value + */ + public Map splitPhaseForRebalancingWork(Phase phase, List existingAgents, + List newAgents, long phaseStartTime) + throws CloneNotSupportedException { + sanityChecks(phase, existingAgents); + + List allAgents = new ArrayList<>(existingAgents); + allAgents.addAll(newAgents); + + Map partitionRunModes = + phase.getRunMode().getRunModeSplitter().distributeRunModeForRebalancingWork(phase.getRunMode(), existingAgents, + newAgents, phaseStartTime); + Map partitionTestSuites = distributeTestSuite(phase.getTestSuite(), allAgents); + + return mapPhaseToAgent(phase, partitionRunModes, partitionTestSuites, allAgents); + } +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/ConstantLoadRunModeSplitter.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/ConstantLoadRunModeSplitter.java new file mode 100644 index 0000000..ba95c53 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/ConstantLoadRunModeSplitter.java @@ -0,0 +1,106 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes; + +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.runmodes.ConstantLoad; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Knows how to split the constant load run mode into multiple run modes to be distributed to the agents + * running in the cluster. + */ +public class ConstantLoadRunModeSplitter implements RunModeSplitter { + + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private ConstantLoad setParamsForDistributedRunMode(ConstantLoad runMode, int nrAgents, int rateRemainder, + int startRemainder, int endRemainder, + int loadRemainder, int agentId) throws CloneNotSupportedException { + ConstantLoad clone = (ConstantLoad) runMode.clone(); + + + if (runMode.isVariableLoad()) { + if (runMode.getRate() >= nrAgents) { + clone.setStart(String.valueOf(runMode.getStart()/ nrAgents + startRemainder)); + clone.setEnd(String.valueOf(runMode.getEnd()/ nrAgents + endRemainder)); + clone.setRate(String.valueOf(runMode.getRate()/ nrAgents + rateRemainder)); + } else { + clone.setInitialDelay(agentId * GlobalArgs.parseDurationToSeconds(runMode.getInterval()) * 1000); + clone.setOneAgentRate(runMode.getRate()); + clone.setStart(String.valueOf(runMode.getStart()+ agentId * runMode.getRate())); + clone.setRate(String.valueOf(nrAgents * runMode.getRate())); + long interval = GlobalArgs.parseDurationToSeconds(runMode.getInterval()); + clone.setInterval(String.valueOf(interval * nrAgents) + 's'); + } + + return clone; + } + + /* we must distribute the load between the agents */ + clone.setLoad(String.valueOf(runMode.getLoad() / nrAgents + loadRemainder)); + + return clone; + } + + @Override + public Map distributeRunMode(ConstantLoad runMode, List agents) + throws CloneNotSupportedException { + int nrAgents = agents.size(); + Map runModes = new HashMap<>(); + + ConstantLoad firstTask = setParamsForDistributedRunMode(runMode, nrAgents, runMode.getRate() % nrAgents, + runMode.getStart() % nrAgents, runMode.getEnd() % nrAgents, runMode.getLoad() % nrAgents, 0); + runModes.put(agents.get(0), firstTask); + + for (int i = 1; i < nrAgents; i++) { + ConstantLoad task = setParamsForDistributedRunMode(runMode, nrAgents, 0, 0, 0, 0, i); + runModes.put(agents.get(i), task); + } + + return runModes; + } + + @Override + public Map distributeRunModeForRebalancingWork(ConstantLoad runMode, List existingAgents, + List newAgents, long phaseStartTime) throws CloneNotSupportedException { + List agents = new ArrayList<>(existingAgents); + agents.addAll(newAgents); + + Map taskRunModes = distributeRunMode(runMode, agents); + newAgents.forEach(agentName -> taskRunModes.get(agentName).setStart("0")); + + if (!runMode.isVariableLoad()) { + return taskRunModes; + } + + // compute the current load to determine the new values for start/current load + long endTime = System.currentTimeMillis(); + long diff = (endTime - phaseStartTime) / 1000; // convert to seconds + int estimatedCurrentLoad = ((int)(diff / GlobalArgs.parseDurationToSeconds(runMode.getInterval()))) + * runMode.getRate() + runMode.getStart(); + + LOG.info("Phase was executed for " + diff + " seconds"); + LOG.info("Estimated current load " + estimatedCurrentLoad); + + // set start property for new agents + newAgents.forEach(agentName -> taskRunModes.get(agentName) + .setStart(String.valueOf(estimatedCurrentLoad / agents.size()))); + + // set current load for old agents + taskRunModes.get(existingAgents.get(0)).setCurrentLoad(estimatedCurrentLoad / agents.size() + + estimatedCurrentLoad % agents.size()); + for (int i = 1; i < existingAgents.size(); i++) { + taskRunModes.get(existingAgents.get(i)).setCurrentLoad(estimatedCurrentLoad / agents.size()); + } + + return taskRunModes; + + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/NormalRunModeSplitter.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/NormalRunModeSplitter.java new file mode 100644 index 0000000..67eac1e --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/NormalRunModeSplitter.java @@ -0,0 +1,113 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes; + +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.runmodes.Normal; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Knows how to split the normal run mode into multiple normal run modes to be distributed to the agents + * running in the cluster. + */ +public class NormalRunModeSplitter implements RunModeSplitter { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private Normal setParamsForDistributedRunMode(Normal runMode, int nrAgents, int rateRemainder, int endRemainder, + int startRemainder, int concurrencyRemainder, int agentId) throws CloneNotSupportedException { + Normal clone = (Normal) runMode.clone(); + + + if (runMode.isVariableConcurrency()) { + if (runMode.getRate() >= nrAgents) { + clone.setRate(String.valueOf(runMode.getRate() / nrAgents + rateRemainder)); + clone.setStart(String.valueOf(runMode.getStart() / nrAgents + startRemainder)); + clone.setEnd(String.valueOf(runMode.getEnd() / nrAgents + endRemainder)); + } else { + clone.setInitialDelay(agentId * GlobalArgs.parseDurationToSeconds(runMode.getInterval()) * 1000); + long interval = GlobalArgs.parseDurationToSeconds(runMode.getInterval()); + clone.setInterval(String.valueOf(interval * nrAgents) + 's'); + + if (agentId > 0) { + clone.setStart("0"); + clone.setEnd(String.valueOf((runMode.getEnd() - runMode.getStart()) / nrAgents)); + } else { + long diff = runMode.getEnd() - runMode.getStart(); + clone.setEnd(String.valueOf(diff / nrAgents + diff % nrAgents + runMode.getStart())); + } + } + } else { + clone.setConcurrency(String.valueOf(runMode.getConcurrency()/ nrAgents + concurrencyRemainder)); + } + + return clone; + } + + + @Override + public Map distributeRunMode(Normal runMode, List agents) throws CloneNotSupportedException { + if (runMode == null) { + throw new IllegalArgumentException("Run mode must not be null."); + } + + Map taskRunModes = new HashMap<>(); + int nrAgents = agents.size(); + + Normal firstTask = setParamsForDistributedRunMode(runMode, nrAgents, runMode.getRate() % nrAgents, runMode.getEnd() % nrAgents, + runMode.getStart() % nrAgents, runMode.getConcurrency() % nrAgents, 0); + taskRunModes.put(agents.get(0), firstTask); + + for (int i = 1; i < nrAgents; i++) { + Normal task = setParamsForDistributedRunMode(runMode, nrAgents, 0, 0, 0, 0, i); + taskRunModes.put(agents.get(i), task); + } + + return taskRunModes; + } + + @Override + public Map distributeRunModeForRebalancingWork(Normal runMode, List existingAgents, + List newAgents, long phaseStartTime) throws CloneNotSupportedException { + if (runMode == null) { + throw new IllegalArgumentException("Run mode must not be null."); + } + + List agents = new ArrayList<>(existingAgents); + agents.addAll(newAgents); + Map taskRunModes = distributeRunMode(runMode, agents); + + // set start property to 'rate' for each new agent + if (!runMode.isVariableConcurrency()) { + return taskRunModes; + } + + // compute the expected current concurrency + long endTime = System.currentTimeMillis(); + long diff = (endTime - phaseStartTime) / 1000; + int estimatedConcurrency = ((int)(diff / GlobalArgs.parseDurationToSeconds(runMode.getInterval()))) + * runMode.getRate() + runMode.getStart(); + + LOG.info("Phase was executed for " + diff + " seconds"); + LOG.info("Estimated concurrency " + estimatedConcurrency); + + // set start property for new agents + newAgents.forEach(agentName -> + taskRunModes.get(agentName).setStart(String.valueOf(estimatedConcurrency / agents.size()))); + + // set desired active threads for old agents + taskRunModes.get(existingAgents.get(0)).setConcurrency(String.valueOf(estimatedConcurrency / agents.size() + + estimatedConcurrency % agents.size())); + for (int i = 1; i < existingAgents.size(); i++) { + taskRunModes.get(existingAgents.get(i)).setConcurrency(String.valueOf(estimatedConcurrency / agents.size())); + } + + return taskRunModes; + + } + +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/RunModeSplitter.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/RunModeSplitter.java new file mode 100644 index 0000000..95015af --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/splitters/runmodes/RunModeSplitter.java @@ -0,0 +1,31 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes; + +import com.adobe.qe.toughday.internal.core.engine.RunMode; + +import java.util.List; +import java.util.Map; + +/** + * Common interface for all run mode splitters. Implement this interface to specify how a certain type of run mode + * must be partitioned into multiple run modes to be distributed to the agents running in the cluster. + * @param Type of the run mode to be partitioned + */ +public interface RunModeSplitter { + /** + * Used for distributing the run mode when before sending TD execution requests to the agents + * running in the cluster. + * @param runMode object to be partitioned + * @param agents names of the agents waiting to receive TD execution requests from the driver. + */ + Map distributeRunMode(T runMode, List agents) throws CloneNotSupportedException; + + /** + * Used for redistributing the run mode when the process of rebalancing the work is triggered. + * @param runMode object to be partitioned + * @param existingAgents list of agents that were running TD before starting the work redistribution process. + * @param newAgents list of agents which recently joined the cluster + * @param phaseStartTime : time when the phase execution started + */ + Map distributeRunModeForRebalancingWork(T runMode, List existingAgents, List newAgents, + long phaseStartTime) throws CloneNotSupportedException; +} diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/HeartbeatTask.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/HeartbeatTask.java new file mode 100644 index 0000000..67ebe19 --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/HeartbeatTask.java @@ -0,0 +1,84 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.tasks; + +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.TaskBalancer; +import com.google.gson.Gson; +import org.apache.http.HttpResponse; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.*; + +import static com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils.HTTP_REQUEST_RETRIES; + +/** + * Class used for implementing the heartbeat protocol for monitoring the state of the agents running in the cluster. + */ +public class HeartbeatTask implements Runnable { + protected static final Logger LOG = LogManager.getLogger(Driver.class); + + private Driver driverInstance; + private final HttpUtils httpUtils = new HttpUtils(); + private final TaskBalancer taskBalancer = TaskBalancer.getInstance(); + + public HeartbeatTask(Driver driverInstance) { + this.driverInstance = driverInstance; + } + + private void processHeartbeatResponse(String agentIp, HttpResponse agentResponse) throws IOException { + // the agent has sent his statistic for executions/test => aggregate counts + Gson gson = new Gson(); + String yamlCounts = EntityUtils.toString(agentResponse.getEntity()); + + // gson treats numbers as double values by default + Map doubleAgentCounts = gson.fromJson(yamlCounts, Map.class); + LOG.info("[Heartbeat] Received execution state from agent " + agentIp + ": " + doubleAgentCounts.toString()); + + // recently added agents might not execute tests yet + if (doubleAgentCounts.isEmpty()) { + return; + } + + this.driverInstance.getDistributedPhaseMonitor().getExecutions().forEach((testName, executionsPerAgent) -> + executionsPerAgent.put(agentIp, doubleAgentCounts.get(testName).longValue())); + } + + /** + * Method used for sending heartbeat messages to all the agents running in the cluster. This method should be + * called periodically during the entire distributed execution of ToughDay. + */ + @Override + public void run() { + List activeAgents = new ArrayList<>(this.driverInstance.getDriverState().getRegisteredAgents()); + // remove agents which previously failed to respond to heartbeat request + this.taskBalancer.getInactiveAgents().forEach(activeAgents::remove); + + for (String agentIp : activeAgents) { + String URI = Agent.getHeartbeatPath(agentIp); + HttpResponse agentResponse = httpUtils.sendHttpRequest(HttpUtils.GET_METHOD, "", URI, HTTP_REQUEST_RETRIES); + if (agentResponse != null) { + try { + processHeartbeatResponse(agentIp, agentResponse); + } catch (IOException e) { + // the system may reach the limit of available IO resources + LOG.warn("Could not process heartbeat information received form agent " + agentIp); + } + continue; + } + + if (!this.driverInstance.getDistributedPhaseMonitor().isPhaseExecuting()) { + this.driverInstance.getDriverState().removeAgent(agentIp); + continue; + } + + this.taskBalancer.scheduleWorkRedistributionProcess(driverInstance, agentIp, false); + } + + LOG.info("Number of executions per test: " + + this.driverInstance.getDistributedPhaseMonitor().getExecutionsPerTest()); + } +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/MasterHeartbeatTask.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/MasterHeartbeatTask.java new file mode 100644 index 0000000..80647db --- /dev/null +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/distributedtd/tasks/MasterHeartbeatTask.java @@ -0,0 +1,146 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.tasks; + +import com.adobe.qe.toughday.internal.core.distributedtd.HttpUtils; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.google.gson.Gson; +import org.apache.http.HttpResponse; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Class used for implementing the heartbeat protocol for monitoring the state of the Master running in the cluster. + */ +public class MasterHeartbeatTask implements Runnable { + protected static final Logger LOG = LogManager.getLogger(Engine.class); + + private final Driver driver; + private final HttpUtils httpUtils = new HttpUtils(); + + /** + * Constructor. + * @param driver : the Driver instance which will execute this task periodically. + */ + public MasterHeartbeatTask(Driver driver) { + this.driver = driver; + } + + private void announceDriversThatMasterDied() { + LOG.info("Starting to announce drivers that the current master just died."); + + for (int i = 0; i < this.driver.getDriverState().getNrDrivers(); i++) { + // skip current driver and the former master(which just died) + if (i == this.driver.getDriverState().getId() || i == this.driver.getDriverState().getMasterId()) { + continue; + } + + String hostname = this.driver.getDriverState().getPathForId(i); + HttpResponse driverResponse = this.httpUtils.sendHttpRequest(HttpUtils.POST_METHOD, + String.valueOf(this.driver.getDriverState().getMasterId()), Driver.getMasterElectionPath(hostname, HttpUtils.SPARK_PORT), + HttpUtils.HTTP_REQUEST_RETRIES); + + if (driverResponse == null) { + /* the assumption is that the driver failed to respond to heartbeat request and he will receive the + * updates after rejoining the cluster. + */ + LOG.warn("Failed to announce driver " + hostname + " that the master failed."); + } + + LOG.info("Successfully announced driver " + hostname + " that the current master died."); + } + + } + + /* Function responsible for determining if the number of tests left to executed by the agents must be updated. This + * operation is required whenever the work is rebalanced by the master. + */ + private void checkIfCountsPerTestMustBeUpdated(Map> masterData) { + Map> executions = driver.getDistributedPhaseMonitor().getExecutions(); + + executions.forEach((testName, execPerAgent) -> { + execPerAgent.forEach((agentIp, counts) -> { + // check if one agent is no longer running TD tests + if (!masterData.get(testName).containsKey(agentIp)) { + LOG.info("Counts must be updated because one agent is no longer running TD tests."); + this.driver.getDistributedPhaseMonitor().updateCountPerTest(); + this.driver.getDistributedPhaseMonitor().resetExecutions(); + return; + } + + // check if work was rebalanced and executions must be reset + if (masterData.get(testName).get(agentIp).longValue() < counts) { + LOG.info("Work was rebalanced so counts must be updated"); + this.driver.getDistributedPhaseMonitor().updateCountPerTest(); + this.driver.getDistributedPhaseMonitor().resetExecutions(); + return; + } + }); + }); + + } + + @Override + public void run() { + DriverState driverState = this.driver.getDriverState(); + Gson gson = new Gson(); + + LOG.info(driverState.getHostname() + ": sending heartbeat message to master: " + + Driver.getHeartbeatPath(driverState.getPathForId(driverState.getMasterId()))); + + /* acquire read lock to prevent modification of the current master when receiving information from a different + * driver while the current one is informing others to trigger the master election process. + * */ + + this.driver.getDriverState().getMasterIdLock().readLock().lock(); + HttpResponse driverResponse = this.httpUtils.sendHttpRequest(HttpUtils.GET_METHOD, "", + Driver.getHeartbeatPath(driverState.getPathForId(driverState.getMasterId())), + HttpUtils.HTTP_REQUEST_RETRIES); + + if (driverResponse != null) { + this.driver.getDriverState().getMasterIdLock().readLock().unlock(); + + /* process the information received from the master regarding the number of tests that were executed by the + * agents running TD. + */ + + try { + String jsonExecutions = EntityUtils.toString(driverResponse.getEntity()); + Map> executions = gson.fromJson(jsonExecutions, Map.class); + + LOG.info("Received execution state from master " + executions.toString()); + checkIfCountsPerTestMustBeUpdated(executions); + + executions.forEach((testName, executionsPerAgent) -> { + Map execPerAgent = new HashMap<>(); + executionsPerAgent.forEach((agentIp, counts) -> execPerAgent.put(agentIp, counts.longValue())); + this.driver.getDistributedPhaseMonitor().getExecutions().put(testName, execPerAgent); + }); + + LOG.info("Successfully updated executions/agent"); + + } catch (IOException e) { + LOG.info("Could not process the updates from the driver regarding the number of executions/test. " + + "Count property will no longer be respected if the current master dies."); + } + return; + } + + LOG.info(driverState.getHostname() + ": master failed to respond to heartbeat message. Master " + + "election process will be triggered soon."); + /* mark candidate as inactive */ + this.driver.getMasterElection().markCandidateAsInvalid(driverState.getMasterId()); + + /* announce all drivers that the current master died */ + announceDriversThatMasterDied(); + + this.driver.getDriverState().getMasterIdLock().readLock().unlock(); + /* elect a new master */ + this.driver.getMasterElection().electMaster(driver); + } +} \ No newline at end of file diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/AsyncTimeoutChecker.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/AsyncTimeoutChecker.java index bda8e69..0b867a2 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/AsyncTimeoutChecker.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/AsyncTimeoutChecker.java @@ -44,7 +44,7 @@ private void interruptWorkerIfTimeout(AsyncTestWorker worker) { return; Long testTimeout = currentTest.getTimeout(); - long timeout = testTimeout >= 0 ? testTimeout : engine.getGlobalArgs().getTimeout(); + long timeout = testTimeout >= 0 ? testTimeout : engine.getGlobalArgs().getTimeoutInSeconds(); if (!worker.getMutex().tryLock()) { /* nothing to interrupt. if the test was running diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Engine.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Engine.java index bcdda9b..48c1c90 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Engine.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Engine.java @@ -23,6 +23,7 @@ import com.adobe.qe.toughday.api.annotations.ConfigArgGet; import com.adobe.qe.toughday.internal.core.config.Configuration; import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.Agent; import com.adobe.qe.toughday.metrics.Metric; import com.adobe.qe.toughday.tests.sequential.AEMTestBase; import com.adobe.qe.toughday.tests.utils.PackageManagerClient; @@ -54,6 +55,7 @@ public class Engine { protected static Random _rnd = new Random(); private final Configuration configuration; + private Agent agent; private GlobalArgs globalArgs; private ExecutorService engineExecutorService = Executors.newFixedThreadPool(2); private final ReentrantReadWriteLock engineSync = new ReentrantReadWriteLock(); @@ -92,6 +94,10 @@ public GlobalArgs getGlobalArgs() { return globalArgs; } + public void setAgent(Agent agent) { + this.agent = agent; + } + public boolean areTestsRunning() { return testsRunning; } /** @@ -370,6 +376,7 @@ public void run() { } finally { currentPhaseLock.readLock().unlock(); } + Engine.logGlobal("Test execution finished at: " + Engine.getCurrentDateTime()); LogManager.shutdown(); } @@ -411,7 +418,7 @@ public void run() { } RunMode currentRunmode = phase.getRunMode(); - Long currentDuration = phase.getDuration(); + Long currentDuration = GlobalArgs.parseDurationToSeconds(phase.getDuration()); currentRunmode.runTests(this); long start = System.currentTimeMillis(); @@ -423,6 +430,10 @@ public void run() { LOG.info("Phase interrupted."); long elapsed = System.currentTimeMillis() - start; + if (this.agent != null) { + agent.announcePhaseCompletion(); + } + // if the phase finishes sooner than its duration, // the remainder is split equally between the remaining phases // whose duration was not given as input @@ -532,7 +543,11 @@ protected void shutdownAndAwaitTermination(ExecutorService pool) { } public Phase getCurrentPhase() { - return currentPhase; + currentPhaseLock.readLock().lock(); + Phase phase = currentPhase; + currentPhaseLock.readLock().unlock(); + + return phase; } public ReadWriteLock getCurrentPhaseLock() { diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Phase.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Phase.java index 53f8385..a0d9a1c 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Phase.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/Phase.java @@ -24,7 +24,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicLong; -public class Phase { +public class Phase implements Cloneable { private static final String DEFAULT_MEASURABILITY = "true"; private String name; @@ -39,6 +39,15 @@ public class Phase { private Map metrics = new LinkedHashMap<>(); private Map counts = new HashMap<>(); + /** + * Creates shallow copy. + * @throws CloneNotSupportedException if the object to be cloned does not implement the Cloneable interface. + */ + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } + @ConfigArgGet public String getName() { return name; @@ -71,8 +80,8 @@ public void setUseconfig(String useconfig) { } @ConfigArgGet - public Long getDuration() { - return duration; + public String getDuration() { + return duration == null ? null : String.valueOf(duration) + 's'; } @ConfigArgSet(required = false, desc = "The duration of the current phase.") diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/RunMode.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/RunMode.java index d29f263..3450a99 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/RunMode.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/RunMode.java @@ -12,6 +12,8 @@ package com.adobe.qe.toughday.internal.core.engine; import com.adobe.qe.toughday.api.core.RunMap; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; import java.util.Collection; import java.util.concurrent.ExecutorService; @@ -22,6 +24,19 @@ public interface RunMode { ExecutorService getExecutorService(); RunContext getRunContext(); + /** + * Returns the RunModeSplitter instance which knows how to split the run mode for running ToughDay distributed. + * @param type of the run mode. + */ + RunModeSplitter getRunModeSplitter(); + + /** + * Returns the RunModeBalancer instance which knows how to process the redistribution instructions received from + * the driver when the number of agents running in the cluster modifies and the work must be redistributed. + * @param type of the run node. + */ + RunModeBalancer getRunModeBalancer(); + interface RunContext { Collection getTestWorkers(); Collection getRunMaps(); diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/ConstantLoad.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/ConstantLoad.java index 36318d5..f2df205 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/ConstantLoad.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/ConstantLoad.java @@ -18,21 +18,26 @@ import com.adobe.qe.toughday.api.annotations.ConfigArgGet; import com.adobe.qe.toughday.api.annotations.ConfigArgSet; import com.adobe.qe.toughday.internal.core.TestSuite; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.ConstantLoadRunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; import com.adobe.qe.toughday.internal.core.engine.*; import com.adobe.qe.toughday.internal.core.config.GlobalArgs; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.ConstantLoadRunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.ConstantLoadRunModeSplitter; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; import org.apache.commons.lang3.mutable.MutableLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; @Description(desc = "Generates a constant load of test executions, regardless of their execution time.") -public class ConstantLoad implements RunMode { +public class ConstantLoad implements RunMode, Cloneable { private static final Logger LOG = LoggerFactory.getLogger(ConstantLoad.class); private static final String DEFAULT_LOAD_STRING = "50"; @@ -52,22 +57,32 @@ public class ConstantLoad implements RunMode { private long interval = DEFAULT_INTERVAL; private int rate; private int currentLoad; + //private int currentLoad; + private long initialDelay = 0; + /* field used for checking the finish condition when running TD distributed with the rate + * smaller than number of agents in the cluster. */ + private int oneAgentRate = 0; + private ScheduledExecutorService runRoundScheduler = Executors.newScheduledThreadPool(1); private TestCache testCache; private Phase phase; + private RunModeSplitter runModeSplitter = new ConstantLoadRunModeSplitter(); + private ScheduledFuture scheduledFuture = null; + private final ConstantLoadRunModeBalancer runModeBalancer = new ConstantLoadRunModeBalancer(); private Boolean measurable = true; - @ConfigArgSet(required = false, defaultValue = DEFAULT_LOAD_STRING, desc = "Set the load, in requests per second for the \"constantload\" runmode.") + @ConfigArgSet(required = false, defaultValue = DEFAULT_LOAD_STRING, + desc = "Set the load, in requests per second for the \"constantload\" runmodes.") public void setLoad(String load) { checkNotNegative(Long.parseLong(load), "load"); this.load = Integer.parseInt(load); } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getLoad() { return this.load; } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getStart() { return start; } @@ -81,12 +96,13 @@ public void setStart(String start) { this.start = Integer.valueOf(start); } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getRate() { return rate; } - @ConfigArgSet(required = false, desc = "The increase in load per time unit. When it equals -1, it means it is not set.", defaultValue = "-1") + @ConfigArgSet(required = false, desc = "The increase in load per time unit. When it equals -1, it means it is not set.", + defaultValue = "-1") public void setRate(String rate) { if (!rate.equals("-1")) { checkNotNegative(Long.parseLong(rate), "rate"); @@ -94,17 +110,18 @@ public void setRate(String rate) { this.rate = Integer.valueOf(rate); } - @ConfigArgGet - public long getInterval() { - return interval; + @ConfigArgGet(redistribute = true) + public String getInterval() { + return String.valueOf(interval / 1000) + 's'; } - @ConfigArgSet(required = false, desc = "Used with rate to specify the time interval to add increase the load.", defaultValue = DEFAULT_INTERVAL_STRING) + @ConfigArgSet(required = false, desc = "Used with rate to specify the time interval to add increase the load.", + defaultValue = DEFAULT_INTERVAL_STRING) public void setInterval(String interval) { - this.interval = GlobalArgs.parseDurationToSeconds(interval); + this.interval = GlobalArgs.parseDurationToSeconds(interval) * 1000; } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getEnd() { return end; } @@ -117,6 +134,31 @@ public void setEnd(String end) { this.end = Integer.valueOf(end); } + public ConstantLoad() { + /* this is required when running TD distributed because scheduled task might be cancelled and + * rescheduled when rebalancing the work between the agents. + */ + ScheduledThreadPoolExecutor scheduledPoolExecutor = (ScheduledThreadPoolExecutor) runRoundScheduler; + scheduledPoolExecutor.setRemoveOnCancelPolicy(true); + } + + public int getOneAgentRate() { + return this.oneAgentRate; + } + + public void setCurrentLoad(int currentLoad) { + this.currentLoad = currentLoad; + } + + public int getCurrentLoad() { + return this.currentLoad; + } + + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } + private static class TestCache { public Map> cache = new HashMap<>(); @@ -135,7 +177,7 @@ public void add(@NotNull AbstractTest test) { } } - private boolean isVariableLoad() { + public boolean isVariableLoad() { return start != -1 && end != -1; } @@ -155,6 +197,22 @@ private void checkInvalidArgs() { } } + public void addRunMaps(long nr) { + for (long i = 0; i < nr; i++) { + synchronized (runMaps) { + runMaps.add(phase.getPublishMode().getRunMap().newInstance()); + } + } + } + + public void removeRunMaps(long nr) { + for (long i = 0; i < nr; i++) { + synchronized (runMaps) { + this.runMaps.remove(0); + } + } + } + @Override public void runTests(Engine engine) { checkInvalidArgs(); @@ -169,11 +227,7 @@ public void runTests(Engine engine) { load = Math.max(start, end); } - for(int i = 0; i < load; i++) { - synchronized (runMaps) { - runMaps.add(phase.getPublishMode().getRunMap().newInstance()); - } - } + addRunMaps(load); this.scheduler = new AsyncTestWorkerScheduler(engine); executorService.execute(scheduler); @@ -193,11 +247,29 @@ public Collection getRunMaps() { @Override public boolean isRunFinished() { - return scheduler.isFinished(); + return scheduler != null && scheduler.isFinished(); } }; } + @Override + public RunModeBalancer getRunModeBalancer() { + return this.runModeBalancer; + } + + public void setInitialDelay(long initialDelay) { + this.initialDelay = initialDelay; + } + + public void setOneAgentRate(int oneAgentRate) { + this.oneAgentRate = oneAgentRate; + } + + @Override + public RunModeSplitter getRunModeSplitter() { + return this.runModeSplitter; + } + @Override public void finishExecutionAndAwait() { scheduler.finishExecution(); @@ -231,6 +303,26 @@ public void finishExecutionAndAwait() { } + /** + * Method used for cancelling the task responsible for updating the current load. + * @return true if the task is successfully cancelled; false otherwise + */ + public boolean cancelPeriodicTask() { + return this.scheduledFuture.cancel(true); + } + + + /** + * Method used for scheduling the task responsible for updating the current load to be executed every 'interval' + * milliseconds. + */ + public void schedulePeriodicTask() { + MutableLong secondsLeft = new MutableLong((initialDelay + interval) / 1000); + + this.runRoundScheduler.scheduleAtFixedRate(this.scheduler.getRunnableToSchedule(secondsLeft), 0, + GlobalArgs.parseDurationToSeconds("1s"), TimeUnit.SECONDS); + } + private class AsyncTestWorkerImpl extends AsyncTestWorker { private AbstractTest test; private RunMap runMap; @@ -269,98 +361,107 @@ public boolean hasExited() { } } - private class AsyncTestWorkerScheduler extends AsyncEngineWorker { private Engine engine; + public AsyncTestWorkerScheduler(Engine engine) { this.engine = engine; } - private void configureRateAndInterval(MutableLong secondsLeft) { + private void configureRateAndInterval() { //the difference from the beginning load to the end one int loadDifference = Math.abs(end - start); // suppose load will increase by second - secondsLeft.setValue(1); - rate = (int)Math.floor(1.0 * secondsLeft.getValue() * loadDifference - / phase.getDuration()); + long newInterval = 1; + rate = (int)Math.floor(1.0 * newInterval* loadDifference + / GlobalArgs.parseDurationToSeconds(phase.getDuration())); // if the rate becomes too small, increase the interval at which the load is increased while (rate < 1) { - secondsLeft.increment(); - rate = (int)Math.floor(1.0 * secondsLeft.getValue() * loadDifference - / phase.getDuration()); + newInterval += 1; + rate = (int)Math.floor(1.0 * newInterval * loadDifference + / GlobalArgs.parseDurationToSeconds(phase.getDuration())); } - interval = secondsLeft.getValue(); + interval = newInterval * 1000; // set interval in milliseconds } - @Override - public void run() { - try { - currentLoad = load; - MutableLong secondsLeft = new MutableLong(interval); + private Runnable getRunnableToSchedule(MutableLong secondsLeft) { + return () -> { + if (!isFinished()) { + try { + // run the current run with the current load + runRound(); + secondsLeft.decrement(); - // if the rate was not specified and start and end were - if (isVariableLoad()) { - if (rate == -1) { - configureRateAndInterval(secondsLeft); - } + rampUp(secondsLeft); - currentLoad = start; - } + rampDown(secondsLeft); - while (!isFinished()) { - // run the current run with the current load - runRound(); + } catch (InterruptedException e) { + finishExecution(); + LOG.warn("Constant load scheduler thread was interrupted."); - secondsLeft.decrement(); + // gracefully shut down scheduler + runRoundScheduler.shutdownNow(); + } + } + }; + } - // ramp up the load if 'start' was specified - rampUp(secondsLeft); + @Override + public void run() { + currentLoad = load; - // ramp down the load if 'end' was specified - rampDown(secondsLeft); + // if the rate was not specified and start and end were + if (isVariableLoad()) { + if (rate == -1) { + configureRateAndInterval(); } - } catch (InterruptedException e) { - finishExecution(); - LOG.warn("Constant load scheduler thread was interrupted."); + + currentLoad = start; } + + MutableLong secondsLeft = new MutableLong((interval + initialDelay) / 1000); + scheduledFuture = runRoundScheduler.scheduleAtFixedRate(getRunnableToSchedule(secondsLeft), + 0, GlobalArgs.parseDurationToSeconds("1s"), TimeUnit.SECONDS); + } private void rampUp(MutableLong secondsLeft) { - if (currentLoad == end) { + if (currentLoad == end || ((oneAgentRate > 0) && (currentLoad + rate >= end + oneAgentRate))) { finishExecution(); } // if 'interval' has passed and the current load is still below 'end', // increase the current load if (secondsLeft.getValue() == 0 && end != -1 && currentLoad < end) { - secondsLeft.setValue(interval); currentLoad += rate; - LOG.debug("Current load: " + currentLoad); if (currentLoad > end) { currentLoad = end; } + + secondsLeft.setValue(interval / 1000); } } private void rampDown(MutableLong secondsLeft) { - if (currentLoad == end) { + if (currentLoad == end || ((oneAgentRate > 0) && (currentLoad - rate <= end - oneAgentRate))) { finishExecution(); } // if 'interval' has passed and the currentLoad is still above 'end', // decrease the current load if (secondsLeft.getValue() == 0 && end != -1 && currentLoad > end) { - secondsLeft.setValue(interval); currentLoad -= rate; - LOG.debug("Current load: " + currentLoad); if (currentLoad < end) { currentLoad = end; } + + secondsLeft.setValue(interval / 1000); } } @@ -399,12 +500,13 @@ private void runRound() throws InterruptedException { testWorkers.add(worker); } } - - //TODO use this - Thread.sleep(1000); } } + public long getInitialDelay() { + return this.initialDelay; + } + @Override public ExecutorService getExecutorService() { return executorService; diff --git a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/Normal.java b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/Normal.java index 1cb0995..60878fd 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/Normal.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/internal/core/engine/runmodes/Normal.java @@ -18,6 +18,11 @@ import com.adobe.qe.toughday.api.annotations.ConfigArgSet; import com.adobe.qe.toughday.internal.core.config.GlobalArgs; import com.adobe.qe.toughday.internal.core.engine.*; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.NormalRunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.NormalRunModeSplitter; + +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +30,7 @@ import java.util.concurrent.*; @Description(desc = "Runs tests normally.") -public class Normal implements RunMode { +public class Normal implements RunMode, Cloneable { private static final Logger LOG = LoggerFactory.getLogger(Normal.class); private static final int EPS = 1; @@ -40,8 +45,8 @@ public class Normal implements RunMode { private static final long DEFAULT_INTERVAL = 1000; private ExecutorService testsExecutorService; - private ScheduledExecutorService addWorkerScheduler = Executors.newSingleThreadScheduledExecutor(); - private ScheduledExecutorService removeWorkerScheduler = Executors.newSingleThreadScheduledExecutor(); + private ScheduledExecutorService rampingScheduler = Executors.newScheduledThreadPool(1); + private ScheduledFuture scheduledFuture = null; private final List testWorkers = Collections.synchronizedList(new LinkedList<>()); private final List runMaps = new ArrayList<>(); @@ -54,10 +59,23 @@ public class Normal implements RunMode { private long waitTime = DEFAULT_WAIT_TIME; private long interval = DEFAULT_INTERVAL; private int activeThreads = 0; + private long initialDelay = 0; private RunContext context = null; + private RunModeSplitter runModeSplitter = new NormalRunModeSplitter(); + private final RunModeBalancer normalRunModeBalancer = new NormalRunModeBalancer(); + private Engine engine; - @ConfigArgGet + + public Normal() { + /* this is required when running TD distributed because scheduled task might be cancelled and + * rescheduled when rebalancing the work between the agents. + */ + ScheduledThreadPoolExecutor scheduledPoolExecutor = (ScheduledThreadPoolExecutor) rampingScheduler; + scheduledPoolExecutor.setRemoveOnCancelPolicy(true); + } + + @ConfigArgGet(redistribute = true) public int getConcurrency() { return concurrency; } @@ -69,6 +87,11 @@ public void setConcurrency(String concurrencyString) { this.concurrency = Integer.parseInt(concurrencyString); } + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } + @ConfigArgGet public long getWaitTime() { return waitTime; @@ -95,12 +118,13 @@ public void setStart(String start) { this.start = Integer.valueOf(start); } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getRate() { return rate; } - @ConfigArgSet(required = false, desc = "The number of threads added per time unit. When it equals -1, it means it is not set.", defaultValue = "-1") + @ConfigArgSet(required = false, desc = "The number of threads added per time unit. When it equals -1, it means it is not set.", + defaultValue = "-1") public void setRate(String rate) { if (!rate.equals("-1")) { checkNotNegative(Long.parseLong(rate), "rate"); @@ -108,17 +132,22 @@ public void setRate(String rate) { this.rate = Integer.valueOf(rate); } - @ConfigArgGet - public long getInterval() { - return interval; + @ConfigArgGet(redistribute = true) + public String getInterval() { + return String.valueOf(this.interval / 1000) + 's'; + } + + public void setInitialDelay(long initialDelay) { + this.initialDelay = initialDelay; } - @ConfigArgSet(required = false, desc = "Used with rate to specify the time interval to add threads.", defaultValue = DEFAULT_INTERVAL_STRING) + @ConfigArgSet(required = false, desc = "Used with rate to specify the time interval to add threads.", + defaultValue = DEFAULT_INTERVAL_STRING) public void setInterval(String interval) { this.interval = GlobalArgs.parseDurationToSeconds(interval) * 1000; } - @ConfigArgGet + @ConfigArgGet(redistribute = true) public int getEnd() { return end; } @@ -135,6 +164,43 @@ public int getActiveThreads() { return activeThreads; } + /** + * Method used for cancelling the execution of the task responsible for updating the number of worker threads. + * @return + */ + public boolean cancelPeriodicTask() { + if (this.scheduledFuture != null) { + return this.scheduledFuture.cancel(true); + } + + return true; + } + + /** + * Method used for scheduling the task responsible for updating the number of worker threads to execute + * periodically, every 'interval' milliseconds. + */ + public void schedulePeriodicTask() { + // added for testing purposes + if (this.engine == null) { + return; + } + + if (this.start < this.end) { + // execute 'rate' workers every 'interval' + this.scheduledFuture = this.rampingScheduler.scheduleAtFixedRate(this.getRampUpRunnable(this.engine, + this.engine.getCurrentPhase().getTestSuite()), this.initialDelay, this.interval, TimeUnit.MILLISECONDS); + } else if (this.end < this.start) { + // interrupt 'rate' workers every 'interval' + this.scheduledFuture = this.rampingScheduler.scheduleAtFixedRate(this.getRampDownRunnable(), this.initialDelay, + this.interval, TimeUnit.MILLISECONDS); + } + } + + public Engine getEngine() { + return this.engine; + } + private void checkNotNegative(long param, String property) { if (param < 0) { throw new IllegalArgumentException("Property " + property + " incorrectly configured as negative."); @@ -151,13 +217,14 @@ private void checkInvalidArgs() { } } - private boolean isVariableConcurrency() { + public boolean isVariableConcurrency() { return start != -1 && end != -1; } @Override public void runTests(Engine engine) { checkInvalidArgs(); + this.engine = engine; this.phase = engine.getCurrentPhase(); TestSuite testSuite = phase.getTestSuite(); @@ -168,7 +235,7 @@ public void runTests(Engine engine) { // namely every 'interval' milliseconds if (isVariableConcurrency()) { // if start and end were provided if (rate == -1) { - interval = (long)Math.floor(1000.0 * (phase.getDuration() - EPS) + interval = (long)Math.floor(1000.0 * (GlobalArgs.parseDurationToSeconds(phase.getDuration()) - EPS) / (start < end? end - start : start - end)); rate = 1; } @@ -184,16 +251,16 @@ public void runTests(Engine engine) { createAndExecuteWorker(engine, testSuite); } - // execute 'rate' workers every 'interval' - rampUp(engine, testSuite); - - // interrupt 'rate' workers every 'interval' - rampDown(); + schedulePeriodicTask(); } + public void finishAndDeleteWorker(AsyncTestWorker worker) { + worker.finishExecution(); + this.getRunContext().getTestWorkers().remove(worker); + activeThreads--; + } - - private void createAndExecuteWorker(Engine engine, TestSuite testSuite) { + public void createAndExecuteWorker(Engine engine, TestSuite testSuite) { AsyncTestWorkerImpl testWorker = new AsyncTestWorkerImpl(engine, phase, testSuite, phase.getPublishMode().getRunMap().newInstance()); synchronized (testWorkers) { testWorkers.add(testWorker); @@ -203,6 +270,8 @@ private void createAndExecuteWorker(Engine engine, TestSuite testSuite) { runMaps.add(testWorker.getLocalRunMap()); } + LOG.info("created test worker "); + try { testsExecutorService.execute(testWorker); } catch (OutOfMemoryError e) { @@ -210,64 +279,52 @@ private void createAndExecuteWorker(Engine engine, TestSuite testSuite) { } } - private void rampUp(Engine engine, TestSuite testSuite) { - // every 'interval' milliseconds, we'll create 'rate' workers - if (start < end) { - addWorkerScheduler.scheduleAtFixedRate(() -> { - for (int i = 0; i < rate; ++i) { - // if all the workers have been created - if (activeThreads >= end) { - addWorkerScheduler.shutdownNow(); + private Runnable getRampUpRunnable(Engine engine, TestSuite testSuite) { + return () -> { + for (int i = 0; i < rate; ++i) { + // if all the workers have been created + if (activeThreads >= end) { + rampingScheduler.shutdownNow(); - // mark workers as finished - for(AsyncTestWorker testWorker : testWorkers) { - testWorker.finishExecution(); - } - break; - } else { - createAndExecuteWorker(engine, testSuite); - } + // mark workers as finished + testWorkers.forEach(AsyncEngineWorker::finishExecution); + + break; + } else { + createAndExecuteWorker(engine, testSuite); } - }, 0, interval, TimeUnit.MILLISECONDS); - } + } + }; } - private void rampDown() { - // every 'interval' milliseconds, we'll stop 'rate' workers - if (end < start) { - ThreadPoolExecutor executor = (ThreadPoolExecutor)testsExecutorService; + private Runnable getRampDownRunnable() { + return () -> { + Iterator testWorkerIterator = testWorkers.iterator(); + int toRemove = rate; - removeWorkerScheduler.scheduleAtFixedRate(() -> { - Iterator testWorkerIterator = testWorkers.iterator(); - int toRemove = rate; + while (testWorkerIterator.hasNext()) { + // if all the workers have been removed + if (activeThreads <= end) { + rampingScheduler.shutdownNow(); - while (testWorkerIterator.hasNext()) { - // if all the workers have been removed - if (activeThreads <= end) { - removeWorkerScheduler.shutdownNow(); + // mark all the workers as finished, so the timeout checker will stop the execution + testWorkers.forEach(AsyncEngineWorker::finishExecution); - // mark all the workers as finished, so the timeout checker will stop the execution - for(AsyncTestWorker testWorker : testWorkers) { - testWorker.finishExecution(); - } + break; + } else { + AsyncTestWorker testWorker = testWorkerIterator.next(); + testWorker.finishExecution(); + + // remove the stopped worker + finishAndDeleteWorker(testWorker); + + // if rate users have been removed + if (toRemove == 0) { break; - } else { - AsyncTestWorker testWorker = testWorkerIterator.next(); - testWorker.finishExecution(); - - // remove the stopped worker - testWorkerIterator.remove(); - --toRemove; - --activeThreads; - - // if rate users have been removed - if (toRemove == 0) { - break; - } } } - }, 0, interval, TimeUnit.MILLISECONDS); - } + } + }; } public RunContext getRunContext() { @@ -298,13 +355,19 @@ public boolean isRunFinished() { } @Override - public void finishExecutionAndAwait() { - if (!addWorkerScheduler.isShutdown()) { - addWorkerScheduler.shutdownNow(); - } + public RunModeBalancer getRunModeBalancer() { + return this.normalRunModeBalancer; + } - if(!removeWorkerScheduler.isShutdown()) { - removeWorkerScheduler.shutdownNow(); + @Override + public RunModeSplitter getRunModeSplitter() { + return this.runModeSplitter; + } + + @Override + public void finishExecutionAndAwait() { + if (!rampingScheduler.isShutdown()) { + rampingScheduler.shutdownNow(); } synchronized (testWorkers) { @@ -426,5 +489,9 @@ public boolean hasExited() { } } + public long getInitialDelay() { + return this.initialDelay; + } + } diff --git a/toughday/src/main/java/com/adobe/qe/toughday/metrics/Percentile.java b/toughday/src/main/java/com/adobe/qe/toughday/metrics/Percentile.java index 7b703be..eebc5b4 100644 --- a/toughday/src/main/java/com/adobe/qe/toughday/metrics/Percentile.java +++ b/toughday/src/main/java/com/adobe/qe/toughday/metrics/Percentile.java @@ -11,6 +11,7 @@ */ package com.adobe.qe.toughday.metrics; +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; import com.adobe.qe.toughday.api.annotations.Description; import com.adobe.qe.toughday.api.annotations.ConfigArgSet; import com.adobe.qe.toughday.api.core.RunMap; @@ -23,11 +24,16 @@ public class Percentile extends Metric { public Percentile setValue(String value) { this.value = Double.valueOf(value.substring(0,value.length() - 1)); if (this.name.equals(getClass().getSimpleName())) { - this.name = value + "p"; + this.name = value; } return this; } + @ConfigArgGet + public String getValue() { + return String.valueOf(this.value) + 'p'; + } + @Override public Object getValue(RunMap.TestStatistics testStatistics) { return testStatistics.getValueAtPercentile(value); diff --git a/toughday/src/test/java/com/adobe/qe/toughday/TestConfiguration.java b/toughday/src/test/java/com/adobe/qe/toughday/TestConfiguration.java index 05d02e1..75acc7c 100644 --- a/toughday/src/test/java/com/adobe/qe/toughday/TestConfiguration.java +++ b/toughday/src/test/java/com/adobe/qe/toughday/TestConfiguration.java @@ -14,8 +14,8 @@ import com.adobe.qe.toughday.internal.core.ReflectionsContainer; import com.adobe.qe.toughday.internal.core.TestSuite; import com.adobe.qe.toughday.internal.core.Timestamp; -import com.adobe.qe.toughday.internal.core.config.ConfigParams; import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.GlobalArgs; import com.adobe.qe.toughday.internal.core.config.PhaseParams; import com.adobe.qe.toughday.internal.core.engine.Phase; import com.adobe.qe.toughday.internal.core.engine.publishmodes.Intervals; @@ -278,7 +278,7 @@ public void testPhaseProperties() throws Exception { Assert.assertEquals(phase.getName(), "phase"); Assert.assertEquals(phase.getMeasurable(), false); - long duration = phase.getDuration(); + long duration = GlobalArgs.parseDurationToSeconds(phase.getDuration()); Assert.assertEquals(duration, 60); } @@ -385,7 +385,7 @@ public void testPhaseWithoutDuration() throws Exception { Assert.assertEquals(configuration.getPhasesWithoutDuration().size(), 1); - long duration = configuration.getPhases().get(1).getDuration(); + long duration = GlobalArgs.parseDurationToSeconds(configuration.getPhases().get(1).getDuration()); Assert.assertEquals(duration, 5L); } @@ -396,10 +396,10 @@ public void testTwoPhasesWithoutDuration() throws Exception { Assert.assertEquals(configuration.getPhasesWithoutDuration().size(), 2); - long duration = configuration.getPhases().get(1).getDuration(); + long duration = GlobalArgs.parseDurationToSeconds(configuration.getPhases().get(1).getDuration()); Assert.assertEquals(duration, 5L); - duration = configuration.getPhases().get(2).getDuration(); + duration = GlobalArgs.parseDurationToSeconds(configuration.getPhases().get(2).getDuration()); Assert.assertEquals(duration, 5L); } diff --git a/toughday/src/test/java/com/adobe/qe/toughday/TestConstantLoadMode.java b/toughday/src/test/java/com/adobe/qe/toughday/TestConstantLoadMode.java index bf9ee5a..c792afa 100644 --- a/toughday/src/test/java/com/adobe/qe/toughday/TestConstantLoadMode.java +++ b/toughday/src/test/java/com/adobe/qe/toughday/TestConstantLoadMode.java @@ -137,7 +137,7 @@ public void testCtLoadStartEndRateInterval() throws Exception { Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getStart(), 10); Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getEnd(), 100); Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getRate(), 5); - Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getInterval(), 60); + Assert.assertEquals(((ConstantLoad)configuration.getRunMode()).getInterval(), "60s"); } @After diff --git a/toughday/src/test/java/com/adobe/qe/toughday/TestNormalMode.java b/toughday/src/test/java/com/adobe/qe/toughday/TestNormalMode.java index b78442a..91377ec 100644 --- a/toughday/src/test/java/com/adobe/qe/toughday/TestNormalMode.java +++ b/toughday/src/test/java/com/adobe/qe/toughday/TestNormalMode.java @@ -138,7 +138,7 @@ public void testNormalStartEndRateInterval() throws Exception { Assert.assertEquals(runMode.getStart(), 10); Assert.assertEquals(runMode.getEnd(), 100); Assert.assertEquals(runMode.getRate(), 5); - Assert.assertEquals(runMode.getInterval(), 60000); + Assert.assertEquals(runMode.getInterval(), "60s"); } @After diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/AbstractRunModeBalancerTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/AbstractRunModeBalancerTest.java new file mode 100644 index 0000000..68e330d --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/AbstractRunModeBalancerTest.java @@ -0,0 +1,59 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class AbstractRunModeBalancerTest { + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullRunModeThrowsException() { + RunMode dummyRunMode = new DummyRunMode(); + dummyRunMode.getRunModeBalancer().getRunModePropertiesToRedistribute(DummyRunMode.class, null); + } + + @Test + public void testAllPropertiesAreCollected() { + DummyRunMode dummyRunMode = new DummyRunMode(); + Map actual = + dummyRunMode.getRunModeBalancer().getRunModePropertiesToRedistribute(DummyRunMode.class, dummyRunMode); + + Map expected = new HashMap<>(); + expected.put("property1", "prop1"); + expected.put("property2", "prop2"); + + Assert.assertEquals(expected, actual); + } + + @Test + public void testNewValuesAreAssignedForAllRunModeProperties() { + DummyRunMode dummyRunMode = new DummyRunMode(); + RedistributionInstructions redistributionInstructions = new RedistributionInstructions(); + + Map runModeChanges = new HashMap<>(); + runModeChanges.put("property1", "prop1_new"); + runModeChanges.put("property2", "prop2_new"); + + redistributionInstructions.setRunModeProperties(runModeChanges); + dummyRunMode.getRunModeBalancer().processRunModeInstructions(redistributionInstructions, dummyRunMode); + + Assert.assertEquals("prop1_new", dummyRunMode.getProperty1()); + Assert.assertEquals("prop2_new", dummyRunMode.getProperty2()); + Assert.assertEquals("prop3", dummyRunMode.getProperty3()); + + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/ConstantLoadRunModeSplitterTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/ConstantLoadRunModeSplitterTest.java new file mode 100644 index 0000000..808398d --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/ConstantLoadRunModeSplitterTest.java @@ -0,0 +1,98 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.engine.runmodes.ConstantLoad; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; + +public class ConstantLoadRunModeSplitterTest { + private ArrayList cmdLineArgs = new ArrayList<>(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test + public void testLoadDistribution() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=constantload", "load=182")); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + Assert.assertEquals(62, ((ConstantLoad) runModes.get("Agent1")).getLoad()); + Assert.assertEquals(60, ((ConstantLoad) runModes.get("Agent2")).getLoad()); + Assert.assertEquals(60, ((ConstantLoad) runModes.get("Agent3")).getLoad()); + } + + @Test + public void testVariableLoadDistribution() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=constantload", "start=3", "end=12", "rate=3")); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + + ConstantLoad firstAgentRunMode = (ConstantLoad) runModes.get("Agent1"); + Assert.assertEquals(2, firstAgentRunMode.getStart()); + Assert.assertEquals(6, firstAgentRunMode.getEnd()); + Assert.assertEquals(2, firstAgentRunMode.getRate()); + + ConstantLoad secondAgentRunMode = (ConstantLoad) runModes.get("Agent2"); + Assert.assertEquals(1, secondAgentRunMode.getStart()); + Assert.assertEquals(6, secondAgentRunMode.getEnd()); + Assert.assertEquals(1, secondAgentRunMode.getRate()); + } + + @Test + public void testVariableLoadWithRateLowerThanTheNumberOfAgents() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=constantload", "start=10", "end=20", "rate=2", "interval=2s")); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + ConstantLoad initialRunMode = (ConstantLoad) configuration.getPhases().get(0).getRunMode(); + + runModes.forEach((key, value) -> { + ConstantLoad taskRunMode = (ConstantLoad) value; + Assert.assertEquals(initialRunMode.getEnd(), taskRunMode.getEnd()); + Assert.assertEquals(initialRunMode.getRate(), taskRunMode.getOneAgentRate()); + Assert.assertEquals(6, taskRunMode.getRate()); + Assert.assertEquals("6s", taskRunMode.getInterval()); + + }); + + /* the values for rate property are checked for the first complete cycle + (nr_agents * interval period of time) */ + ConstantLoad firstAgentRunMode = (ConstantLoad) runModes.get("Agent1"); + Assert.assertEquals(10, firstAgentRunMode.getStart()); + Assert.assertEquals(0, firstAgentRunMode.getInitialDelay()); + + ConstantLoad secondAgentRunMode = (ConstantLoad) runModes.get("Agent2"); + Assert.assertEquals(12, secondAgentRunMode.getStart()); + Assert.assertEquals(2000, secondAgentRunMode.getInitialDelay()); + + ConstantLoad thirdAgentRunMode = (ConstantLoad) runModes.get("Agent3"); + Assert.assertEquals(14, thirdAgentRunMode.getStart()); + Assert.assertEquals(4000, thirdAgentRunMode.getInitialDelay()); + + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitorTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitorTest.java new file mode 100644 index 0000000..d0507dd --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DistributedPhaseMonitorTest.java @@ -0,0 +1,119 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; + +public class DistributedPhaseMonitorTest { + + private List cmdLineArgs; + private final DistributedPhaseMonitor distributedPhaseMonitor = new DistributedPhaseMonitor(); + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + + reflections.getTestClasses().put("MockTest", MockTest.class); + } + + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test + public void testIsPhaseExecuting() throws Exception { + // test when phase is null + Assert.assertFalse(this.distributedPhaseMonitor.isPhaseExecuting()); + + // test when no agent is executing TD + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + this.distributedPhaseMonitor.setPhase(configuration.getPhases().get(0)); + + Assert.assertFalse(this.distributedPhaseMonitor.isPhaseExecuting()); + + // test when phase is executed by agents + this.distributedPhaseMonitor.registerAgentRunningTD("Agent1"); + Assert.assertTrue(this.distributedPhaseMonitor.isPhaseExecuting()); + + this.distributedPhaseMonitor.addAgentWhichCompletedTheCurrentPhase("Agent1"); + } + + @Test + public void testUpdateCountPerTest() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest", "count=400")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + this.distributedPhaseMonitor.setPhase(configuration.getPhases().get(0)); + + // simulate executions per test + Map> executionsPerTest = this.distributedPhaseMonitor.getExecutions(); + Map executionsPerAgent = new HashMap() {{ + put(mockAgents.get(0), 100L); + put(mockAgents.get(1), 100L); + }}; + + executionsPerTest.put("MockTest", executionsPerAgent); + Assert.assertEquals(200L, (long) this.distributedPhaseMonitor.getExecutionsPerTest().get("MockTest")); + } + + @Test + public void testResetExecutions() { + this.distributedPhaseMonitor.getExecutions().clear(); + Map executionsPerAgent = new HashMap() {{ + put("Agent1", 100L); + }}; + + this.distributedPhaseMonitor.getExecutions().put("MockTest", executionsPerAgent); + this.distributedPhaseMonitor.resetExecutions(); + + Assert.assertEquals(0L, + (long) this.distributedPhaseMonitor.getExecutionsPerTest().get("MockTest")); + + this.distributedPhaseMonitor.getExecutions().clear(); + } + + + @Test + public void testGetExecutionsPerTest() { + this.distributedPhaseMonitor.getExecutions().clear(); + + Map executionsPerAgent = new HashMap() {{ + put("Agent1", 100L); + put("Agent2", 60L); + put("Agent3", 34L); + }}; + + this.distributedPhaseMonitor.getExecutions().put("MockTest", executionsPerAgent); + Assert.assertEquals(194L, (long) this.distributedPhaseMonitor.getExecutionsPerTest().get("MockTest")); + } + + @Test + public void testRemoveAgentFromActiveTDRunners() { + this.distributedPhaseMonitor.getExecutions().clear(); + + this.distributedPhaseMonitor.registerAgentRunningTD("Agent1"); + this.distributedPhaseMonitor.registerAgentRunningTD("Agent2"); + + this.distributedPhaseMonitor.getExecutions() + .forEach((key, value) -> Assert.assertTrue(value.containsKey("Agent2"))); + + this.distributedPhaseMonitor.removeAgentFromActiveTDRunners("Agent2"); + + this.distributedPhaseMonitor.getExecutions() + .forEach((key, value) -> Assert.assertFalse(value.containsKey("Agent2"))); + } + + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunMode.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunMode.java new file mode 100644 index 0000000..4619cec --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunMode.java @@ -0,0 +1,77 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.api.annotations.ConfigArgGet; +import com.adobe.qe.toughday.api.annotations.ConfigArgSet; +import com.adobe.qe.toughday.internal.core.engine.Engine; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.RunModeBalancer; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.runmodes.RunModeSplitter; + +import java.util.concurrent.ExecutorService; + +public class DummyRunMode implements RunMode { + private RunModeBalancer dummyRunModeBalancer = new DummyRunModeBalancer(); + private String property1 = "prop1"; + private String property2 = "prop2"; + private String property3 = "prop3"; + + @ConfigArgSet(required = false, defaultValue = "p1") + public void setProperty1(String property1) { + this.property1 = property1; + } + + @ConfigArgSet(required = false, defaultValue = "p2") + public void setProperty2(String property2) { + this.property2 = property2; + } + + @ConfigArgSet(required = false, defaultValue = "p3") + public void setProperty3(String property3) { + this.property3 = property3; + } + + @ConfigArgGet(redistribute = true) + public String getProperty1() { + return this.property1; + } + + @ConfigArgGet(redistribute = true) + public String getProperty2() { + return this.property2; + } + + @ConfigArgGet + public String getProperty3() { + return this.property3; + } + + @Override + public void runTests(Engine engine) throws Exception { + + } + + @Override + public void finishExecutionAndAwait() { + + } + + @Override + public ExecutorService getExecutorService() { + return null; + } + + @Override + public RunContext getRunContext() { + return null; + } + + @Override + public RunModeSplitter getRunModeSplitter() { + return null; + } + + @Override + public RunModeBalancer getRunModeBalancer() { + return this.dummyRunModeBalancer; + } +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunModeBalancer.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunModeBalancer.java new file mode 100644 index 0000000..c744a07 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/DummyRunModeBalancer.java @@ -0,0 +1,16 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.runmodes.AbstractRunModeBalancer; + +public class DummyRunModeBalancer extends AbstractRunModeBalancer { + @Override + public void before(RedistributionInstructions redistributionInstructions, DummyRunMode runMode) { + + } + + @Override + public void after(RedistributionInstructions redistributionInstructions, DummyRunMode runMode) { + + } +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/MasterElectionTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/MasterElectionTest.java new file mode 100644 index 0000000..ff12249 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/MasterElectionTest.java @@ -0,0 +1,122 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.MasterElection; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.*; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import java.util.LinkedList; +import java.util.Queue; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class MasterElectionTest { + private MasterElection masterElection = MasterElection.getInstance(3); + + @Mock Driver mockDriver; + @Mock DriverState driverStateMock; + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + + } + + @Before + public void setup() { + this.masterElection.resetInvalidCandidates(); + } + + @Test + public void testAllCandidatesAreConsideredValidAfterTheInstanceIsCreated() { + Assert.assertTrue( MasterElection.getInstance(3).getInvalidCandidates().isEmpty()); + } + + @Test + public void testMarkCandidateAsInvalid() { + this.masterElection.markCandidateAsInvalid(0); + Queue expectedInvalidCandidates = new LinkedList() {{ + add(0); + }}; + Assert.assertEquals(expectedInvalidCandidates, masterElection.getInvalidCandidates()); + } + + @Test + public void testIsCandidateInvalid() { + this.masterElection.markCandidateAsInvalid(0); + + Assert.assertTrue(this.masterElection.isCandidateInvalid(0)); + Assert.assertFalse(this.masterElection.isCandidateInvalid(1)); + Assert.assertFalse(this.masterElection.isCandidateInvalid(2)); + } + + @Test + public void testResetInvalidCandidates() { + this.masterElection.markCandidateAsInvalid(0); + Assert.assertEquals(1, this.masterElection.getInvalidCandidates().size()); + + this.masterElection.resetInvalidCandidates(); + Assert.assertTrue(this.masterElection.getInvalidCandidates().isEmpty()); + } + + @Test + public void checkListOfInvalidCandidatesIsClearedWhenNoOptionAvailable() { + int currentDriverId = 2; + + Mockito.when(mockDriver.getDriverState()).thenReturn(driverStateMock); + Mockito.when(driverStateMock.getId()).thenReturn(currentDriverId); + Mockito.when(driverStateMock.getMasterIdLock()).thenReturn(new ReentrantReadWriteLock()); + + masterElection.markCandidateAsInvalid(0); + masterElection.markCandidateAsInvalid(1); + masterElection.markCandidateAsInvalid(2); + + // before starting the election process all candidates should be considered invalid + Assert.assertEquals(3, masterElection.getInvalidCandidates().size()); + + masterElection.electMaster(mockDriver); + + // after the election process, all candidates should be considered eligible to become the new master + Assert.assertTrue(masterElection.getInvalidCandidates().isEmpty()); + } + + @Test + public void testElectMasterFromCandidatePerspective() { + int expectedMaster = 1; + int currentDriverId = 2; + + Mockito.when(mockDriver.getDriverState()).thenReturn(driverStateMock); + Mockito.when(driverStateMock.getId()).thenReturn(currentDriverId); + Mockito.when(driverStateMock.getMasterIdLock()).thenReturn(new ReentrantReadWriteLock()); + + masterElection.markCandidateAsInvalid(0); + masterElection.electMaster(mockDriver); + + // check that the list of invalid candidates was not modified + Queue expectedInvalidCandidates = new LinkedList() {{ + add(0); + }}; + Assert.assertEquals(expectedInvalidCandidates, masterElection.getInvalidCandidates()); + + // check that current master (with id 2) is running as candidate + Mockito.verify(driverStateMock, Mockito.times(1)) + .setCurrentState(DriverState.Role.CANDIDATE); + + // check that driver-1 was elected as master + Mockito.verify(driverStateMock, Mockito.times(1)).setMasterId(expectedMaster); + + // check that the master election heartbeat task was scheduled + Mockito.verify(mockDriver, Mockito.times(1)).scheduleMasterHeartbeatTask(); + } + + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/NormalRunModeSplitterTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/NormalRunModeSplitterTest.java new file mode 100644 index 0000000..e723bab --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/NormalRunModeSplitterTest.java @@ -0,0 +1,111 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.RunMode; +import com.adobe.qe.toughday.internal.core.engine.runmodes.Normal; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; + +public class NormalRunModeSplitterTest { + private ArrayList cmdLineArgs = new ArrayList<>(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test + public void testConcurrencyDistribution() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=normal", "concurrency=320")); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + + Assert.assertEquals(108, ((Normal) runModes.get("Agent1")).getConcurrency()); + Assert.assertEquals(106, ((Normal) runModes.get("Agent2")).getConcurrency()); + Assert.assertEquals(106, ((Normal) runModes.get("Agent3")).getConcurrency()); + } + + @Test + public void testVariableConcurrencyDistribution() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=normal", "start=3", "end=12", "rate=3")); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode runMode = configuration.getRunMode(); + + Map runModes = runMode.getRunModeSplitter().distributeRunMode(runMode, mockAgents); + Normal initialRunMode = (Normal) configuration.getRunMode(); + + Normal firstAgentRunMode = (Normal) runModes.get("Agent1"); + Assert.assertEquals(2, firstAgentRunMode.getStart()); + Assert.assertEquals(6, firstAgentRunMode.getEnd()); + Assert.assertEquals(2, firstAgentRunMode.getRate()); + // check that the interval property does not change + Assert.assertEquals(initialRunMode.getInterval(), firstAgentRunMode.getInterval()); + + Normal secondAgentRunMode = (Normal) runModes.get("Agent2"); + Assert.assertEquals(1, secondAgentRunMode.getStart()); + Assert.assertEquals(6, secondAgentRunMode.getEnd()); + Assert.assertEquals(1, secondAgentRunMode.getRate()); + // check that the interval property does not change + Assert.assertEquals(initialRunMode.getInterval(), secondAgentRunMode.getInterval()); + } + + + @Test + public void testVariableConcurrencyWithRateLowerThanNumberOfAgents() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=normal", "start=3", "end=9", "interval=1s", "rate=1")); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode initialRunMode = configuration.getRunMode(); + Map runModes = initialRunMode.getRunModeSplitter().distributeRunMode(initialRunMode, mockAgents); + + Normal firstAgentRunMode = (Normal) runModes.get("Agent1"); + Assert.assertEquals(3, firstAgentRunMode.getStart()); + Assert.assertEquals(6, firstAgentRunMode.getEnd()); + Assert.assertEquals("2s", firstAgentRunMode.getInterval()); + Assert.assertEquals(0, firstAgentRunMode.getInitialDelay()); + + Normal secondAgentRunMode = (Normal) runModes.get("Agent2"); + Assert.assertEquals(0, secondAgentRunMode.getStart()); + Assert.assertEquals(3, secondAgentRunMode.getEnd()); + Assert.assertEquals("2s", secondAgentRunMode.getInterval()); + Assert.assertEquals(1000, secondAgentRunMode.getInitialDelay()); + + // check that the initial rate is not modified + Assert.assertEquals(((Normal)initialRunMode).getRate(), firstAgentRunMode.getRate()); + Assert.assertEquals(((Normal)initialRunMode).getRate(), secondAgentRunMode.getRate()); + } + + @Test + public void testWaitTimeIsNotModified() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=normal", "waittime=1000")); + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + RunMode initialRunMode = configuration.getPhases().get(0).getRunMode(); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + + Map runModes = initialRunMode.getRunModeSplitter().distributeRunMode(initialRunMode, mockAgents); + runModes.forEach((key, value) -> + Assert.assertEquals(((Normal)initialRunMode).getWaitTime(), + ((Normal) value).getWaitTime())); + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/PhaseSplitterTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/PhaseSplitterTest.java new file mode 100644 index 0000000..0331a51 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/PhaseSplitterTest.java @@ -0,0 +1,103 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.api.core.AbstractTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.distributedtd.splitters.PhaseSplitter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.*; + +import java.util.*; +import java.util.stream.Collectors; + +public class PhaseSplitterTest { + private final PhaseSplitter phaseSplitter = new PhaseSplitter(); + private List cmdLineArgs; + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + + reflections.getTestClasses().put("MockTest", MockTest.class); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullPhaseThrowsException() throws CloneNotSupportedException { + phaseSplitter.splitPhase(null, new ArrayList<>()); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullListOfAgentsThrowsException() throws CloneNotSupportedException { + phaseSplitter.splitPhase(new Phase(), null); + } + + @Test(expected = IllegalStateException.class) + public void testNoAvailableAgentsThrowsException() throws CloneNotSupportedException { + phaseSplitter.splitPhase(new Phase(), new ArrayList<>()); + } + + @Test + public void testNumberOfTasksIsEqualToTheNumberOfAgents() throws Exception { + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + configuration.getPhases().forEach(phase -> { + try { + Map taskMap = phaseSplitter.splitPhase(phase, mockAgents); + Assert.assertEquals(mockAgents.size(), taskMap.keySet().size()); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + } + }); + } + + @Test + public void testCountIsDistributedForTestsInTestSuite() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest", "name=Test1", "count=201")); + List mockAgents = Arrays.asList("Agent1", "Agent2"); + + Phase phase = new Configuration(cmdLineArgs.toArray(new String[0])).getPhases().get(0); + Map taskMap = phaseSplitter.splitPhase(phase, mockAgents); + + taskMap.get("Agent1").getTestSuite().getTests().forEach(test -> { + Assert.assertEquals(101, test.getCount()); + }); + + taskMap.get("Agent2").getTestSuite().getTests().forEach(test -> { + Assert.assertEquals(100, test.getCount()); + }); + } + + @Test + public void testEachAgentRunsTheCompleteTestSuite() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest", "name=Test1", "--add", "MockTest", "name=Test2")); + List mockAgents = Arrays.asList("Agent1", "Agent2", "Agent3"); + + Phase phase = new Configuration(cmdLineArgs.toArray(new String[0])).getPhases().get(0); + Set testNames = phase.getTestSuite().getTests().stream() + .map(AbstractTest::getName) + .collect(Collectors.toSet()); + + Map taskMap = phaseSplitter.splitPhase(phase, mockAgents); + taskMap.forEach((key, value) -> { + Set namesDiff = value.getTestSuite().getTests().stream() + .map(AbstractTest::getName) + .collect(Collectors.toSet()); + namesDiff.removeAll(testNames); + + Assert.assertEquals(0, namesDiff.size()); + }); + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/RedistributionInstructionsProcessorTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/RedistributionInstructionsProcessorTest.java new file mode 100644 index 0000000..4e7f97b --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/RedistributionInstructionsProcessorTest.java @@ -0,0 +1,86 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructions; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.RedistributionInstructionsProcessor; +import com.adobe.qe.toughday.internal.core.engine.Phase; +import com.adobe.qe.toughday.internal.core.engine.runmodes.Normal; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.*; + +public class RedistributionInstructionsProcessorTest { + private List cmdLineArgs; + private final RedistributionInstructionsProcessor instructionsProcessor = new RedistributionInstructionsProcessor(); + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + + reflections.getTestClasses().put("MockTest", MockTest.class); + } + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullPhaseThrowsExceptions() throws IOException { + this.instructionsProcessor.processInstructions("", null); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullInstructionsThrowsException() throws Exception{ + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + this.instructionsProcessor.processInstructions(null, configuration.getPhases().get(0)); + } + + @Test + public void testProcessInstructions() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest", "--runmode", "type=normal", "rate=20", "interval=2s", + "end=20", "start=15")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + Phase phase = configuration.getPhases().get(0); + RedistributionInstructions instructions = new RedistributionInstructions(); + + Map runModeProperties = new HashMap() {{ + put("rate", "10"); + put("interval","5s"); + put("end", "200"); + }}; + Map counts = new HashMap() {{ + put("MockTest", 1200L); + }}; + + instructions.setRunModeProperties(runModeProperties); + instructions.setCounts(counts); + + ObjectMapper mapper = new ObjectMapper(); + String jsonInstructions = mapper.writeValueAsString(instructions); + + this.instructionsProcessor.processInstructions(jsonInstructions, phase); + + // test run mode properties were changed + Normal runmode = (Normal)phase.getRunMode(); + Assert.assertEquals(10, runmode.getRate()); + Assert.assertEquals("5s", runmode.getInterval()); + Assert.assertEquals(200, runmode.getEnd()); + + // test count properties were changed + Assert.assertEquals(1200, phase.getTestSuite().getTest("MockTest").getCount()); + + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgentsTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgentsTest.java new file mode 100644 index 0000000..fe46b05 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/YamlDumpConfigurationAsTaskForTDAgentsTest.java @@ -0,0 +1,152 @@ +package com.adobe.qe.toughday.internal.core.distributedtd; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.distributedtd.DummyRunMode; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.mocks.MockMetric; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.Assert; +import org.junit.rules.ExpectedException; + +import java.util.*; + +public class YamlDumpConfigurationAsTaskForTDAgentsTest { + private List cmdLineArgs; + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + + reflections.getTestClasses().put("MockTest", MockTest.class); + reflections.getMetricClasses().put("MockMetric", MockMetric.class); + reflections.getRunModeClasses().put("DummyRunMode", DummyRunMode.class); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + @Before + public void before() { + cmdLineArgs = new ArrayList<>(Collections.singletonList("--host=localhost")); + } + + @Test + public void testNullCofigurationThrowsException() { + expectedEx.expect(IllegalArgumentException.class); + expectedEx.expectMessage("Configuration must not be null."); + + new YamlDumpConfigurationAsTaskForTDAgents(null); + } + + @Test() + public void testCollectConfigurablePropertiesWithIncompatibleTypeThrowsException() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--add", "MockTest")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + expectedEx.expect(IllegalArgumentException.class); + expectedEx.expectMessage("The object must have the specified type."); + + yamlDumpConfiguration.collectConfigurableProperties(DummyRunMode.class, configuration.getTestSuite().getTest("MockTest")); + } + + @Test + public void testCollectConfigurablePropertiesContainsOnlyModifiedProperties() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--runmode", "type=dummyrunmode", "property1=value1", "property2=value2")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + + Map actualCollectedPropsForDummyRunMode = + yamlDumpConfiguration.collectConfigurableProperties(DummyRunMode.class, configuration.getRunMode()); + + Map expectedCollectedPropsForMockTest = + new HashMap() {{ + put("property1", "value1"); + put("property2", "value2"); + }}; + + Assert.assertEquals(expectedCollectedPropsForMockTest, actualCollectedPropsForDummyRunMode); + } + + @Test + public void testDumpedConfiguration() throws Exception{ + cmdLineArgs.addAll(Arrays.asList("--duration=10s", "--phase", "name=phase123", "--add", "MockTest", "name=Test1", "count=201", + "--add" , "MockMetric", "name=Metric123", "--runmode", "type=dummyrunmode", "--add", "CSVPublisher")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + String expectedYaml = + "globals:\n" + + " duration: 10s\n" + + " host: localhost\n" + + "phases:\n" + + "- duration: 10s\n" + + " metrics:\n" + + " - add: MockMetric\n" + + " properties:\n" + + " name: Metric123\n" + + " name: phase123\n" + + " publishers:\n" + + " - add: CSVPublisher\n" + + " properties:\n" + + " name: CSVPublisher\n" + + " aggregatedpublish: false\n" + + " publishmode:\n" + + " type: simple\n" + + " runmode:\n" + + " type: dummyrunmode\n" + + " tests:\n" + + " - add: MockTest\n" + + " properties:\n" + + " name: Test1\n" + + " count: 201\n"; + + Assert.assertEquals(expectedYaml, yamlDumpConfiguration.generateConfigurationObject()); + } + + @Test + public void testGetGlobals() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--duration=10s", "--port=1000", "--user=testUser", "--timeout=10")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + Map actualGlobals = yamlDumpConfiguration.getGlobals(); + + Map expectedGlobals = new HashMap() {{ + put("host", "localhost"); + put("duration", "10s"); + put("port", 1000); + put("user", "testUser"); + put("timeout", 10L); + }}; + + Assert.assertEquals(expectedGlobals, actualGlobals); + } + + @Test + public void testGetDistributedConfig() throws Exception { + cmdLineArgs.addAll(Arrays.asList("--distributedconfig", "driverip=10.0.0.1", "heartbeatinterval=10s")); + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + + Map actualDistributedConfig = yamlDumpConfiguration.getDistributedConfig(); + Map expectedDistributedConfig = new HashMap() {{ + put("driverip", "10.0.0.1"); + put("heartbeatinterval", "10s"); + }}; + + Assert.assertEquals(expectedDistributedConfig, actualDistributedConfig); + } + + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/CandidateRequestProcessorTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/CandidateRequestProcessorTest.java new file mode 100644 index 0000000..9512944 --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/CandidateRequestProcessorTest.java @@ -0,0 +1,146 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.requests; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.config.Configuration; +import com.adobe.qe.toughday.internal.core.config.PhaseParams; +import com.adobe.qe.toughday.internal.core.distributedtd.DistributedPhaseMonitor; +import com.adobe.qe.toughday.internal.core.distributedtd.YamlDumpConfigurationAsTaskForTDAgents; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverUpdateInfo; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.MasterElection; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.RequestProcessor; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.RequestProcessorDispatcher; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.*; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.mockito.stubbing.Answer; +import spark.Request; + +import java.util.*; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class CandidateRequestProcessorTest { + @Mock Request request; + @Mock Driver mockDriver; + @Mock DriverState driverStateMock; + private DistributedPhaseMonitor distributedPhaseMonitor= new DistributedPhaseMonitor(); + private final MasterElection masterElection = MasterElection.getInstance(3); + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + + reflections.getTestClasses().put("MockTest", MockTest.class); + } + + @Before + public void setup() { + Mockito.when(mockDriver.getDriverState()).thenReturn(driverStateMock); + Mockito.when(driverStateMock.getCurrentState()).thenReturn(DriverState.Role.CANDIDATE); + Mockito.when(mockDriver.getMasterElection()).thenReturn(this.masterElection); + Mockito.when(driverStateMock.getId()).thenReturn(1); + + masterElection.resetInvalidCandidates(); + } + + @Test + public void testProcessUpdatesRequests() throws JsonProcessingException { + Queue registeredAgents = new LinkedList<>(Collections.singleton("10.0.0.1")); + + masterElection.markCandidateAsInvalid(0); + masterElection.markCandidateAsInvalid(1); + Mockito.when(driverStateMock.getRegisteredAgents()).thenReturn(registeredAgents); + + DriverUpdateInfo updates = new DriverUpdateInfo(1, DriverState.Role.CANDIDATE, + masterElection.getInvalidCandidates(), registeredAgents, "", ""); + ObjectMapper objectMapper = new ObjectMapper(); + String expectedYaml = objectMapper.writeValueAsString(updates); + + RequestProcessor processor = RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver); + + Assert.assertEquals(expectedYaml, processor.processUpdatesRequest(request, mockDriver)); + } + + @Test + public void testProcessMasterElectionRequest() { + Mockito.when(request.body()).then((Answer) answer -> "1"); + Mockito.when(driverStateMock.getMasterIdLock()).thenReturn(new ReentrantReadWriteLock()); + RequestProcessor processor = RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver); + + masterElection.markCandidateAsInvalid(0); + processor.processMasterElectionRequest(request, mockDriver); + + // test that driver-1 was added as invalid candidate + Queue expectedInvalidCandidates = new LinkedList<>(Arrays.asList(0, 1)); + Assert.assertEquals(expectedInvalidCandidates, masterElection.getInvalidCandidates()); + + // test that driver-2 was elected as master + Mockito.verify(driverStateMock).setMasterId(2); + + // test that driver-1 is running as a candidate after this process + Mockito.verify(driverStateMock).setCurrentState(DriverState.Role.CANDIDATE); + } + + @Test + public void testProcessPhaseCompletionAnnouncement() { + Mockito.when(request.body()).thenReturn("10.0.0.1"); + Mockito.when(request.queryParams("forward")).thenReturn("false"); + Mockito.when(mockDriver.getDistributedPhaseMonitor()).thenReturn(distributedPhaseMonitor); + + RequestProcessor processor = RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver); + processor.processPhaseCompletionAnnouncement(request); + + // test that the agent was added to the list of agents which finished executing the current phase + Assert.assertEquals(Collections.singletonList("10.0.0.1"), mockDriver.getDistributedPhaseMonitor().getAgentsWhichCompletedCurrentPhase()); + } + + @Test + public void processExecutionRequest() throws Exception { + List cmdLineArgs = new ArrayList() {{ + add("--phase"); + add("name=testPhase"); + add("--host=localhost"); + add("--add"); + add("MockTest"); + add("count=400"); + }}; + + Configuration configuration = new Configuration(cmdLineArgs.toArray(new String[0])); + PhaseParams.namedPhases.clear(); + YamlDumpConfigurationAsTaskForTDAgents yamlDumpConfiguration + = new YamlDumpConfigurationAsTaskForTDAgents(configuration); + String yamlConfig = yamlDumpConfiguration.generateConfigurationObject(); + + Mockito.when(request.body()).thenReturn(yamlConfig); + Mockito.when(request.queryParams("forward")).thenReturn("false"); + Mockito.when(mockDriver.getDistributedPhaseMonitor()).thenReturn(distributedPhaseMonitor); + Mockito.when(mockDriver.getConfiguration()).thenReturn(configuration); + + RequestProcessor processor = RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver); + processor.processExecutionRequest(request, mockDriver); + + // test that the configuration was set + Mockito.verify(mockDriver, Mockito.times(1)) + .setConfiguration(Mockito.any(Configuration.class)); + + // test that the first phase of the configuration to be executed is being monitored + Assert.assertEquals("testPhase", mockDriver.getDistributedPhaseMonitor().getPhase().getName()); + + // test that the sample content package should not be installed by the candidates + Assert.assertFalse(mockDriver.getConfiguration().getGlobalArgs().getInstallSampleContent()); + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/MasterRequestProcessorTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/MasterRequestProcessorTest.java new file mode 100644 index 0000000..952b90f --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/MasterRequestProcessorTest.java @@ -0,0 +1,135 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.requests; + +import com.adobe.qe.toughday.MockTest; +import com.adobe.qe.toughday.internal.core.ReflectionsContainer; +import com.adobe.qe.toughday.internal.core.distributedtd.DistributedPhaseMonitor; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.RequestProcessor; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.RequestProcessorDispatcher; +import com.adobe.qe.toughday.internal.core.distributedtd.redistribution.TaskBalancer; +import com.google.gson.Gson; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.*; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import spark.Request; +import spark.Response; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; + + +public class MasterRequestProcessorTest { + @Mock Request requestMock; + @Mock Response responseMock; + @Mock Driver mockDriver; + @Mock DistributedPhaseMonitor distributedPhaseMonitorMock; + @Mock DriverState driverStateMock; + @Mock TaskBalancer taskBalancerMock; + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + private static ReflectionsContainer reflections = ReflectionsContainer.getInstance(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + + reflections.getTestClasses().put("MockTest", MockTest.class); + } + + @Before + public void setup() { + Mockito.when(mockDriver.getDriverState()).thenReturn(driverStateMock); + Mockito.when(driverStateMock.getCurrentState()).thenReturn(DriverState.Role.MASTER); + } + + @Test + public void testSampleContentInstallationFailureForcesExecutionToStop() { + Mockito.when(requestMock.body()).thenReturn("false"); + RequestProcessor requestProcessor = RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver); + + requestProcessor.acknowledgeSampleContentSuccessfulInstallation(requestMock, mockDriver, responseMock); + + /* check that the driver finishes the distributed execution if TD sample content package was not successfully + /* installed + */ + Mockito.verify(mockDriver).finishDistributedExecution(); + } + + @Test + public void testProcessHeartbeatRequest() { + Map> executions = new HashMap<>(); + Map testsPerAgent = new HashMap() {{ + put("Agent1", 200L); + put("Agent2", 231L); + }}; + executions.put("MockTest", testsPerAgent); + + Mockito.when(mockDriver.getDistributedPhaseMonitor()).thenReturn(distributedPhaseMonitorMock); + Mockito.when(distributedPhaseMonitorMock.getExecutions()).thenReturn(executions); + + Gson gson = new Gson(); + String expected = gson.toJson(executions); + String actual = RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver) + .processHeartbeatRequest(requestMock, mockDriver); + + Assert.assertEquals(expected, actual); + } + + @Test(expected = IllegalStateException.class) + public void testProcessAgentFailureAnnouncementThrowsExceptionForMaster() { + /* the master is the one sending heartbeat messages to the agents running in the cluster and should therefore + * never receive this type of request from another driver. + */ + RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver) + .processAgentFailureAnnouncement(requestMock, mockDriver); + } + + private void mockitoSetupForRegisterRequest(String newAgentIP) { + Mockito.when(requestMock.body()).thenReturn(newAgentIP); + Mockito.when(requestMock.queryParams("forward")).thenReturn("false"); + Mockito.when(mockDriver.getDistributedPhaseMonitor()).thenReturn(distributedPhaseMonitorMock); + Mockito.when(mockDriver.getTaskBalancer()).thenReturn(taskBalancerMock); + } + + @Test + public void testProcessRegisterRequestWhenExecutionDidNotStarted() { + String newAgentIP = "10.0.0.1"; + + mockitoSetupForRegisterRequest(newAgentIP); + Mockito.when(distributedPhaseMonitorMock.isPhaseExecuting()).thenReturn(false); + Mockito.when(driverStateMock.getRegisteredAgents()).thenReturn(new LinkedList<>()); + + RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver) + .processRegisterRequest(requestMock, mockDriver); + + // check that the new agent was added to the list of active agents + Mockito.verify(driverStateMock, Mockito.times(1)).registerAgent(newAgentIP); + + // check that the work redistribution process is not scheduled when the execution did not started + Mockito.verify(taskBalancerMock, Mockito.times(0)). + scheduleWorkRedistributionProcess(mockDriver, newAgentIP, true); + } + + @Test + public void testRegisterRequestTriggersWorkRedistributionWhenTDIsExecuted() { + String newAgentIP = "10.0.0.1"; + + mockitoSetupForRegisterRequest(newAgentIP); + Mockito.when(distributedPhaseMonitorMock.isPhaseExecuting()).thenReturn(true); + + RequestProcessorDispatcher.getInstance().getRequestProcessor(mockDriver) + .processRegisterRequest(requestMock, mockDriver); + Mockito.verify(taskBalancerMock, Mockito.times(1)) + .scheduleWorkRedistributionProcess(mockDriver, newAgentIP, true); + + } +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/RequestProcessorDispatcherTest.java b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/RequestProcessorDispatcherTest.java new file mode 100644 index 0000000..885ce9e --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/internal/core/distributedtd/requests/RequestProcessorDispatcherTest.java @@ -0,0 +1,56 @@ +package com.adobe.qe.toughday.internal.core.distributedtd.requests; + +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.Driver; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.DriverState; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.MasterRequestProcessor; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.RequestProcessorDispatcher; +import com.adobe.qe.toughday.internal.core.distributedtd.cluster.driver.requests.CandidateRequestProcessor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.junit.*; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.mockito.stubbing.Answer; + +public class RequestProcessorDispatcherTest { + @Mock Driver mockDriver; + @Mock DriverState driverStateMock; + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @BeforeClass + public static void onlyOnce() { + System.setProperty("logFileName", "."); + ((LoggerContext) LogManager.getContext(false)).reconfigure(); + } + + + @Test + public void testGetInstance() { + RequestProcessorDispatcher requestProcessorDispatcher = RequestProcessorDispatcher.getInstance(); + + Assert.assertNotNull(requestProcessorDispatcher); + } + + @Test + public void testMasterRequestProcessorIsReturnedForMaster() { + Mockito.when(mockDriver.getDriverState()).then((Answer) invocationOnMocl -> driverStateMock); + Mockito.when(driverStateMock.getCurrentState()).then((Answer) invocationOnMock -> DriverState.Role.MASTER); + + RequestProcessorDispatcher requestProcessorDispatcher = RequestProcessorDispatcher.getInstance(); + Assert.assertTrue(requestProcessorDispatcher.getRequestProcessor(mockDriver) instanceof MasterRequestProcessor); + } + + @Test + public void testCandidateRequestProcessorIsReturnedForCandidate() { + Mockito.when(mockDriver.getDriverState()).then((Answer) invocationOnMocl -> driverStateMock); + Mockito.when(driverStateMock.getCurrentState()).then((Answer) invocationOnMock -> DriverState.Role.CANDIDATE); + + RequestProcessorDispatcher requestProcessorDispatcher = RequestProcessorDispatcher.getInstance(); + Assert.assertTrue(requestProcessorDispatcher.getRequestProcessor(mockDriver) instanceof CandidateRequestProcessor); + } + +} diff --git a/toughday/src/test/java/com/adobe/qe/toughday/mocks/MockMetric.java b/toughday/src/test/java/com/adobe/qe/toughday/mocks/MockMetric.java new file mode 100644 index 0000000..8b5f9ae --- /dev/null +++ b/toughday/src/test/java/com/adobe/qe/toughday/mocks/MockMetric.java @@ -0,0 +1,21 @@ +package com.adobe.qe.toughday.mocks; + +import com.adobe.qe.toughday.api.core.RunMap; +import com.adobe.qe.toughday.metrics.Metric; + +public class MockMetric extends Metric { + @Override + public Object getValue(RunMap.TestStatistics testStatistics) { + return null; + } + + @Override + public String getFormat() { + return null; + } + + @Override + public String getUnitOfMeasure() { + return null; + } +} diff --git a/toughday2-api/src/main/java/com/adobe/qe/toughday/api/annotations/ConfigArgGet.java b/toughday2-api/src/main/java/com/adobe/qe/toughday/api/annotations/ConfigArgGet.java index 45b22ec..f379cd8 100644 --- a/toughday2-api/src/main/java/com/adobe/qe/toughday/api/annotations/ConfigArgGet.java +++ b/toughday2-api/src/main/java/com/adobe/qe/toughday/api/annotations/ConfigArgGet.java @@ -18,7 +18,7 @@ /** * Use this annotation on a getter to expose it as a configuration property. - * These properties will be automatically picked up, shown in logging or in "dry" runmode + * These properties will be automatically picked up, shown in logging or in "dry" runmodes * Supported classes: subtypes of AbstractTest, subtypes of Publisher and * GlobalArgs. */ @@ -26,4 +26,14 @@ @Target(value = ElementType.METHOD) public @interface ConfigArgGet { String name() default ""; + + /** + * Use this field when you want to specify that a certain property must be taken into + * consideration when redistributing the work between the agents running in the cluster. + * The property will be automatically collected by the AbstractRunModeBalancer. + * Currently, this annotations is supported only for classes implementing the RunMode + * interface. + * @return + */ + boolean redistribute() default false; }