diff --git a/__pycache__/glados_cli.cpython-311.pyc b/__pycache__/glados_cli.cpython-311.pyc index 13599f1..c854a49 100644 Binary files a/__pycache__/glados_cli.cpython-311.pyc and b/__pycache__/glados_cli.cpython-311.pyc differ diff --git a/empty-experiment.zip b/empty-experiment.zip deleted file mode 100644 index 0a1fe9c..0000000 Binary files a/empty-experiment.zip and /dev/null differ diff --git a/invalid-experiment.zip b/invalid-experiment.zip deleted file mode 100644 index 5298558..0000000 --- a/invalid-experiment.zip +++ /dev/null @@ -1 +0,0 @@ -Hehehe yup \ No newline at end of file diff --git a/manifest.yml b/manifest.yml index 706751a..3c6ad06 100644 --- a/manifest.yml +++ b/manifest.yml @@ -44,13 +44,13 @@ hyperparameters: useDefault: false # The experiment name. # Example: "Evolutionary Aglorithm Experiment" -name: "Good3" +name: "Test AddNums" # This is the description of the experiment. # Example: "Experiment for CSSE490 Homework 4." -description: "Hi" +description: "This is to test AddNums!" # These are the tags which can be defined for an experiment, allowing for better filtering on the web app. # Example: ["Neural Network", "ECE497"] -tags: ["Neural Network", "ECE497"] +tags: ["Test", "AddNums"] # A folder or file included in the downloadable Project Zip after experiment completion. # Example: "test.csv" trialExtraFile: "AddNumResult.csv" @@ -83,7 +83,7 @@ sendEmail: false workers: 1 # If using a zip experiment, specify the main executable filename. # Example: "test.py" if using zip, "" if not -experimentExecutable: "test.py" +experimentExecutable: "addNumber.py" # This attribute, set to either ini or yaml, defines the format of the config file generated for each trial. # Example: "ini" or "yaml" configFileFormat: "ini" diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..3fea2f7 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,8 @@ +# GLADOS CLI Test Suite + +This directory has unit and integration tests for the GLADOS CLI. + +## Directories + +- The unit directory has an automated test suite that deals with the business logic of the CLI, with the data subdirectory holding files that allow the tests to run properly. The test file can be run with the command `python tests/unit/glados_cli_tests.py` from the project root. +- The integration directory tests the flow of API calls to the NextJS endpoints of GLADOS. It has a partially automated test suite; it requires a tester to first authenticate via the CLI and then ensure they have no experiments named Test AddNums. The test file can be run with the command `python tests/integration/glados_workflow_tests.py` from the project root. \ No newline at end of file diff --git a/tests/integration/data/addNumbers.py b/tests/integration/data/addNumbers.py new file mode 100644 index 0000000..a6a269e --- /dev/null +++ b/tests/integration/data/addNumbers.py @@ -0,0 +1,53 @@ +import csv +import sys +import configparser + +# pylint: disable-next=pointless-string-statement +""" +This experiment demonstrates outputting additional information to a file +and telling the system to gather the data from that file. + +There are two different ways to gather additional information from an experiment + - Trial's Extra File: Gathers the designated file that a run of the file generates and places in a zip to be uploaded when + the experiment completes + - Trial Result: Integrates the information from a 2 line csv of headers and values that the file run generates + and adds them to the result csv that is being uploaded + +How to tell if they worked + - Trial's Extra File: You can download a zip file that contains the different output files specified + - Trial Result: The result csv downloaded has been expanded with information from the specified file + + +Example settings for a run that demonstrates this: (Any Fields not specified can be left blank or to whatever their default is) + +Info: +Trial Result: AddNumResult.csv +If you want a collection of each CSV this experiment runs: + Trial's Extra File: AddNumResult.csv +Both can be used at the same time + +Parameters: +x, 1, 1, 10, 1 +y, 1, 1, 10, 1 +""" + +# pylint: disable=glados-print-used + +def main(): + config = configparser.ConfigParser() + args = sys.argv[1:] + configFile = args[0] + config.read(configFile) + x = int(config["DEFAULT"]["x"]) + y = int(config["DEFAULT"]["y"]) + with open('AddNumResult.csv', 'w', encoding="utf8") as result: + writer = csv.writer(result) + writer.writerow(['Addition', 'Subtraction']) + writer.writerow([x + y, x - y]) + + print("done") + return 0 + + +if __name__ == "__main__": + main() diff --git a/tests/integration/data/addNumbersExpected.csv b/tests/integration/data/addNumbersExpected.csv new file mode 100644 index 0000000..ea0964d --- /dev/null +++ b/tests/integration/data/addNumbersExpected.csv @@ -0,0 +1,101 @@ +Experiment Run,Addition,Subtraction,x,y +0,2,0,1,1 +1,3,-1,1,2 +2,4,-2,1,3 +3,5,-3,1,4 +4,6,-4,1,5 +5,7,-5,1,6 +6,8,-6,1,7 +7,9,-7,1,8 +8,10,-8,1,9 +9,11,-9,1,10 +10,3,1,2,1 +11,4,0,2,2 +12,5,-1,2,3 +13,6,-2,2,4 +14,7,-3,2,5 +15,8,-4,2,6 +16,9,-5,2,7 +17,10,-6,2,8 +18,11,-7,2,9 +19,12,-8,2,10 +20,4,2,3,1 +21,5,1,3,2 +22,6,0,3,3 +23,7,-1,3,4 +24,8,-2,3,5 +25,9,-3,3,6 +26,10,-4,3,7 +27,11,-5,3,8 +28,12,-6,3,9 +29,13,-7,3,10 +30,5,3,4,1 +31,6,2,4,2 +32,7,1,4,3 +33,8,0,4,4 +34,9,-1,4,5 +35,10,-2,4,6 +36,11,-3,4,7 +37,12,-4,4,8 +38,13,-5,4,9 +39,14,-6,4,10 +40,6,4,5,1 +41,7,3,5,2 +42,8,2,5,3 +43,9,1,5,4 +44,10,0,5,5 +45,11,-1,5,6 +46,12,-2,5,7 +47,13,-3,5,8 +48,14,-4,5,9 +49,15,-5,5,10 +50,7,5,6,1 +51,8,4,6,2 +52,9,3,6,3 +53,10,2,6,4 +54,11,1,6,5 +55,12,0,6,6 +56,13,-1,6,7 +57,14,-2,6,8 +58,15,-3,6,9 +59,16,-4,6,10 +60,8,6,7,1 +61,9,5,7,2 +62,10,4,7,3 +63,11,3,7,4 +64,12,2,7,5 +65,13,1,7,6 +66,14,0,7,7 +67,15,-1,7,8 +68,16,-2,7,9 +69,17,-3,7,10 +70,9,7,8,1 +71,10,6,8,2 +72,11,5,8,3 +73,12,4,8,4 +74,13,3,8,5 +75,14,2,8,6 +76,15,1,8,7 +77,16,0,8,8 +78,17,-1,8,9 +79,18,-2,8,10 +80,10,8,9,1 +81,11,7,9,2 +82,12,6,9,3 +83,13,5,9,4 +84,14,4,9,5 +85,15,3,9,6 +86,16,2,9,7 +87,17,1,9,8 +88,18,0,9,9 +89,19,-1,9,10 +90,11,9,10,1 +91,12,8,10,2 +92,13,7,10,3 +93,14,6,10,4 +94,15,5,10,5 +95,16,4,10,6 +96,17,3,10,7 +97,18,2,10,8 +98,19,1,10,9 +99,20,0,10,10 diff --git a/tests/integration/data/manifest.yml b/tests/integration/data/manifest.yml new file mode 100644 index 0000000..bf0e540 --- /dev/null +++ b/tests/integration/data/manifest.yml @@ -0,0 +1,91 @@ +# This is the config file required to be in the same directory as as an +# experiment that is submitted from the command line. Consult documentation +# on experiment compatability and more information on these parameters a +# https://automatingsciencepipeline.github.io/Monorepo/tutorial/usage/ + +#CONFIGURATION: + +# This allows you to define hyperparameters for each run. Multiple can be defined. +# Example: +# - name: x +# default: "-1" +# min: "1" +# max: "10" +# step: "1" +# type: integer +# useDefault: false + +# - name: "values", +# default: "-1", +# values: [ +# "a", +# "b" +# ], +# type: "stringlist", +# useDefault: false +# See https://automatingsciencepipeline.github.io/Monorepo/tutorial/usage/#information-tab +# for full list of hyperparameters + +hyperparameters: + - name: x + default: "-1" + min: "1" + max: "10" + step: "1" + type: integer + useDefault: false + + - name: y + default: "-1" + min: "1" + max: "10" + step: "1" + type: integer + useDefault: false +# The experiment name. +# Example: "Evolutionary Aglorithm Experiment" +name: "Test AddNums" +# This is the description of the experiment. +# Example: "Experiment for CSSE490 Homework 4." +description: "This is to test AddNums!" +# These are the tags which can be defined for an experiment, allowing for better filtering on the web app. +# Example: ["Neural Network", "ECE497"] +tags: ["Neural Network", "ECE497"] +# A folder or file included in the downloadable Project Zip after experiment completion. +# Example: "test.csv" +trialExtraFile: "AddNumResult.csv" +# The CSV file captured as the experiment result. +# Example: "test.csv" +trialResult: "AddNumResult.csv" +# This allows you to specific the line number of a trial result. +# Example: 0 +trialResultLineNumber: 0 +# This allows you to include a scatter plot in the downloadable Project Zip +# Example: true +scatter: true +# If a scatter plot is included, this defined the independent variable. Leave blank if not included. +# Example: "y" for included, "" for not +scatterIndVar: "x" +# If a scatter plot is included, this defined the dependent variable. Leave blank if not included. +# Example: "x" for included, "" for not +scatterDepVar: "y" +# This tab allows defining a text block appended to every generated .ini config file. +# Example: "" for default behavior +dumbTextArea: "" +# The duration before the experiment automatically times out (seconds) +# Example: 600 +timeout: 100 +# This allows an email to be sent following experiment completion with number of passed and failed trials +# Example: true +sendEmail: false +# Define number of workers used for experiment +# Example: 1 +workers: 1 +# If using a zip experiment, specify the main executable filename. +# Example: "test.py" if using zip, "" if not +experimentExecutable: "" +# This attribute, set to either ini or yaml, defines the format of the config file generated for each trial. +# Example: "ini" or "yaml" +configFileFormat: "ini" + + diff --git a/tests/integration/glados_workflow_tests.py b/tests/integration/glados_workflow_tests.py new file mode 100644 index 0000000..14ab8ad --- /dev/null +++ b/tests/integration/glados_workflow_tests.py @@ -0,0 +1,108 @@ +import unittest +import subprocess +import os +import glob +import time +import pandas as pd + +GLADOS_CLI_PATH = "glados_cli.py" +CSV_FILE_PATH = "tests/integration/data/addNumbersExpected.csv" +EXPERIMENT_FILE = "tests/integration/data/addNumbers.py" + +class TestGladosCLI(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Clean up environment before starting tests.""" + cls.experiment_id = None + cls._cleanup_files() + + @staticmethod + def _cleanup_files(): + for f in glob.glob("Test_AddNums*"): + os.remove(f) + + def _run_cli(self, args): + cmd = ["python", GLADOS_CLI_PATH] + args + result = subprocess.run(cmd, capture_output=True, text=True) + return result + + def _assert_in_output(self, expected, actual, message=None): + self.assertIn(expected, actual, message or f"Expected '{expected}' not found in output.") + + def _filter_output(self, text): + # ID and Time Started lines will vary, so they must be ignored + skip_prefixes = ("ID:", "Time Started:") + return "\n".join([ + line.strip() for line in text.splitlines() + if not line.strip().startswith(skip_prefixes) + ]) + + def test_01_experiment_creation(self): + result = self._run_cli(["-z", EXPERIMENT_FILE]) + + self.assertEqual(result.returncode, 0, f"CLI exited with error: {result.stderr}") + self._assert_in_output("Experiment started successfully", result.stdout) + + # Parse and store the experiment ID for subsequent tests + try: + parts = result.stdout.strip().split('=') + TestGladosCLI.experiment_id = parts[1].strip(' ).').split()[0] + except (IndexError, AttributeError): + self.fail("Failed to parse Experiment ID from output.") + + def test_02_experiment_download(self): + if not self.experiment_id: + self.skipTest("No experiment ID available from previous step.") + + # Give the system a moment to register the experiment + time.sleep(10) + + result = self._run_cli(["-d", self.experiment_id]) + + self.assertEqual(result.returncode, 0) + self.assertRegex(result.stdout, r"Experiment results Test_AddNums_.*\.csv downloaded successfully\.") + + downloaded_files = glob.glob("Test_AddNums*.csv") + if downloaded_files: + df_actual = pd.read_csv(downloaded_files[0]) + df_expected = pd.read_csv(CSV_FILE_PATH) + pd.testing.assert_frame_equal(df_actual, df_expected) + else: + self.fail("Results CSV file was not found after download.") + TestGladosCLI._cleanup_files() # Remove csv file download from previous test to ensure this test is valid + + def test_03_experiment_download_all(self): + if not self.experiment_id: + self.skipTest("No experiment ID available from previous step.") + + # Give the system a moment to register the experiment + time.sleep(5) + + result = self._run_cli(["-da", self.experiment_id]) + + self.assertEqual(result.returncode, 0) + self._assert_in_output("All experiment artifacts downloaded successfully.", result.stdout) + + downloaded_files = glob.glob("Test_AddNums*.csv") + if downloaded_files: + df_actual = pd.read_csv(downloaded_files[0]) + df_expected = pd.read_csv(CSV_FILE_PATH) + pd.testing.assert_frame_equal(df_actual, df_expected) + else: + self.fail("Results CSV file was not found after download.") + TestGladosCLI._cleanup_files() # Remove csv file download from previous test to ensure this test is valid + + def test_04_experiment_query(self): + result = self._run_cli(["-q", "Test AddNums"]) + + expected_output_fragment = ( + "Matches:\n***********************************************\nExperiment 1: Test AddNums\n***********************************************\nTags: ['Test', 'AddNums']\nStatus: COMPLETED\nTrials: 100/100 Completed\n" + ) + + actual_filtered = self._filter_output(result.stdout) + expected_filtered = self._filter_output(expected_output_fragment) + + self._assert_in_output(expected_filtered, actual_filtered) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/empty-experiment/requirements.txt b/tests/unit/data/empty-experiment/requirements.txt similarity index 100% rename from empty-experiment/requirements.txt rename to tests/unit/data/empty-experiment/requirements.txt diff --git a/test_manifests/test_manifest_bool_errors.yml b/tests/unit/data/test_manifests/test_manifest_bool_errors.yml similarity index 100% rename from test_manifests/test_manifest_bool_errors.yml rename to tests/unit/data/test_manifests/test_manifest_bool_errors.yml diff --git a/test_manifests/test_manifest_int_errors.yml b/tests/unit/data/test_manifests/test_manifest_int_errors.yml similarity index 94% rename from test_manifests/test_manifest_int_errors.yml rename to tests/unit/data/test_manifests/test_manifest_int_errors.yml index e08e723..fcfcc33 100644 --- a/test_manifests/test_manifest_int_errors.yml +++ b/tests/unit/data/test_manifests/test_manifest_int_errors.yml @@ -1,33 +1,33 @@ -hyperparameters: - - name: x - default: "-1" - min: "1.5" - max: "10.5" - step: "1.0" - type: float - useDefault: false - - - name: y - default: "-1" - min: "1.5" - max: "10.5" - step: "1.0" - type: float - useDefault: false -name: "Int test" -description: "Hi" -tags: ["Neural Network", "ECE497"] -trialExtraFile: "AddNumResult.csv" -trialResult: "IntTest.csv" -trialResultLineNumber: -scatter: false -scatterIndVar: "" -scatterDepVar: "" -dumbTextArea: "" -timeout: -1 -keepLogs: true -sendEmail: false -workers: -2 -experimentExecutable: "testint.py" -configFileFormat: "ini" - +hyperparameters: + - name: x + default: "-1" + min: "1.5" + max: "10.5" + step: "1.0" + type: float + useDefault: false + + - name: y + default: "-1" + min: "1.5" + max: "10.5" + step: "1.0" + type: float + useDefault: false +name: "Int test" +description: "Hi" +tags: ["Neural Network", "ECE497"] +trialExtraFile: "AddNumResult.csv" +trialResult: "IntTest.csv" +trialResultLineNumber: +scatter: false +scatterIndVar: "" +scatterDepVar: "" +dumbTextArea: "" +timeout: -1 +keepLogs: true +sendEmail: false +workers: -2 +experimentExecutable: "testint.py" +configFileFormat: "ini" + diff --git a/test_manifests/test_manifest_no_errors.yml b/tests/unit/data/test_manifests/test_manifest_no_errors.yml similarity index 94% rename from test_manifests/test_manifest_no_errors.yml rename to tests/unit/data/test_manifests/test_manifest_no_errors.yml index ad88d3f..1daf61e 100644 --- a/test_manifests/test_manifest_no_errors.yml +++ b/tests/unit/data/test_manifests/test_manifest_no_errors.yml @@ -1,33 +1,33 @@ -hyperparameters: - - name: x - default: "-1" - min: "1.5" - max: "10.5" - step: "1.0" - type: float - useDefault: false - - - name: y - default: "-1" - min: "1.5" - max: "10.5" - step: "1.0" - type: float - useDefault: false -name: "New Manifest Test" -description: "Hi" -tags: ["Neural Network", "ECE497"] -trialExtraFile: "AddNumResult.csv" -trialResult: "AddNumResult.csv" -trialResultLineNumber: 0 -scatter: false -scatterIndVar: "" -scatterDepVar: "" -dumbTextArea: "" -timeout: 100 -keepLogs: true -sendEmail: false -workers: 1 -experimentExecutable: "test.py" -configFileFormat: "ini" - +hyperparameters: + - name: x + default: "-1" + min: "1.5" + max: "10.5" + step: "1.0" + type: float + useDefault: false + + - name: y + default: "-1" + min: "1.5" + max: "10.5" + step: "1.0" + type: float + useDefault: false +name: "New Manifest Test" +description: "Hi" +tags: ["Neural Network", "ECE497"] +trialExtraFile: "AddNumResult.csv" +trialResult: "AddNumResult.csv" +trialResultLineNumber: 0 +scatter: false +scatterIndVar: "" +scatterDepVar: "" +dumbTextArea: "" +timeout: 100 +keepLogs: true +sendEmail: false +workers: 1 +experimentExecutable: "test.py" +configFileFormat: "ini" + diff --git a/test_manifests/test_manifest_param_errors.yml b/tests/unit/data/test_manifests/test_manifest_param_errors.yml similarity index 94% rename from test_manifests/test_manifest_param_errors.yml rename to tests/unit/data/test_manifests/test_manifest_param_errors.yml index 062891d..c45eaf1 100644 --- a/test_manifests/test_manifest_param_errors.yml +++ b/tests/unit/data/test_manifests/test_manifest_param_errors.yml @@ -1,57 +1,57 @@ -hyperparameters: - - name: x - default: "-1" - min: "hi" - max: "10.5" - step: "1.0" - type: float - useDefault: false - - - name: values1 - default: "-1" - values: "a" - type: stringlist - useDefault: false - - - name: y - default: "-1" - min: "1.5" - max: "10" - step: "1" - type: integer - useDefault: false - - - name: z - default: "-1" - min: "12" - max: "10" - step: "1" - type: integer - useDefault: false - - - name: values2 - default: 3 - useDefault: true - type: bool - - - name: values3 - default: true - useDefault: true - type: truthy -name: "Int test" -description: "Hi" -tags: ["Neural Network", "ECE497"] -trialExtraFile: "AddNumResult.csv" -trialResult: "IntTest.csv" -trialResultLineNumber: 0 -scatter: false -scatterIndVar: "" -scatterDepVar: "" -dumbTextArea: "" -timeout: 1 -keepLogs: true -sendEmail: false -workers: 1 -experimentExecutable: "testint.py" -configFileFormat: "ini" - +hyperparameters: + - name: x + default: "-1" + min: "hi" + max: "10.5" + step: "1.0" + type: float + useDefault: false + + - name: values1 + default: "-1" + values: "a" + type: stringlist + useDefault: false + + - name: y + default: "-1" + min: "1.5" + max: "10" + step: "1" + type: integer + useDefault: false + + - name: z + default: "-1" + min: "12" + max: "10" + step: "1" + type: integer + useDefault: false + + - name: values2 + default: 3 + useDefault: true + type: bool + + - name: values3 + default: true + useDefault: true + type: truthy +name: "Int test" +description: "Hi" +tags: ["Neural Network", "ECE497"] +trialExtraFile: "AddNumResult.csv" +trialResult: "IntTest.csv" +trialResultLineNumber: 0 +scatter: false +scatterIndVar: "" +scatterDepVar: "" +dumbTextArea: "" +timeout: 1 +keepLogs: true +sendEmail: false +workers: 1 +experimentExecutable: "testint.py" +configFileFormat: "ini" + diff --git a/test_manifests/test_manifest_string_errors.yml b/tests/unit/data/test_manifests/test_manifest_string_errors.yml similarity index 94% rename from test_manifests/test_manifest_string_errors.yml rename to tests/unit/data/test_manifests/test_manifest_string_errors.yml index 8bc909e..6816a8a 100644 --- a/test_manifests/test_manifest_string_errors.yml +++ b/tests/unit/data/test_manifests/test_manifest_string_errors.yml @@ -1,33 +1,33 @@ -hyperparameters: - - name: x - default: "-1" - min: "1.5" - max: "10.5" - step: "1.0" - type: float - useDefault: false - - - name: y - default: "-1" - min: "1.5" - max: "10.5" - step: "1.0" - type: float - useDefault: false -name: -description: "Hi" -tags: ["Neural Network", "ECE497"] -trialExtraFile: "AddNumResult.csv" -trialResult: -trialResultLineNumber: 0 -scatter: true -scatterIndVar: "" -scatterDepVar: "" -dumbTextArea: "" -timeout: 100 -keepLogs: true -sendEmail: false -workers: 1 -experimentExecutable: "" -configFileFormat: "ini" - +hyperparameters: + - name: x + default: "-1" + min: "1.5" + max: "10.5" + step: "1.0" + type: float + useDefault: false + + - name: y + default: "-1" + min: "1.5" + max: "10.5" + step: "1.0" + type: float + useDefault: false +name: +description: "Hi" +tags: ["Neural Network", "ECE497"] +trialExtraFile: "AddNumResult.csv" +trialResult: +trialResultLineNumber: 0 +scatter: true +scatterIndVar: "" +scatterDepVar: "" +dumbTextArea: "" +timeout: 100 +keepLogs: true +sendEmail: false +workers: 1 +experimentExecutable: "" +configFileFormat: "ini" + diff --git a/valid-experiment/exp_template.py b/tests/unit/data/valid-experiment/exp_template.py similarity index 95% rename from valid-experiment/exp_template.py rename to tests/unit/data/valid-experiment/exp_template.py index 4fe7e01..0819672 100644 --- a/valid-experiment/exp_template.py +++ b/tests/unit/data/valid-experiment/exp_template.py @@ -1,29 +1,29 @@ -from core.experiment import Experiment - -class MyExperiment(Experiment): - def __init__(self): - hyperparams = { - #variable : (start, stop, step_size) - - } - - super().__init__(hyperparams, - csv_name="output.csv") - - - def process_trial(self, data): - # your defined function here - # e.g. compute f(x, y, z) - pass - - def graph_result(self, data, save_path): - # your defined graph function here - pass - - -def main(): - new_exp = MyExperiment() - new_exp.doExperiment() - -if __name__ == "__main__": +from core.experiment import Experiment + +class MyExperiment(Experiment): + def __init__(self): + hyperparams = { + #variable : (start, stop, step_size) + + } + + super().__init__(hyperparams, + csv_name="output.csv") + + + def process_trial(self, data): + # your defined function here + # e.g. compute f(x, y, z) + pass + + def graph_result(self, data, save_path): + # your defined graph function here + pass + + +def main(): + new_exp = MyExperiment() + new_exp.doExperiment() + +if __name__ == "__main__": main() \ No newline at end of file diff --git a/valid-experiment/manifest.yaml b/tests/unit/data/valid-experiment/manifest.yaml similarity index 100% rename from valid-experiment/manifest.yaml rename to tests/unit/data/valid-experiment/manifest.yaml diff --git a/valid-experiment/requirements.txt b/tests/unit/data/valid-experiment/requirements.txt similarity index 100% rename from valid-experiment/requirements.txt rename to tests/unit/data/valid-experiment/requirements.txt diff --git a/glados_cli_tests.py b/tests/unit/glados_cli_tests.py similarity index 91% rename from glados_cli_tests.py rename to tests/unit/glados_cli_tests.py index 33ba68b..96e537f 100644 --- a/glados_cli_tests.py +++ b/tests/unit/glados_cli_tests.py @@ -1,380 +1,393 @@ -import glados_cli as gcli - -from unittest import mock -from typing import * -from io import StringIO -from contextlib import redirect_stdout - -import os -import io -import unittest -import zipfile - -class GladosCliTests(unittest.TestCase): - - def setUp(self): - self.request_manager: gcli.RequestManager = mock.MagicMock() - self._makeZipFile('valid-experiment') - self._makeZipFile('empty-experiment') - # Allow for testing what's printed to stdout and stderr - self.out = io.StringIO() - self.err = io.StringIO() - # Create a stored token file for tests that require authentication - with open('.token.glados', 'w') as f: - f.write('new_valid_token') - - def tearDown(self): - # Clean up the token file - if os.path.exists('.token.glados'): - os.remove('.token.glados') - - def _makeZipFile(self, dirname: str) -> None: - with zipfile.ZipFile(f'{dirname}.zip', 'w') as zf: - for dirpath, _, filenames in os.walk(dirname): - for filename in filenames: - filepath = os.path.join(dirpath, filename) - zf.write(filepath, os.path.relpath(filepath, dirname)) - - def parse_args(self, args: List[str]) -> int: - return gcli.parse_args(self.request_manager, args, stdout=self.out, stderr=self.err) - - def _assert_status_code(self, args: List[str], expected_code: int) -> None: - status = self.parse_args(args) - self.assertEqual(status, expected_code) - - def _assert_in_output(self, substring: str) -> None: - output = self.out.getvalue() - if (substring not in output): - print("OUTPUT:") - print(output) - self.assertIn(substring, output) - - def _assert_in_error(self, substring: str) -> None: - error = self.err.getvalue() - if (substring not in error): - print("ERROR:") - print(error) - self.assertIn(substring, error) - - def test_mutually_exclusive_parameters(self) -> None: - # Test that mutually exclusive parameters cannot be used together - self._assert_status_code(['-q', 'some_value', '-z', 'another_value'], gcli.EX_PARSE_ERROR) - self._assert_status_code(['-d', 'some_value', '-q', 'another_value'], gcli.EX_PARSE_ERROR) - self._assert_status_code(['-z', 'some_value', '-d', 'another_value'], gcli.EX_PARSE_ERROR) - self._assert_status_code(['-da', 'some_value', '-z', 'another_value'], gcli.EX_PARSE_ERROR) - self._assert_in_error("Invalid flags") - - def test_with_invalid_token(self) -> None: - # Test with an invalid stored token - self.request_manager.authenticate.return_value = {"uid": None, "error": "invalid"} - self._assert_status_code(['-z', 'experiment.zip'], gcli.EX_INVALID_TOKEN) - self._assert_in_error("Cannot authenticate token") - self._assert_status_code(['-q', 'experiment_name'], gcli.EX_INVALID_TOKEN) - self._assert_in_error("Cannot authenticate token") - self._assert_status_code(['-d', 'experiment.zip'], gcli.EX_INVALID_TOKEN) - self._assert_in_error("Cannot authenticate token") - self.request_manager.authenticate.assert_has_calls([ - mock.call('new_valid_token'), - mock.call('new_valid_token'), - mock.call('new_valid_token')], any_order=False) - - def test_run_experiment(self) -> None: - # Test running an experiment with a valid token - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.upload_and_start_experiment.return_value = { - 'success': True, - 'error': '', - 'exp_id': 'exp123' - } - self._assert_status_code(['-z', 'valid-experiment.zip'], gcli.EX_SUCCESS) - - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.upload_and_start_experiment.assert_called_with('valid-experiment.zip') - self._assert_in_output('exp123') - - def test_with_stored_token(self) -> None: - # Test running an experiment with a stored token - with open('.token.glados', 'w') as f: - f.write('valid_token') - - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.upload_and_start_experiment.return_value = { - 'success': True, - 'error': '', - 'exp_id': 'expabc' - } - self._assert_status_code(['-z', 'valid-experiment.zip'], gcli.EX_SUCCESS) - self.request_manager.authenticate.assert_called_with('valid_token') - self.request_manager.upload_and_start_experiment.assert_called_with('valid-experiment.zip') - self._assert_in_output('expabc') - - os.remove('.token.glados') - - def test_generate_token(self) -> None: - # Test running an experiment without any token - self.request_manager.generate_token.return_value = { - "access_token": "new_valid_token", - "error": None - } - self._assert_status_code(['--generate-token'], gcli.EX_SUCCESS) - self.request_manager.generate_token.assert_called_once() - self._assert_in_output('new_valid_token') - - def test_run_missing_experiment(self) -> None: - # Test running a non-existent experiment file - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self._assert_status_code(['-z', 'missing_experiment.zip'], gcli.EX_NOTFOUND) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self._assert_in_error('missing_experiment.zip') - self._assert_in_error('not found') - - def test_run_experiment_backend_format_failure(self) -> None: - # Test running an experiment where the backend's format validation fails - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.upload_and_start_experiment.return_value = { - 'success': False, - 'error': 'bad_format', - 'exp_id': '' - } - self._assert_status_code(['-z', 'valid-experiment.zip'], gcli.EX_INVALID_EXP_FORMAT) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.upload_and_start_experiment.assert_called_with('valid-experiment.zip') - self._assert_in_error('format') - - def test_run_experiment_other_backend_failure(self) -> None: - # Test running an experiment where the backend fails for other reasons - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.upload_and_start_experiment.return_value = { - 'success': False, - 'error': 'other', - 'exp_id': '' - } - self._assert_status_code(['-z', 'valid-experiment.zip'], gcli.EX_UNKNOWN) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.upload_and_start_experiment.assert_called_with('valid-experiment.zip') - self._assert_in_error('other') - - def test_query_one_experiment(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.query_experiments.return_value = { - 'success': True, - 'matches': [ - {'id': 'exp1', - 'name': 'Test Experiment', - 'tags': ['tag1', 'tag2'], - 'status': 'completed', - 'started_on': 1762488593221, - 'current_permutation': 50, - 'total_permutations': 100}, - ] - } - self._assert_status_code(['-q', 'Test Experiment'], gcli.EX_SUCCESS) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.query_experiments.assert_called_with('Test Experiment') - self._assert_in_output('Test Experiment') - - def test_query_multiple_experiments(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.query_experiments.return_value = { - 'success': True, - 'matches': [ - {'id': 'exp1', - 'name': 'Test Experiment 1', - 'tags': ['tag1'], - 'status': 'running', - 'started_on': 1762488593221, - 'current_permutation': 70, - 'total_permutations': 100}, - {'id': 'exp2', - 'name': 'Test Experiment 2', - 'tags': ['tag2'], - 'status': 'completed', - 'started_on': 1762489593221, - 'current_permutation': 80, - 'total_permutations': 100}, - ] - } - self._assert_status_code(['-q', 'Test Experiment'], gcli.EX_SUCCESS) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.query_experiments.assert_called_with('Test Experiment') - self._assert_in_output('Test Experiment 1') - self._assert_in_output('Test Experiment 2') - - def test_query_no_experiments(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.query_experiments.return_value = { - 'success': True, - 'matches': [] - } - self._assert_status_code(['-q', 'Nonexistent Experiment'], gcli.EX_NOTFOUND) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.query_experiments.assert_called_with('Nonexistent Experiment') - self._assert_in_error('No experiments found') - - def test_download_experiment_results(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_experiment_results.return_value = { - 'success': True, - 'files': [ - { - 'name': 'downloaded_results.zip', - 'content': b'PK\x03\x04...' # Simulated binary content of a zip file - } - ] - } - self._assert_status_code(['-d', 'exp123'], gcli.EX_SUCCESS) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_experiment_results.assert_called_with('exp123') - self._assert_in_output('downloaded_results.zip') - - def test_download_experiment_not_found(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_experiment_results.return_value = { - 'success': False, - 'error': 'not_found' - } - self._assert_status_code(['-d', 'exp123'], gcli.EX_NOTFOUND) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_experiment_results.assert_called_with('exp123') - self._assert_in_error("not found") - - def test_download_experiment_still_running(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_experiment_results.return_value = { - 'success': False, - 'error': 'not_done' - } - self._assert_status_code(['-d', 'exp123'], gcli.EX_NOT_DONE) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_experiment_results.assert_called_with('exp123') - self._assert_in_error("still running") - - def test_download_experiment_failed(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_experiment_results.return_value = { - 'success': False, - 'error': 'exp_failed' - } - self._assert_status_code(['-d', 'exp123'], gcli.EX_EXP_FAILED) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_experiment_results.assert_called_with('exp123') - self._assert_in_error("did not complete successfully") - - def test_download_all_experiment_results(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_all.return_value = { - 'success': True, - 'files': [ - { - 'name': 'downloaded_results.zip', - 'content': b'PK\x03\x04...' # Simulated binary content of a zip file - } - ] - } - self._assert_status_code(['-da', 'exp123'], gcli.EX_SUCCESS) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_all.assert_called_with('exp123') - self._assert_in_output('All experiment artifacts downloaded successfully.') - - def test_download_all_experiment_not_found(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_all.return_value = { - 'success': False, - 'error': 'not_found' - } - self._assert_status_code(['-da', 'exp123'], gcli.EX_NOTFOUND) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_all.assert_called_with('exp123') - self._assert_in_error("not found") - - def test_download_all_experiment_still_running(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_all.return_value = { - 'success': False, - 'error': 'not_done' - } - self._assert_status_code(['-da', 'exp123'], gcli.EX_NOT_DONE) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_all.assert_called_with('exp123') - self._assert_in_error("still running") - - def test_download_all_experiment_failed(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_all.return_value = { - 'success': False, - 'error': 'exp_failed' - } - self._assert_status_code(['-da', 'exp123'], gcli.EX_EXP_FAILED) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_all.assert_called_with('exp123') - self._assert_in_error("did not complete successfully") - - def test_cli_update_success(self): - self.request_manager.update.return_value = { - "success": True, - "error": False - } - self._assert_status_code(['--update'], gcli.UPDATE_SUCCEED) - self.request_manager.update.assert_called_once() - self._assert_in_output("Downloaded most up-to-date CLI successfully") - - def test_cli_update_failure(self): - self.request_manager.update.return_value = { - "success": False, - "error": "network" - } - self._assert_status_code(['-u'], gcli.UPDATE_FAIL) - self.request_manager.update.assert_called_once() - self._assert_in_output("Unable to download most up-to-date version") - - def test_manifest_no_errors(self): - gcli.check_manifest_format("test_manifests/test_manifest_no_errors.yml", False) - self._assert_in_output("") - - def test_manifest_string_errors(self): - buf = StringIO() - with redirect_stdout(buf): - result = gcli.check_manifest_format("test_manifests/test_manifest_string_errors.yml", True) - output = buf.getvalue() - self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) - self.assertIn("name attribute in manifest.yml is empty, missing, or not a string.", output) - self.assertIn("trialResult attribute in manifest.yml is empty, missing, or not a string.", output) - self.assertIn("scatterIndVar attribute in manifest.yml is empty, missing, or not a string.", output) - self.assertIn("scatterDepVar attribute in manifest.yml is empty, missing, or not a string.", output) - self.assertIn("experimentExecutable attribute in manifest.yml is empty, missing, or not a string.", output) - - def test_manifest_int_errors(self): - buf = StringIO() - with redirect_stdout(buf): - result = gcli.check_manifest_format("test_manifests/test_manifest_int_errors.yml", True) - output = buf.getvalue() - self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) - self.assertIn("trialResultLineNumber attribute in manifest.yml is empty or missing.", output) - self.assertIn("timeout attribute in manifest.yml is not greater than 0.", output) - self.assertIn("workers attribute in manifest.yml is not greater than 0", output) - - def test_manifest_bool_errors(self): - buf = StringIO() - with redirect_stdout(buf): - result = gcli.check_manifest_format("test_manifests/test_manifest_bool_errors.yml", True) - output = buf.getvalue() - self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) - self.assertIn("sendEmail attribute in manifest.yml is empty, missing, or not true or false.", output) - self.assertIn("scatter attribute in manifest.yml is empty, missing, or not true or false.", output) - - def test_manifest_param_errors(self): - buf = StringIO() - with redirect_stdout(buf): - result = gcli.check_manifest_format("test_manifests/test_manifest_param_errors.yml", True) - output = buf.getvalue() - self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) - self.assertIn("min attribute in hyperparameter x is not a float.", output) - self.assertIn("values attribute in hyperparameter values1 is not a list.", output) - self.assertIn("min attribute in hyperparameter y is not an integer.", output) - self.assertIn("max attribute in hyperparameter z is not greater than 12.", output) - self.assertIn("default attribute in hyperparameter values2 is empty, missing, or not true or false.", output) - self.assertIn("Type specified in hyperparameter values3 is not integer, float, bool, stringlist, or paramgroup.", output) - -if __name__ == '__main__': - unittest.main() - +from unittest import mock +from typing import * +from io import StringIO +from contextlib import redirect_stdout + +import sys +import os +import io +import unittest +import zipfile + +# Import glados_cli.py, adjust if necessary +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) + +import glados_cli as gcli + +VALID_EXPERIMENT_ZIP = 'tests/unit/data/valid-experiment.zip' +EMPTY_EXPERIMENT_ZIP = 'tests/unit/data/empty-experiment.zip' +MANIFEST_DIRECTORY = 'tests/unit/data/test_manifests/' + +class GladosCliTests(unittest.TestCase): + + def setUp(self): + self.request_manager: gcli.RequestManager = mock.MagicMock() + self._makeZipFile(VALID_EXPERIMENT_ZIP.replace('.zip', '')) + self._makeZipFile(EMPTY_EXPERIMENT_ZIP.replace('.zip', '')) + # Allow for testing what's printed to stdout and stderr + self.out = io.StringIO() + self.err = io.StringIO() + # Create a stored token file for tests that require authentication + with open('.token.glados', 'w') as f: + f.write('new_valid_token') + + def tearDown(self): + # Clean up the token file + if os.path.exists('.token.glados'): + os.remove('.token.glados') + + # Clean up the generated zips + for path in [VALID_EXPERIMENT_ZIP, EMPTY_EXPERIMENT_ZIP]: + if os.path.exists(path): + os.remove(path) + + def _makeZipFile(self, dirname: str) -> None: + with zipfile.ZipFile(f'{dirname}.zip', 'w') as zf: + for dirpath, _, filenames in os.walk(dirname): + for filename in filenames: + filepath = os.path.join(dirpath, filename) + zf.write(filepath, os.path.relpath(filepath, dirname)) + + def parse_args(self, args: List[str]) -> int: + return gcli.parse_args(self.request_manager, args, stdout=self.out, stderr=self.err) + + def _assert_status_code(self, args: List[str], expected_code: int) -> None: + status = self.parse_args(args) + self.assertEqual(status, expected_code) + + def _assert_in_output(self, substring: str) -> None: + output = self.out.getvalue() + if (substring not in output): + print("OUTPUT:") + print(output) + self.assertIn(substring, output) + + def _assert_in_error(self, substring: str) -> None: + error = self.err.getvalue() + if (substring not in error): + print("ERROR:") + print(error) + self.assertIn(substring, error) + + def test_mutually_exclusive_parameters(self) -> None: + # Test that mutually exclusive parameters cannot be used together + self._assert_status_code(['-q', 'some_value', '-z', 'another_value'], gcli.EX_PARSE_ERROR) + self._assert_status_code(['-d', 'some_value', '-q', 'another_value'], gcli.EX_PARSE_ERROR) + self._assert_status_code(['-z', 'some_value', '-d', 'another_value'], gcli.EX_PARSE_ERROR) + self._assert_status_code(['-da', 'some_value', '-z', 'another_value'], gcli.EX_PARSE_ERROR) + self._assert_in_error("Invalid flags") + + def test_with_invalid_token(self) -> None: + # Test with an invalid stored token + self.request_manager.authenticate.return_value = {"uid": None, "error": "invalid"} + self._assert_status_code(['-z', 'experiment.zip'], gcli.EX_INVALID_TOKEN) + self._assert_in_error("Cannot authenticate token") + self._assert_status_code(['-q', 'experiment_name'], gcli.EX_INVALID_TOKEN) + self._assert_in_error("Cannot authenticate token") + self._assert_status_code(['-d', 'experiment.zip'], gcli.EX_INVALID_TOKEN) + self._assert_in_error("Cannot authenticate token") + self.request_manager.authenticate.assert_has_calls([ + mock.call('new_valid_token'), + mock.call('new_valid_token'), + mock.call('new_valid_token')], any_order=False) + + def test_run_experiment(self) -> None: + # Test running an experiment with a valid token + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.upload_and_start_experiment.return_value = { + 'success': True, + 'error': '', + 'exp_id': 'exp123' + } + self._assert_status_code(['-z', VALID_EXPERIMENT_ZIP], gcli.EX_SUCCESS) + + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.upload_and_start_experiment.assert_called_with(VALID_EXPERIMENT_ZIP) + self._assert_in_output('exp123') + + def test_with_stored_token(self) -> None: + # Test running an experiment with a stored token + with open('.token.glados', 'w') as f: + f.write('valid_token') + + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.upload_and_start_experiment.return_value = { + 'success': True, + 'error': '', + 'exp_id': 'expabc' + } + self._assert_status_code(['-z', VALID_EXPERIMENT_ZIP], gcli.EX_SUCCESS) + self.request_manager.authenticate.assert_called_with('valid_token') + self.request_manager.upload_and_start_experiment.assert_called_with(VALID_EXPERIMENT_ZIP) + self._assert_in_output('expabc') + + os.remove('.token.glados') + + def test_generate_token(self) -> None: + # Test running an experiment without any token + self.request_manager.generate_token.return_value = { + "access_token": "new_valid_token", + "error": None + } + self._assert_status_code(['--generate-token'], gcli.EX_SUCCESS) + self.request_manager.generate_token.assert_called_once() + self._assert_in_output('new_valid_token') + + def test_run_missing_experiment(self) -> None: + # Test running a non-existent experiment file + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self._assert_status_code(['-z', 'missing_experiment.zip'], gcli.EX_NOTFOUND) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self._assert_in_error('missing_experiment.zip') + self._assert_in_error('not found') + + def test_run_experiment_backend_format_failure(self) -> None: + # Test running an experiment where the backend's format validation fails + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.upload_and_start_experiment.return_value = { + 'success': False, + 'error': 'bad_format', + 'exp_id': '' + } + self._assert_status_code(['-z', VALID_EXPERIMENT_ZIP], gcli.EX_INVALID_EXP_FORMAT) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.upload_and_start_experiment.assert_called_with(VALID_EXPERIMENT_ZIP) + self._assert_in_error('format') + + def test_run_experiment_other_backend_failure(self) -> None: + # Test running an experiment where the backend fails for other reasons + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.upload_and_start_experiment.return_value = { + 'success': False, + 'error': 'other', + 'exp_id': '' + } + self._assert_status_code(['-z', VALID_EXPERIMENT_ZIP], gcli.EX_UNKNOWN) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.upload_and_start_experiment.assert_called_with(VALID_EXPERIMENT_ZIP) + self._assert_in_error('other') + + def test_query_one_experiment(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.query_experiments.return_value = { + 'success': True, + 'matches': [ + {'id': 'exp1', + 'name': 'Test Experiment', + 'tags': ['tag1', 'tag2'], + 'status': 'completed', + 'started_on': 1762488593221, + 'current_permutation': 50, + 'total_permutations': 100}, + ] + } + self._assert_status_code(['-q', 'Test Experiment'], gcli.EX_SUCCESS) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.query_experiments.assert_called_with('Test Experiment') + self._assert_in_output('Test Experiment') + + def test_query_multiple_experiments(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.query_experiments.return_value = { + 'success': True, + 'matches': [ + {'id': 'exp1', + 'name': 'Test Experiment 1', + 'tags': ['tag1'], + 'status': 'running', + 'started_on': 1762488593221, + 'current_permutation': 70, + 'total_permutations': 100}, + {'id': 'exp2', + 'name': 'Test Experiment 2', + 'tags': ['tag2'], + 'status': 'completed', + 'started_on': 1762489593221, + 'current_permutation': 80, + 'total_permutations': 100}, + ] + } + self._assert_status_code(['-q', 'Test Experiment'], gcli.EX_SUCCESS) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.query_experiments.assert_called_with('Test Experiment') + self._assert_in_output('Test Experiment 1') + self._assert_in_output('Test Experiment 2') + + def test_query_no_experiments(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.query_experiments.return_value = { + 'success': True, + 'matches': [] + } + self._assert_status_code(['-q', 'Nonexistent Experiment'], gcli.EX_NOTFOUND) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.query_experiments.assert_called_with('Nonexistent Experiment') + self._assert_in_error('No experiments found') + + def test_download_experiment_results(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_experiment_results.return_value = { + 'success': True, + 'files': [ + { + 'name': 'downloaded_results.zip', + 'content': b'PK\x03\x04...' # Simulated binary content of a zip file + } + ] + } + self._assert_status_code(['-d', 'exp123'], gcli.EX_SUCCESS) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_experiment_results.assert_called_with('exp123') + self._assert_in_output('downloaded_results.zip') + + def test_download_experiment_not_found(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_experiment_results.return_value = { + 'success': False, + 'error': 'not_found' + } + self._assert_status_code(['-d', 'exp123'], gcli.EX_NOTFOUND) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_experiment_results.assert_called_with('exp123') + self._assert_in_error("not found") + + def test_download_experiment_still_running(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_experiment_results.return_value = { + 'success': False, + 'error': 'not_done' + } + self._assert_status_code(['-d', 'exp123'], gcli.EX_NOT_DONE) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_experiment_results.assert_called_with('exp123') + self._assert_in_error("still running") + + def test_download_experiment_failed(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_experiment_results.return_value = { + 'success': False, + 'error': 'exp_failed' + } + self._assert_status_code(['-d', 'exp123'], gcli.EX_EXP_FAILED) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_experiment_results.assert_called_with('exp123') + self._assert_in_error("did not complete successfully") + + def test_download_all_experiment_results(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_all.return_value = { + 'success': True, + 'files': [ + { + 'name': 'downloaded_results.zip', + 'content': b'PK\x03\x04...' # Simulated binary content of a zip file + } + ] + } + self._assert_status_code(['-da', 'exp123'], gcli.EX_SUCCESS) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_all.assert_called_with('exp123') + self._assert_in_output('All experiment artifacts downloaded successfully.') + + def test_download_all_experiment_not_found(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_all.return_value = { + 'success': False, + 'error': 'not_found' + } + self._assert_status_code(['-da', 'exp123'], gcli.EX_NOTFOUND) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_all.assert_called_with('exp123') + self._assert_in_error("not found") + + def test_download_all_experiment_still_running(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_all.return_value = { + 'success': False, + 'error': 'not_done' + } + self._assert_status_code(['-da', 'exp123'], gcli.EX_NOT_DONE) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_all.assert_called_with('exp123') + self._assert_in_error("still running") + + def test_download_all_experiment_failed(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_all.return_value = { + 'success': False, + 'error': 'exp_failed' + } + self._assert_status_code(['-da', 'exp123'], gcli.EX_EXP_FAILED) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_all.assert_called_with('exp123') + self._assert_in_error("did not complete successfully") + + def test_cli_update_success(self): + self.request_manager.update.return_value = { + "success": True, + "error": False + } + self._assert_status_code(['--update'], gcli.UPDATE_SUCCEED) + self.request_manager.update.assert_called_once() + self._assert_in_output("Downloaded most up-to-date CLI successfully") + + def test_cli_update_failure(self): + self.request_manager.update.return_value = { + "success": False, + "error": "network" + } + self._assert_status_code(['-u'], gcli.UPDATE_FAIL) + self.request_manager.update.assert_called_once() + self._assert_in_output("Unable to download most up-to-date version") + + def test_manifest_no_errors(self): + gcli.check_manifest_format(MANIFEST_DIRECTORY + "/test_manifest_no_errors.yml", False) + self._assert_in_output("") + + def test_manifest_string_errors(self): + buf = StringIO() + with redirect_stdout(buf): + result = gcli.check_manifest_format(MANIFEST_DIRECTORY + "/test_manifest_string_errors.yml", True) + output = buf.getvalue() + self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) + self.assertIn("name attribute in manifest.yml is empty, missing, or not a string.", output) + self.assertIn("trialResult attribute in manifest.yml is empty, missing, or not a string.", output) + self.assertIn("scatterIndVar attribute in manifest.yml is empty, missing, or not a string.", output) + self.assertIn("scatterDepVar attribute in manifest.yml is empty, missing, or not a string.", output) + self.assertIn("experimentExecutable attribute in manifest.yml is empty, missing, or not a string.", output) + + def test_manifest_int_errors(self): + buf = StringIO() + with redirect_stdout(buf): + result = gcli.check_manifest_format(MANIFEST_DIRECTORY + "/test_manifest_int_errors.yml", True) + output = buf.getvalue() + self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) + self.assertIn("trialResultLineNumber attribute in manifest.yml is empty or missing.", output) + self.assertIn("timeout attribute in manifest.yml is not greater than 0.", output) + self.assertIn("workers attribute in manifest.yml is not greater than 0", output) + + def test_manifest_bool_errors(self): + buf = StringIO() + with redirect_stdout(buf): + result = gcli.check_manifest_format(MANIFEST_DIRECTORY + "/test_manifest_bool_errors.yml", True) + output = buf.getvalue() + self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) + self.assertIn("sendEmail attribute in manifest.yml is empty, missing, or not true or false.", output) + self.assertIn("scatter attribute in manifest.yml is empty, missing, or not true or false.", output) + + def test_manifest_param_errors(self): + buf = StringIO() + with redirect_stdout(buf): + result = gcli.check_manifest_format(MANIFEST_DIRECTORY + "/test_manifest_param_errors.yml", True) + output = buf.getvalue() + self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) + self.assertIn("min attribute in hyperparameter x is not a float.", output) + self.assertIn("values attribute in hyperparameter values1 is not a list.", output) + self.assertIn("min attribute in hyperparameter y is not an integer.", output) + self.assertIn("max attribute in hyperparameter z is not greater than 12.", output) + self.assertIn("default attribute in hyperparameter values2 is empty, missing, or not true or false.", output) + self.assertIn("Type specified in hyperparameter values3 is not integer, float, bool, stringlist, or paramgroup.", output) + +if __name__ == '__main__': + unittest.main() + diff --git a/valid-experiment.zip b/valid-experiment.zip deleted file mode 100644 index 4245ce7..0000000 Binary files a/valid-experiment.zip and /dev/null differ