From c8575165ba7acb5c9422268b645bf37b27703c48 Mon Sep 17 00:00:00 2001 From: helena-donaldson Date: Sat, 4 Apr 2026 16:36:58 -0400 Subject: [PATCH 01/10] Automated test script for GLADOS CLI experiment submission and result verification. --- glados_cli_tests_script.py | 42 ++++++++ manifest.yml | 6 +- .../test_add_nums/addNumbers.py | 53 +++++++++ .../test_add_nums/addNumbersExpected.csv | 101 ++++++++++++++++++ .../test_add_nums/manifest.yml | 91 ++++++++++++++++ 5 files changed, 290 insertions(+), 3 deletions(-) create mode 100644 glados_cli_tests_script.py create mode 100644 test_submission_results/test_add_nums/addNumbers.py create mode 100644 test_submission_results/test_add_nums/addNumbersExpected.csv create mode 100644 test_submission_results/test_add_nums/manifest.yml diff --git a/glados_cli_tests_script.py b/glados_cli_tests_script.py new file mode 100644 index 0000000..7c9920d --- /dev/null +++ b/glados_cli_tests_script.py @@ -0,0 +1,42 @@ +# To run this test suite, ensure that you are first authenticated with the CLI, +# as it expects there is a valid token stored in the .token.glados file. + +import subprocess +import time +import pandas as pd + +def compare_result_files(file1, file2): + df1 = pd.read_csv(file1) + df2 = pd.read_csv(file2) + + if df1.equals(df2): + print("Test passed: The downloaded results match the expected results.") + else: + print("Test failed: The downloaded results do not match the expected results.") + +print("Starting experiment creation test...\n") +try: + result = subprocess.run(["python", "glados_cli.py", "-z", "test_submission_results/test_add_nums/addNumbers.py"], capture_output=True, text=True) + print("Output:\n", result.stdout.strip()) + experiment_id = result.stdout.strip().split('=')[1].strip(' ).') + if result.stderr: + print("Errors:\n", result.stderr.strip()) +except Exception as e: + print(f"Test failed with error: {e}") + +print("\nExperiment creation test completed.") + +time.sleep(10) # Wait for a moment to ensure the experiment is fully registered before attempting to download + +print("\nStarting experiment download test...\n") +try: + result = subprocess.run(["python", "glados_cli.py", "-d", experiment_id], capture_output=True, text=True) + print("Output:\n", result.stdout.strip()) + if result.stderr: + print("Errors:\n", result.stderr.strip()) + else: + words = result.stdout.strip().split() + file_name = next((w for w in words if w.endswith('.csv')), None) + compare_result_files("test_submission_results/test_add_nums/addNumbersExpected.csv", file_name) +except Exception as e: + print(f"Test failed with error: {e}") diff --git a/manifest.yml b/manifest.yml index 706751a..66f8ade 100644 --- a/manifest.yml +++ b/manifest.yml @@ -44,10 +44,10 @@ hyperparameters: useDefault: false # The experiment name. # Example: "Evolutionary Aglorithm Experiment" -name: "Good3" +name: "Demo" # This is the description of the experiment. # Example: "Experiment for CSSE490 Homework 4." -description: "Hi" +description: "This is a demo for Dr. Wilson!" # These are the tags which can be defined for an experiment, allowing for better filtering on the web app. # Example: ["Neural Network", "ECE497"] tags: ["Neural Network", "ECE497"] @@ -83,7 +83,7 @@ sendEmail: false workers: 1 # If using a zip experiment, specify the main executable filename. # Example: "test.py" if using zip, "" if not -experimentExecutable: "test.py" +experimentExecutable: "" # This attribute, set to either ini or yaml, defines the format of the config file generated for each trial. # Example: "ini" or "yaml" configFileFormat: "ini" diff --git a/test_submission_results/test_add_nums/addNumbers.py b/test_submission_results/test_add_nums/addNumbers.py new file mode 100644 index 0000000..a6a269e --- /dev/null +++ b/test_submission_results/test_add_nums/addNumbers.py @@ -0,0 +1,53 @@ +import csv +import sys +import configparser + +# pylint: disable-next=pointless-string-statement +""" +This experiment demonstrates outputting additional information to a file +and telling the system to gather the data from that file. + +There are two different ways to gather additional information from an experiment + - Trial's Extra File: Gathers the designated file that a run of the file generates and places in a zip to be uploaded when + the experiment completes + - Trial Result: Integrates the information from a 2 line csv of headers and values that the file run generates + and adds them to the result csv that is being uploaded + +How to tell if they worked + - Trial's Extra File: You can download a zip file that contains the different output files specified + - Trial Result: The result csv downloaded has been expanded with information from the specified file + + +Example settings for a run that demonstrates this: (Any Fields not specified can be left blank or to whatever their default is) + +Info: +Trial Result: AddNumResult.csv +If you want a collection of each CSV this experiment runs: + Trial's Extra File: AddNumResult.csv +Both can be used at the same time + +Parameters: +x, 1, 1, 10, 1 +y, 1, 1, 10, 1 +""" + +# pylint: disable=glados-print-used + +def main(): + config = configparser.ConfigParser() + args = sys.argv[1:] + configFile = args[0] + config.read(configFile) + x = int(config["DEFAULT"]["x"]) + y = int(config["DEFAULT"]["y"]) + with open('AddNumResult.csv', 'w', encoding="utf8") as result: + writer = csv.writer(result) + writer.writerow(['Addition', 'Subtraction']) + writer.writerow([x + y, x - y]) + + print("done") + return 0 + + +if __name__ == "__main__": + main() diff --git a/test_submission_results/test_add_nums/addNumbersExpected.csv b/test_submission_results/test_add_nums/addNumbersExpected.csv new file mode 100644 index 0000000..ea0964d --- /dev/null +++ b/test_submission_results/test_add_nums/addNumbersExpected.csv @@ -0,0 +1,101 @@ +Experiment Run,Addition,Subtraction,x,y +0,2,0,1,1 +1,3,-1,1,2 +2,4,-2,1,3 +3,5,-3,1,4 +4,6,-4,1,5 +5,7,-5,1,6 +6,8,-6,1,7 +7,9,-7,1,8 +8,10,-8,1,9 +9,11,-9,1,10 +10,3,1,2,1 +11,4,0,2,2 +12,5,-1,2,3 +13,6,-2,2,4 +14,7,-3,2,5 +15,8,-4,2,6 +16,9,-5,2,7 +17,10,-6,2,8 +18,11,-7,2,9 +19,12,-8,2,10 +20,4,2,3,1 +21,5,1,3,2 +22,6,0,3,3 +23,7,-1,3,4 +24,8,-2,3,5 +25,9,-3,3,6 +26,10,-4,3,7 +27,11,-5,3,8 +28,12,-6,3,9 +29,13,-7,3,10 +30,5,3,4,1 +31,6,2,4,2 +32,7,1,4,3 +33,8,0,4,4 +34,9,-1,4,5 +35,10,-2,4,6 +36,11,-3,4,7 +37,12,-4,4,8 +38,13,-5,4,9 +39,14,-6,4,10 +40,6,4,5,1 +41,7,3,5,2 +42,8,2,5,3 +43,9,1,5,4 +44,10,0,5,5 +45,11,-1,5,6 +46,12,-2,5,7 +47,13,-3,5,8 +48,14,-4,5,9 +49,15,-5,5,10 +50,7,5,6,1 +51,8,4,6,2 +52,9,3,6,3 +53,10,2,6,4 +54,11,1,6,5 +55,12,0,6,6 +56,13,-1,6,7 +57,14,-2,6,8 +58,15,-3,6,9 +59,16,-4,6,10 +60,8,6,7,1 +61,9,5,7,2 +62,10,4,7,3 +63,11,3,7,4 +64,12,2,7,5 +65,13,1,7,6 +66,14,0,7,7 +67,15,-1,7,8 +68,16,-2,7,9 +69,17,-3,7,10 +70,9,7,8,1 +71,10,6,8,2 +72,11,5,8,3 +73,12,4,8,4 +74,13,3,8,5 +75,14,2,8,6 +76,15,1,8,7 +77,16,0,8,8 +78,17,-1,8,9 +79,18,-2,8,10 +80,10,8,9,1 +81,11,7,9,2 +82,12,6,9,3 +83,13,5,9,4 +84,14,4,9,5 +85,15,3,9,6 +86,16,2,9,7 +87,17,1,9,8 +88,18,0,9,9 +89,19,-1,9,10 +90,11,9,10,1 +91,12,8,10,2 +92,13,7,10,3 +93,14,6,10,4 +94,15,5,10,5 +95,16,4,10,6 +96,17,3,10,7 +97,18,2,10,8 +98,19,1,10,9 +99,20,0,10,10 diff --git a/test_submission_results/test_add_nums/manifest.yml b/test_submission_results/test_add_nums/manifest.yml new file mode 100644 index 0000000..bf0e540 --- /dev/null +++ b/test_submission_results/test_add_nums/manifest.yml @@ -0,0 +1,91 @@ +# This is the config file required to be in the same directory as as an +# experiment that is submitted from the command line. Consult documentation +# on experiment compatability and more information on these parameters a +# https://automatingsciencepipeline.github.io/Monorepo/tutorial/usage/ + +#CONFIGURATION: + +# This allows you to define hyperparameters for each run. Multiple can be defined. +# Example: +# - name: x +# default: "-1" +# min: "1" +# max: "10" +# step: "1" +# type: integer +# useDefault: false + +# - name: "values", +# default: "-1", +# values: [ +# "a", +# "b" +# ], +# type: "stringlist", +# useDefault: false +# See https://automatingsciencepipeline.github.io/Monorepo/tutorial/usage/#information-tab +# for full list of hyperparameters + +hyperparameters: + - name: x + default: "-1" + min: "1" + max: "10" + step: "1" + type: integer + useDefault: false + + - name: y + default: "-1" + min: "1" + max: "10" + step: "1" + type: integer + useDefault: false +# The experiment name. +# Example: "Evolutionary Aglorithm Experiment" +name: "Test AddNums" +# This is the description of the experiment. +# Example: "Experiment for CSSE490 Homework 4." +description: "This is to test AddNums!" +# These are the tags which can be defined for an experiment, allowing for better filtering on the web app. +# Example: ["Neural Network", "ECE497"] +tags: ["Neural Network", "ECE497"] +# A folder or file included in the downloadable Project Zip after experiment completion. +# Example: "test.csv" +trialExtraFile: "AddNumResult.csv" +# The CSV file captured as the experiment result. +# Example: "test.csv" +trialResult: "AddNumResult.csv" +# This allows you to specific the line number of a trial result. +# Example: 0 +trialResultLineNumber: 0 +# This allows you to include a scatter plot in the downloadable Project Zip +# Example: true +scatter: true +# If a scatter plot is included, this defined the independent variable. Leave blank if not included. +# Example: "y" for included, "" for not +scatterIndVar: "x" +# If a scatter plot is included, this defined the dependent variable. Leave blank if not included. +# Example: "x" for included, "" for not +scatterDepVar: "y" +# This tab allows defining a text block appended to every generated .ini config file. +# Example: "" for default behavior +dumbTextArea: "" +# The duration before the experiment automatically times out (seconds) +# Example: 600 +timeout: 100 +# This allows an email to be sent following experiment completion with number of passed and failed trials +# Example: true +sendEmail: false +# Define number of workers used for experiment +# Example: 1 +workers: 1 +# If using a zip experiment, specify the main executable filename. +# Example: "test.py" if using zip, "" if not +experimentExecutable: "" +# This attribute, set to either ini or yaml, defines the format of the config file generated for each trial. +# Example: "ini" or "yaml" +configFileFormat: "ini" + + From 052d7d348bb7cdce97bcf56714c26844940b760f Mon Sep 17 00:00:00 2001 From: helena-donaldson Date: Mon, 6 Apr 2026 01:32:45 -0400 Subject: [PATCH 02/10] Added query to test --- glados_cli_tests_script.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/glados_cli_tests_script.py b/glados_cli_tests_script.py index 7c9920d..10e467f 100644 --- a/glados_cli_tests_script.py +++ b/glados_cli_tests_script.py @@ -1,5 +1,9 @@ # To run this test suite, ensure that you are first authenticated with the CLI, # as it expects there is a valid token stored in the .token.glados file. +# There should also be no existing experiment with the same name as the one +# specified in the manifest.yml file of the experiment being tested, as this test +# suite expects to create a new experiment and will fail if an experiment with the +# same name already exists. import subprocess import time @@ -13,6 +17,14 @@ def compare_result_files(file1, file2): print("Test passed: The downloaded results match the expected results.") else: print("Test failed: The downloaded results do not match the expected results.") + +def compare_filtered(s1, s2): + def filter_lines(text): + skip_prefixes = ("ID:", "Time Started:") + return [line.strip() for line in text.splitlines() + if not line.strip().startswith(skip_prefixes)] + + return filter_lines(s1) == filter_lines(s2) print("Starting experiment creation test...\n") try: @@ -40,3 +52,21 @@ def compare_result_files(file1, file2): compare_result_files("test_submission_results/test_add_nums/addNumbersExpected.csv", file_name) except Exception as e: print(f"Test failed with error: {e}") + +print("\nStarting experiment query test...\n") +try: + result = subprocess.run(["python", "glados_cli.py", "-q", "Test AddNums"], capture_output=True, text=True) + print("Output:\n", result.stdout.strip()) + if result.stderr: + print("Errors:\n", result.stderr.strip()) + else: + # Compare expected results with actual results from query output + expected_output = "Matches:\n***********************************************\nExperiment 1: Test AddNums\n*********************************************** \nID: 69d342be8bb268f5b2add93d\nTags: ['Neural Network', 'ECE497']\nStatus: COMPLETED\nTime Started: 2026-04-06 01:21:14.109000\nTrials: 100/100 Completed" + if compare_filtered(result.stdout.strip(), expected_output): + print("\nTest passed: The query output matches the expected output.") + else: + print("\nTest failed: The query output does not match the expected output.") + print("\nFinished querying experiment.") +except Exception as e: + print(f"\nTest failed with error: {e}") + From 31ee67f316a506654279e8ffa3075d23593fe084 Mon Sep 17 00:00:00 2001 From: helena-donaldson Date: Mon, 6 Apr 2026 01:35:40 -0400 Subject: [PATCH 03/10] Added download all test --- glados_cli_tests_script.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/glados_cli_tests_script.py b/glados_cli_tests_script.py index 10e467f..e10d119 100644 --- a/glados_cli_tests_script.py +++ b/glados_cli_tests_script.py @@ -53,6 +53,17 @@ def filter_lines(text): except Exception as e: print(f"Test failed with error: {e}") +print("\nStarting experiment download all test...\n") +try: + result = subprocess.run(["python", "glados_cli.py", "-da", experiment_id], capture_output=True, text=True) + print("Output:\n", result.stdout.strip()) + if result.stderr: + print("Errors:\n", result.stderr.strip()) + else: + print("Test passed: Experiment artifacts downloaded successfully.") +except Exception as e: + print(f"Test failed with error: {e}") + print("\nStarting experiment query test...\n") try: result = subprocess.run(["python", "glados_cli.py", "-q", "Test AddNums"], capture_output=True, text=True) From ed600c7e3d6fa0a486266920f60097a1b83db370 Mon Sep 17 00:00:00 2001 From: helena-donaldson Date: Mon, 6 Apr 2026 22:18:19 -0400 Subject: [PATCH 04/10] Manifest changes --- manifest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manifest.yml b/manifest.yml index 66f8ade..05e9647 100644 --- a/manifest.yml +++ b/manifest.yml @@ -44,13 +44,13 @@ hyperparameters: useDefault: false # The experiment name. # Example: "Evolutionary Aglorithm Experiment" -name: "Demo" +name: "Test AddNums" # This is the description of the experiment. # Example: "Experiment for CSSE490 Homework 4." -description: "This is a demo for Dr. Wilson!" +description: "This is to test AddNums!" # These are the tags which can be defined for an experiment, allowing for better filtering on the web app. # Example: ["Neural Network", "ECE497"] -tags: ["Neural Network", "ECE497"] +tags: ["Test", "AddNums"] # A folder or file included in the downloadable Project Zip after experiment completion. # Example: "test.csv" trialExtraFile: "AddNumResult.csv" From 76e194e52ee95469af8d7211470a5f56e1fa9bf5 Mon Sep 17 00:00:00 2001 From: helena-donaldson Date: Tue, 7 Apr 2026 00:15:40 -0400 Subject: [PATCH 05/10] Restructuring for easier nav and removal of generated data files during tests --- __pycache__/glados_cli.cpython-311.pyc | Bin 9229 -> 37856 bytes empty-experiment.zip | Bin 130 -> 0 bytes invalid-experiment.zip | 1 - manifest.yml | 2 +- tests/README.md | 8 + .../integration/data}/addNumbers.py | 0 .../integration/data}/addNumbersExpected.csv | 0 .../integration/data}/manifest.yml | 0 .../integration/glados_workflow_tests.py | 27 +- .../data/empty-experiment}/requirements.txt | 0 .../test_manifest_bool_errors.yml | 0 .../test_manifest_int_errors.yml | 66 +- .../test_manifest_no_errors.yml | 66 +- .../test_manifest_param_errors.yml | 114 +-- .../test_manifest_string_errors.yml | 66 +- .../data/valid-experiment}/exp_template.py | 56 +- .../unit/data/valid-experiment}/manifest.yaml | 0 .../data/valid-experiment}/requirements.txt | 0 .../unit/glados_cli_tests.py | 768 +++++++++--------- valid-experiment.zip | Bin 1015 -> 0 bytes 20 files changed, 600 insertions(+), 574 deletions(-) delete mode 100644 empty-experiment.zip delete mode 100644 invalid-experiment.zip create mode 100644 tests/README.md rename {test_submission_results/test_add_nums => tests/integration/data}/addNumbers.py (100%) rename {test_submission_results/test_add_nums => tests/integration/data}/addNumbersExpected.csv (100%) rename {test_submission_results/test_add_nums => tests/integration/data}/manifest.yml (100%) rename glados_cli_tests_script.py => tests/integration/glados_workflow_tests.py (74%) rename {empty-experiment => tests/unit/data/empty-experiment}/requirements.txt (100%) rename {test_manifests => tests/unit/data/test_manifests}/test_manifest_bool_errors.yml (100%) rename {test_manifests => tests/unit/data/test_manifests}/test_manifest_int_errors.yml (94%) rename {test_manifests => tests/unit/data/test_manifests}/test_manifest_no_errors.yml (94%) rename {test_manifests => tests/unit/data/test_manifests}/test_manifest_param_errors.yml (94%) rename {test_manifests => tests/unit/data/test_manifests}/test_manifest_string_errors.yml (94%) rename {valid-experiment => tests/unit/data/valid-experiment}/exp_template.py (95%) rename {valid-experiment => tests/unit/data/valid-experiment}/manifest.yaml (100%) rename {valid-experiment => tests/unit/data/valid-experiment}/requirements.txt (100%) rename glados_cli_tests.py => tests/unit/glados_cli_tests.py (91%) delete mode 100644 valid-experiment.zip diff --git a/__pycache__/glados_cli.cpython-311.pyc b/__pycache__/glados_cli.cpython-311.pyc index 13599f16310f47271758f711dea35488791b1a04..c854a49f0018eb15dcf1e16d1198d3c002284935 100644 GIT binary patch literal 37856 zcmd_T3sf9enkE>T8JQXRP96yeBuD{5LJ}Zb4e8N5eNH4Ne4LvzoK^3R&$jn* zmO526Q^&j8wcmdu9}x-IQcYLS?rakL_r{I?KH@(9```cm_m4kzx$GQ%Kl_$4`p>U% z-2X&3nV&Q5`GsiWxVJcw6AfcrKe@&+!?>~EXi&pU{U-K|@8_9o?l&{n(r;m|wcpBI zpe#|}Y?suo3dHOw~x!)^V`U^yBf1xP! z7X`h4$ceUhjQz#TXNS+nd=B_Zn9m7cDf7ADD`P%4e4Cih17A7wdEu)N3;H*Uh5cK^ zqW(&;xW7u|`u(D>f2&y1UoDpQ*NA2Pwc@7!ZDM(UomkPoUEJKi!^j0aSL-vC1^>>B z|LU%c?_C4Oy@!AGXW)$7Nlx7IDkoM3cfX5TeGmWYufIX;6#cLA{f*+*V1rl{Y#QRl zYTWN(_cg&Lggr525o;0lBn#Wd!uBGp4q^LP*mf3g2k!T?`}!=-PJ|uE3)_XTW)`-a z@I(XBYCzaQ7FHG9i!wCg{*c&&=X-)rpbV7X69{V&p9~%r_pB7qMBIUN!fh6k9jK zXYIs`ZOHeO*#0Wl-xWM9c3d-3f|3)xrt4k!-pi(sUT)}N|Me%oq|okqLyA8?5E-dA zrA)08w^Drj=wKuzoSTe{h9(Ba@K_KBBEiV$craQ!5{XQPTbi1N#|FeuxN$HX4mL`m zaIj%yYHWO9qA@5=HKmHrU4L@#*y#1s*E)t;BVA8FJ#nM!ye95wWMt|}<6vmKX)H86 zI?*Hsua6D}n+8K-FuE(7Lq1k$U@9`wG%z?A42J`e(6!)1w4iBVaQXVu#yY`?=%>Tm>E;Q z^`3Fico{V}ZN6;=@nr4|!wv4H=`wf25aZ*TyfdkD+`I7R2=BBdW)X#4|DLwu-ln-1 zaBtV#+i>sD+}lOxv?Jomr7yZ-c9hNyJWmeq54m^vd`2MVhza1g@S8J8)e(0tjcKQ@ zHm#ypEVyTyHbn|@I9F{Mf6SJj@&>i;%+z|!wzk$$8e0_aG3|K7nsbSTG1qlYa>d}w zz3H?&<_7zQ*6HD5t~g^YGbPQqVv)YYVsUzEG~D^ej~vIBj$?`0#1d9=+r-UBj$N9L zO>An}W0rQ4CT{s-#;wrA-Tatww`k&4rsHb*&|)>-rAiI2_jgB&rQm?rLKBUjKuZwQ z$5_2FWyAD@*(xw9rp)2N&}1-WX~q0K8HsuaCMU;62Ql4+CYr8>Lla*XqshR6@ZdED zQq~b92wIe~65A1)iXbtkcIr==gOU`IQVx}O2oUd(vSa2Rm4ac=qLeU-iBh^gFqX1W zkr;wRgd8iAY1_)?~0nQ`CBsREe@k+W}H` zh7%whX&Z&sQK*mDoe&acbrhHkPKYRB$`(dZ1H^1d9zxqZIx!fMq~KtL=4qlt6p8D0Ne$CUAORX<4%}sfGiK+6BhW+XAlq+53NN+SKM!nQ{;V@ySAt_%j zcg@>HeYtGaU{uT!=w4H132wc2yHf6)~($U=; z=xR?1k~*!0Q#?4NNXna@Tsm%2Cqmar@zi%x=1B?7o$|J}wRQCL1bWY%>F8!@@zj0L z_r`+3$&_h07^(M2WmIh*O>xnDV^S4`I;isnPL4)SOyFp&w7&Hx>@r|ZNrjTE;)*8*0E|d*^aFgmfvlNdls)fI3@Ru$Y)07 z@RVG5JyCdFDZD=0Hrw{=6;G++*|EwQY{zmSS-WE~xY!~W9i_{efBhQ2RTE;sedsBC zcDc&|~OO}?+I#&dj z?7p#h3xKW{2IMOd<%KErHX+)UR@G*Lg#-YWsp)scN2cCU@*x zipZtyba~s8h2?MUeslNyjYQ#grEt5v>%?;5iB--)QM_fzf=zF2eslBuaH612DX3f6 zwOp`&m9wc2=b!kFb558O@{r`6rVo9&XXo>M{Mq+-f&{$Q$7O&~64ynb~@5P{X@%#E>#h&(78aHY`hHhEd}1H2mw; ztY6EiiAHdKruX?gj?NS_#k0I0QqTVKcrhM)As5JUdmHf~qFK*9%+MTeNzaE^b>w)F z3?Bl1GMgv(NqR%}dmMhUUg(xO5v!9?_#c^47B<=3k`5wJss%`yFr`M_#IE}%LJ>@0 zQxhVlNYm7)$VgVbRoYL;#C4~H){DKTh#aa3P}0&<1eyWrEsP+t3{!SCRkDVXjuS>Z zfjWRNy@^V)R8r-n>4~Np%;CXkMINEjKDl)@eiN!>uKteI`Ub2{lMeIxdIY`!Dmsa8DI)JczHSk}mmN9EFE zba{^@D{A5$_l_<0%0(btEwZykL)#7zZBs$AqH5vd!cMtpKV8oJD?{53z!RqW{&uUc zwTAn!%ucYzbHZr)@JVy)Q>G7_4dj2y47h&JE=GTUi^J+9+l$dFvVL8UHoR|q7ufG* zN60k#_@ne@0|z`F#a#1y7a^Jz2u6o!%Jtc4KH|tli5Wy*pZ|WWb7JN=wfvf2UdkkT z#h*UDftzsDa8Gby{)SOqDH*uSSdKyjWscAsnEu5q`cZ~8jpZFvoJ0s~etG#?V^~E+ zfi*2eigL(TvulBvpieuldE*>@UaGd3Eyia#V5?{q1zKZ>wu~=tlxIZO97efk+8(n5 z)1F;3+Hc$0nh|bp%~&1AsPfdao2$1+K zWera;?yK`hMwKh+L-cx=I?ic0ryPA97kawRshp2j8W;@+14AJx02x+fD$E9diU$)8 zu8H{dlxYlJ3nWd%J6lHv!Xsm&S0ED@8F(UP8wuVNM~5L&P^pANWt#+Wk%37F3sv$^ zP1)L|;CKiu3%!A1C}mB{no<^(hX?l+2}!|}Z7ei6Fh*D^L7|4w@IWe6v4YZt!PiAg z@(4;gJcLMLk`@px@!1X#1=;=1%Zn%E!sbL_vr^bh^SS49r+?Y$PdKX;XSM9CeppoU z*1UljP}M&f*;V4G5~jmSyOLP4^;;`?=P*L`oty&ZE_ zus5z6_JDz(6Ur|MYg)xF;TWlDckjV+5Y9S{fV+>rL1|j>!GJ?{`9h^Zk02+jwb7N zE`l35LYMc*!@6Bdyj=PeUEZgD{Y93q)6%{z)eBRPMESZ<+23xhXshQw+~R9J3YzC6 z|55&g(fX0m1ph~7&_C-(mewt}`M8!RSnp}Knm#_rw;N3#A2GoHiO~$G6Y&s9VH5wq zK~w(L1hPxQsx?17IgxxD2vdwP zQ$E3r_;QJX!7=NfgCO%@Rn{e8F~ij?8Pb=N?Q1;_aR!5<61+SzmlZuRGKZ-FtPlv} z*NMjJZQas&ypr?`z*jE4(2x&=#>ABI#+A-q=@OpTyEVkj+kWm+_t|r;?Idkv%*$m; zhk8xQb_23=khS1As1-p=@aAYZ5>6RIkQhLkJ~l8Il==zPd_x+=GGC;)T6X6+J>-d! zr392hrD=nNRR^$!+p$bd5)qF2^2m}J{EsL~OPVTq(^ToRs-~qvx$sD$@Q6})WIeL< z1y)6#M9~hVXa~qrX~kQ&-@F|ce>l2Wm#90KC_SW<9-2M%&{Oor%dfxu+Vq`iqAI&q zyoGn|^ZVju3GYtDyHob=BswO}wzmej(>+JRTdR0$Wp6E! zGe+lBK;kuyygi=7A30_>QQYR@J}Rj=@s#PK zC!FMeif=PmKQ@@)|Cl$B-)tqn%hR^Y^zmlCt45v%jRBnHgur)Z{TRUI)0RkS_Vu`! zy9_%8ZT@dQZg@UWxk}GFEFKO7Q z6-l(0{%zcfk6356|M;zqb-5Rk?}A*tpf2qj^k9GH%{fjVhbB;P#+WqTFU(uDn}2sb zRV)(i`f{i}w!XMK#SdX0Sn5H!qGeifbZ8K}-3{P;g9GEL^q2`oJHOlk)Oubbz9?m; z$YE79NJ~Uj8fy+=Q7ok($d-sbXOjudPLo)#rz{kuOH5hEAw)!0f!0o2mzWIQx*_Th}6}5n0(+61^ap~o_M;eH=vK4 zD(>lRy^tSTuPzg$8x&EX*JWjjQ698bP*)8!!={SVUH3>(4A2V)2s!0a_vfEh>nG*X z;!LuFh1Xp+RH*R+0}~>xcBBZ>%uEB(oonW7+P_~H_idCc{2C?zNK}h_cbh*is#z|o ziJwap?Nf^O%?b~lg>%oo6N#T&>`hd)C{-=;kv_Tj(yGZ=V4pRw3Y@EKzW81_?!ER| ze$~R;s+f%9Gyndj;Di10rRU_}uv|EjC>&7=M`qhpQRPdoJ$>itS^Q~)6;+spZ7Ntn zrop+EgtJm{R?2IxjIg2#o`()MCcF_?pvcA$h)-fXlt= z=3GVJZJ)1PC{7eND#eYtS}3o)SDq|ySQt^tn^(;yFG>J_3K6Wfs^sRTMVqp@W%lV6 zXTe;9UQD}DMI|>^egD+kr@nXk-f4qvC$%WxWVOS{)=gH20yCy@YDMMqGCQEAKCzX=jXfdiU71-?I zQ|OXEd9w9M?!zZNCn`)I8N4S-Odpk);s3`LKk>8{nf~0{>Nb7oHp9zy7c;E*sJMPt z`vY`{zyGclGvyRHMDsP)Wu-Gp5Xx#$JDoG<)V94mNOEz=qf zq9O0>Xs_gc+(7$l)dqM!tSM-1Hhs9yNd5x`@;6(_f2_3%&c~G%?Pk-*hn(<#V&>az z)=z9E@;eRWcX`QQ>FE$mpEP(oOr}pwX86~igMP+uJO{uJQ_V*omOMBEyZhjM;y197WS?gU?$a~pJ*`Q8cNR?k5F zQJ#1Z#+7Io8i@oSs!U@#Jv-XZMo z17vuV1qxuoBc^{sf+cmWB$4j`K&oOr=H=|UlT0S{PYI1NEI%OMKO?|IT3Lo<{rLS| z=P7=@upW=XSP_*oA@ai5ldu3^hBHa4v}Bo$fw8eQDlOWF{|<4(FJfJhS%s}(J+*%WI8{(npL&6mj~P3pB+oG3V;6wpq;YunP+ z-#uIS_fe1BmaX#sGxD>~DbEhdyG6Qo1!d1LUBYnqZ!pg8G#uvs!qZv{Jwk75mFdGO zGyETUcXU{|j}A4oZRb9&Gr<4x_JX#<&^s`azr{fQ!&dk|F?k6VRkXL5K52B4zlCo< zYW?J>3I0zF2KYZUGQY*s(PaA6=k3^O`gEt6{Cjtt+Rgpd=7y7Y?x!{b{6Do9oUAnc zw8BXKEe7&eTFGBmMDS3>DZlAKu@n9We*RRg^+BzP{M!xW-@*L5JzYN2gTvk~uj#+@ zn&I!p`5l~}p`~9Skg^8?;~{ZsjQq|(;H9a7G4+X5PNa)ONGTH*Y7)3sPU@gr3pB@} z8ZLEH$OwV!1l}O^|KzfV5e@so(lVH8gy*$o3kti#_1qdJrQTQwTYE_9|0u+*8` z4RJo+vrrR%_J@!G)BQT|g28S+5QmNooQ2wj@M7l&rx#vPb{=CP`JlW6$om#d3w`hR zFL;!igR2}u^1+@0^MQGOz9(My!=3X1r3^$9A^D(gpV_{&ca?*4f8PVsgT9~jKkz7R z&$5twFlMls%NL8FjcYDnDqb4Af8djrrI2!{n}y_ql1=8SB_PAOUwnV?!GWK(+z%m3@>&5<>|#&lqXNBL0=Em=3yFkeMpZLsQ&_s$2 zY&2Nr7lZPwP8O0~jf_H1=7)y=!T?T`xC2wP;5=!|hy9U|KODg+3#n1Mg=D(1$8-oT zy4S66%A!7k5*j}s4dedW4RXVjtlzJ8pIP|X?CwWF!qvE&;lAK)v-@Oz*LMf+UWF6y z1khgW^->;4N9nv;06So74y@5zVs7PrLH(AbgK@ZPfBKceG~QA)%oS+<{m4N z8J|{sEkb_>+#&x?o<vsYwX6icl*HwMkFWY(yPQohXKG?N7an z^|0gcY}~`vtnx|eT+_!y;}o^LDY_dHwNX+KLaX3BQM#5zzeWujvOu-P8W}f=Q`d}M zqoS!^RRQ0h!xyG2vUm97(&?;c4xFE^t z!(4t5X9)+PyL_uL`rO(q1_vhS=*5+wKdYtS2kjgk^AB7JT@U)TN#i6TMC(_kh(NHE z@TQ>AL!&o?Vk29pcSgPGEUpX;;uN0P;#ZYN>{p@aNTKs z2&WX`lq{TD@s!I#c`jCC!cng{Ak4R# z>shRhgwUx7owCrm;-NX@i>(J1Z-072IeA&`A5EORs+_!<*m_ObdQBE8la8&QZEKMo zEy&bX46h2cc?0j2tY>^X=lAskYKL$Yuvxua>(s_Zzp zbar{iDcSSvXDFek_OGGF@ho^Q*?wxVdFi0Cw;ew9O7L9@-zD>1D-NH``&7ou%r?Ds za{*C7HW&OEn+rB#x7JXryvYz$nNmsr^2OeuiG1c6he=aMz1A>bT7_->l&cr6wsSB1 z`{z;WX%i_%>xJGb6$=acaGc<>;9S3T+A^_S|3nm`5D-sW*Vk){S@k*NWDKvRn%FAM zip8<@YS&}q?X^!Cy^Nwdh*BaQcaE5@cY0qJ_v=&kAT_TpwKddUms$bv3m*f&i1b9) z&LFxpi^*ofH{ke?Z;3Wvu@oU?x{yupteIEu{93Cai&><+8`^XO5l||ebxsq|?vQ4R z2&-q~8l%F~Dq2g*o+h0E?9QU^a8XltQ)|l7+1hosqkWil(&KL&rv(rln~65(@GFZ4=uk{g&s znM1~yAIGPF50>&PLsP0S5{lqpjV3f4b@~2~#C)@`V_W?#5=ME{3EjcRQkPsRbp-~nZRnonJ z#pTWf?^Af9Xe(t~BQ zXR&QtFfWWLI}S5{vciuuzrHQAowJ<}y}r3I#aj~}N_cnAwyspvC>2jEZcS9|Q!4fm zR{8wFyMftGc3}BfLO8An$7SI-yFEDH5pVrom+U@>Yr(g0{rxSfKTEsUi&Z^$aem+3 z=VW&^u7psd2sN@$!!YaSrsn(ZPRs5(TnS;jB5aq1?d;Lk`4ZW^6<0#2R)lIQ#S2~7Z(nOAHAR)>{0Ik6T%fmxFWMlGfPN+fVR@j5oQFi zIpR1P@9`E)al}~r6W3_Vhfje!*US;H|HEtb%5ZPit|4%5xkpEE)=xi=Ugs!FX1>tU zc^t$+D!ENs@Wg!H%0a+9q_NhbwbY`u(gKR^T!;77Y#M@Uxw=)J?<=2%RAkTbHh#5P z6O4s^7)<4uUz(bj0I$^eRlVk^I4Uv@7q+z6XxFiWjW{}NL$cu9a6C9AkQcUnR67}g zA(-gEsI5cWM2j>@rr9rl{>ITJ$QM-8S9x4#Bgn)8Qc<^kh`?mn%Ru|N?v9iT3?O4G z8G&W|rz%$<67wUoN@A$TsU*e(O%g^+^3~;vGWl!tMx=j>cp};^{3q~0)R5)48WQeP ziu=^M9GBHx^054gA0L)C9ZPO%{#O-pSx2&>^RHcUc^}j-OG*6_K;Xcu?RtIJ+;GBP zsn{!jY2!=%uQ_H-vwgp^b2iUx_%+8J2aFqde6udbg7w9%3swb|1_~7a-o;MEe^|En zsaJx(r0|zy{*q>x=|&j!d^W<0@izbe9AWumFmJBMQL;W%Qm6Fa1473h;9nE?Hw0KW zeuDNWgSk~8=1_H?^6R=TlZ&$*OExz?q}2X~0Glsqs$_Ga^gqIv*CizumjUX6bHGS< z!JW(QoeNJT+=mqRp?}aWcr?KuQ}|;te@xTSqydyn*W0r6ocrDNwp|<2W}Jx9OEX|^ zib|CJ(U}?0QJewK?-3+u-7r;a{)wM(fiBfG=c81$KhWcB&!S$GvR<6HHUUx>t-m-i za0PN%$Q^K|d1zD`_ot5``$I#197}ApcaZKIxU@;Xo#eY9)({zl#$hx~1eJtn755Dp zy2Ydy;X&xwMbfebEZK?cVh?TvuRvuGhKDj1{8Tyc`X(0b(nSNG5Wo`D1FB4sxSFc%YtG)E4E+EeMz~v-lgP@SfvX?~6eK;cZg9O|WaD3WleQOW^Z3&lH(^$wCqgUpJCq_&SB>!H1p_Izl;rWWks4G%21Y z>}uI24LE)v7fvLcA;lSzGZ*&Y@km8rD`wrf|oe`i2RJI@+P0&lViC*3j)V zu`~$0M%{^yY^A$X?XtTz;jUBMbsKc1O2!`_P6$U7;fO38Ny~tUJ3f~X&MU%sSvbGq z*&+*DR0)YIEAV2cgB_}ri=9u>kxDhJ@2`7&2x>28u6iXF5>X%yXaxLqo+EtjEN3jej|=^ zU7V~Y9%;^rt~F)Ri0tt^PwRZxiX*W46g>GUgiB-Q_0Q&u-ZhhomxM>spCc!Y8AXwY zn7?bIRPt)GAV2^7JRUKf!f#4vVkobaUX=2A2>&%>J<|)o7#HN5WPuUNjc^Os;F5vL zqV#5f1qWl({&20<)QVXAsPDSA=CQ9zwOsp%wd4Dy+CklDgi2y*THAC_UU*qLT>l=M ze*5>(S#8J@T8Y#`*1XLVjdrRk#4<=x^HgN0;UHOCW)@j6A@?$&5!<#%Ik0~)Fm{27 z##27c%~==-?w%UI0^t{JIs~%eFIYVwmmqBTlE;j^EwoD-$sq3VrjQIFc)M9DV=Whk56HS!Dh+! z?fx6E8|6v;xL=Fol@EE$0qxBq_n*emUQ~5{{Bw@)Y!mKFhIvdQ~q^#K_K*! z0wcjOROUt)P_f8hY+XN%E1}R>24&a!DEj8gAX{)+8BMC0R87!Wl#wWrc_@a2-9sDh4OXvyMRTLrROln=~EILW+t5 zvo_c4_@`Kv+ZR!gs?>sh`UIZSHVQ1wS-iQ>S&i9TTh$gyi9^g z!B&@a7QNB^diQ4)hnG6_PmnlfDr-qLYz#O*Nd z?NxMM7{!Ls!HqY**3699sQ(sbQqh>69SgL4UB0>k^sXRGgnpLBTp%{C{vEUBgJP`x z!LCl8DK*+|imydP2yFG_<65-q0&J_mN>9uj1#TWks98M#3+F!8#nSSX*z#H*YuJR1 zGc5<1Cw`S~gO|uC78L5ZHy$S~4EmBy8}G~k-;u7}wfOuR`~~1U)A&9ecgBOiT=&h8 z%g2c2&bFbJ>qM#YL=?t5DPVij*oHfQqib_7GZbF1j>v*XKL7sf>Lvt3GT;IVyL;7H z_PCztLscN7JV0~WcI~9Lof+#LpN!n1iEMl{OzZf@PU%-@Y{oobdkGW%S9#OEpBZ{= z4-O4Lm5T^|BnYGNFlm7qFOuSEULxwRKE`$#)CGsfU|=a_XPm>Z6q=fhZtX?N{_tdQ z5O$CuTR_?L1@}h}Qt?P8kP(5vEATu4eU_OCa=VJ7W)PsFuaAhik;sE&qlC*cc9dB`(&`SRo{%y_3u_8y zgxLeNUe%gVZ8rZo3b@WBX`qw|{MYyoF9KlA3VW6OW@x>=ar*Vsb75F*a&A?eThsb) zo6P&?&GEvw9i&mZpOM1z3E>$-1Y__GP`WyO1W zWrNTs7Pl>JmQOw_KNm=x3?#h2r+9x4vhfo4Y$t@@-lDk!uf2Td!89@;4@Tr0FDC|HPBg`ordSU3$g&Tj91EfL$(Np&U%ZlN8dRDF^D+2>{V+TH zMe!!NJhmhPs8^zRS}C5Ei>H%K`w->VD-QSU_16yHIV?N&$`yNMelHUUNM+QHw0*0a zKr9Gg6UhGsk2-Ida<&&XO&fI8tJ8JXS7{&j7DnBjauV-K zB}jw}X^w(2Vq+U$Rgo6eAYv+i)%H`)>Js`ybV)wxHE8Eo8kdTCT4gk~FJiEZis+P)~FMK&rL{ zPKu}cQ#aFyHcCV)p;4Ocv8mb&P1BOXeEOzSIhA~r5P{F|AN~L-f^6+-lKCCjQJ4$P zpSm0Rqe#|TOqxt}Ew;;U?10mi5Kb$?X<0Z;<4V|#YsKYNUvzQ*QrQPE)Z3Quwkd$F zHpSIOe4k3}dMTA&^-2gA72%>RTvSs@Ixy5ZXRGnEeZBIqzc&^Ln0S4x7LjqKuc)NyebM zX_U^zOj5nBmtrVu2Oa6?Z$jqkPd_|2N_nDsO^TX#$yc+2+;9Ez@e|$amQIt_@4j>! zy(2hbQ>Z1zhYsewE7V>$iWADO7vZ;QH;^w|cgi%5FDLO0qZ31Hbm4yBW-l`>IjIU& zXy9R7{RRyudEh7MLxe29z|W=zsfI!wnbC&yG&ch`^Y;ylDIS6U0srCu8vyHP92DHY zwCz*h{p%lY`DBZH{#m*Inmi#T`X$w3!OWSY)k6!O>!wvB*J@-I@5@3aT5*||ram2f zuuVRH@n^deotKnOOu?FovoxlZ#^ll%_`(ZDiH1nnc$1u{@f9NiTTPv$Xn)f5v5Id~ldHxZmpl-^cKZ_A~(QG#B4)d%sWj8Sr8##R#o+f1h@&o0wB%CpUM zo-!i`ffw){Kse8v#>ts9MF{12)#)`(5hgj9z{Hv<7yNzfGpc(-sIb|H7n^EHgIg`d)RBJCAk5k^QrjwNS%f?fbDLDu{ zZMr~CuPI2*h-r*adR6C?ahx#8!87x?NhMa!VHli)C@v5g8JI{p(?;Uhxv)VEkB}o@ zljt`aZ$d%1n!hlF$<}ZEXSCc>7ct=(GKnL3v;zmr9;BMngqd*b2w2`zvJW0JUPBD@ zXWcmf^%Ou1W(%^%Z1SvpV_A%EDNS622^Wp<8i)0tAdPgrhP@^=%i6CSP3RYV#6iKl z4xn`{E^JyN*Nhn+EkC8nLs%P+Vu?1!?XXqaS8hFz6ZK{0?Q|<*hZV}~Ul|%ZqeYoV z2uQDl8l&E9pt>k(R8t7%FXnndX4qP22WpuSA9cR}XQ*=6?$0h?q+Rr!$dI&~Kq)|& zm{dResi$)o0VD7~)6+@-%t9(oFT_Y@+ol|Vy?+op*I7e4xu@llrztsdaV5Oninm*4*B5Grr{$8< zgh4K@g!hc%JtKS1V0|>Ve{TP(1L;GI_$xPObI%pY?vuFUXH^$4t0vA?T1kgzc3-#v zbrnC_x|REIYx9ZCrjIHNfI4O(yAF68sE;)oNr+SdAyQ^E>JF#Hbh>bQ1~*)~PQRt2 zY^HVkEksY$ck31)lYL8tv*E(bIhK>bLPZ7kK})f64r>!ik19rJ;=l zU>!`Y)rX5(i(Fg@;gTX;l7&mT)?#Gz|ND(o|22&hk*4qgB^(T7TPAreDfrWhTj?yp?Bf4B-?v8}!!wFlBVyhv6xj!CF3Wc8wWy?a@ye}bC zDFVr_X?KL|L8Srhj`(SJL=%>Kn)9W1NAj`0U?)GAO#C|~drkk6Z?^uohGy#^ij@6j zo8$)wdW0sS<80a{%_j6Nd^xR`^)2`4aqv0E$I^3%;mfsXDrq0`m;MiAmuaHs;Ftc0 z0*ulDdPwc0Ya1KB{grN+LqKp*%e)Gtby{X8TE<)cmgh}R!n;-RZvEW5ec8J`;jLG^ z^)T{^27(kT1E_&^QUhtia!+%DP5-pgXr z$1T^ltq2VfGWl%Pi;ckIZG;a&Q-n5g(!aGfrM@PvQ*`O#y7TQ(eN9}C=+(t7c#OD( zVv#Ox@ngjGi6y9^Qq&OI`n_y>vyy7|UNZH8#ZkT~de~sh9c2YOPfSbtmro|4Ggg#& zhLrF@N_{)xppsAX+?unpg*>vY0)KX|1pcgY`*5BPT6hbddNS!j)VVd|$G+7XeTztS z?kV=mv@uUbYID!7X5S!QoD0qQ#B%)-CNJL#{ZqX0&Rks7PeblbPpx9sKxyWgwsg{o zNL41B>QtY;Hdf>b4W9LCHZ^^WX(1*=p3GsUwN*U2+S{y;gWkf$%X4vZK9qE$d~A_s z4W2DRW|z`GeXo!zU_yNh^jP#QSeai2eN9g6irL~@GAU|Zu}U9v=CRV%)(z@P(?{{8 z72;RZzZmtQte%+ujaa;Fe@47xnOM|+cLK>Wn{x+<;uP`ebqG}T(|9z0BmjCPO*h2RSMRLGmgh(aHKaSG5PpOLM(qt*Epik~ zj6Rc5%Vk+)j_jaJ1Jugu!9Pv;9wiHRE{g`Te`bOvaP(|8A1ZY2JO$HQ*B#b>lq^_F zvhU~^8>XUlnY6?H^)!pqO5}~wRSdCcclvN=*95bO0htMoO*Z<|Mp1A!%3rPO%QxT) zWsQ@!{0$A%d8%3Oo}#ZUpvS8x&e5|P84DT3W)M@~$B9wauQi4{V3J^h%;TV&liECe z7nhx)%4V4v5vXzXaaxByZ>TgzkEAncoq%}&>N5Dcgs55p$bJSFDsj+nW7%Q9nYAfP z6vCsk>FDUhY;~^r>S8+M2318(HRQsE^+Yg2Cl)4<6<(GNJ~cV%4C=CetMo77FbhBU z7O(UH-3T!5gMFcJFy(HQhS^u&&(p1h&n}DjSVB6CNdcG|0KP3p_JCCF?!ToZ@NHa- z+p$|XSBvjFt3x(rW@kftk#LOa-V1Pya1^^>$UUoOo*KI7MV{jW1A5 zLE#~QI>zTO)b~|gQKDV_l!Gj|Wb7c4ex)i+BdE$b+9@pph2Vw5$4#`NfHbSncenhV zo_qbXwKMI>Ej95znJ@qSwpUMo*L2tZh0CY7YLl)_$>Jjrf4PtToCEkJftk)#3uktG z&X+FprE=NM#Ul_9pENvcl=)J~kIk1!E)7UxL>pQb`v#%k5*xJi(7E{J6}IKXes+?y6jNRnA8eu3FU1Oy@(Z>s5U3%49yc;_=Oe z5}rE6Qzr{`UkJF1zmgD|6`@%cn!j+A%vcTgPC;%E?JCtCgHrGI4{V~3l9soeqPwPT-b=?!!h-nVKZk| ztOaws-iaivwTcz8RSWbA@xt3=O2cTYWl0PqoL3a*71?=3EgHI%trp$L?t7!^^(u%K zY*h+&22|TNgAA?D^vx$OMrJz-P_6j{aiD$L(z~aycp+rHaQqaksDXMy4J05PC4=r3* znocAN+LVGeD1zb(V%2fj3d@cRRM5=RsuT|+D*K}~5e63biz5K}5(-j+_1^L;B&WY%D<6)ytNr4u=d#Om309hpk zf#g;HoEN#Y3W7}}fdzOAAayg_d$=K~d&)$~Qz98U;zww-QbR%4(JJUA60;qV(pm-X z9kP4JTu;0W&g@G7c^9immN~S_P2s6=*qkB)7^ce2*z1B{>HxSviPI{J%&NH6l;*J= zza!z^p#U;tB@3;~js}@;P}>N|+27Z+k#>7YzoOEw0(#x`-n8QIR(z3_={<{h!}Xc&ZdnRf6{`ykE{-EGF4kq-cQv8h~PS*PoGEkwnT$ zEFwO8%04Wp&OS6bO84K|YN&1L<^p4B`qePMz=|uwj3D7beRO6#OdpjR460mqs-z1aq9>zq>2Hl(Qjt=Y z(3Pv;7}QFirNkBqu(_JetA9baY#T~HxvFzI@c=3}@F9FDUK~YQ1}d?S0Ne7WttY0W zqU!Up8Jtb%#|h;FiYN4tWkAx?VN)x0P1e}bpHf_XU-(KmqT0$lD*YbtaB*SUYKM%@ zU`TSF8TOat%4GeOEpgL<1t%gZM{3y&+!CgXPdF`TRL?rlTVU`m^uqS5!MlVC zhVuXggCqC9sJas`sqS@Xb|5?-)S1-;;7|f^C;>Qf&n4#O12;uLNhv}Ra~`0!C_*o6 z%fihEZj+&84pjhWUUisX1MK)z$Q*KolMn1%pP}C%i^GaIA>WFDIgc9J!E+xq6fzKZ zGiPyR>H1&3@*u1n>roD%I@n{bi)jUAP;-%1j~k7K#<}ZvUx|m6QaXHLXk0|I!pR4M r!LSwlTn1P;-Z>jxHeqMpU~nXPVW$0Cov(Jzus8;2Q_x4SnZ%lQ=|*{qr1q*eq?|SP+)ceD&Qglu?DhObb`wqy)bh zyyK>*dvKX09uisv?j{ot3qdg=1jM>2MhL-ky$}{dLWHz!neqyC(AFT--=yM=!WPmR z5oi^71^fo+YZ7SDHM_M;7wRxCvrM@Otwzy3yRF<%CcjJ4SZhGP^LZjH~ zz%cpR3h!wXnt_KG#CBoZWfzi=f-XvY;SSX9*%WD@IZm#w)+g9HqBhMa%6wmMZzh*Y zW@d7-(zpMW{jWUv(+@xV@C2?Op1@*LPmtmhT$~hKc44jBQw=H~QzC`C06RbfCvL$d zGJ;$5z&bF%pI7jZXP?N3?5w{`W6D*TFJO8B6L#kRfXMz6asTTmA#&PO%dgODei#8Zp%uLCM zz(3oQ>brPO7A5)Ol(ZnGW@P!|Tc-yGN5>P#PM^3qokBADnWs>t43Cn#EM^U_Bq|G1Hbr(HJD_aBrNlwI6a>)al0sPD z0zk%5JTHaeW(1Q9%8Zy*(y62(s%fv13Ey<7M_xb{y=rEzDPTYJFm2Ka+pslqW*TXTA7OU8lRlbJL4H!t|IZLZ7&i_BS zYOm$0XR%N1RzI-&>gxs|(+5M_FN}{}wQREGK|0B4BynS#-y22=ZxZ8<@5lAHFa>7Ce z>%m0g%0e<@_DB(6#c<1tWH6JtTt;fbXGTb;6vGRflFm*W?t$#0ggQ-n0l{_zI}xD7 zkpo6#OhOGv05TzaVNLS(Z{0bsxQ508+RT0oG;P2`COJ0D#AN`26)zdlxrM|eTx z`zB@WtT{-#a8bMT7uw{EKAF)jW%LVkn~t=nVlEL`od#H$6kK-kKn=Ix=>Y(sLGU11 zWeKdBQl=>(5H*tRnMDj+Yfdl%$x!OrON_{j7x56-h z>c2Tj)VmIwf;{7Ce~Mc8V+!YF+H&CArFNfBX3_#^WMF+p1af4}r zF^C!5Rm4t^CsW(grlQA1v-EtJ-~dGn4VN_vmyC;?r`Ekxu>SU$hun)A^P;62CDS4% z=1h|)p>lR8NYsmD4*vl!a%%Zmd%p_y?f|x3mgqZxanHt*TNSi*RuNzye+_&1iQ9=q zQ%czn0nBkqLHlNvT8cMZORitEIH)p9u3uB1xHiy4aka`g>_nl8lX5#nZJ_U7au=$& zsbC5o!Fva0eb1h`eAK~*@3|vcCvTf^yW|$wiXSETEf3JARLvsmRq()B1=b0bmsNxS zP~a*QV4Z`TtTS%5Rp3O1)Klk(A(nZt#2`yxiLuI8G&4V)zAD0z#N!I?0E!&es}SNR z&hjXHF?Ab}q&|Mgb|FXyAAJ^#^4XliPvsV}LTp6!S%`*DUPz@xSss2?XEjYGQ*ii& zKHl(<79%hTe!^5vnoBC`t}4oeS&$uz!qh@0v&ip0G04Bp8(v@!jF{@`igg7IS57t< zWZv+mW#p8|y1{{;GBP?bJbG?q(AZ|nQDtTc#5I}#?Zn9Wfzu}j6GIoyCWc4H&J0Wt z`(U`iuo8dg+{mes(RW6q7_6;?kw6zC9mG1Ww^3&iM^kD}%4gX-nSp+=jHWc3MhEw> z8!q{O0DvOmLbncn*#ANQN^gj@U;(KyS-R=smSftx!p+SU7FB2REU1|@^4?hJ6+syK;LmdiyVS~ zkv*)lhc)(aX?O3cPv3p${^^Iik86<&{|>K+bpG3{#$Eu`ss)a(9=v}@-!}*~^Iv36 z=*$U?IkCnyX-pGQRt~j<28F04G#aQ{3(&B!EYTGu`7Z{C^x%*d99nDW z(1IP`AjXlx^kVpRO7lW_s!Pm9m>uU||TCjZ$CFT0ptGH|6^theUw4b^`3(OK-K>6KJ?m){uyY(tKIo3NopSV+) z41%AN9t}={pR{l6$#OfXpHVV%&2`=VE_Dr@#&;oH0oM`6ba_Xf(NVQE%BN*MokdHR z&JyJerU@-f%%CMT1fEfP3c{b*Vsb8{wiE2Rv@An#z(*aO(O4H__|)_~F%^NdoG90Q z@$~#K+7A{3k4#!tAQ(Oivou`lRJ8*t9fL*!PC`ZgPXG|ia$9t+ePyP|MRhKU?)ITl zlkEcW;SnE$l$O{0}{!_Gnx$jP)NaHT2)U zt~KIfOSnH^47kf&*0DpRfun=*Kt@{aM>xC_ju??G!tulO-CEMVA`<8sOw!xH_tNqDVb5`M z_4Yg^rtGYchLy6T0jOiBO$yW-tLk=)FwHnuKz6QAv$Gf02c1x>kod{YdY4=AUcFyV z3N`e%IeRy(|0@-$H~iIelbRH??p9gCH`?=d?h+Yi&t0lh_LoNipD=C%HnZgUY5HNw z4@RVAL=Bi3U>-8S>aGchhf=vYFbo2pNoPf{DT*jfB~v0=72ZURVG;rY{%M1Q%vwrH z&!fMi`g(due6PBCAiNouC4B1il4zY|zRq63SMHMM&fR%x6ReIIc|S&%Z;CsiByejWP55>YU~2p3R6!+0Nh~ z@0;atV7-aY(4+822$}vUuE*pepOQpyc)_1Ynjtc-qrgWUFN?;caX7&0VcWY-rW6Qr zb8y69y?StgVKMAQ=I0h2_?*?bl)t2BBga8SCt4)d` za)4lBqf9bc(otxY-oP#h*}+GW&CtBAD(0RTJvB69Fu8g7KElAN35Ex`pW1_lOOJ{(3E=$#!H8y`vxjg5_tk)$sP)6Iw*z5lL&8L*=dBE5%z zOpi&xObrIZb}NREq){XuAp2>i2;j2~$zG#@!eHJkW|DXkNg)`6Oa3eHH1gCRz0{Vb zk6J$&|4n?kGe1~r-&r`XF|B`n?B>Z|xjzbg6Kc{!ouyDqsqqN-YT)MC!fAIcFt{92wkr+6S{!WObfMDY> z7qAx^L}~z`1>jo4PA#}|EmVKI7aZVFR1cw-?B9R4uhjO^-yi+Eqs6vvy{-FE+unz5 zdy8$a>TRzs58pby?gku!NWix^?w^swH9gD*!IU<>mx4ukc-~+ei1Bk`*jZ9@AB{j z`BIU6PiNoL*!R{V+ckDO9G395{BX$^x_JY5^PHjA>bvhI?oa*8T(SPNUVnPoYxlhY zeY*-jzwa&9AJyxRE_>m$bnGZV9@{=#C9XbyMn(hr)zR*ugYMrSWQJbz{l1?D=*VQd z0R^w8djP=KQThf1euLB{KgqyC;zMw&kaV~~-wI&e&T=#A;W6?RQ|2e&dnuou0+m}) zTxEqo!Y|MR7>S{A8UZPeN$7}=Na$9Z32!%v2@FpP(hQK(N5z3Wfq@IO(+}Gw3WW(S7<;mrt?@L_W?ST*a>`W63wF^4xWa^mb za7M-owp+#N)YZB-7;l=NL|sJw&Cf-c%{4zAU~0u+=91~GbQP+5zwig**XLzmrH#7L~nG1&{5)e_M*EiDDAtm5$YEx9v67HEa0l;96kB(26Ns$aT zVscUv5WI00`nq(ak$j+3-;fWNLSd*iG-^~`DICdjrEM)5)wIsk)9^8FBe;G3TWtNV(zVM) zcf-MP0MGf5kN5Z`a<2uQ0eb5?WrJ;Ax~?Fv zQ*gU0uTFffl4>oeqsHlXXe}w|$;;Z+MNov-Xw<-aXi$&<3UIpNwmS7??m{=Wu!W{y1mP(ImSTd->O*%6ws9qf V%@6)!_~vk){BQ#~(m{^u{{p0zL`(nx diff --git a/empty-experiment.zip b/empty-experiment.zip deleted file mode 100644 index 0a1fe9c3e3904d9cb8ab84ee41334e94d6182eec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 130 zcmWIWW@Zs#00D=UY0+Q?ln?;YMX80QnMJ9&sd**EdL None: - with zipfile.ZipFile(f'{dirname}.zip', 'w') as zf: - for dirpath, _, filenames in os.walk(dirname): - for filename in filenames: - filepath = os.path.join(dirpath, filename) - zf.write(filepath, os.path.relpath(filepath, dirname)) - - def parse_args(self, args: List[str]) -> int: - return gcli.parse_args(self.request_manager, args, stdout=self.out, stderr=self.err) - - def _assert_status_code(self, args: List[str], expected_code: int) -> None: - status = self.parse_args(args) - self.assertEqual(status, expected_code) - - def _assert_in_output(self, substring: str) -> None: - output = self.out.getvalue() - if (substring not in output): - print("OUTPUT:") - print(output) - self.assertIn(substring, output) - - def _assert_in_error(self, substring: str) -> None: - error = self.err.getvalue() - if (substring not in error): - print("ERROR:") - print(error) - self.assertIn(substring, error) - - def test_mutually_exclusive_parameters(self) -> None: - # Test that mutually exclusive parameters cannot be used together - self._assert_status_code(['-q', 'some_value', '-z', 'another_value'], gcli.EX_PARSE_ERROR) - self._assert_status_code(['-d', 'some_value', '-q', 'another_value'], gcli.EX_PARSE_ERROR) - self._assert_status_code(['-z', 'some_value', '-d', 'another_value'], gcli.EX_PARSE_ERROR) - self._assert_status_code(['-da', 'some_value', '-z', 'another_value'], gcli.EX_PARSE_ERROR) - self._assert_in_error("Invalid flags") - - def test_with_invalid_token(self) -> None: - # Test with an invalid stored token - self.request_manager.authenticate.return_value = {"uid": None, "error": "invalid"} - self._assert_status_code(['-z', 'experiment.zip'], gcli.EX_INVALID_TOKEN) - self._assert_in_error("Cannot authenticate token") - self._assert_status_code(['-q', 'experiment_name'], gcli.EX_INVALID_TOKEN) - self._assert_in_error("Cannot authenticate token") - self._assert_status_code(['-d', 'experiment.zip'], gcli.EX_INVALID_TOKEN) - self._assert_in_error("Cannot authenticate token") - self.request_manager.authenticate.assert_has_calls([ - mock.call('new_valid_token'), - mock.call('new_valid_token'), - mock.call('new_valid_token')], any_order=False) - - def test_run_experiment(self) -> None: - # Test running an experiment with a valid token - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.upload_and_start_experiment.return_value = { - 'success': True, - 'error': '', - 'exp_id': 'exp123' - } - self._assert_status_code(['-z', 'valid-experiment.zip'], gcli.EX_SUCCESS) - - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.upload_and_start_experiment.assert_called_with('valid-experiment.zip') - self._assert_in_output('exp123') - - def test_with_stored_token(self) -> None: - # Test running an experiment with a stored token - with open('.token.glados', 'w') as f: - f.write('valid_token') - - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.upload_and_start_experiment.return_value = { - 'success': True, - 'error': '', - 'exp_id': 'expabc' - } - self._assert_status_code(['-z', 'valid-experiment.zip'], gcli.EX_SUCCESS) - self.request_manager.authenticate.assert_called_with('valid_token') - self.request_manager.upload_and_start_experiment.assert_called_with('valid-experiment.zip') - self._assert_in_output('expabc') - - os.remove('.token.glados') - - def test_generate_token(self) -> None: - # Test running an experiment without any token - self.request_manager.generate_token.return_value = { - "access_token": "new_valid_token", - "error": None - } - self._assert_status_code(['--generate-token'], gcli.EX_SUCCESS) - self.request_manager.generate_token.assert_called_once() - self._assert_in_output('new_valid_token') - - def test_run_missing_experiment(self) -> None: - # Test running a non-existent experiment file - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self._assert_status_code(['-z', 'missing_experiment.zip'], gcli.EX_NOTFOUND) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self._assert_in_error('missing_experiment.zip') - self._assert_in_error('not found') - - def test_run_experiment_backend_format_failure(self) -> None: - # Test running an experiment where the backend's format validation fails - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.upload_and_start_experiment.return_value = { - 'success': False, - 'error': 'bad_format', - 'exp_id': '' - } - self._assert_status_code(['-z', 'valid-experiment.zip'], gcli.EX_INVALID_EXP_FORMAT) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.upload_and_start_experiment.assert_called_with('valid-experiment.zip') - self._assert_in_error('format') - - def test_run_experiment_other_backend_failure(self) -> None: - # Test running an experiment where the backend fails for other reasons - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.upload_and_start_experiment.return_value = { - 'success': False, - 'error': 'other', - 'exp_id': '' - } - self._assert_status_code(['-z', 'valid-experiment.zip'], gcli.EX_UNKNOWN) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.upload_and_start_experiment.assert_called_with('valid-experiment.zip') - self._assert_in_error('other') - - def test_query_one_experiment(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.query_experiments.return_value = { - 'success': True, - 'matches': [ - {'id': 'exp1', - 'name': 'Test Experiment', - 'tags': ['tag1', 'tag2'], - 'status': 'completed', - 'started_on': 1762488593221, - 'current_permutation': 50, - 'total_permutations': 100}, - ] - } - self._assert_status_code(['-q', 'Test Experiment'], gcli.EX_SUCCESS) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.query_experiments.assert_called_with('Test Experiment') - self._assert_in_output('Test Experiment') - - def test_query_multiple_experiments(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.query_experiments.return_value = { - 'success': True, - 'matches': [ - {'id': 'exp1', - 'name': 'Test Experiment 1', - 'tags': ['tag1'], - 'status': 'running', - 'started_on': 1762488593221, - 'current_permutation': 70, - 'total_permutations': 100}, - {'id': 'exp2', - 'name': 'Test Experiment 2', - 'tags': ['tag2'], - 'status': 'completed', - 'started_on': 1762489593221, - 'current_permutation': 80, - 'total_permutations': 100}, - ] - } - self._assert_status_code(['-q', 'Test Experiment'], gcli.EX_SUCCESS) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.query_experiments.assert_called_with('Test Experiment') - self._assert_in_output('Test Experiment 1') - self._assert_in_output('Test Experiment 2') - - def test_query_no_experiments(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.query_experiments.return_value = { - 'success': True, - 'matches': [] - } - self._assert_status_code(['-q', 'Nonexistent Experiment'], gcli.EX_NOTFOUND) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.query_experiments.assert_called_with('Nonexistent Experiment') - self._assert_in_error('No experiments found') - - def test_download_experiment_results(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_experiment_results.return_value = { - 'success': True, - 'files': [ - { - 'name': 'downloaded_results.zip', - 'content': b'PK\x03\x04...' # Simulated binary content of a zip file - } - ] - } - self._assert_status_code(['-d', 'exp123'], gcli.EX_SUCCESS) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_experiment_results.assert_called_with('exp123') - self._assert_in_output('downloaded_results.zip') - - def test_download_experiment_not_found(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_experiment_results.return_value = { - 'success': False, - 'error': 'not_found' - } - self._assert_status_code(['-d', 'exp123'], gcli.EX_NOTFOUND) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_experiment_results.assert_called_with('exp123') - self._assert_in_error("not found") - - def test_download_experiment_still_running(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_experiment_results.return_value = { - 'success': False, - 'error': 'not_done' - } - self._assert_status_code(['-d', 'exp123'], gcli.EX_NOT_DONE) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_experiment_results.assert_called_with('exp123') - self._assert_in_error("still running") - - def test_download_experiment_failed(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_experiment_results.return_value = { - 'success': False, - 'error': 'exp_failed' - } - self._assert_status_code(['-d', 'exp123'], gcli.EX_EXP_FAILED) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_experiment_results.assert_called_with('exp123') - self._assert_in_error("did not complete successfully") - - def test_download_all_experiment_results(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_all.return_value = { - 'success': True, - 'files': [ - { - 'name': 'downloaded_results.zip', - 'content': b'PK\x03\x04...' # Simulated binary content of a zip file - } - ] - } - self._assert_status_code(['-da', 'exp123'], gcli.EX_SUCCESS) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_all.assert_called_with('exp123') - self._assert_in_output('All experiment artifacts downloaded successfully.') - - def test_download_all_experiment_not_found(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_all.return_value = { - 'success': False, - 'error': 'not_found' - } - self._assert_status_code(['-da', 'exp123'], gcli.EX_NOTFOUND) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_all.assert_called_with('exp123') - self._assert_in_error("not found") - - def test_download_all_experiment_still_running(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_all.return_value = { - 'success': False, - 'error': 'not_done' - } - self._assert_status_code(['-da', 'exp123'], gcli.EX_NOT_DONE) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_all.assert_called_with('exp123') - self._assert_in_error("still running") - - def test_download_all_experiment_failed(self): - self.request_manager.authenticate.return_value = {"uid": "test", "error": None} - self.request_manager.download_all.return_value = { - 'success': False, - 'error': 'exp_failed' - } - self._assert_status_code(['-da', 'exp123'], gcli.EX_EXP_FAILED) - self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.download_all.assert_called_with('exp123') - self._assert_in_error("did not complete successfully") - - def test_cli_update_success(self): - self.request_manager.update.return_value = { - "success": True, - "error": False - } - self._assert_status_code(['--update'], gcli.UPDATE_SUCCEED) - self.request_manager.update.assert_called_once() - self._assert_in_output("Downloaded most up-to-date CLI successfully") - - def test_cli_update_failure(self): - self.request_manager.update.return_value = { - "success": False, - "error": "network" - } - self._assert_status_code(['-u'], gcli.UPDATE_FAIL) - self.request_manager.update.assert_called_once() - self._assert_in_output("Unable to download most up-to-date version") - - def test_manifest_no_errors(self): - gcli.check_manifest_format("test_manifests/test_manifest_no_errors.yml", False) - self._assert_in_output("") - - def test_manifest_string_errors(self): - buf = StringIO() - with redirect_stdout(buf): - result = gcli.check_manifest_format("test_manifests/test_manifest_string_errors.yml", True) - output = buf.getvalue() - self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) - self.assertIn("name attribute in manifest.yml is empty, missing, or not a string.", output) - self.assertIn("trialResult attribute in manifest.yml is empty, missing, or not a string.", output) - self.assertIn("scatterIndVar attribute in manifest.yml is empty, missing, or not a string.", output) - self.assertIn("scatterDepVar attribute in manifest.yml is empty, missing, or not a string.", output) - self.assertIn("experimentExecutable attribute in manifest.yml is empty, missing, or not a string.", output) - - def test_manifest_int_errors(self): - buf = StringIO() - with redirect_stdout(buf): - result = gcli.check_manifest_format("test_manifests/test_manifest_int_errors.yml", True) - output = buf.getvalue() - self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) - self.assertIn("trialResultLineNumber attribute in manifest.yml is empty or missing.", output) - self.assertIn("timeout attribute in manifest.yml is not greater than 0.", output) - self.assertIn("workers attribute in manifest.yml is not greater than 0", output) - - def test_manifest_bool_errors(self): - buf = StringIO() - with redirect_stdout(buf): - result = gcli.check_manifest_format("test_manifests/test_manifest_bool_errors.yml", True) - output = buf.getvalue() - self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) - self.assertIn("sendEmail attribute in manifest.yml is empty, missing, or not true or false.", output) - self.assertIn("scatter attribute in manifest.yml is empty, missing, or not true or false.", output) - - def test_manifest_param_errors(self): - buf = StringIO() - with redirect_stdout(buf): - result = gcli.check_manifest_format("test_manifests/test_manifest_param_errors.yml", True) - output = buf.getvalue() - self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) - self.assertIn("min attribute in hyperparameter x is not a float.", output) - self.assertIn("values attribute in hyperparameter values1 is not a list.", output) - self.assertIn("min attribute in hyperparameter y is not an integer.", output) - self.assertIn("max attribute in hyperparameter z is not greater than 12.", output) - self.assertIn("default attribute in hyperparameter values2 is empty, missing, or not true or false.", output) - self.assertIn("Type specified in hyperparameter values3 is not integer, float, bool, stringlist, or paramgroup.", output) - -if __name__ == '__main__': - unittest.main() - +from unittest import mock +from typing import * +from io import StringIO +from contextlib import redirect_stdout + +import sys +import os +import io +import unittest +import zipfile + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) + +import glados_cli as gcli + +class GladosCliTests(unittest.TestCase): + + def setUp(self): + self.request_manager: gcli.RequestManager = mock.MagicMock() + self._makeZipFile('tests/unit/data/valid-experiment') + self._makeZipFile('tests/unit/data/empty-experiment') + # Allow for testing what's printed to stdout and stderr + self.out = io.StringIO() + self.err = io.StringIO() + # Create a stored token file for tests that require authentication + with open('.token.glados', 'w') as f: + f.write('new_valid_token') + + def tearDown(self): + # Clean up the token file + if os.path.exists('.token.glados'): + os.remove('.token.glados') + + # Clean up the generated zips + for path in ['tests/unit/data/valid-experiment.zip', 'tests/unit/data/empty-experiment.zip']: + if os.path.exists(path): + os.remove(path) + + def _makeZipFile(self, dirname: str) -> None: + with zipfile.ZipFile(f'{dirname}.zip', 'w') as zf: + for dirpath, _, filenames in os.walk(dirname): + for filename in filenames: + filepath = os.path.join(dirpath, filename) + zf.write(filepath, os.path.relpath(filepath, dirname)) + + def parse_args(self, args: List[str]) -> int: + return gcli.parse_args(self.request_manager, args, stdout=self.out, stderr=self.err) + + def _assert_status_code(self, args: List[str], expected_code: int) -> None: + status = self.parse_args(args) + self.assertEqual(status, expected_code) + + def _assert_in_output(self, substring: str) -> None: + output = self.out.getvalue() + if (substring not in output): + print("OUTPUT:") + print(output) + self.assertIn(substring, output) + + def _assert_in_error(self, substring: str) -> None: + error = self.err.getvalue() + if (substring not in error): + print("ERROR:") + print(error) + self.assertIn(substring, error) + + def test_mutually_exclusive_parameters(self) -> None: + # Test that mutually exclusive parameters cannot be used together + self._assert_status_code(['-q', 'some_value', '-z', 'another_value'], gcli.EX_PARSE_ERROR) + self._assert_status_code(['-d', 'some_value', '-q', 'another_value'], gcli.EX_PARSE_ERROR) + self._assert_status_code(['-z', 'some_value', '-d', 'another_value'], gcli.EX_PARSE_ERROR) + self._assert_status_code(['-da', 'some_value', '-z', 'another_value'], gcli.EX_PARSE_ERROR) + self._assert_in_error("Invalid flags") + + def test_with_invalid_token(self) -> None: + # Test with an invalid stored token + self.request_manager.authenticate.return_value = {"uid": None, "error": "invalid"} + self._assert_status_code(['-z', 'experiment.zip'], gcli.EX_INVALID_TOKEN) + self._assert_in_error("Cannot authenticate token") + self._assert_status_code(['-q', 'experiment_name'], gcli.EX_INVALID_TOKEN) + self._assert_in_error("Cannot authenticate token") + self._assert_status_code(['-d', 'experiment.zip'], gcli.EX_INVALID_TOKEN) + self._assert_in_error("Cannot authenticate token") + self.request_manager.authenticate.assert_has_calls([ + mock.call('new_valid_token'), + mock.call('new_valid_token'), + mock.call('new_valid_token')], any_order=False) + + def test_run_experiment(self) -> None: + # Test running an experiment with a valid token + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.upload_and_start_experiment.return_value = { + 'success': True, + 'error': '', + 'exp_id': 'exp123' + } + self._assert_status_code(['-z', 'tests/unit/data/valid-experiment.zip'], gcli.EX_SUCCESS) + + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.upload_and_start_experiment.assert_called_with('tests/unit/data/valid-experiment.zip') + self._assert_in_output('exp123') + + def test_with_stored_token(self) -> None: + # Test running an experiment with a stored token + with open('.token.glados', 'w') as f: + f.write('valid_token') + + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.upload_and_start_experiment.return_value = { + 'success': True, + 'error': '', + 'exp_id': 'expabc' + } + self._assert_status_code(['-z', 'tests/unit/data/valid-experiment.zip'], gcli.EX_SUCCESS) + self.request_manager.authenticate.assert_called_with('valid_token') + self.request_manager.upload_and_start_experiment.assert_called_with('tests/unit/data/valid-experiment.zip') + self._assert_in_output('expabc') + + os.remove('.token.glados') + + def test_generate_token(self) -> None: + # Test running an experiment without any token + self.request_manager.generate_token.return_value = { + "access_token": "new_valid_token", + "error": None + } + self._assert_status_code(['--generate-token'], gcli.EX_SUCCESS) + self.request_manager.generate_token.assert_called_once() + self._assert_in_output('new_valid_token') + + def test_run_missing_experiment(self) -> None: + # Test running a non-existent experiment file + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self._assert_status_code(['-z', 'missing_experiment.zip'], gcli.EX_NOTFOUND) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self._assert_in_error('missing_experiment.zip') + self._assert_in_error('not found') + + def test_run_experiment_backend_format_failure(self) -> None: + # Test running an experiment where the backend's format validation fails + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.upload_and_start_experiment.return_value = { + 'success': False, + 'error': 'bad_format', + 'exp_id': '' + } + self._assert_status_code(['-z', 'tests/unit/data/valid-experiment.zip'], gcli.EX_INVALID_EXP_FORMAT) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.upload_and_start_experiment.assert_called_with('tests/unit/data/valid-experiment.zip') + self._assert_in_error('format') + + def test_run_experiment_other_backend_failure(self) -> None: + # Test running an experiment where the backend fails for other reasons + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.upload_and_start_experiment.return_value = { + 'success': False, + 'error': 'other', + 'exp_id': '' + } + self._assert_status_code(['-z', 'tests/unit/data/valid-experiment.zip'], gcli.EX_UNKNOWN) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.upload_and_start_experiment.assert_called_with('tests/unit/data/valid-experiment.zip') + self._assert_in_error('other') + + def test_query_one_experiment(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.query_experiments.return_value = { + 'success': True, + 'matches': [ + {'id': 'exp1', + 'name': 'Test Experiment', + 'tags': ['tag1', 'tag2'], + 'status': 'completed', + 'started_on': 1762488593221, + 'current_permutation': 50, + 'total_permutations': 100}, + ] + } + self._assert_status_code(['-q', 'Test Experiment'], gcli.EX_SUCCESS) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.query_experiments.assert_called_with('Test Experiment') + self._assert_in_output('Test Experiment') + + def test_query_multiple_experiments(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.query_experiments.return_value = { + 'success': True, + 'matches': [ + {'id': 'exp1', + 'name': 'Test Experiment 1', + 'tags': ['tag1'], + 'status': 'running', + 'started_on': 1762488593221, + 'current_permutation': 70, + 'total_permutations': 100}, + {'id': 'exp2', + 'name': 'Test Experiment 2', + 'tags': ['tag2'], + 'status': 'completed', + 'started_on': 1762489593221, + 'current_permutation': 80, + 'total_permutations': 100}, + ] + } + self._assert_status_code(['-q', 'Test Experiment'], gcli.EX_SUCCESS) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.query_experiments.assert_called_with('Test Experiment') + self._assert_in_output('Test Experiment 1') + self._assert_in_output('Test Experiment 2') + + def test_query_no_experiments(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.query_experiments.return_value = { + 'success': True, + 'matches': [] + } + self._assert_status_code(['-q', 'Nonexistent Experiment'], gcli.EX_NOTFOUND) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.query_experiments.assert_called_with('Nonexistent Experiment') + self._assert_in_error('No experiments found') + + def test_download_experiment_results(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_experiment_results.return_value = { + 'success': True, + 'files': [ + { + 'name': 'downloaded_results.zip', + 'content': b'PK\x03\x04...' # Simulated binary content of a zip file + } + ] + } + self._assert_status_code(['-d', 'exp123'], gcli.EX_SUCCESS) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_experiment_results.assert_called_with('exp123') + self._assert_in_output('downloaded_results.zip') + + def test_download_experiment_not_found(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_experiment_results.return_value = { + 'success': False, + 'error': 'not_found' + } + self._assert_status_code(['-d', 'exp123'], gcli.EX_NOTFOUND) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_experiment_results.assert_called_with('exp123') + self._assert_in_error("not found") + + def test_download_experiment_still_running(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_experiment_results.return_value = { + 'success': False, + 'error': 'not_done' + } + self._assert_status_code(['-d', 'exp123'], gcli.EX_NOT_DONE) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_experiment_results.assert_called_with('exp123') + self._assert_in_error("still running") + + def test_download_experiment_failed(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_experiment_results.return_value = { + 'success': False, + 'error': 'exp_failed' + } + self._assert_status_code(['-d', 'exp123'], gcli.EX_EXP_FAILED) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_experiment_results.assert_called_with('exp123') + self._assert_in_error("did not complete successfully") + + def test_download_all_experiment_results(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_all.return_value = { + 'success': True, + 'files': [ + { + 'name': 'downloaded_results.zip', + 'content': b'PK\x03\x04...' # Simulated binary content of a zip file + } + ] + } + self._assert_status_code(['-da', 'exp123'], gcli.EX_SUCCESS) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_all.assert_called_with('exp123') + self._assert_in_output('All experiment artifacts downloaded successfully.') + + def test_download_all_experiment_not_found(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_all.return_value = { + 'success': False, + 'error': 'not_found' + } + self._assert_status_code(['-da', 'exp123'], gcli.EX_NOTFOUND) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_all.assert_called_with('exp123') + self._assert_in_error("not found") + + def test_download_all_experiment_still_running(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_all.return_value = { + 'success': False, + 'error': 'not_done' + } + self._assert_status_code(['-da', 'exp123'], gcli.EX_NOT_DONE) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_all.assert_called_with('exp123') + self._assert_in_error("still running") + + def test_download_all_experiment_failed(self): + self.request_manager.authenticate.return_value = {"uid": "test", "error": None} + self.request_manager.download_all.return_value = { + 'success': False, + 'error': 'exp_failed' + } + self._assert_status_code(['-da', 'exp123'], gcli.EX_EXP_FAILED) + self.request_manager.authenticate.assert_called_with('new_valid_token') + self.request_manager.download_all.assert_called_with('exp123') + self._assert_in_error("did not complete successfully") + + def test_cli_update_success(self): + self.request_manager.update.return_value = { + "success": True, + "error": False + } + self._assert_status_code(['--update'], gcli.UPDATE_SUCCEED) + self.request_manager.update.assert_called_once() + self._assert_in_output("Downloaded most up-to-date CLI successfully") + + def test_cli_update_failure(self): + self.request_manager.update.return_value = { + "success": False, + "error": "network" + } + self._assert_status_code(['-u'], gcli.UPDATE_FAIL) + self.request_manager.update.assert_called_once() + self._assert_in_output("Unable to download most up-to-date version") + + def test_manifest_no_errors(self): + gcli.check_manifest_format("tests/unit/data/test_manifests/test_manifest_no_errors.yml", False) + self._assert_in_output("") + + def test_manifest_string_errors(self): + buf = StringIO() + with redirect_stdout(buf): + result = gcli.check_manifest_format("tests/unit/data/test_manifests/test_manifest_string_errors.yml", True) + output = buf.getvalue() + self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) + self.assertIn("name attribute in manifest.yml is empty, missing, or not a string.", output) + self.assertIn("trialResult attribute in manifest.yml is empty, missing, or not a string.", output) + self.assertIn("scatterIndVar attribute in manifest.yml is empty, missing, or not a string.", output) + self.assertIn("scatterDepVar attribute in manifest.yml is empty, missing, or not a string.", output) + self.assertIn("experimentExecutable attribute in manifest.yml is empty, missing, or not a string.", output) + + def test_manifest_int_errors(self): + buf = StringIO() + with redirect_stdout(buf): + result = gcli.check_manifest_format("tests/unit/data/test_manifests/test_manifest_int_errors.yml", True) + output = buf.getvalue() + self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) + self.assertIn("trialResultLineNumber attribute in manifest.yml is empty or missing.", output) + self.assertIn("timeout attribute in manifest.yml is not greater than 0.", output) + self.assertIn("workers attribute in manifest.yml is not greater than 0", output) + + def test_manifest_bool_errors(self): + buf = StringIO() + with redirect_stdout(buf): + result = gcli.check_manifest_format("tests/unit/data/test_manifests/test_manifest_bool_errors.yml", True) + output = buf.getvalue() + self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) + self.assertIn("sendEmail attribute in manifest.yml is empty, missing, or not true or false.", output) + self.assertIn("scatter attribute in manifest.yml is empty, missing, or not true or false.", output) + + def test_manifest_param_errors(self): + buf = StringIO() + with redirect_stdout(buf): + result = gcli.check_manifest_format("tests/unit/data/test_manifests/test_manifest_param_errors.yml", True) + output = buf.getvalue() + self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) + self.assertIn("min attribute in hyperparameter x is not a float.", output) + self.assertIn("values attribute in hyperparameter values1 is not a list.", output) + self.assertIn("min attribute in hyperparameter y is not an integer.", output) + self.assertIn("max attribute in hyperparameter z is not greater than 12.", output) + self.assertIn("default attribute in hyperparameter values2 is empty, missing, or not true or false.", output) + self.assertIn("Type specified in hyperparameter values3 is not integer, float, bool, stringlist, or paramgroup.", output) + +if __name__ == '__main__': + unittest.main() + diff --git a/valid-experiment.zip b/valid-experiment.zip deleted file mode 100644 index 4245ce771d116859a4183b1bc57bf36d5d8b0db5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1015 zcmaJ=&59F25FR(`As7_d2PhMS2^$*4gRn5*#ZwS+6g)J&lbTJ#^mN7r7@}yDDHlhk zKcBo9e?LnS9#6>>H%>-0u2d0)YYJ}-dJ`rK6N-z_QpYSh;El!6~myfDFF29;{%Qtd-a5szkViG(_d&3_`Sy zXLK^?8=M|{bf<63uYCp})P#~w#eqf-cGj8g%@1oH*0NFtv$3tCtD~Tc@eXqTK8NSL z%`pTSiEg=#aRy5jRX@@RG`9Af4PD_9in_|9wiT?a=Y#v3g&x_;-7POCt#j;p+B~gelHA>1(3bMi fZ>3Y*Q|6uXU+TrvI2Q9qkJ(~C-!ED(27LMlgpdC! From 81c82ec508795be0b327290c7661d3165e64d967 Mon Sep 17 00:00:00 2001 From: helena-donaldson Date: Tue, 7 Apr 2026 00:19:05 -0400 Subject: [PATCH 06/10] Run command correction --- tests/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index 9fd39ca..3fea2f7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,5 +4,5 @@ This directory has unit and integration tests for the GLADOS CLI. ## Directories -- The unit directory has an automated test suite that deals with the business logic of the CLI, with the data subdirectory holding files that allow the tests to run properly. The test file can be run with the command `python glados_cli_tests.py`. -- The integration directory tests the flow of API calls to the NextJS endpoints of GLADOS. It has a partially automated test suite; it requires a tester to first authenticate via the CLI and then ensure they have no experiments named Test AddNums. The test file can be run with the command `python glados_workflow_tests.py`. \ No newline at end of file +- The unit directory has an automated test suite that deals with the business logic of the CLI, with the data subdirectory holding files that allow the tests to run properly. The test file can be run with the command `python tests/unit/glados_cli_tests.py` from the project root. +- The integration directory tests the flow of API calls to the NextJS endpoints of GLADOS. It has a partially automated test suite; it requires a tester to first authenticate via the CLI and then ensure they have no experiments named Test AddNums. The test file can be run with the command `python tests/integration/glados_workflow_tests.py` from the project root. \ No newline at end of file From 9d98c3dbe838fb620de69e59890e0aa48b10e79b Mon Sep 17 00:00:00 2001 From: helena-donaldson Date: Wed, 8 Apr 2026 13:53:50 -0400 Subject: [PATCH 07/10] Refactored to remove redundances and add comments for clarity --- tests/integration/glados_workflow_tests.py | 133 ++++++++++++--------- tests/unit/glados_cli_tests.py | 1 + 2 files changed, 79 insertions(+), 55 deletions(-) diff --git a/tests/integration/glados_workflow_tests.py b/tests/integration/glados_workflow_tests.py index e261da4..62b14ff 100644 --- a/tests/integration/glados_workflow_tests.py +++ b/tests/integration/glados_workflow_tests.py @@ -11,9 +11,9 @@ import glob import pandas as pd -GLADOS_CLI_PATH = "glados_cli.py" # Adjust the path to your glados_cli.py if necessary -CSV_FILE_PATH = "tests/integration/data/addNumbersExpected.csv" # Adjust the path to your expected CSV file if necessary -EXPERIMENT_FILE = "tests/integration/data/addNumbers.py" +GLADOS_CLI_PATH = "glados_cli.py" # glados_cli.py file path from root of the repository, adjust if necessary +CSV_FILE_PATH = "tests/integration/data/addNumbersExpected.csv" # csv file path from root of the repository, adjust if necessary +EXPERIMENT_FILE = "tests/integration/data/addNumbers.py" # executable file path from root of the repository, adjust if necessary def compare_result_files(file1, file2): df1 = pd.read_csv(file1) @@ -35,60 +35,83 @@ def filter_lines(text): def teardown(): for f in glob.glob("Test_AddNums*"): os.remove(f) - -print("Starting experiment creation test...\n") -try: - result = subprocess.run(["python", GLADOS_CLI_PATH, "-z", EXPERIMENT_FILE], capture_output=True, text=True) - print("Output:\n", result.stdout.strip()) - experiment_id = result.stdout.strip().split('=')[1].strip(' ).') - if result.stderr: - print("Errors:\n", result.stderr.strip()) -except Exception as e: - print(f"Test failed with error: {e}") + +def start_test_printout(test_name): + print(f"\n{'='*10} Starting {test_name} {'='*10}\n") + +def end_test_printout(test_name): + print(f"\n{'='*10} Finished {test_name} {'='*10}\n") + +def experiment_creation_test(): + start_test_printout("Experiment Creation Test") + try: + result = subprocess.run(["python", GLADOS_CLI_PATH, "-z", EXPERIMENT_FILE], capture_output=True, text=True) + print("Output:\n", result.stdout.strip()) + experiment_id = result.stdout.strip().split('=')[1].strip(' ).') + if result.stderr: + print("Errors:\n", result.stderr.strip()) + except Exception as e: + print(f"Test failed with error: {e}") + + end_test_printout("Experiment Creation Test") -print("\nExperiment creation test completed.") + return experiment_id -time.sleep(10) # Wait for a moment to ensure the experiment is fully registered before attempting to download +def experiment_download(experiment_id): + start_test_printout("Experiment Download Test") + try: + result = subprocess.run(["python", GLADOS_CLI_PATH, "-da", experiment_id], capture_output=True, text=True) + print("Output:\n", result.stdout.strip()) + if result.stderr: + print("Errors:\n", result.stderr.strip()) + else: + print("Test passed: Experiment artifacts downloaded successfully.") + except Exception as e: + print(f"Test failed with error: {e}") + end_test_printout("Experiment Download Test") -print("\nStarting experiment download test...\n") -try: - result = subprocess.run(["python", GLADOS_CLI_PATH, "-d", experiment_id], capture_output=True, text=True) - print("Output:\n", result.stdout.strip()) - if result.stderr: - print("Errors:\n", result.stderr.strip()) - else: - words = result.stdout.strip().split() - file_name = next((w for w in words if w.endswith('.csv')), None) - compare_result_files(CSV_FILE_PATH, file_name) -except Exception as e: - print(f"Test failed with error: {e}") - -print("\nStarting experiment download all test...\n") -try: - result = subprocess.run(["python", GLADOS_CLI_PATH, "-da", experiment_id], capture_output=True, text=True) - print("Output:\n", result.stdout.strip()) - if result.stderr: - print("Errors:\n", result.stderr.strip()) - else: - print("Test passed: Experiment artifacts downloaded successfully.") -except Exception as e: - print(f"Test failed with error: {e}") - -print("\nStarting experiment query test...\n") -try: - result = subprocess.run(["python", GLADOS_CLI_PATH, "-q", "Test AddNums"], capture_output=True, text=True) - print("Output:\n", result.stdout.strip()) - if result.stderr: - print("Errors:\n", result.stderr.strip()) - else: - # Compare expected results with actual results from query output - expected_output = "Matches:\n***********************************************\nExperiment 1: Test AddNums\n*********************************************** \nID: 69d342be8bb268f5b2add93d\nTags: ['Test', 'AddNums']\nStatus: COMPLETED\nTime Started: 2026-04-06 01:21:14.109000\nTrials: 100/100 Completed" - if compare_filtered(result.stdout.strip(), expected_output): - print("\nTest passed: The query output matches the expected output.") +def experiment_download_all(experiment_id): + start_test_printout("Experiment Download All Test") + try: + result = subprocess.run(["python", GLADOS_CLI_PATH, "-da", experiment_id], capture_output=True, text=True) + print("Output:\n", result.stdout.strip()) + if result.stderr: + print("Errors:\n", result.stderr.strip()) + else: + print("Test passed: Experiment artifacts downloaded successfully.") + except Exception as e: + print(f"Test failed with error: {e}") + end_test_printout("Experiment Download All Test") + +def experiment_query(): + start_test_printout("Experiment Query Test") + try: + result = subprocess.run(["python", GLADOS_CLI_PATH, "-q", "Test AddNums"], capture_output=True, text=True) + print("Output:\n", result.stdout.strip()) + if result.stderr: + print("Errors:\n", result.stderr.strip()) else: - print("\nTest failed: The query output does not match the expected output.") - print("\nFinished querying experiment.") -except Exception as e: - print(f"\nTest failed with error: {e}") + # Compare expected results with actual results from query output + expected_output = "Matches:\n***********************************************\nExperiment 1: Test AddNums\n*********************************************** \nID: 69d342be8bb268f5b2add93d\nTags: ['Test', 'AddNums']\nStatus: COMPLETED\nTime Started: 2026-04-06 01:21:14.109000\nTrials: 100/100 Completed" + if compare_filtered(result.stdout.strip(), expected_output): + print("\nTest passed: The query output matches the expected output.") + else: + print("\nTest failed: The query output does not match the expected output.") + print("\nFinished querying experiment.") + except Exception as e: + print(f"\nTest failed with error: {e}") + end_test_printout("Experiment Query Test") + +def main(): + experiment_id = experiment_creation_test() + + time.sleep(10) # Wait for a moment to ensure the experiment is fully registered before attempting to download + + experiment_download(experiment_id) + experiment_download_all(experiment_id) + experiment_query() + + teardown() -teardown() \ No newline at end of file +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/unit/glados_cli_tests.py b/tests/unit/glados_cli_tests.py index 2c30381..34bd68d 100644 --- a/tests/unit/glados_cli_tests.py +++ b/tests/unit/glados_cli_tests.py @@ -9,6 +9,7 @@ import unittest import zipfile +# Imports GLADOS CLI sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) import glados_cli as gcli From 83756078c0385d8daa1dc242bb5e9b21523fa3ac Mon Sep 17 00:00:00 2001 From: helena-donaldson Date: Wed, 8 Apr 2026 14:01:46 -0400 Subject: [PATCH 08/10] Adjusted comments and printouts for consistency --- tests/integration/glados_workflow_tests.py | 3 ++- tests/unit/glados_cli_tests.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/glados_workflow_tests.py b/tests/integration/glados_workflow_tests.py index 62b14ff..e92832c 100644 --- a/tests/integration/glados_workflow_tests.py +++ b/tests/integration/glados_workflow_tests.py @@ -50,6 +50,8 @@ def experiment_creation_test(): experiment_id = result.stdout.strip().split('=')[1].strip(' ).') if result.stderr: print("Errors:\n", result.stderr.strip()) + else: + print(f"Test passed: Experiment created successfully.") except Exception as e: print(f"Test failed with error: {e}") @@ -97,7 +99,6 @@ def experiment_query(): print("\nTest passed: The query output matches the expected output.") else: print("\nTest failed: The query output does not match the expected output.") - print("\nFinished querying experiment.") except Exception as e: print(f"\nTest failed with error: {e}") end_test_printout("Experiment Query Test") diff --git a/tests/unit/glados_cli_tests.py b/tests/unit/glados_cli_tests.py index 34bd68d..348261c 100644 --- a/tests/unit/glados_cli_tests.py +++ b/tests/unit/glados_cli_tests.py @@ -9,7 +9,7 @@ import unittest import zipfile -# Imports GLADOS CLI +# Import glados_cli.py, adjust if necessary sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) import glados_cli as gcli From 57012d71365c43627a886f24ecb787f41f771875 Mon Sep 17 00:00:00 2001 From: helena-donaldson Date: Thu, 9 Apr 2026 03:53:46 -0400 Subject: [PATCH 09/10] Added file path constants --- tests/unit/glados_cli_tests.py | 36 +++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/tests/unit/glados_cli_tests.py b/tests/unit/glados_cli_tests.py index 348261c..96e537f 100644 --- a/tests/unit/glados_cli_tests.py +++ b/tests/unit/glados_cli_tests.py @@ -14,12 +14,16 @@ import glados_cli as gcli +VALID_EXPERIMENT_ZIP = 'tests/unit/data/valid-experiment.zip' +EMPTY_EXPERIMENT_ZIP = 'tests/unit/data/empty-experiment.zip' +MANIFEST_DIRECTORY = 'tests/unit/data/test_manifests/' + class GladosCliTests(unittest.TestCase): def setUp(self): self.request_manager: gcli.RequestManager = mock.MagicMock() - self._makeZipFile('tests/unit/data/valid-experiment') - self._makeZipFile('tests/unit/data/empty-experiment') + self._makeZipFile(VALID_EXPERIMENT_ZIP.replace('.zip', '')) + self._makeZipFile(EMPTY_EXPERIMENT_ZIP.replace('.zip', '')) # Allow for testing what's printed to stdout and stderr self.out = io.StringIO() self.err = io.StringIO() @@ -33,7 +37,7 @@ def tearDown(self): os.remove('.token.glados') # Clean up the generated zips - for path in ['tests/unit/data/valid-experiment.zip', 'tests/unit/data/empty-experiment.zip']: + for path in [VALID_EXPERIMENT_ZIP, EMPTY_EXPERIMENT_ZIP]: if os.path.exists(path): os.remove(path) @@ -95,10 +99,10 @@ def test_run_experiment(self) -> None: 'error': '', 'exp_id': 'exp123' } - self._assert_status_code(['-z', 'tests/unit/data/valid-experiment.zip'], gcli.EX_SUCCESS) + self._assert_status_code(['-z', VALID_EXPERIMENT_ZIP], gcli.EX_SUCCESS) self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.upload_and_start_experiment.assert_called_with('tests/unit/data/valid-experiment.zip') + self.request_manager.upload_and_start_experiment.assert_called_with(VALID_EXPERIMENT_ZIP) self._assert_in_output('exp123') def test_with_stored_token(self) -> None: @@ -112,9 +116,9 @@ def test_with_stored_token(self) -> None: 'error': '', 'exp_id': 'expabc' } - self._assert_status_code(['-z', 'tests/unit/data/valid-experiment.zip'], gcli.EX_SUCCESS) + self._assert_status_code(['-z', VALID_EXPERIMENT_ZIP], gcli.EX_SUCCESS) self.request_manager.authenticate.assert_called_with('valid_token') - self.request_manager.upload_and_start_experiment.assert_called_with('tests/unit/data/valid-experiment.zip') + self.request_manager.upload_and_start_experiment.assert_called_with(VALID_EXPERIMENT_ZIP) self._assert_in_output('expabc') os.remove('.token.glados') @@ -145,9 +149,9 @@ def test_run_experiment_backend_format_failure(self) -> None: 'error': 'bad_format', 'exp_id': '' } - self._assert_status_code(['-z', 'tests/unit/data/valid-experiment.zip'], gcli.EX_INVALID_EXP_FORMAT) + self._assert_status_code(['-z', VALID_EXPERIMENT_ZIP], gcli.EX_INVALID_EXP_FORMAT) self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.upload_and_start_experiment.assert_called_with('tests/unit/data/valid-experiment.zip') + self.request_manager.upload_and_start_experiment.assert_called_with(VALID_EXPERIMENT_ZIP) self._assert_in_error('format') def test_run_experiment_other_backend_failure(self) -> None: @@ -158,9 +162,9 @@ def test_run_experiment_other_backend_failure(self) -> None: 'error': 'other', 'exp_id': '' } - self._assert_status_code(['-z', 'tests/unit/data/valid-experiment.zip'], gcli.EX_UNKNOWN) + self._assert_status_code(['-z', VALID_EXPERIMENT_ZIP], gcli.EX_UNKNOWN) self.request_manager.authenticate.assert_called_with('new_valid_token') - self.request_manager.upload_and_start_experiment.assert_called_with('tests/unit/data/valid-experiment.zip') + self.request_manager.upload_and_start_experiment.assert_called_with(VALID_EXPERIMENT_ZIP) self._assert_in_error('other') def test_query_one_experiment(self): @@ -337,13 +341,13 @@ def test_cli_update_failure(self): self._assert_in_output("Unable to download most up-to-date version") def test_manifest_no_errors(self): - gcli.check_manifest_format("tests/unit/data/test_manifests/test_manifest_no_errors.yml", False) + gcli.check_manifest_format(MANIFEST_DIRECTORY + "/test_manifest_no_errors.yml", False) self._assert_in_output("") def test_manifest_string_errors(self): buf = StringIO() with redirect_stdout(buf): - result = gcli.check_manifest_format("tests/unit/data/test_manifests/test_manifest_string_errors.yml", True) + result = gcli.check_manifest_format(MANIFEST_DIRECTORY + "/test_manifest_string_errors.yml", True) output = buf.getvalue() self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) self.assertIn("name attribute in manifest.yml is empty, missing, or not a string.", output) @@ -355,7 +359,7 @@ def test_manifest_string_errors(self): def test_manifest_int_errors(self): buf = StringIO() with redirect_stdout(buf): - result = gcli.check_manifest_format("tests/unit/data/test_manifests/test_manifest_int_errors.yml", True) + result = gcli.check_manifest_format(MANIFEST_DIRECTORY + "/test_manifest_int_errors.yml", True) output = buf.getvalue() self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) self.assertIn("trialResultLineNumber attribute in manifest.yml is empty or missing.", output) @@ -365,7 +369,7 @@ def test_manifest_int_errors(self): def test_manifest_bool_errors(self): buf = StringIO() with redirect_stdout(buf): - result = gcli.check_manifest_format("tests/unit/data/test_manifests/test_manifest_bool_errors.yml", True) + result = gcli.check_manifest_format(MANIFEST_DIRECTORY + "/test_manifest_bool_errors.yml", True) output = buf.getvalue() self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) self.assertIn("sendEmail attribute in manifest.yml is empty, missing, or not true or false.", output) @@ -374,7 +378,7 @@ def test_manifest_bool_errors(self): def test_manifest_param_errors(self): buf = StringIO() with redirect_stdout(buf): - result = gcli.check_manifest_format("tests/unit/data/test_manifests/test_manifest_param_errors.yml", True) + result = gcli.check_manifest_format(MANIFEST_DIRECTORY + "/test_manifest_param_errors.yml", True) output = buf.getvalue() self.assertEqual(result, gcli.EX_INVALID_EXP_FORMAT) self.assertIn("min attribute in hyperparameter x is not a float.", output) From 4abda724243b27a751d01268cac6ef9435ae67e8 Mon Sep 17 00:00:00 2001 From: helena-donaldson Date: Wed, 15 Apr 2026 13:39:18 -0400 Subject: [PATCH 10/10] Unit tests and assert in refactoring --- tests/integration/glados_workflow_tests.py | 188 ++++++++++----------- 1 file changed, 89 insertions(+), 99 deletions(-) diff --git a/tests/integration/glados_workflow_tests.py b/tests/integration/glados_workflow_tests.py index e92832c..14ab8ad 100644 --- a/tests/integration/glados_workflow_tests.py +++ b/tests/integration/glados_workflow_tests.py @@ -1,118 +1,108 @@ -# To run this test script, ensure that you are first authenticated with the CLI, -# as it expects there is a valid token stored in the .token.glados file. -# There should also be no existing experiment with the same name as the one -# specified in the manifest.yml file of the experiment being tested, as this test -# script expects to create a new experiment and will fail if an experiment with the -# same name already exists. - -import os +import unittest import subprocess -import time +import os import glob +import time import pandas as pd -GLADOS_CLI_PATH = "glados_cli.py" # glados_cli.py file path from root of the repository, adjust if necessary -CSV_FILE_PATH = "tests/integration/data/addNumbersExpected.csv" # csv file path from root of the repository, adjust if necessary -EXPERIMENT_FILE = "tests/integration/data/addNumbers.py" # executable file path from root of the repository, adjust if necessary +GLADOS_CLI_PATH = "glados_cli.py" +CSV_FILE_PATH = "tests/integration/data/addNumbersExpected.csv" +EXPERIMENT_FILE = "tests/integration/data/addNumbers.py" -def compare_result_files(file1, file2): - df1 = pd.read_csv(file1) - df2 = pd.read_csv(file2) - - if df1.equals(df2): - print("Test passed: The downloaded results match the expected results.") - else: - print("Test failed: The downloaded results do not match the expected results.") - -def compare_filtered(s1, s2): - def filter_lines(text): +class TestGladosCLI(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Clean up environment before starting tests.""" + cls.experiment_id = None + cls._cleanup_files() + + @staticmethod + def _cleanup_files(): + for f in glob.glob("Test_AddNums*"): + os.remove(f) + + def _run_cli(self, args): + cmd = ["python", GLADOS_CLI_PATH] + args + result = subprocess.run(cmd, capture_output=True, text=True) + return result + + def _assert_in_output(self, expected, actual, message=None): + self.assertIn(expected, actual, message or f"Expected '{expected}' not found in output.") + + def _filter_output(self, text): + # ID and Time Started lines will vary, so they must be ignored skip_prefixes = ("ID:", "Time Started:") - return [line.strip() for line in text.splitlines() - if not line.strip().startswith(skip_prefixes)] + return "\n".join([ + line.strip() for line in text.splitlines() + if not line.strip().startswith(skip_prefixes) + ]) - return filter_lines(s1) == filter_lines(s2) + def test_01_experiment_creation(self): + result = self._run_cli(["-z", EXPERIMENT_FILE]) + + self.assertEqual(result.returncode, 0, f"CLI exited with error: {result.stderr}") + self._assert_in_output("Experiment started successfully", result.stdout) + + # Parse and store the experiment ID for subsequent tests + try: + parts = result.stdout.strip().split('=') + TestGladosCLI.experiment_id = parts[1].strip(' ).').split()[0] + except (IndexError, AttributeError): + self.fail("Failed to parse Experiment ID from output.") + + def test_02_experiment_download(self): + if not self.experiment_id: + self.skipTest("No experiment ID available from previous step.") -def teardown(): - for f in glob.glob("Test_AddNums*"): - os.remove(f) + # Give the system a moment to register the experiment + time.sleep(10) -def start_test_printout(test_name): - print(f"\n{'='*10} Starting {test_name} {'='*10}\n") - -def end_test_printout(test_name): - print(f"\n{'='*10} Finished {test_name} {'='*10}\n") + result = self._run_cli(["-d", self.experiment_id]) -def experiment_creation_test(): - start_test_printout("Experiment Creation Test") - try: - result = subprocess.run(["python", GLADOS_CLI_PATH, "-z", EXPERIMENT_FILE], capture_output=True, text=True) - print("Output:\n", result.stdout.strip()) - experiment_id = result.stdout.strip().split('=')[1].strip(' ).') - if result.stderr: - print("Errors:\n", result.stderr.strip()) - else: - print(f"Test passed: Experiment created successfully.") - except Exception as e: - print(f"Test failed with error: {e}") + self.assertEqual(result.returncode, 0) + self.assertRegex(result.stdout, r"Experiment results Test_AddNums_.*\.csv downloaded successfully\.") - end_test_printout("Experiment Creation Test") - - return experiment_id - -def experiment_download(experiment_id): - start_test_printout("Experiment Download Test") - try: - result = subprocess.run(["python", GLADOS_CLI_PATH, "-da", experiment_id], capture_output=True, text=True) - print("Output:\n", result.stdout.strip()) - if result.stderr: - print("Errors:\n", result.stderr.strip()) + downloaded_files = glob.glob("Test_AddNums*.csv") + if downloaded_files: + df_actual = pd.read_csv(downloaded_files[0]) + df_expected = pd.read_csv(CSV_FILE_PATH) + pd.testing.assert_frame_equal(df_actual, df_expected) else: - print("Test passed: Experiment artifacts downloaded successfully.") - except Exception as e: - print(f"Test failed with error: {e}") - end_test_printout("Experiment Download Test") + self.fail("Results CSV file was not found after download.") + TestGladosCLI._cleanup_files() # Remove csv file download from previous test to ensure this test is valid + + def test_03_experiment_download_all(self): + if not self.experiment_id: + self.skipTest("No experiment ID available from previous step.") -def experiment_download_all(experiment_id): - start_test_printout("Experiment Download All Test") - try: - result = subprocess.run(["python", GLADOS_CLI_PATH, "-da", experiment_id], capture_output=True, text=True) - print("Output:\n", result.stdout.strip()) - if result.stderr: - print("Errors:\n", result.stderr.strip()) - else: - print("Test passed: Experiment artifacts downloaded successfully.") - except Exception as e: - print(f"Test failed with error: {e}") - end_test_printout("Experiment Download All Test") + # Give the system a moment to register the experiment + time.sleep(5) -def experiment_query(): - start_test_printout("Experiment Query Test") - try: - result = subprocess.run(["python", GLADOS_CLI_PATH, "-q", "Test AddNums"], capture_output=True, text=True) - print("Output:\n", result.stdout.strip()) - if result.stderr: - print("Errors:\n", result.stderr.strip()) + result = self._run_cli(["-da", self.experiment_id]) + + self.assertEqual(result.returncode, 0) + self._assert_in_output("All experiment artifacts downloaded successfully.", result.stdout) + + downloaded_files = glob.glob("Test_AddNums*.csv") + if downloaded_files: + df_actual = pd.read_csv(downloaded_files[0]) + df_expected = pd.read_csv(CSV_FILE_PATH) + pd.testing.assert_frame_equal(df_actual, df_expected) else: - # Compare expected results with actual results from query output - expected_output = "Matches:\n***********************************************\nExperiment 1: Test AddNums\n*********************************************** \nID: 69d342be8bb268f5b2add93d\nTags: ['Test', 'AddNums']\nStatus: COMPLETED\nTime Started: 2026-04-06 01:21:14.109000\nTrials: 100/100 Completed" - if compare_filtered(result.stdout.strip(), expected_output): - print("\nTest passed: The query output matches the expected output.") - else: - print("\nTest failed: The query output does not match the expected output.") - except Exception as e: - print(f"\nTest failed with error: {e}") - end_test_printout("Experiment Query Test") - -def main(): - experiment_id = experiment_creation_test() + self.fail("Results CSV file was not found after download.") + TestGladosCLI._cleanup_files() # Remove csv file download from previous test to ensure this test is valid - time.sleep(10) # Wait for a moment to ensure the experiment is fully registered before attempting to download - - experiment_download(experiment_id) - experiment_download_all(experiment_id) - experiment_query() + def test_04_experiment_query(self): + result = self._run_cli(["-q", "Test AddNums"]) + + expected_output_fragment = ( + "Matches:\n***********************************************\nExperiment 1: Test AddNums\n***********************************************\nTags: ['Test', 'AddNums']\nStatus: COMPLETED\nTrials: 100/100 Completed\n" + ) - teardown() + actual_filtered = self._filter_output(result.stdout) + expected_filtered = self._filter_output(expected_output_fragment) + + self._assert_in_output(expected_filtered, actual_filtered) if __name__ == "__main__": - main() \ No newline at end of file + unittest.main() \ No newline at end of file