Integration tests and Refactoring

manifest.yml

@@ -44,13 +44,13 @@ hyperparameters:
     useDefault: false
 # The experiment name. 
 # Example: "Evolutionary Aglorithm Experiment"
-name: "Good3"
+name: "Test AddNums"
 # This is the description of the experiment. 
 # Example: "Experiment for CSSE490 Homework 4."
-description: "Hi"
+description: "This is to test AddNums!"
 # These are the tags which can be defined for an experiment, allowing for better filtering on the web app.
 # Example: ["Neural Network", "ECE497"]
-tags: ["Neural Network", "ECE497"]
+tags: ["Test", "AddNums"]
 # A folder or file included in the downloadable Project Zip after experiment completion.
 # Example: "test.csv"
 trialExtraFile: "AddNumResult.csv"
@@ -83,7 +83,7 @@ sendEmail: false
 workers: 1
 # If using a zip experiment, specify the main executable filename.
 # Example: "test.py" if using zip, "" if not
-experimentExecutable: "test.py"
+experimentExecutable: "addNumber.py"
 # This attribute, set to either ini or yaml, defines the format of the config file generated for each trial.
 # Example: "ini" or "yaml"
 configFileFormat: "ini"

tests/README.md

@@ -0,0 +1,8 @@
+# GLADOS CLI Test Suite
+
+This directory has unit and integration tests for the GLADOS CLI.
+
+## Directories
+
+- The unit directory has an automated test suite that deals with the business logic of the CLI, with the data subdirectory holding files that allow the tests to run properly. The test file can be run with the command `python tests/unit/glados_cli_tests.py` from the project root.
+- The integration directory tests the flow of API calls to the NextJS endpoints of GLADOS. It has a partially automated test suite; it requires a tester to first authenticate via the CLI and then ensure they have no experiments named Test AddNums. The test file can be run with the command `python tests/integration/glados_workflow_tests.py` from the project root.

tests/integration/data/addNumbers.py

@@ -0,0 +1,53 @@
+import csv
+import sys
+import configparser
+
+# pylint: disable-next=pointless-string-statement
+"""
+This experiment demonstrates outputting additional information to a file
+and telling the system to gather the data from that file.
+
+There are two different ways to gather additional information from an experiment
+    - Trial's Extra File: Gathers the designated file that a run of the file generates and places in a zip to be uploaded when
+        the experiment completes
+    - Trial Result: Integrates the information from a 2 line csv of headers and values that the file run generates
+        and adds them to the result csv that is being uploaded
+
+How to tell if they worked
+    - Trial's Extra File: You can download a zip file that contains the different output files specified
+    - Trial Result: The result csv downloaded has been expanded with information from the specified file
+
+
+Example settings for a run that demonstrates this: (Any Fields not specified can be left blank or to whatever their default is)
+
+Info:
+Trial Result: AddNumResult.csv
+If you want a collection of each CSV this experiment runs:
+    Trial's Extra File: AddNumResult.csv
+Both can be used at the same time
+
+Parameters:
+x, 1, 1, 10, 1
+y, 1, 1, 10, 1
+"""
+
+# pylint: disable=glados-print-used
+
+def main():
+    config = configparser.ConfigParser()
+    args = sys.argv[1:]
+    configFile = args[0]
+    config.read(configFile)
+    x = int(config["DEFAULT"]["x"])
+    y = int(config["DEFAULT"]["y"])
+    with open('AddNumResult.csv', 'w', encoding="utf8") as result:
+        writer = csv.writer(result)
+        writer.writerow(['Addition', 'Subtraction'])
+        writer.writerow([x + y, x - y])
+
+    print("done")
+    return 0
+
+
+if __name__ == "__main__":
+    main()

tests/integration/data/addNumbersExpected.csv

@@ -0,0 +1,101 @@
+Experiment Run,Addition,Subtraction,x,y
+0,2,0,1,1
+1,3,-1,1,2
+2,4,-2,1,3
+3,5,-3,1,4
+4,6,-4,1,5
+5,7,-5,1,6
+6,8,-6,1,7
+7,9,-7,1,8
+8,10,-8,1,9
+9,11,-9,1,10
+10,3,1,2,1
+11,4,0,2,2
+12,5,-1,2,3
+13,6,-2,2,4
+14,7,-3,2,5
+15,8,-4,2,6
+16,9,-5,2,7
+17,10,-6,2,8
+18,11,-7,2,9
+19,12,-8,2,10
+20,4,2,3,1
+21,5,1,3,2
+22,6,0,3,3
+23,7,-1,3,4
+24,8,-2,3,5
+25,9,-3,3,6
+26,10,-4,3,7
+27,11,-5,3,8
+28,12,-6,3,9
+29,13,-7,3,10
+30,5,3,4,1
+31,6,2,4,2
+32,7,1,4,3
+33,8,0,4,4
+34,9,-1,4,5
+35,10,-2,4,6
+36,11,-3,4,7
+37,12,-4,4,8
+38,13,-5,4,9
+39,14,-6,4,10
+40,6,4,5,1
+41,7,3,5,2
+42,8,2,5,3
+43,9,1,5,4
+44,10,0,5,5
+45,11,-1,5,6
+46,12,-2,5,7
+47,13,-3,5,8
+48,14,-4,5,9
+49,15,-5,5,10
+50,7,5,6,1
+51,8,4,6,2
+52,9,3,6,3
+53,10,2,6,4
+54,11,1,6,5
+55,12,0,6,6
+56,13,-1,6,7
+57,14,-2,6,8
+58,15,-3,6,9
+59,16,-4,6,10
+60,8,6,7,1
+61,9,5,7,2
+62,10,4,7,3
+63,11,3,7,4
+64,12,2,7,5
+65,13,1,7,6
+66,14,0,7,7
+67,15,-1,7,8
+68,16,-2,7,9
+69,17,-3,7,10
+70,9,7,8,1
+71,10,6,8,2
+72,11,5,8,3
+73,12,4,8,4
+74,13,3,8,5
+75,14,2,8,6
+76,15,1,8,7
+77,16,0,8,8
+78,17,-1,8,9
+79,18,-2,8,10
+80,10,8,9,1
+81,11,7,9,2
+82,12,6,9,3
+83,13,5,9,4
+84,14,4,9,5
+85,15,3,9,6
+86,16,2,9,7
+87,17,1,9,8
+88,18,0,9,9
+89,19,-1,9,10
+90,11,9,10,1
+91,12,8,10,2
+92,13,7,10,3
+93,14,6,10,4
+94,15,5,10,5
+95,16,4,10,6
+96,17,3,10,7
+97,18,2,10,8
+98,19,1,10,9
+99,20,0,10,10

tests/integration/data/manifest.yml

@@ -0,0 +1,91 @@
+# This is the config file required to be in the same directory as as an
+# experiment that is submitted from the command line. Consult documentation
+# on experiment compatability and more information on these parameters a
+# https://automatingsciencepipeline.github.io/Monorepo/tutorial/usage/
+
+#CONFIGURATION:
+
+# This allows you to define hyperparameters for each run. Multiple can be defined.
+# Example:
+# - name: x
+#     default: "-1"
+#     min: "1"
+#     max: "10"
+#     step: "1"
+#     type: integer
+#     useDefault: false
+
+# - name: "values",
+#       default: "-1",
+#       values: [
+#         "a",
+#         "b"
+#       ],
+#       type: "stringlist",
+#       useDefault: false
+# See https://automatingsciencepipeline.github.io/Monorepo/tutorial/usage/#information-tab
+# for full list of hyperparameters
+
+hyperparameters:
+  - name: x
+    default: "-1"
+    min: "1"
+    max: "10"
+    step: "1"
+    type: integer
+    useDefault: false
+
+  - name: y
+    default: "-1"
+    min: "1"
+    max: "10"
+    step: "1"
+    type: integer
+    useDefault: false
+# The experiment name. 
+# Example: "Evolutionary Aglorithm Experiment"
+name: "Test AddNums"
+# This is the description of the experiment. 
+# Example: "Experiment for CSSE490 Homework 4."
+description: "This is to test AddNums!"
+# These are the tags which can be defined for an experiment, allowing for better filtering on the web app.
+# Example: ["Neural Network", "ECE497"]
+tags: ["Neural Network", "ECE497"]
+# A folder or file included in the downloadable Project Zip after experiment completion.
+# Example: "test.csv"
+trialExtraFile: "AddNumResult.csv"
+# The CSV file captured as the experiment result.
+# Example: "test.csv"
+trialResult: "AddNumResult.csv"
+# This allows you to specific the line number of a trial result.
+# Example: 0
+trialResultLineNumber: 0
+# This allows you to include a scatter plot in the downloadable Project Zip
+# Example: true
+scatter: true
+# If a scatter plot is included, this defined the independent variable. Leave blank if not included.
+# Example: "y" for included, "" for not
+scatterIndVar: "x"
+# If a scatter plot is included, this defined the dependent variable. Leave blank if not included.
+# Example: "x" for included, "" for not
+scatterDepVar: "y"
+# This tab allows defining a text block appended to every generated .ini config file. 
+# Example: "" for default behavior
+dumbTextArea: ""
+# The duration before the experiment automatically times out (seconds)
+# Example: 600
+timeout: 100
+# This allows an email to be sent following experiment completion with number of passed and failed trials
+# Example: true
+sendEmail: false
+# Define number of workers used for experiment
+# Example: 1
+workers: 1
+# If using a zip experiment, specify the main executable filename.
+# Example: "test.py" if using zip, "" if not
+experimentExecutable: ""
+# This attribute, set to either ini or yaml, defines the format of the config file generated for each trial.
+# Example: "ini" or "yaml"
+configFileFormat: "ini"
+
+

tests/integration/glados_workflow_tests.py

@@ -0,0 +1,108 @@
+import unittest
+import subprocess
+import os
+import glob
+import time
+import pandas as pd
+
+GLADOS_CLI_PATH = "glados_cli.py"
+CSV_FILE_PATH = "tests/integration/data/addNumbersExpected.csv"
+EXPERIMENT_FILE = "tests/integration/data/addNumbers.py"
+
+class TestGladosCLI(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        """Clean up environment before starting tests."""
+        cls.experiment_id = None
+        cls._cleanup_files()
+
+    @staticmethod
+    def _cleanup_files():
+        for f in glob.glob("Test_AddNums*"):
+            os.remove(f)
+
+    def _run_cli(self, args):
+        cmd = ["python", GLADOS_CLI_PATH] + args
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        return result
+
+    def _assert_in_output(self, expected, actual, message=None):
+        self.assertIn(expected, actual, message or f"Expected '{expected}' not found in output.")
+
+    def _filter_output(self, text):
+        # ID and Time Started lines will vary, so they must be ignored
+        skip_prefixes = ("ID:", "Time Started:")
+        return "\n".join([
+            line.strip() for line in text.splitlines() 
+            if not line.strip().startswith(skip_prefixes)
+        ])
+
+    def test_01_experiment_creation(self):
+        result = self._run_cli(["-z", EXPERIMENT_FILE])
+
+        self.assertEqual(result.returncode, 0, f"CLI exited with error: {result.stderr}")
+        self._assert_in_output("Experiment started successfully", result.stdout)
+
+        # Parse and store the experiment ID for subsequent tests
+        try:
+            parts = result.stdout.strip().split('=')
+            TestGladosCLI.experiment_id = parts[1].strip(' ).').split()[0]
+        except (IndexError, AttributeError):
+            self.fail("Failed to parse Experiment ID from output.")
+
+    def test_02_experiment_download(self):
+        if not self.experiment_id:
+            self.skipTest("No experiment ID available from previous step.")
+
+        # Give the system a moment to register the experiment
+        time.sleep(10)
+
+        result = self._run_cli(["-d", self.experiment_id])
+
+        self.assertEqual(result.returncode, 0)
+        self.assertRegex(result.stdout, r"Experiment results Test_AddNums_.*\.csv downloaded successfully\.")
+
+        downloaded_files = glob.glob("Test_AddNums*.csv")
+        if downloaded_files:
+            df_actual = pd.read_csv(downloaded_files[0])
+            df_expected = pd.read_csv(CSV_FILE_PATH)
+            pd.testing.assert_frame_equal(df_actual, df_expected)
+        else:
+            self.fail("Results CSV file was not found after download.")
+        TestGladosCLI._cleanup_files()  # Remove csv file download from previous test to ensure this test is valid
+
+    def test_03_experiment_download_all(self):
+        if not self.experiment_id:
+            self.skipTest("No experiment ID available from previous step.")
+
+        # Give the system a moment to register the experiment
+        time.sleep(5)
+
+        result = self._run_cli(["-da", self.experiment_id])
+
+        self.assertEqual(result.returncode, 0)
+        self._assert_in_output("All experiment artifacts downloaded successfully.", result.stdout)
+
+        downloaded_files = glob.glob("Test_AddNums*.csv")
+        if downloaded_files:
+            df_actual = pd.read_csv(downloaded_files[0])
+            df_expected = pd.read_csv(CSV_FILE_PATH)
+            pd.testing.assert_frame_equal(df_actual, df_expected)
+        else:
+            self.fail("Results CSV file was not found after download.")
+        TestGladosCLI._cleanup_files()  # Remove csv file download from previous test to ensure this test is valid
+
+    def test_04_experiment_query(self):
+        result = self._run_cli(["-q", "Test AddNums"])
+
+        expected_output_fragment = (
+            "Matches:\n***********************************************\nExperiment 1: Test AddNums\n***********************************************\nTags: ['Test', 'AddNums']\nStatus: COMPLETED\nTrials: 100/100 Completed\n"
+        )
+
+        actual_filtered = self._filter_output(result.stdout)
+        expected_filtered = self._filter_output(expected_output_fragment)
+
+        self._assert_in_output(expected_filtered, actual_filtered)
+
+if __name__ == "__main__":
+    unittest.main()