Add Ramble application and experiment definitions

tests/ramble/README.md

@@ -0,0 +1,100 @@
+# Testing with Ramble
+
+## High-level steps
+- Create and activate a python environment
+- Install Ramble following the instructions in https://ramble.readthedocs.io/en/latest/getting_started.html#installation
+- Install Spack following the instructions in https://spack.io/about/#install-spack
+
+SPATTER_GIT - refers to the top-level Git repo for Spatter
+
+## Creating the Ramble Application:
+Run the following commands to setup a new experiment for spatter. `application.py` into `spatter/applications/spatter`
+```
+SPATTER_GIT$ ramble repo create spatter
+SPATTER_GIT$ ramble repo add spatter
+
+SPATTER_GIT$ mkdir -p spatter/applications/spatter
+
+SPATTER_GIT$ cp tests/ramble/application.py spatter/applications/spatter/.
+```
+
+## Create workspace and experiments:
+Run the following commands:
+```
+$ ramble workspace create -d tests -a
+==> Created and activated workspace in /workspaces/spatter/tests
+$ ramble workspace manage experiments spatter --overwrite -e UniformStride -v f=$PWD/tests/inputs/cpu-ustride.json 
+$ ramble workspace manage experiments spatter --overwrite -e Stream -v f=$PWD/tests/inputs/cpu-stream.json
+```
+
+```
+$ ramble config add "variants:package_manager:spack"
+$ ramble config add "software:packages:spatter:pkg_spec:'spatter@develop backend=openmp'"
+$ ramble config add "software:environments:spatter:packages:[spatter]"
+```
+
+```
+//Run experiment
+$ ramble workspace setup
+$ ramble workspace concretize
+$ ramble on
+
+```
+
+The python file `example_ramble_test.py` automatically creates a workspace and runs both experiments.
+
+## Spack Installation details
+
+### Prerequisites
+
+If you are building Spatter in a container, you will need to following base dependencies, along with a compiler (GCC).
+```
+apt install python3.12 python3.12-dev python3.12-venv python3-pip cmake
+```
+
+
+### Spack Install
+
+. spack/share/spack/setup-env.sh
+
+```
+spack external find python
+...
+==> The following specs have been detected on this system and added to /home/vscode/.spack/packages.yaml
+-- no arch / no compilers ---------------------------------------
+python@3.12.3
+
+spack compiler find
+==> Added 1 new compiler to /home/vscode/.spack/packages.yaml
+    gcc@13.3.0
+==> Compilers are defined in the following files:
+    /home/vscode/.spack/packages.yaml
+
+spack external find cmake
+==> The following specs have been detected on this system and added to /home/vscode/.spack/packages.yaml
+-- no arch / no compilers ---------------------------------------
+cmake@3.28.3
+```
+
+```
+$ spack env create spatter
+==> Created environment spatter in: /workspaces/spatter/spack_tmp/spack/var/spack/environments/spatter
+==> Activate with: spack env activate spatter
+```
+
+It also is likely useful to create a Python environment to isolate any pip installed packages
+```
+$ python3 -m venv pyenv_spatter
+vscode ➜ /workspaces/spatter (benchpark) $ . pyenv_spatter/bin/activate
+```
+
+### Install Ramble
+
+```
+git clone -c feature.manyFiles=true https://github.com/GoogleCloudPlatform/ramble.git
+pip install -r ramble/requirements.txt
+# Source the path for the ramble executable
+$ . ramble/share/ramble/setup-env.sh
+$ ramble -V
+0.6.0 (ff9672ec8267ef4de83387f26f4d4d62e11d824a)
+```

tests/ramble/application.py

@@ -0,0 +1,118 @@
+from ramble.appkit import *
+
+
+class Spatter(ExecutableApplication):
+    name = 'spatter'
+    maintainers('plavin', 'jyoung3131')
+
+
+    # Load from spack
+    with when("package_manager_family=spack"):
+        software_spec("spatter_openmp", pkg_spec="spatter@main backend=openmp")
+
+    required_package('spatter')
+
+    input_file(
+        "cpu_stream_patterns",
+        url='https://raw.githubusercontent.com/hpcgarage/spatter/refs/heads/main/standard-suite/basic-tests/cpu-stream.json',
+        description="CPU Patterns for stream",
+        expand=False
+    )
+
+    input_file(
+        "cpu_uniform_patterns",
+        url='https://raw.githubusercontent.com/hpcgarage/spatter/refs/heads/spatter-devel/standard-suite/basic-tests/cpu-ustride.json',
+        description="CPU Patterns for Uniform Test",
+        expand=False
+    )
+
+    input_file(
+        "gpu_stream_patterns",
+        url='https://raw.githubusercontent.com/hpcgarage/spatter/refs/heads/main/standard-suite/basic-tests/gpu-stream.json',
+        description="GPU Patterns for stream",
+        expand=False
+    )
+
+    input_file(
+        "gpu_uniform_patterns",
+        url='https://raw.githubusercontent.com/hpcgarage/spatter/refs/heads/spatter-devel/standard-suite/basic-tests/gpu-ustride.json',
+        description="GPU Patterns for Uniform Test",
+        expand=False
+    )
+
+    # Executable
+    executable('spatter_openmp', 
+                'spatter {a} -b openmp -f {f}  -r {r}  -t {t} -l {l} -s {s} -v {v} -w {w} -x {x} -y {y} -z {z} {args}', 
+                use_mpi=False)
+
+    # Workloads
+    workload("spatter", executable="spatter_openmp", inputs=["cpu_stream_patterns", "cpu_uniform_patterns"])
+
+
+    workload_variable('a', default='-a', description='Aggregate (default on)', workloads=['spatter'])
+    workload_variable('b', default='serial', description='Backend', workloads=['spatter'], values = ['serial', 'openmp', 'cuda'])
+    workload_variable('d', default='8', description='Delta', workloads=['spatter'])
+    workload_variable('k', default='gather', description='Kernel', workloads=['spatter'], values = ['gather', 'scatter', 'gs', 'multigather', 'multiscatter'])
+    workload_variable('l', default='1024', description='Set Number of Gathers or Scatters to Perform', workloads=['spatter'])
+    workload_variable('r', default='10', description='Number of Runs', workloads=['spatter'])
+    workload_variable('s', default='random', description='Set Random Seed', workloads=['spatter'])
+    workload_variable('t', default='1', description='Number of Threads', workloads=['spatter'])
+    workload_variable('v', default='1', description='Set Verbosity Level', workloads=['spatter'])
+    workload_variable('w', default='1', description='Set Wrap', workloads=['spatter'])
+    workload_variable('x', default='8', description='Delta Gather', workloads=['spatter'])
+    workload_variable('y', default='8', description='Delta Scatter', workloads=['spatter'])
+    workload_variable('z', default='1024', description='Local Work Size', workloads=['spatter'])
+    workload_variable('args', default='', description='Additional arguments', workloads=['spatter'])
+
+    # Context for summary output
+    figure_of_merit_context('config_row',
+                            regex=r'^\s*(?P<config_idx>\d+)\s+(?P<bytes>\d+)',
+                            output_format='Config {config_idx}')
+
+    # Categorical FOMs
+    figure_of_merit('Backend',
+                    log_file='{log_file}',
+                    fom_regex=r'Backend:\s+(?P<backend>.+)',
+                    group_name='backend',
+                    units='')
+
+    figure_of_merit('Compiler',
+                    log_file='{log_file}',
+                    fom_regex=r'Compiler:\s+(?P<compiler>.+)',
+                    group_name='compiler',
+                    units='')
+
+
+    figure_of_merit('Bytes',
+                    log_file='{log_file}',
+                    fom_regex=r'^\s*(?P<config>\d+)\s+(?P<bytes>\d+)\s+[0-9.eE+\-]+\s+[0-9.eE+\-]+',
+                    group_name='bytes',
+                    units='bytes',
+                    contexts=['config_row'])
+
+    figure_of_merit('Time',
+                    log_file='{log_file}',
+                    fom_regex=r'^\s*(?P<config>\d+)\s+(?P<bytes>\d+)\s+(?P<time>[0-9.eE+\-]+)\s+[0-9.eE+\-]+',
+                    group_name='time',
+                    units='s',
+                    contexts=['config_row'])
+
+    figure_of_merit('Bandwidth',
+                    log_file='{log_file}',
+                    fom_regex=r'^\s*(?P<config>\d+)\s+(?P<bytes>\d+)\s+[0-9.eE+\-]+\s+(?P<bw>[0-9.eE+\-]+)',
+                    group_name='bw',
+                    units='MB/s',
+                    contexts=['config_row'])
+
+    # Aggregate Summary FOM
+    figure_of_merit('Mean Bandwidth',
+                    log_file='{log_file}',
+                    fom_regex=r'Aggregate Summary - Mean Bandwidth:\s+(?P<summary_bw>[0-9.eE+\-]+)',
+                    group_name='summary_bw',
+                    units='MB/s')
+
+    # Success Criteria
+    success_criteria('mean_bytes',
+                        mode='fom_comparison',
+                        fom_name='Mean Bandwidth',
+                        formula='{value} > 0.0')

tests/ramble/benchpark/README.md

@@ -0,0 +1,10 @@
+# Uploading to Benchpark
+1. Create a directory `repos/ramble_applications/spatter`
+2. Copy application.py to the folder
+3. Create a directory `experiments/spatter`
+4. Copy experiment.py to the folder
+
+## Sources:
+https://software.llnl.gov/benchpark/add-a-benchmark.html
+https://software.llnl.gov/benchpark/add-an-experiment.html
+https://github.com/llnl/benchpark/

tests/ramble/benchpark/experiment.py

@@ -0,0 +1,103 @@
+from benchpark.directives import variant, maintainers
+from benchpark.experiment import Experiment
+from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
+
+class Spatter(
+    Experiment,
+    ProgrammingModel(
+        ProgrammingModelType.Mpionly,
+        ),
+    ):
+
+
+    variant(
+        "pattern",
+        default="stream",
+        values=('stream', 'uniform'),
+        description="Which ramble workload to execute"
+    )
+
+    variant(
+        "backend",
+        default="openmp",
+        values = ("openmp", "cuda", "serial"),
+        description="Configuration String."
+    )
+    variant(
+        "cuda_arch",
+        default="none",
+        values=lambda x: True, # Allow custom input
+        description="Which cuda architecture to use"
+    )
+    variant(
+        "version",
+        default="main",
+        values = ("main", "devel"),
+        description="Which Version to use."
+    )
+
+    maintainers("plavin", "jyoung3131")
+
+
+    def compute_applications_section(self):
+
+
+        # Define pattern file
+        if self.spec.satisfies("pattern=stream"):
+            if self.spec.satisfies("backend=cuda"):
+                self.add_experiment_variable("f", "{input_path}/gpu-stream.json")
+            else:
+                self.add_experiment_variable("f", "{input_path}/cpu-stream.json")
+        elif self.spec.satisfies("pattern=uniform"):
+            if self.spec.satisfies("backend=cuda"):
+                self.add_experiment_variable("f", "{input_path}/gpu-ustride.json")
+            else:
+                self.add_experiment_variable("f", "{input_path}/cpu-ustride.json")
+
+        # n_gpus
+        if self.spec.satisfies("backend=cuda"):
+            self.add_experiment_variable("n_gpus", "{n_resources}", True)  
+        else: 
+            self.add_experiment_variable("n_gpus", "0") 
+
+
+        self.add_experiment_variable("n_nodes", "1")
+        self.add_experiment_variable("n_ranks", "1")
+        self.add_experiment_variable("processes_per_node", "1")
+
+
+
+        # Satisfy the base class required variables
+        self.set_required_variables(
+            n_resources="{n_ranks}",
+            process_problem_size="4194304",
+            total_problem_size="4194304"
+        )
+
+        # # 2. Write the file to the current working directory.
+        # # When 'benchpark setup' runs, it executes this Python script 
+        # # from the root of the experiment's staging area.
+
+    def compute_package_section(self):
+
+        version = self.spec.variants['version'][0]
+
+        # Backend
+        if self.spec.satisfies("backend=openmp"):
+            backend = "backend=openmp"
+        elif self.spec.satisfies("backend=cuda") and not self.spec.satisfies("cuda_arch=none"):
+            backend = f"backend=cuda cuda_arch={self.spec.variants['cuda_arch'][0]}"
+        else:
+            backend = "backend=serial"
+
+        if self.spec.satisfies("+mpi"):
+            mpi_flag = '+mpi'
+        else:
+            mpi_flag = '~mpi'
+
+        self.add_package_spec(
+            self.name,
+            [
+                f"spatter@{version} {backend} {mpi_flag}"
+            ]
+        )