From 9f1b0366a476347d493941c3a00b70da6b700916 Mon Sep 17 00:00:00 2001 From: Andrew Duller Date: Fri, 20 Mar 2026 16:40:57 +0000 Subject: [PATCH 1/7] #482: Removed numpy deprecated calls which are errors on python 3.11. --- python/src/odin_data/control/live_view_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/src/odin_data/control/live_view_adapter.py b/python/src/odin_data/control/live_view_adapter.py index 1ddf5a88c..089baab41 100644 --- a/python/src/odin_data/control/live_view_adapter.py +++ b/python/src/odin_data/control/live_view_adapter.py @@ -244,7 +244,7 @@ def create_image_from_socket(self, msg): dtype = 'float32' # create a np array of the image data, of type specified in the frame header - img_data = np.fromstring(msg[1], dtype=np.dtype(dtype)) + img_data = np.frombuffer(msg[1], dtype=np.dtype(dtype)) self.img_data = img_data.reshape([int(header["shape"][0]), int(header["shape"][1])]) self.header = header @@ -289,7 +289,7 @@ def render_image(self, colormap=None, clip_min=None, clip_max=None): # Most time consuming step, depending on image size and the type of image img_encode = cv2.imencode( '.png', img_colormapped, params=[cv2.IMWRITE_PNG_COMPRESSION, 0])[1] - return img_encode.tostring() + return img_encode.tobytes() @staticmethod def scale_array(src, tmin, tmax): From 7fb42617dcca948a8e791ea0ac60fbe084c58757 Mon Sep 17 00:00:00 2001 From: Andrew Duller Date: Wed, 25 Mar 2026 17:50:10 +0000 Subject: [PATCH 2/7] #482: Refactored OdinDataClient to be more amenable to use with pytest. Original operation should not have changed! --- python/src/odin_data/client.py | 52 +++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/python/src/odin_data/client.py b/python/src/odin_data/client.py index a6184c0f1..b6a85e862 100644 --- a/python/src/odin_data/client.py +++ b/python/src/odin_data/client.py @@ -26,20 +26,26 @@ class OdinDataClient(object): This class implements the odin-data command-line client. """ - def __init__(self): + def __init__(self, is_cli=False, ctrl_endpoint=None, log_level=logging.WARNING): """Initialise the odin-data client.""" + self.is_cli = is_cli + prog_name = os.path.basename(sys.argv[0]) - # Parse command line arguments - self.args = self._parse_arguments(prog_name) + if is_cli: + # Parse command line arguments + self.args = self._parse_arguments(prog_name) + log_level = logging.DEBUG + else: + self.args = None # create logger self.logger = logging.getLogger(prog_name) - self.logger.setLevel(logging.DEBUG) + self.logger.setLevel(log_level) - # create console handler and set level to debug + # create console handler ch = logging.StreamHandler(sys.stdout) - ch.setLevel(logging.DEBUG) + ch.setLevel(log_level) # create formatter formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s - %(message)s') @@ -50,12 +56,20 @@ def __init__(self): # add ch to logger self.logger.addHandler(ch) + # When not in CLI mode the ctrl_endpoint must be given + if is_cli: + self.ctrl_endpoint = self.args.ctrl_endpoint + else: + assert ctrl_endpoint, "The ctrl_endpoint must be given" + self.ctrl_endpoint = ctrl_endpoint + # Create the appropriate IPC channels self.ctrl_channel = IpcChannel(IpcChannel.CHANNEL_TYPE_DEALER) - self.ctrl_channel.connect(self.args.ctrl_endpoint) + self.ctrl_channel.connect(self.ctrl_endpoint) self._msg_id = 0 self._run = True + self._response = None def _parse_arguments(self, prog_name=sys.argv[0]): """Parse arguments from the command line.""" @@ -118,16 +132,19 @@ def do_config_cmd(self, config_file): ) self.ctrl_channel.send(config_msg.encode()) self.await_response() + return self._response except JSONDecodeError as e: self.logger.error("Failed to parse configuration file: {}".format(e)) def do_status_cmd(self): """Send a status command to odin-data.""" - status_msg = IpcMessage('cmd', 'status', id=self._next_msg_id()) - self.logger.info("Sending status request to the odin-data application") + id = self._next_msg_id() + status_msg = IpcMessage('cmd', 'status', id=id) + self.logger.info(f"Sending status request to the odin-data application {id}") self.ctrl_channel.send(status_msg.encode()) self.await_response() + return self._response def do_request_config_cmd(self): """Send a request configuration command to odin-data.""" @@ -135,6 +152,7 @@ def do_request_config_cmd(self): self.logger.info("Sending configuration request to the odin-data application") self.ctrl_channel.send(status_msg.encode()) self.await_response() + return self._response def do_request_command_cmd(self): """Send a request commands command to odin-data""" @@ -142,6 +160,7 @@ def do_request_command_cmd(self): self.logger.info("Sending command request to the odin-data application") self.ctrl_channel.send(status_msg.encode()) self.await_response() + return self._response def do_shutdown_cmd(self): """Send a shutdown command to odin-data.""" @@ -149,18 +168,25 @@ def do_shutdown_cmd(self): self.logger.info("Sending shutdown command to the odin-data application") self.ctrl_channel.send(shutdown_msg.encode()) self.await_response() + return self._response def await_response(self, timeout_ms=1000): """Await a response to a client command.""" pollevts = self.ctrl_channel.poll(1000) if pollevts == IpcChannel.POLLIN: - reply = IpcMessage(from_str=self.ctrl_channel.recv()) - self.logger.info("Got response: {}".format(reply)) - + self._response = IpcMessage(from_str=self.ctrl_channel.recv()) + if self.is_cli: + self.logger.info("Got response: {}".format(self._response)) + else: + # For non CLI return the dict of attributes from the response message + assert isinstance(self._response, IpcMessage), self._response + self._response = self._response.attrs + else: + self._response = None def main(): """Run the odin-data client.""" - app = OdinDataClient() + app = OdinDataClient(is_cli=True) app.run() From 2b9891bc6d393bf01d47f4daf98d722cd6fb4527 Mon Sep 17 00:00:00 2001 From: Andrew Duller Date: Wed, 25 Mar 2026 17:50:53 +0000 Subject: [PATCH 3/7] #482: First attempt at pytest fixtures to start apps (and teardown). --- systest/conftest.py | 132 ++++++++++++++++++++++++++++++++++++ systest/test_integration.py | 31 +++++++++ 2 files changed, 163 insertions(+) create mode 100644 systest/conftest.py create mode 100644 systest/test_integration.py diff --git a/systest/conftest.py b/systest/conftest.py new file mode 100644 index 000000000..ec6b03a83 --- /dev/null +++ b/systest/conftest.py @@ -0,0 +1,132 @@ +import os +import pytest +import shlex +import subprocess +from pathlib import Path +from time import sleep + +from odin_data.client import OdinDataClient + +@pytest.fixture +def install_prefix(): + try: + iprefix = os.environ["INSTALL_PREFIX"] + except KeyError: + raise KeyError("INSTALL_PREFIX is not set") + assert Path(iprefix).exists(), f"INSTALL_PREFIX: {iprefix} not found" + return iprefix + +def app_startup(install_path, app, param, msg_prefix): + app_path = install_path / "bin" / app + cmd = [app_path.as_posix()] + ctrl_endpoint = None + for k, v in param.items(): + assert k in ["ctrl", "config", "debug-level", "decoder", "lib-path", "frames", "dest-ip", "ports", "packet-gap", "packet-len", "width", "height", "drop-fraction", "drop-packets", "interval", "pcap-file"], k + if k == "ctrl": + assert isinstance(v, int), f"{msg_prefix} {k} argument must be type int" + ctrl_endpoint = f"tcp://127.0.0.1:{v}" + cmd.append(f"--{k} {ctrl_endpoint}") + elif k == "config": + assert isinstance(v, str), f"{msg_prefix} {k} argument must be type str" + cpath = install_path / "test_config" / v + assert cpath.exists(), f"{msg_prefix} {k} path {cpath} not found" + cmd.append(f"--{k} {cpath.as_posix()}") + elif k == "lib-path": + assert isinstance(v, str), f"{msg_prefix} {k} argument must be type str" + lpath = install_path / v + assert lpath.exists(), f"{msg_prefix} {k} path {lpath} not found" + cmd.append(f"--{k} {lpath.as_posix()}") + elif k == "decoder": + assert isinstance(v, str), f"{msg_prefix} {k} argument must be type str" + # Insert a positional arg + cmd.insert(1, v) + elif k in ["debug-level", "frames", "packet-gap", "packet-len", "width", "height"]: # Generic int parameter + assert isinstance(v, int), f"{msg_prefix} {k} argument must be type int" + cmd.append(f"--{k} {v}") + elif k in ["dest-ip", "drop-packets", "pcap-file", "ports"]: # Generic str parameter + assert isinstance(v, str), f"{msg_prefix} {k} argument must be type str" + cmd.append(f"--{k} {v}") + elif k in ["drop-fraction", "interval"]: # Generic float parameter + assert isinstance(v, str), f"{msg_prefix} {k} argument must be type str" + cmd.append(f"--{k} {v}") + else: + assert False, f"{msg_prefix} Unknown parameter name {k}" + + cmdstring = " ".join(cmd) + print(f"Launching: {cmdstring}") + proc = subprocess.Popen(shlex.split(cmdstring), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + client = None + if ctrl_endpoint is not None: + client = OdinDataClient(is_cli=False, ctrl_endpoint=ctrl_endpoint) + # Poll for app being up by checking that status is not None + retries = 5 + is_up = False + while retries != 0: + if client.do_status_cmd() is not None: + is_up = True + break + print(f"Waiting for {app_path.name} to start up") + retries -= 1 + sleep(1) + return proc, client + +def app_teardown(proc, client): + if client is not None: + client.do_shutdown_cmd() + proc.wait() + assert proc.returncode == 0, f"Error: {proc.stderr}" + # Need to check the stdout for ERROR as at least frameSimulator produces errors but still + # exits with status 0. + # DIAMOND_TODO frameSimulator should be changed so this is not required as this is horrible. + error_lines = [] + for line in proc.stdout: + linestr = line.decode("utf-8").rstrip() + print(linestr) + if " ERROR " in linestr: + error_lines.append(linestr) + assert len(error_lines) == 0, f"Error: found {len(error_lines)} error in stdout\n{error_lines}" + +@pytest.fixture +def frame_processor(request, install_prefix): + install_path = Path(install_prefix) + msg_prefix = f"Fixture {request.fixturename}:" + assert hasattr(request, "param"), f"{msg_prefix} requires param" + assert isinstance(request.param, dict), f"{msg_prefix} param must be a dict" + assert "ctrl" in request.param, f"{msg_prefix} requires ctrl to be set" + print(f"{msg_prefix} using {request.param}") + proc, client = app_startup(install_path, "frameProcessor", request.param, msg_prefix) + # These must be yielded rather than returned so that the teardown can happen once the test + # is complete + yield proc, client, request.param + app_teardown(proc, client) + +@pytest.fixture +def frame_receiver(request, install_prefix): + install_path = Path(install_prefix) + msg_prefix = f"Fixture {request.fixturename}:" + assert hasattr(request, "param"), f"{msg_prefix} requires param" + assert isinstance(request.param, dict), f"{msg_prefix} param must be a dict" + assert "ctrl" in request.param, f"{msg_prefix} requires ctrl to be set" + print(f"{msg_prefix} using {request.param}") + # These must be yielded rather than returned so that the teardown can happen once the test + # is complete + proc, client = app_startup(install_path, "frameReceiver", request.param, msg_prefix) + yield proc, client, request.param + app_teardown(proc, client) + +@pytest.fixture +def frame_simulator(request, install_prefix): + install_path = Path(install_prefix) + msg_prefix = f"Fixture {request.fixturename}:" + assert hasattr(request, "param"), f"{msg_prefix} requires param" + assert isinstance(request.param, dict), f"{msg_prefix} param must be a dict" + # This must have a decoder, this is a positional arg + assert "decoder" in request.param, f"{msg_prefix} requires decoder to be set" + # This must set the number of frames, otherwise the app will not terminate + assert "frames" in request.param, f"{msg_prefix} requires frames to be set" + assert "lib-path" not in request.param + request.param["lib-path"] = "lib" + print(f"{msg_prefix} using {request.param}") + proc, client = app_startup(install_path, "frameSimulator", request.param, msg_prefix) + yield proc, client, request.param + app_teardown(proc, client) diff --git a/systest/test_integration.py b/systest/test_integration.py new file mode 100644 index 000000000..38cdbb783 --- /dev/null +++ b/systest/test_integration.py @@ -0,0 +1,31 @@ +import pytest +from time import sleep +from odin_data.client import OdinDataClient + +@pytest.mark.parametrize("frame_processor", [{"ctrl": 5004}], indirect=True) +def test_can_start_frame_processor(frame_processor): + fp, fpclient, fpparams = frame_processor + status = fpclient.do_status_cmd() + assert status["msg_val"] == "status" + +@pytest.mark.parametrize("frame_receiver", [{"ctrl": 5000}], indirect=True) +def test_can_start_frame_receiver(frame_receiver): + fr, frclient, frparams = frame_receiver + status = frclient.do_status_cmd() + assert status["msg_val"] == "status" + +@pytest.mark.parametrize("frame_simulator", [{"decoder": "DummyUDP", "dest-ip": "127.0.0.1", "frames": 5}], indirect=True) +def test_can_start_frame_simulator(frame_simulator): + fs, _, fsparams = frame_simulator + +@pytest.mark.parametrize("frame_simulator", [{"decoder": "DummyUDP", "dest-ip": "127.0.0.1", "frames": 10, "ports": "61649", "packet-gap": 10}], indirect=True) +@pytest.mark.parametrize("frame_receiver", [{"ctrl": 5000, "config": "dummyUDP-fr.json"}], indirect=True) +@pytest.mark.parametrize("frame_processor", [{"ctrl": 5004, "config": "dummyUDP-fp.json"}], indirect=True) +def test_simple_pipe(frame_simulator, frame_receiver, frame_processor): + # Trying to mimic odinDataTest, but not working yet! + fr, frclient, frparams = frame_receiver + fp, fpclient, fpparams = frame_processor + fs, _, fsparams = frame_simulator + + +# Running /atsw-work/odin-dev-release/bin/frameSimulator DummyUDP --lib-path=/atsw-work/odin-dev-release/lib --frames=10 --dest-ip=127.0.0.1 --ports=61649 --packet-gap=10 From e08287ef129b9d7253f5203372487dfb80adc952 Mon Sep 17 00:00:00 2001 From: Andrew Duller Date: Thu, 26 Mar 2026 18:11:36 +0000 Subject: [PATCH 4/7] #482: Switched to using fixture factory and creating a single "system under test" fixture to allow cross checking of ports etc. --- systest/conftest.py | 117 +++++++++++++++++++++++++++++++++++ systest/system_under_test.py | 62 +++++++++++++++++++ systest/test_sut.py | 43 +++++++++++++ 3 files changed, 222 insertions(+) create mode 100644 systest/system_under_test.py create mode 100644 systest/test_sut.py diff --git a/systest/conftest.py b/systest/conftest.py index ec6b03a83..78e8983ef 100644 --- a/systest/conftest.py +++ b/systest/conftest.py @@ -6,6 +6,7 @@ from time import sleep from odin_data.client import OdinDataClient +from system_under_test import SystemUnderTest @pytest.fixture def install_prefix(): @@ -16,6 +17,9 @@ def install_prefix(): assert Path(iprefix).exists(), f"INSTALL_PREFIX: {iprefix} not found" return iprefix +################################################################################ +# Separate fixtures for each of frameReciver, frameProcessor and frameSimulator +################################################################################ def app_startup(install_path, app, param, msg_prefix): app_path = install_path / "bin" / app cmd = [app_path.as_posix()] @@ -130,3 +134,116 @@ def frame_simulator(request, install_prefix): proc, client = app_startup(install_path, "frameSimulator", request.param, msg_prefix) yield proc, client, request.param app_teardown(proc, client) + +####################################################################################### +# END OF Separate fixtures for each of frameReciver, frameProcessor and frameSimulator +####################################################################################### + +####################################################################################### +# System Under Test fixture, combining frameReciver, frameProcessor and frameSimulator +####################################################################################### + +def launch_app(app, sut): + cfg = sut.cfg(app) + if cfg is None: + print(f"No {app} required") + return + msg_prefix = f"{app}:" + assert isinstance(cfg, dict), f"{msg_prefix} cfg must be a dict" + app_path = sut.install_path / "bin" / app + cmd = [app_path.as_posix()] + ctrl_endpoint = None + for k, v in cfg.items(): + assert k in ["ctrl", "config", "debug-level"], k + if k == "ctrl": + assert isinstance(v, int), f"{msg_prefix} {k} argument must be type int" + ctrl_endpoint = f"tcp://127.0.0.1:{v}" + cmd.append(f"--{k} {ctrl_endpoint}") + elif k == "config": + assert isinstance(v, str), f"{msg_prefix} {k} argument must be type str" + cpath = sut.install_path / "test_config" / v + assert cpath.exists(), f"{msg_prefix} {k} path {cpath} not found" + cmd.append(f"--{k} {cpath.as_posix()}") + + assert ctrl_endpoint is not None, f"{msg_prefix} ctrl has not been specified" + cmdstring = " ".join(cmd) + print(f"Launching: {cmdstring}") + proc = subprocess.Popen(shlex.split(cmdstring), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + sut.set_proc(app, proc) + sut.set_client(app, OdinDataClient(ctrl_endpoint=ctrl_endpoint)) + # Poll for app being up by checking that status is not None + retries = 5 + is_up = False + while retries != 0: + if sut.client(app).do_status_cmd() is not None: + is_up = True + break + print(f"Waiting for {app_path.name} to start up") + retries -= 1 + sleep(1) + +def launch_simulator(app, sut): + cfg = sut.cfg(app) + if cfg is None: + print(f"No {app} required") + return + msg_prefix = f"{app}:" + assert isinstance(cfg, dict), f"{msg_prefix} cfg must be a dict" + app_path = sut.install_path / "bin" / app + cmd = [app_path.as_posix()] + + # This must have a decoder, this is a positional arg + assert "decoder" in cfg, f"{msg_prefix} requires decoder to be set" + # This must set the number of frames, otherwise the app will not terminate + assert "frames" in cfg, f"{msg_prefix} requires frames to be set" + assert "lib-path" not in cfg + cfg["lib-path"] = "lib" + + for k, v in cfg.items(): + assert k in ["debug-level", "decoder", "lib-path", "frames", "dest-ip", "ports", "packet-gap", "packet-len", "width", "height", "drop-fraction", "drop-packets", "interval", "pcap-file"], k + if k == "lib-path": + assert isinstance(v, str), f"{msg_prefix} {k} argument must be type str" + lpath = sut.install_path / v + assert lpath.exists(), f"{msg_prefix} {k} path {lpath} not found" + cmd.append(f"--{k} {lpath.as_posix()}") + elif k == "decoder": + assert isinstance(v, str), f"{msg_prefix} {k} argument must be type str" + # Insert a positional arg + cmd.insert(1, v) + elif k in ["debug-level", "frames", "packet-gap", "packet-len", "width", "height"]: # Generic int parameter + assert isinstance(v, int), f"{msg_prefix} {k} argument must be type int" + cmd.append(f"--{k} {v}") + elif k in ["dest-ip", "drop-packets", "pcap-file", "ports"]: # Generic str parameter + assert isinstance(v, str), f"{msg_prefix} {k} argument must be type str" + cmd.append(f"--{k} {v}") + elif k in ["drop-fraction", "interval"]: # Generic float parameter + assert isinstance(v, str), f"{msg_prefix} {k} argument must be type str" + cmd.append(f"--{k} {v}") + else: + assert False, f"{msg_prefix} Unknown parameter name {k}" + + cmdstring = " ".join(cmd) + print(f"Launching: {cmdstring}") + proc = subprocess.Popen(shlex.split(cmdstring), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + sut.set_proc(app, proc) + +@pytest.fixture +def sut_launch(request, install_prefix): + + def _sut_launch(sut): + assert isinstance(sut, SystemUnderTest) + sut.install_path = Path(install_prefix) + # Launch the parts of the system + launch_app("frameProcessor", sut) + launch_app("frameReceiver", sut) + launch_simulator("frameSimulator", sut) + + # Register function which will clean up the launched apps as appropriate + request.addfinalizer(sut.cleanup) + return + + return _sut_launch + +############################################################################################## +# END OF System Under Test fixture, combining frameReciver, frameProcessor and frameSimulator +############################################################################################## diff --git a/systest/system_under_test.py b/systest/system_under_test.py new file mode 100644 index 000000000..9e1ba533e --- /dev/null +++ b/systest/system_under_test.py @@ -0,0 +1,62 @@ +import subprocess + +from pathlib import Path +from odin_data.client import OdinDataClient + +class SystemUnderTest: + def __init__(self): + self._install_path = None + self._cfg = {"frameProcessor": None, "frameReceiver": None, "frameSimulator": None} + self._proc = {"frameProcessor": None, "frameReceiver": None, "frameSimulator": None} + self._client = {"frameProcessor": None, "frameReceiver": None, "frameSimulator": None} + + @property + def install_path(self): + assert self._install_path is not None + return self._install_path + + @install_path.setter + def install_path(self, path): + assert isinstance(path, Path), "The install path must be a Path" + self._install_path = path + + def cfg(self, app): + return self._cfg[app] + + def set_cfg(self, app, cfg): + assert app in self._cfg, app + assert isinstance(cfg, dict), "The cfg must be a dict" + self._cfg[app] = cfg + + def proc(self, app): + return self._proc[app] + + def set_proc(self, app, proc): + assert app in self._proc, app + assert self._proc[app] is None, "Process can only be launched once" + assert isinstance(proc, subprocess.Popen), "The proc must be a subprocess.Popen" + self._proc[app] = proc + + def client(self, app): + return self._client[app] + + def set_client(self, app, client): + assert app in self._client, app + assert isinstance(client, OdinDataClient), "The client must be an OdinDataClient" + self._client[app] = client + + def cleanup(self): + print("Test complete, cleaning up") + cleanup_ok = True + for app, client in self._client.items(): + if client is not None: + print(f"Instructing {app} to shutdown") + status = client.do_shutdown_cmd() + if status['msg_type'] != 'ack' or status['msg_val'] != 'shutdown': + print(f"Warning: {app} shutdown failed: {status}") + cleanup_ok = False + for app, proc in self._proc.items(): + if proc is not None: + print(f"Waiting for {app} to complete") + proc.wait() + assert cleanup_ok, "Test cleanup failed" diff --git a/systest/test_sut.py b/systest/test_sut.py new file mode 100644 index 000000000..e7252e3f9 --- /dev/null +++ b/systest/test_sut.py @@ -0,0 +1,43 @@ +import pytest +from time import sleep +from odin_data.client import OdinDataClient +from system_under_test import SystemUnderTest + +def test_can_start_frame_processor(sut_launch): + sut = SystemUnderTest() + sut.set_cfg("frameProcessor", {"ctrl": 5004}) + sut_launch(sut) + status = sut.client("frameProcessor").do_status_cmd() + assert status["msg_val"] == "status" + +def test_can_start_frame_receiver(sut_launch): + sut = SystemUnderTest() + sut.set_cfg("frameReceiver", {"ctrl": 5000}) + sut_launch(sut) + status = sut.client("frameReceiver").do_status_cmd() + assert status["msg_val"] == "status" + +def test_can_start_frame_simulator(sut_launch): + sut = SystemUnderTest() + sut.set_cfg("frameSimulator", {"decoder": "DummyUDP", "dest-ip": "127.0.0.1", "frames": 10, "ports": "61649", "packet-gap": 10}) + sut_launch(sut) + +def test_can_start_receiver_and_processor(sut_launch): + sut = SystemUnderTest() + sut.set_cfg("frameReceiver", {"ctrl": 5000}) + sut.set_cfg("frameProcessor", {"ctrl": 5004}) + sut_launch(sut) + status = sut.client("frameReceiver").do_status_cmd() + assert status["msg_val"] == "status" + status = sut.client("frameProcessor").do_status_cmd() + assert status["msg_val"] == "status" + +def test_simple_pipe(sut_launch): + # Trying to mimic odinDataTest, but not working yet! + sut = SystemUnderTest() + sut.set_cfg("frameReceiver", {"ctrl": 5000, "config": "dummyUDP-fr.json"}) + sut.set_cfg("frameProcessor", {"ctrl": 5004, "config": "dummyUDP-fp.json"}) + sut.set_cfg("frameSimulator", {"decoder": "DummyUDP", "dest-ip": "127.0.0.1", "frames": 10, "ports": "61649", "packet-gap": 10}) + sut_launch(sut) + +# Running /atsw-work/odin-dev-release/bin/frameSimulator DummyUDP --lib-path=/atsw-work/odin-dev-release/lib --frames=10 --dest-ip=127.0.0.1 --ports=61649 --packet-gap=10 From b5408beb006d3163d42165f38cee405502cd845f Mon Sep 17 00:00:00 2001 From: Andrew Duller Date: Fri, 27 Mar 2026 08:38:24 +0000 Subject: [PATCH 5/7] #482: Added setting of the IpcMessage type for frameProcessor replies (previously reporting "illegal") --- cpp/frameProcessor/src/FrameProcessorController.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/frameProcessor/src/FrameProcessorController.cpp b/cpp/frameProcessor/src/FrameProcessorController.cpp index 09df13438..c816b1c04 100644 --- a/cpp/frameProcessor/src/FrameProcessorController.cpp +++ b/cpp/frameProcessor/src/FrameProcessorController.cpp @@ -118,6 +118,7 @@ void FrameProcessorController::handleCtrlChannel() try { OdinData::IpcMessage ctrlMsg(ctrlMsgEncoded.c_str()); OdinData::IpcMessage replyMsg; // Instantiate default IpmMessage + replyMsg.set_msg_type(OdinData::IpcMessage::MsgTypeAck); replyMsg.set_msg_val(ctrlMsg.get_msg_val()); msg_id = ctrlMsg.get_msg_id(); replyMsg.set_msg_id(msg_id); From c9748aebb6d95fb461fbcdf06779b0b9f1b19363 Mon Sep 17 00:00:00 2001 From: Andrew Duller Date: Fri, 27 Mar 2026 08:40:33 +0000 Subject: [PATCH 6/7] #482: Added timeout_ms as a settable parameter on all client commands. This is needed as polling from the app causes responses to be out of sync. --- python/src/odin_data/client.py | 24 ++++++++++++------------ systest/conftest.py | 29 ++++++++++++++++------------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/python/src/odin_data/client.py b/python/src/odin_data/client.py index b6a85e862..739471a75 100644 --- a/python/src/odin_data/client.py +++ b/python/src/odin_data/client.py @@ -118,7 +118,7 @@ def run(self): if self.args.shutdown: self.do_shutdown_cmd() - def do_config_cmd(self, config_file): + def do_config_cmd(self, config_file, timeout_ms=1000): """Send a configure command to odin-data with the specified JSON configuration file.""" try: config_params = json.load(config_file) @@ -131,48 +131,48 @@ def do_config_cmd(self, config_file): "Sending configure command to the odin-data application with specified parameters" ) self.ctrl_channel.send(config_msg.encode()) - self.await_response() + self.await_response(timeout_ms) return self._response except JSONDecodeError as e: self.logger.error("Failed to parse configuration file: {}".format(e)) - def do_status_cmd(self): + def do_status_cmd(self, timeout_ms=1000): """Send a status command to odin-data.""" id = self._next_msg_id() status_msg = IpcMessage('cmd', 'status', id=id) self.logger.info(f"Sending status request to the odin-data application {id}") self.ctrl_channel.send(status_msg.encode()) - self.await_response() + self.await_response(timeout_ms) return self._response - def do_request_config_cmd(self): + def do_request_config_cmd(self, timeout_ms=1000): """Send a request configuration command to odin-data.""" status_msg = IpcMessage('cmd', 'request_configuration', id=self._next_msg_id()) self.logger.info("Sending configuration request to the odin-data application") self.ctrl_channel.send(status_msg.encode()) - self.await_response() + self.await_response(timeout_ms) return self._response - def do_request_command_cmd(self): + def do_request_command_cmd(self, timeout_ms=1000): """Send a request commands command to odin-data""" status_msg = IpcMessage('cmd', 'request_commands', id=self._next_msg_id()) self.logger.info("Sending command request to the odin-data application") self.ctrl_channel.send(status_msg.encode()) - self.await_response() + self.await_response(timeout_ms) return self._response - def do_shutdown_cmd(self): + def do_shutdown_cmd(self, timeout_ms=1000): """Send a shutdown command to odin-data.""" shutdown_msg = IpcMessage('cmd', 'shutdown', id=self._next_msg_id()) self.logger.info("Sending shutdown command to the odin-data application") self.ctrl_channel.send(shutdown_msg.encode()) - self.await_response() + self.await_response(timeout_ms) return self._response - def await_response(self, timeout_ms=1000): + def await_response(self, timeout_ms): """Await a response to a client command.""" - pollevts = self.ctrl_channel.poll(1000) + pollevts = self.ctrl_channel.poll(timeout_ms) if pollevts == IpcChannel.POLLIN: self._response = IpcMessage(from_str=self.ctrl_channel.recv()) if self.is_cli: diff --git a/systest/conftest.py b/systest/conftest.py index 78e8983ef..dfb2ac9cf 100644 --- a/systest/conftest.py +++ b/systest/conftest.py @@ -164,6 +164,11 @@ def launch_app(app, sut): cpath = sut.install_path / "test_config" / v assert cpath.exists(), f"{msg_prefix} {k} path {cpath} not found" cmd.append(f"--{k} {cpath.as_posix()}") + elif k == "debug-level": + assert isinstance(v, int), f"{msg_prefix} {k} argument must be type int" + cmd.append(f"--{k} {v}") + else: + assert False, f"{msg_prefix} Unknown parameter name {k}" assert ctrl_endpoint is not None, f"{msg_prefix} ctrl has not been specified" cmdstring = " ".join(cmd) @@ -172,15 +177,9 @@ def launch_app(app, sut): sut.set_proc(app, proc) sut.set_client(app, OdinDataClient(ctrl_endpoint=ctrl_endpoint)) # Poll for app being up by checking that status is not None - retries = 5 - is_up = False - while retries != 0: - if sut.client(app).do_status_cmd() is not None: - is_up = True - break - print(f"Waiting for {app_path.name} to start up") - retries -= 1 - sleep(1) + # The polling is done by the client as sending multiple status requests meant that the + # response messages got out of sync + assert sut.client(app).do_status_cmd(timeout_ms=5000) is not None, f"{msg_prefix}: cannot contact the app" def launch_simulator(app, sut): cfg = sut.cfg(app) @@ -233,10 +232,14 @@ def sut_launch(request, install_prefix): def _sut_launch(sut): assert isinstance(sut, SystemUnderTest) sut.install_path = Path(install_prefix) - # Launch the parts of the system - launch_app("frameProcessor", sut) - launch_app("frameReceiver", sut) - launch_simulator("frameSimulator", sut) + # Launch the parts of the system, any problems in the launch must run the cleanup + try: + launch_app("frameProcessor", sut) + launch_app("frameReceiver", sut) + launch_simulator("frameSimulator", sut) + except AssertionError as e: + sut.cleanup() + raise e # Register function which will clean up the launched apps as appropriate request.addfinalizer(sut.cleanup) From 081c7ac33a83c995e5f10caa654510a504b3b1e9 Mon Sep 17 00:00:00 2001 From: Andrew Duller Date: Fri, 27 Mar 2026 08:41:46 +0000 Subject: [PATCH 7/7] #482: Added the checking of the return codes from the app subprocesses. --- systest/system_under_test.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/systest/system_under_test.py b/systest/system_under_test.py index 9e1ba533e..c2e13bf49 100644 --- a/systest/system_under_test.py +++ b/systest/system_under_test.py @@ -46,6 +46,24 @@ def set_client(self, app, client): self._client[app] = client def cleanup(self): + def check_proc_status(app, proc): + if proc.returncode != 0: + for line in proc.stderr: + linestr = line.decode("utf-8").rstrip() + print(f"{app}:stderr:{linestr}") + return False + # Need to check the stdout for ERROR as at least frameSimulator + # produces errors but still exits with status 0. + # DIAMOND_TODO #490: frameSimulator should be changed so this is not + # required as this is horrible. + error_lines = [] + for line in proc.stdout: + linestr = line.decode("utf-8").rstrip() + print(linestr) + if " ERROR " in linestr: + error_lines.append(linestr) + return len(error_lines) == 0 + print("Test complete, cleaning up") cleanup_ok = True for app, client in self._client.items(): @@ -59,4 +77,12 @@ def cleanup(self): if proc is not None: print(f"Waiting for {app} to complete") proc.wait() + + procs_ok = True + for app, proc in self._proc.items(): + if proc is not None: + if not check_proc_status(app, proc): + procs_ok = False + assert procs_ok, "Test processes failed" + assert cleanup_ok, "Test cleanup failed"