From 98192a66eab9785f07836249b455ec6f9131a37a Mon Sep 17 00:00:00 2001 From: yehuda arav Date: Thu, 2 Apr 2026 11:49:40 +0300 Subject: [PATCH] Remove GraphQL/web experiment support Removed webExperimentFactory, webExperiment, and all GraphQL dependencies from the codebase: Code changes: - dataObjectsFactory.py: removed webExperimentFactory class and gql imports - dataObjects.py: removed webExperiment class, removed client/url properties from Experiment/TrialSet/Trial/EntityType, removed requests/BytesIO imports, base getImage now raises NotImplementedError, base _init_ImageMaps is a no-op (overridden by ExperimentZipFile) - __init__.py (experimentSetup): removed WEB constant and getExperimentSetup function, removed webExperimentFactory import - __init__.py (argos): removed WEB import Deleted files: - argos/examples/web/ (web experiment examples) - argos/experimentSetup/.ipynb_checkpoints/ (stale checkpoint) Documentation updates: - Removed all webExperimentFactory/webExperiment/GraphQL references from 9 doc pages (API, architecture, data model, user guide) - Removed "Loading from Web" swimlane diagram - Updated class diagrams, dependency matrix, module maps - Changelog left as-is (historical) Tests: 99 passed, 10 xfailed (unchanged) Co-Authored-By: Claude Opus 4.6 (1M context) --- .coverage | Bin 0 -> 53248 bytes argos/__init__.py | 2 +- argos/examples/web/expConf.json | 50 - argos/examples/web/runExperiment.sh | 13 - .../dataObjects-checkpoint.py | 1126 ----------------- argos/experimentSetup/__init__.py | 60 +- argos/experimentSetup/dataObjects.py | 135 +- argos/experimentSetup/dataObjectsFactory.py | 368 +----- docs/developer_guide/api/experiment_setup.md | 112 +- docs/developer_guide/api/index.md | 8 +- .../architecture/core_concepts.md | 22 +- .../developer_guide/architecture/data_flow.md | 26 - .../architecture/experiment_setup.md | 42 +- docs/developer_guide/data_model.md | 2 +- .../developer_guide/experiment_setup_index.md | 10 +- docs/user_guide/concepts.md | 5 +- docs/user_guide/experiment_setup.md | 12 - 17 files changed, 45 insertions(+), 1948 deletions(-) create mode 100644 .coverage delete mode 100644 argos/examples/web/expConf.json delete mode 100644 argos/examples/web/runExperiment.sh delete mode 100644 argos/experimentSetup/.ipynb_checkpoints/dataObjects-checkpoint.py diff --git a/.coverage b/.coverage new file mode 100644 index 0000000000000000000000000000000000000000..b5d30d2ec3c8af2a549aa71ba80885d8f01fec70 GIT binary patch literal 53248 zcmeI4&u<&Y6~||GSEMPCTD7V`$%5fQi-r*9&L{8JVNS(M%fEJ~fA@-5_x6V<%2p5v zF;WZPF}a-GnfKJ>NAYQqmYPNXi1DIuj;Q%VRiLGKd1vy`Wdm7UP1 zJd{5!J0X_0eq6G@EsD8Ug#EqJS$nCtS^Ve3FN#kTzBS?ItA%AMKmq{}009st=6gUkXB*)UzFRov@wyv&S`s$T+m0VK`@LjxWCU!|{B1dRo4&W1pO$L9dAsZ|K6% zmFG4{QNs_KY<)d&yhclhRplIb*zn2Dohb1(@H;D!uiCB`yXl2)QczUto$yARzB<*MUgXE;HR}A@DjkZqK655tK7U@mej^SP##5`iQFxFkj0DZW z0G%a`Zp<;_AIMrm)QANi3G%6jrX(`Z)I(N5S4vWeuQCdo`LHleo4$K0(cBR2U*W=y|BlRm%A5tI4K(B$X!?$VairrD(HxB(4WnuC)SKyk&U zLzSJn-;3h-<h6nTSSg~K3wfo5s?igY-*82BNEe44btGh)*1 z^#om3EB%5rzQ`MVY2Ov}LIMF0009sH0T2KI5C8!X009sH0T4KS1WakjJn#Pv`;M^x zMh7Gi009sH0T2KI5C8!X009sH0T2Lzhmt_iFwfc9pM_kQlIHwe`ZoYyT3)@l{P;L| zW!Ucu`(68A52b@J8U#Q91V8`;KmY_l00ck)1V8`;K;VEt(L5)!p8*z+P$j9lf%vTfL^^ ztURN)bldNCv=^;(cj<8iez=kx^$tB7i2irK7rmsTUUwzvd7cjFaRFB>QgiO4YUmLG z?WgH+JPtiArAgYpZj!dSlWODV1-wvyRX6BKD&5_y2I;DN0=h0cjmQsnN#(PWR8F5% zWv%9VZd9w0BEJ7WHhD|fzq99TJ^2U9B7p!1fB*=900@8p2!H?xfB*=9zyl$0*&5jL zqxTI6wp^3K65_(o-r9NM&okAV!W@%@Tf#Ix{JkZ9E`|8c*uE(3 zi=t?W&k3>Th|-??uDMgXQxN0(9XTt`&_CMdirdTd(We{UeCSId?`$GIu&hE*xeL1^kE{N^N1s#7iC-!sKMXuG)QA_*cjmNm`&Y0K}FV6^0 zWFkq^i`$xt+gao5Z2VqMymwJlC%9EJN8;iU@u$*lQM@fmq^JMng}zTk`-b?S-sd*< z2nG83@ucZ7Y{e&7BpWspDs1V8`;KmY_l00ck)1V8`;KmY_DQUW>p zSw&h_{9&5$hhbRs6F_|bpMR}D0s#;J0T2KI5C8!X009sH0T2KI5IDUA`2Iik|EITq z;TZ^k00@8p2!H?xfB*=900@8p2rvQu|Nq$kGXx|M009sH0T2KI5C8!X009sH0T4L7 G1pWsPMVnCo literal 0 HcmV?d00001 diff --git a/argos/__init__.py b/argos/__init__.py index d82a615..a5c32d5 100755 --- a/argos/__init__.py +++ b/argos/__init__.py @@ -1,5 +1,5 @@ __version__ = '1.3.0' -from .experimentSetup import WEB,FILE +from .experimentSetup import FILE from .manager import experimentManager diff --git a/argos/examples/web/expConf.json b/argos/examples/web/expConf.json deleted file mode 100644 index 756168e..0000000 --- a/argos/examples/web/expConf.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "name": "edenTest", - "graphql": { - "url": "XXXX", - "token": "YYYY" - }, - "thingsboard": { - "login": { - "username": "tenant@thingsboard.org", - "password": "tenant" - }, - "server": { - "ip": "127.0.0.1", - "port": "8080" - } - }, - "kafka": { - "consumers": { - "deviceType1": { - "slide": "60", - "toParquet": ["argos.kafka.processes.to_parquet_type1", {}], - "processes": { - "180": { - "argos.kafka.processes.function1": { - "arg1": "value1", - "arg2": "value2" - } - }, - "300": { - "argos.kafka.processes.function2": { - "arg3": "value3" - } - } - } - }, - "deviceType2": { - "slide": 10, - "toParquet": ["argos.kafka.processes.to_parquet_type2", {}], - "processes": { - "60": { - "argos.kafka.processes.function3": { - "arg4": "value4" - } - } - } - } - }, - "ip": "127.0.0.1" - } -} diff --git a/argos/examples/web/runExperiment.sh b/argos/examples/web/runExperiment.sh deleted file mode 100644 index 7c8f5f5..0000000 --- a/argos/examples/web/runExperiment.sh +++ /dev/null @@ -1,13 +0,0 @@ -# Experiment configuration example can be found in this directory as expConf.json - -# Experiment setup -python argos-experiment-web.py --expConf {The experiment configuration} --setup - -# Trial load -python argos-experiment-web.py --expConf {The experiment configuration} --load {trial set name} {trial name} - -# Run consumers -python argos-experiment-web.py --expConf {The experiment configuration} --runConsumers {default folder for saving data} - -# Finalize (Updating the database documents desc with the information from the trials) -python argos-exeperiment-web.py --expConf {The experiments configuration} --finalize \ No newline at end of file diff --git a/argos/experimentSetup/.ipynb_checkpoints/dataObjects-checkpoint.py b/argos/experimentSetup/.ipynb_checkpoints/dataObjects-checkpoint.py deleted file mode 100644 index a45cbce..0000000 --- a/argos/experimentSetup/.ipynb_checkpoints/dataObjects-checkpoint.py +++ /dev/null @@ -1,1126 +0,0 @@ -import zipfile -import os -import json -import pandas -import requests -from io import BytesIO -import matplotlib.pyplot as plt -from ..utils.jsonutils import loadJSON -from ..utils.logging import get_logger as argos_get_logger - -class Experiment: - """ - Interface to the WEB experiment object. - - """ - - _setupFileNameOrData = None # experiment description holds its name, descrition and ect. - _experimentSetup = None - - _trialSetsDict = None # A dictionary of the trial sets. - _entitiesTypesDict = None # A dictionary of the devices types. - - _client = None # The client of the connection to the WEB. - - _imagesMap = None - - def refresh(self): - """ - Loads the experiment setup and rebuilds all the trial sets and entity types. - """ - self._experimentSetup = loadJSON(self._setupFileNameOrData) - self._initTrialSets() - self._initEntitiesTypes() - - @property - def setup(self): - return self._experimentSetup - - @property - def url(self): - return self.setup['experimentsWithData']['url'] - - @property - def client(self): - return self._client - - @property - def id(self): - ## DO NOT USE the self._experimentDescription['id']. It is not useful here. - return self.setup['project']['id'] - - @property - def name(self): - return self.setup['name'] - - @property - def description(self): - return self.setup['description'] - - @property - def trialSet(self): - return self._trialSetsDict - - @property - def entityType(self): - return self._entitiesTypesDict - - @property - def entityTypeTable(self): - entityTypeList = [] - for entityTypeName, entityTypeData in self.entityType.items(): - entityTypeList.append(entityTypeData.propertiesTable.assign(entityType=entityTypeName)) - - return pandas.concat(entityTypeList, ignore_index=True) - - @property - def entitiesTable(self): - return self.entitiesTableFull.drop(columns=["key","entitiesTypeKey"]) - - def trialsTable(self,trialsetName): - return self.trialSet[trialsetName].trialsTable - - @property - def entitiesTableFull(self): - entityTypeList = [] - for entityTypeName, entityTypeData in self.entityType.items(): - for entityTypeDataName, entityData in entityTypeData.items(): - entityTypeList.append( - pandas.DataFrame(entityData.propertiesList, index=[0]).assign(entityType=entityTypeName)) - - return pandas.concat(entityTypeList, ignore_index=True) - - def __init__(self, setupFileOrData): - """ - Experiment object contains information on a specific experiment - - Parameters - ----------- - - setupFileOrData: str,dict - - - A file name that contains a dictionary with all the information on the experiment, that was downloaded from - argosWEB (using the download metadata). - - dict that includes all the data. - - - """ - self.logger = argos_get_logger(self) - self._trialSetsDict = dict() - self._entitiesTypesDict = dict() - - self._setupFileNameOrData = setupFileOrData - self.refresh() - - self.logger.execution("Loading images") - self._init_ImageMaps() - - def _init_ImageMaps(self): - ## Initializing the images map - self._imagesMap = dict() - if 'experimentsWithData' in self.setup: - for imgs in self.setup['experimentsWithData']['maps']: - imgName = imgs['imageName'] - imageFullURL = f"{self.url}/{imgs['imageUrl']}" - imgs['imageURL'] = imageFullURL - - self._imagesMap[imgName] = imgs - - - @property - def imageMap(self): - return self._imagesMap - - def getImageURL(self, imageName: str): - return self._imagesMap[imageName]['imageURL'] - - def getImageJSMappingFunction(self, imageName: str): - """ - Return the javascript mapping function that maps the - coordindates of the image to from the coordinates to 0..1 coordinats - - Used in thingsboard dashboards. Therefore the inputs to the function are - (origXpos,origYpos). - - - Parameters - ---------- - imageName: str - The name of the image - - - Returns - ------- - A string with the mapping function (in javascript). - - """ - metadata = self.getImageMetadata(imageName) - - xdim = metadata['right'] - metadata['left'] - ydim = metadata['upper'] - metadata['lower'] - - Xconvert = f"image_x = (origXPos - ({metadata['left']}))/{xdim};" - Yconvert = f"image_y = (({metadata['upper']})-origYPos)/{ydim};" - - return_string = "return {x: image_x, y: image_y};" - - return "\n".join([Xconvert, Yconvert, return_string]) - - def getImageMetadata(self, imageName: str): - return self._imagesMap[imageName] - - def _initTrialSets(self): - - for trialset in self.setup['trialSets']: - self._trialSetsDict[trialset['name']] = TrialSet(experiment=self, metadata=trialset) - - def _initEntitiesTypes(self): - - for entityType in self.setup['entityTypes']: - self._entitiesTypesDict[entityType['name']] = EntityType(experiment=self, metadata=entityType) - - def toJSON(self): - """ - Create a single JSON with the data of the experiment. - - - :return: - """ - ret = dict() - - entityTypeMap = dict() - for entityName, entityType in self.entityType.items(): - entityTypeMap[entityName] = entityType.toJSON() - - trialMap = dict() - for trialName, trialData in self.trialSet.items(): - trialMap[trialName] = trialData.toJSON() - - ret['entityType'] = entityTypeMap - ret['trialSet'] = trialMap - - expr = dict() - - for field in ['maps', 'begin', 'end', 'description']: - expr[field] = self._setupFileNameOrData['experimentsWithData'][field] - - ret['experiment'] = expr - return ret - - def getExperimentEntities(self): - """ - Returns the list of all the entities - - :return: dict - Return a list of the entities. - """ - retList = [] - - for entitytypeName, entityTypeObj in self._entitiesTypesDict.items(): - for entityName, entityData in entityTypeObj.items(): - retList.append(dict(entityName=entityName, entityTypeName=entityData.entityType.name)) - - return retList - - def getEntitiesTypeByID(self, entityTypeID): - - ret = None - for entityTypeName, entityTypeData in self.entityType.items(): - if entityTypeID == entityTypeData.keyID: - ret = entityTypeData - break - - return ret - - def getImage(self, imageName: str): - - imgUrl = os.path.join(self.setup['experimentsWithData']['url'], "images", f"{imageName}.png") - - # maybe we can skip the open(...), didn't want to risk it - try: - with open(imgUrl) as imageFile: - img = plt.imread(imageFile) - except UnicodeDecodeError: - img = plt.imread(imgUrl) - return img - -class ExperimentZipFile(Experiment): - - def __init__(self, setupFileOrData): - super().__init__(setupFileOrData=setupFileOrData) - - def getImage(self, imageName: str): - - imageurl = self._imagesMap[imageName]['imageURL'] - - with zipfile.ZipFile(self._setupFileNameOrData) as archive: - imageFile = archive.open(os.path.join("images",imageurl)) - - return plt.imread(imageFile) - - - def refresh(self): - """ - Loads the experiment setup and rebuilds all the trial sets and entity types. - """ - self.logger.execution("------- Start ----") - self.logger.debug(f"Loading file {self._setupFileNameOrData}") - with zipfile.ZipFile(self._setupFileNameOrData) as archive: - experimentDict =loadJSON("\n".join([x.decode() for x in archive.open("data.json").readlines()])) - - - fileVersion = experimentDict.get("version","1.0.0.").replace(".","_") - self.logger.debug(f"Got file version {fileVersion}") - - - experimentDict = getattr(self,f"_fix_json_version_{fileVersion}")(experimentDict) - - self.logger.execution("Experiemnt dict") - self._experimentSetup = experimentDict - - self.logger.execution("Init trial sets") - self._initTrialSets() - - self.logger.execution("Init entity type") - self._initEntitiesTypes() - - def _init_ImageMaps(self): - ## Initializing the images map - with zipfile.ZipFile(self._setupFileNameOrData) as archive: - experimentDict =loadJSON("\n".join([x.decode() for x in archive.open("data.json").readlines()])) - - self._imagesMap = dict() - - experimentDatakey = 'experiment' if 'experiment' in experimentDict else 'experimentWithData' - - if 'experiment' in experimentDict: - for imgs in experimentDict[experimentDatakey]['maps']: - imgName = imgs['imageName'] - self._imagesMap[imgName] = imgs - - def _fix_json_version_1_0_0_(self,jsonFile): - return jsonFile - - def _fix_json_version_2_0_0_(self,jsonFile): - - oldFormat = dict(experiment=jsonFile['experiment'], - entityTypes=jsonFile['entityTypes'], - trialSets=jsonFile['trialSets']) - - - for trialSet in oldFormat['trialSets']: - trialSet['trials'] = [] - currentKey = trialSet['key'] - for trial in jsonFile['trials']: - if trial['trialSetKey'] == currentKey: - trialSet['trials'].append(trial) - - - for entityType in oldFormat['entityTypes']: - entityType['entities'] = [] - currentKey = entityType['key'] - for entity in jsonFile['entities']: - if entity['entitiesTypeKey'] == currentKey: - entityType['entities'].append(entity) - - return oldFormat - - - -class webExperiment(Experiment): - - def getImage(self, imageName: str): - imgUrl = self.getImageURL(imageName) - response = requests.get(imgUrl) - - if response.status_code != 200: - raise ValueError(f"Image {imageName} not found on the server.") - - imageFile = BytesIO(response.content) - return plt.imread(imageFile) - - -class TrialSet(dict): - """ - Interface to the web trial set object. - - """ - _experiment = None - _metadata = None - - _trialsDict = None - - @property - def client(self): - return self.experiment.client - - @property - def experiment(self): - return self._experiment - - @property - def keyID(self): - return self._metadata['key'] - - @property - def id(self): - return self._metadata['id'] - - @property - def name(self): - return self._metadata['name'] - - @property - def description(self): - return self._metadata['description'] - - @property - def numberOfTrials(self): - return self._metadata['numberOfTrials'] - - @property - def propertiesTable(self): - if 'properties' in self._metadata: - ret = pandas.DataFrame(self._metadata['properties']) .set_index('key') - else: - ret = pandas.DataFrame() - - return ret - - @property - def properties(self): - ret = dict() - for prop in self._metadata['properties']: - ret[prop['label']] = prop - - return ret - - def toJSON(self): - ret = dict() - ret['name'] = self.name - ret['properties'] = self.properties - - trialsJSON = {} - for trialName, trialData in self.items(): - trialsJSON[trialName] = trialData.toJSON() - - ret['trials'] = trialsJSON - - return ret - - @property - def trials(self): - retList = [] - for trialName, trialData in self.items(): - trialProps = trialData.propertiesTable.assign(trialName=trialName, key=trialData.key) - retList.append(trialProps) - - return retList - - @property - def trialsTable(self): - return pandas.DataFrame(self.toJSON()['trials']).T - - def __init__(self, experiment: Experiment, metadata: dict): - """ - Trial set object contains information on a specific trial set - - :param experimentId: The experiment id - :param desc: A dictionary with information on the trial set - :param client: GraphQL client - """ - self._experiment = experiment - self._metadata = metadata - - self._initTrials() - - def _initTrials(self): - - for trial in self._metadata['trials']: - self[trial['name']] = Trial(trialSet=self, metadata=trial) - - -class Trial: - """ - Interface to the WEB trials. - - """ - - _trialSet = None - _metadata = None - - @property - def experiment(self): - return self._trialSet.experiment - - @property - def client(self): - return self._trialSet.experiment.client - - @property - def experiment(self): - return self._trialSet.experiment - - @property - def key(self): - return self._metadata['key'] - - @property - def id(self): - return self._metadata['id'] - - @property - def name(self): - return self._metadata['name'] - - @property - def trialSetKey(self): - return self._metadata['trialSetKey'] - - @property - def created(self): - return self._metadata['created'] - - @property - def status(self): - return self._metadata['status'] - - @property - def cloneFrom(self): - return self._metadata['cloneFrom'] - - @property - def numberOfEntities(self): - return self._metadata['numberOfEntities'] - - @property - def state(self): - return self._metadata['state'] - - @property - def properties(self): - return self._properties - - @property - def trialSet(self): - return self._trialSet - - @property - def propertiesTable(self): - return pandas.DataFrame(self.properties, index=[0]) - - def __init__(self, trialSet: TrialSet, metadata: dict): - """ - Trial object contains information on a specific trial - - Parameters - ---------- - - experiment: The experiment object - trialSet: The trial set object - desc: A dictionary with information on the trial - client: GraphQL client - """ - self._trialSet = trialSet - self._metadata = metadata - if metadata['properties']: - propertiesPandas = pandas.DataFrame(metadata['properties']).set_index('key') - - properties = propertiesPandas.merge(trialSet.propertiesTable, left_index=True, right_index=True)[ - ['val', 'type', 'label', 'description']] - getParser = lambda x: getattr(self, f"_parseProperty_{x.replace('-', '_')}") - - - # this wont work well for location property in the trial because it has 2 fields. - # we have to get the list, and the change all the lists with size 1 to the object itself (like we do now) - # and leave all the lists with size 2 as is. - parsedValuesList = [] - for key, data in properties.T.to_dict().items(): - parsed_data = getParser(data['type'])(data, data) - if len(parsed_data[1]) > 0: - val = parsed_data[1][0] - else: - val = None - parsedValuesList.append((data['label'], val)) - - self._properties = dict(parsedValuesList) - else: - - self._properties = metadata['properties'] - - def toJSON(self): - val = self.properties - val['name'] = self.name - val['status'] = self.status - val['state'] = self.status - return val - - def __str__(self): - return json.dumps(self.toJSON()) - - def __repr__(self): - return json.dumps(dict(name=self.name, status=self.status, state=self.state)) - - def _parseProperty_location(self, property, propertyMetadata): - """ - Parse the location property. - - Returns 3 values: location name, latitude and longitude. - The column names are at this order. - - :param property: - property - :param propertyMetadata: - The data that describes the property. - - :return: list,list - Returns list of column names and list of values. - """ - try: - if isinstance(property['val'],dict): - locationDict = property['val'] - else: - locationDict = json.loads(property['val']) - - locationName = locationDict['name'] - coords = locationDict['coordinates'] - if isinstance(coords,str): - coords = eval(coords) - - latitude = float(coords[0]) - longitude = float(coords[1]) - data = [locationName, latitude, longitude] - columns = ['locationName', 'latitude', 'longitude'] - except KeyError: - # data = [None] * 3 - data = [] - columns = [] - except TypeError: - # data = [None] * 3 - data = [] - columns = [] - - return columns, data - - def _parseProperty_text(self, property, propertyMetadata): - """ - Parse the text property. - - Returns 2 values: location name, latitude and longitude. - :param property: - :param propertyMetadata: - - :return: list,list - Returns list of column names and list of values. - """ - propertyLabel = propertyMetadata['label'] - data = [property['val']] - columns = [propertyLabel] - - return columns, data - - def _parseProperty_textArea(self, property, propertyMetadata): - """ - Parse the text property. - - Returns 2 values: location name, latitude and longitude. - :param property: - :param propertyMetadata: - - :return: list,list - Returns list of column names and list of values. - """ - propertyLabel = propertyMetadata['label'] - - data = [property['val']] - columns = [propertyLabel] - - return columns, data - - def _parseProperty_boolean(self, property, propertyMetadata): - """ - Parse the text property. - - Returns 2 values: location name, latitude and longitude. - :param property: - :param propertyMetadata: - - :return: list,list - Returns list of column names and list of values. - """ - propertyLabel = propertyMetadata['label'] - data = [bool(property['val'])] - columns = [propertyLabel] - - return columns, data - - def _parseProperty_number(self, property, propertyMetadata): - """ - Parse the text property. - - Returns 2 values: location name, latitude and longitude. - :param property: - :param propertyMetadata: - - :return: list,list - Returns list of column names and list of values. - """ - try: - propertyLabel = propertyMetadata['label'] - flt = property['val'] - if flt is None: - data = [None] - else: - data = [float(property['val'])] - columns = [propertyLabel] - except ValueError: - #print(f"\tCannot convert to float property {propertyLabel}. Got value '{property['val']}'") - data = [] - columns = [] - - return columns, data - - def _parseProperty_datetime_local(self, property, propertyMetadata): - """ - Parse the text property. - - Returns 2 values: location name, latitude and longitude. - :param property: - :param propertyMetadata: - - :return: list,list - Returns list of column names and list of values. - """ - propertyLabel = propertyMetadata['label'] - localdata = pandas.to_datetime(property['val'], utc=False) - if localdata is None: - data = [None] - else: - data = [localdata.tz_localize("israel")] - columns = [propertyLabel] - - return columns, data - - def _parseProperty_selectList(self, property, propertyMetadata): - propertyLabel = propertyMetadata['label'] - data = [property['val']] - columns = [propertyLabel] - - return columns, data - - def _composeEntityProperties(self, entityType, properties): - """ - Just resolves the properties names. - If it is location, split into 3 coordinates. - :param properties: - :return: - """ - data = [] - columns = [] - for property in properties: - propertyKey = property['key'] - propertyMetadata = entityType.propertiesTable.loc[propertyKey] - propertyType = propertyMetadata['type'] - - prop_type_handler = getattr(self, f"_parseProperty_{propertyType}") - - pcolumns, pdata = prop_type_handler(property, propertyMetadata) - columns += pcolumns - data += pdata - - entity_trial_properties = pandas.DataFrame(data=[data], columns=columns, index=[0]) - return entity_trial_properties - - def _composeProperties(self, entities): - - fullData = self.experiment.entitiesTableFull.set_index("key").join(entities, rsuffix="_r", how="inner").reset_index() - dfList = [] - for indx, (entitykey, entitydata) in enumerate(fullData.iterrows()): - - properties = entitydata['properties'] - entityType = self.experiment.getEntitiesTypeByID(entityTypeID=entitydata.entitiesTypeKey) - - entity_trial_properties = self._composeEntityProperties(entityType, properties) - - entityProperties = entityType[entitydata['name']].propertiesTable.copy() - entity_total_properties = entity_trial_properties.join(entityProperties, - how='left',rsuffix='_prop') # .assign(trialSet = self.trialSet.name, - - dfList.append(entity_total_properties) - new_df = pandas.concat(dfList, sort=False, ignore_index=True).drop(columns=["key","entitiesTypeKey"]) - - return new_df - - @property - def designEntities(self): - ret = self.designEntitiesTable - if ret.empty: - return dict() - else: - return ret.set_index("entityName").T.to_dict() - - @property - def deployEntities(self): - ret = self.deployEntitiesTable - if ret.empty: - return dict() - else: - return ret.set_index("entityName").T.to_dict() - - def entities(self, status): - """ - - :param status: str - design or deploy - :return: - """ - return getattr(self, f"{status}Entities") - - def entitiesTable(self, status): - """ - - :param status: str - design or deploy - :return: - """ - return getattr(self, f"{status}EntitiesTable") - - def getDesignPropertiesTableByEntityID(self, entityID): - try: - entity = pandas.DataFrame(self._metadata['entities']).set_index('key').loc[entityID] - entityType = self.experiment.getEntitiesTypeByID(entityTypeID=entity.entitiesTypeKey) - ret = self._composeEntityProperties(entityType, entity.propertiesList) - except KeyError: - ret = pandas.DataFrame() - return ret - - def getDesignPropertiesByEntityID(self, entityID): - ret = self.getDesignPropertiesTableByEntityID(entityID) - if ret.empty: - return dict() - else: - return ret.loc[0].T.to_dict() - - def _prepareEntitiesMetadata(self,metadata): - - retList = [] - for entityData in metadata: - for propData in entityData['properties']: - properties = pandas.DataFrame(propData,index=[0]) - itm = pandas.DataFrame(properties).assign(entitiesTypeKey=entityData['entitiesTypeKey'],containsEntities=entityData['containsEntities']) - retList.append(itm) - - - return pandas.concat(retList,ignore_index=True) - - @property - def designEntitiesTable(self): - entities = pandas.DataFrame(self._metadata['entities']).set_index('key') - return self._composeProperties(entities) - - @property - def deployEntitiesTable(self): - if not self._metadata['deployedEntities']: - return pandas.DataFrame() - else: - entities = pandas.DataFrame(self._metadata['deployedEntities']).set_index('key') - return self._composeProperties(entities) - - def getDeployPropertiesTableByEntityID(self, entityID): - try: - entity = pandas.DataFrame(self._metadata['deployedEntities']).set_index('key').loc[entityID] - entityType = self.experiment.getEntitiesTypeByID(entityTypeID=entity.entitiesTypeKey) - ret = self._composeEntityProperties(entityType, entity.propertiesList) - except KeyError: - ret = pandas.DataFrame() - return ret - - def getDeployPropertiesByEntityID(self, entityID): - ret = self.getDeployPropertiesTableByEntityID(entityID) - if ret.empty: - return dict() - else: - return ret.loc[0].T.to_dict() - - def __getitem__(self, item): - return self._properties.loc[item].val - - -class EntityType(dict): - _experiment = None - _metadata = None - - @property - def experiment(self): - return self._experiment - - @property - def client(self): - return self.experiment.client - - @property - def keyID(self): - return self._metadata['key'] - - @property - def id(self): - return self._metadata['id'] - - @property - def name(self): - return self._metadata['name'] - - @property - def numberOfEntities(self): - return self._metadata['numberOfEntities'] - - @property - def state(self): - return self._metadata['state'] - - @property - def propertiesTable(self): - if 'properties' in self._metadata: - ret = pandas.DataFrame(self._metadata['properties']).set_index('key') - else: - ret = pandas.DataFrame() - - return ret - - @property - def properties(self): - ret = dict() - for prop in self._metadata['properties']: - ret[prop['label']] = prop - - return ret - - def toJSON(self): - ret = dict() - ret['name'] = self.name - ret['properties'] = self.properties - - entityJSON = {} - for entityName, entityData in self.items(): - entityJSON[entityName] = entityData.toJSON() - - ret['entities'] = entityJSON - - return ret - - def __init__(self, experiment: Experiment, metadata: dict): - """ - EntityType object contains information on a specific entity type - - :param experiment: The experiment object - :param metadata: dict - The metadata of the entity type (the properties and ect). - """ - self._experiment = experiment - self._metadata = metadata - - self._initEntities() - - def _initEntities(self): - for entity in self._metadata['entities']: - self[entity['name']] = Entity(entityType=self, metadata=entity) - - @property - def entities(self): - retList = [] - for entityName, entityData in self.items(): - trialProps = entityData.propertiesTable.assign(entityName=entityName, key=entityData.key) - retList.append(trialProps) - return pandas.concat(retList, ignore_index=True).drop(columns=["key","entitiesTypeKey"]) - - -class Entity: - _entityType = None - _metadata = None - - @property - def entityType(self): - return self._entityType - - @property - def experiment(self): - return self.entityType.experiment - - @property - def client(self): - return self.experiment.client - - @property - def key(self): - return self._metadata['key'] - - @property - def id(self): - return self._metadata['id'] - - @property - def name(self): - return self._metadata['name'] - - @property - def entityTypeKey(self): - return self._metadata['entitiesTypeKey'] - - @property - def properties(self): - - ret = dict(self._properties) - - ret['key'] = self.key - ret['entitiesTypeKey'] = self._metadata['entitiesTypeKey'] - ret['name'] = self.name - ret['entityType'] = self.entityType.name - - return ret - - @property - def allProperties(self): - trialsetdict = dict() - - for trialsSetsName in self.experiment.trialSet.keys(): - trialsetdict[trialsSetsName] = dict() - for trialName in self.experiment.trialSet[trialsSetsName].keys(): - ddp = trialsetdict[trialsSetsName].setdefault(trialName, dict()) - - ddp['design'] = self.trialDesign(trialsSetsName, trialName) - ddp['deploy'] = self.trialDeploy(trialsSetsName, trialName) - - return trialsetdict - - @property - def allPropertiesList(self): - - trialsetlist = [] - - for trialsSetsName in self.experiment.trialSet.keys(): - for trialName in self.experiment.trialSet[trialsSetsName].keys(): - design = self.trialDesign(trialsSetsName, trialName) - deploy = self.trialDeploy(trialsSetsName, trialName) - - design['trialSetName'] = trialsSetsName - design['trialName'] = trialName - design['state'] = 'design' - - deploy['trialSetName'] = trialsSetsName - deploy['trialName'] = trialName - deploy['state'] = 'deploy' - - trialsetlist.append(design) - trialsetlist.append(deploy) - - return trialsetlist - - @property - def allPropertiesTable(self): - return pandas.DataFrame(self.allPropertiesList) - - @property - def propertiesTable(self): - val = pandas.DataFrame(self.properties, index=[0]) - val = val.assign(entityName=self.name).drop("name", axis=1) - return val - - def toJSON(self): - ret = dict() - ret['properties'] = dict(self._properties) - ret['name'] = self.name - ret['entityType'] = self.entityType.name - ret['trialProperties'] = self.allPropertiesList - return ret - - def __str__(self): - return json.dumps(self.toJSON()) - - def __repr__(self): - props = self.properties - props['name'] = self.name - return json.dumps(props) - - @property - def designProperties(self): - - trialsetdict = dict() - - for trialsSetsName in self.experiment.trialSet.keys(): - trialsetdict[trialsSetsName] = dict() - for trialName in self.experiment.trialSet[trialsSetsName].keys(): - trialsetdict[trialsSetsName][trialName] = self.trialDesign(trialsSetsName, trialName) - - return trialsetdict - - @property - def deployProperties(self): - trialsetdict = dict() - - for trialsSetsName in self.experiment.trialSet.keys(): - trialsetdict[trialsSetsName] = dict() - for trialName in self.experiment.trialSet[trialsSetsName].keys(): - trialsetdict[trialsSetsName][trialName] = self.trialDeploy(trialsSetsName, trialName) - - return trialsetdict - - def trialDesign(self, trialSet, trialName): - ret = self.experiment.trialSet[trialSet][trialName].getDesignPropertiesByEntityID(self.key) - for fld in ['key', 'name', 'entityType', 'entitiesTypeKey']: - if fld in ret: - del ret[fld] - - return ret - - def trialDeploy(self, trialSet, trialName): - properties = self.experiment.trialSet[trialSet][trialName].deployEntities - ret = properties.get(self.name, dict()) - - for fld in ['key', 'name', 'entityType', 'entitiesTypeKey']: - if fld in ret: - del ret[fld] - - return ret - - def trial(self, trialSet, trialName, state): - """ - Gets the properties of the trial useng the state - - Parameters - ----------- - trialSet: str - The trialset name - trialName: str - The trial name - state: str - 'design' or 'deploy' - - Returns - ------- - dict - """ - - return getattr(self, f"trial{state.title()}")(trialSet, trialName) - - def __init__(self, entityType: EntityType, metadata: dict): - """ - - :param entityType: EntityType - The entity type - - :param metadata : dict - The metadata of the - """ - self._entityType = entityType - self._metadata = metadata - if 'properties' in metadata: - propertiesPandas = pandas.DataFrame(metadata['properties']).set_index('key') - - properties = propertiesPandas.merge(entityType.propertiesTable.query("trialField==False"), left_index=True, - right_index=True)[['val', 'type', 'label', 'description']] \ - .set_index("label") - else: - properties = pandas.DataFrame() - - self._properties = dict([(key, data['val']) for key, data in properties.T.to_dict().items()]) - diff --git a/argos/experimentSetup/__init__.py b/argos/experimentSetup/__init__.py index 525bc1c..1d636f3 100644 --- a/argos/experimentSetup/__init__.py +++ b/argos/experimentSetup/__init__.py @@ -2,65 +2,9 @@ Experiment setup module for pyArgos. Provides factory classes and data objects for loading and working with -Argos experiment configurations from local files or the ArgosWEB server. - -Constants ---------- -WEB : str - Source type for web-based experiments (``"web"``). -FILE : str - Source type for file-based experiments (``"json"``). +Argos experiment configurations from local files. """ -from .dataObjectsFactory import fileExperimentFactory,webExperimentFactory +from .dataObjectsFactory import fileExperimentFactory -WEB = "web" FILE = "json" - - - -def getExperimentSetup(experimentType,experimentName, **kwargs): - """ - Load an experiment using the appropriate factory based on source type. - - A convenience function that routes to either ``fileExperimentFactory`` - or ``webExperimentFactory`` depending on the ``experimentType`` parameter. - - Parameters - ---------- - experimentType : str - The source type: ``"web"`` for ArgosWEB or ``"json"`` for local files. - Use the module constants ``WEB`` and ``FILE``. - experimentName : str - The name of the experiment to load. - **kwargs : dict - Additional keyword arguments passed to the factory: - - - For ``WEB``: ``url`` (str) and ``token`` (str) are required. - - For ``FILE``: ``experimentPath`` (str, optional) specifies the directory. - - Returns - ------- - Experiment - The loaded experiment object. - - Raises - ------ - ValueError - If ``experimentType`` is not ``WEB`` or ``FILE``. - - Examples - -------- - >>> from argos.experimentSetup import getExperimentSetup, FILE, WEB - >>> experiment = getExperimentSetup(FILE, "MyExp", experimentPath="/path/to/exp") - >>> experiment = getExperimentSetup(WEB, "MyExp", url="http://server", token="abc") - """ - if experimentType not in [WEB, FILE]: - raise ValueError(f"experimentType must be {FILE} or {WEB}. Got {experimentType}") - - if experimentType==WEB: - experiment = webExperimentFactory(url=kwargs['url'],token=kwargs['token']) - else: - experiment= fileExperimentFactory(**kwargs) - - return experiment.getExperiment(experimentName) diff --git a/argos/experimentSetup/dataObjects.py b/argos/experimentSetup/dataObjects.py index 69b72e4..6230afc 100644 --- a/argos/experimentSetup/dataObjects.py +++ b/argos/experimentSetup/dataObjects.py @@ -11,8 +11,6 @@ import os import json import pandas -import requests -from io import BytesIO import matplotlib.pyplot as plt import warnings @@ -75,8 +73,6 @@ class Experiment: _trialSetsDict = None # A dictionary of the trial sets. _entitiesTypesDict = None # A dictionary of the devices types. - _client = None # The client of the connection to the WEB. - _imagesMap = None def refresh(self): @@ -103,30 +99,6 @@ def setup(self): """ return self._experimentSetup - @property - def url(self): - """ - The base URL of the experiment on the ArgosWEB server. - - Returns - ------- - str - The URL string from the experiment's ``experimentsWithData`` section. - """ - return self.setup['experimentsWithData']['url'] - - @property - def client(self): - """ - The GraphQL client used for web-based experiments. - - Returns - ------- - Client or None - The GQL client instance, or None for file-based experiments. - """ - return self._client - @property def name(self): """ @@ -136,7 +108,6 @@ def name(self): - v3.0.0 ZIP (after migration): ``setup['experiment']['name']`` - v2.0.0 ZIP (after migration): ``setup['experiment']['name']`` - - Web (GraphQL): ``setup['experimentsWithData']['name']`` - Legacy/direct: ``setup['name']`` Returns @@ -169,7 +140,6 @@ def _get_experiment_field(self, field): 1. ``setup[field]`` (legacy/direct) 2. ``setup['experiment'][field]`` (v2/v3 ZIP after migration) - 3. ``setup['experimentsWithData'][field]`` (web/GraphQL) Parameters ---------- @@ -190,8 +160,6 @@ def _get_experiment_field(self, field): return self.setup[field] if 'experiment' in self.setup and field in self.setup['experiment']: return self.setup['experiment'][field] - if 'experimentsWithData' in self.setup and field in self.setup['experimentsWithData']: - return self.setup['experimentsWithData'][field] raise KeyError(f"Field '{field}' not found in experiment setup") @property @@ -310,15 +278,8 @@ def __init__(self, setupFileOrData): self._init_ImageMaps() def _init_ImageMaps(self): - ## Initializing the images map + """Initialize the images map. Overridden by ExperimentZipFile.""" self._imagesMap = dict() - if 'experimentsWithData' in self.setup: - for imgs in self.setup['experimentsWithData']['maps']: - imgName = imgs['imageName'] - imageFullURL = f"{self.url}/{imgs['imageUrl']}" - imgs['imageURL'] = imageFullURL - - self._imagesMap[imgName] = imgs @property def imageMap(self): @@ -486,7 +447,10 @@ def getEntitiesTypeByID(self, entityTypeID): def getImage(self, imageName: str): """ - Load an experiment image from the local filesystem. + Load an experiment image. + + Override in subclasses to provide image loading from the + appropriate source (e.g., ZIP archive). Parameters ---------- @@ -496,16 +460,14 @@ def getImage(self, imageName: str): Returns ------- numpy.ndarray - The image data as a NumPy array (via matplotlib.pyplot.imread). + The image data as a NumPy array. + + Raises + ------ + NotImplementedError + If not overridden by a subclass. """ - imgUrl = os.path.join(self.setup['experimentsWithData']['url'], "images", f"{imageName}.png") - # maybe we can skip the open(...), didn't want to risk it - try: - with open(imgUrl) as imageFile: - img = plt.imread(imageFile) - except UnicodeDecodeError: - img = plt.imread(imgUrl) - return img + raise NotImplementedError("getImage must be called on ExperimentZipFile, not base Experiment") class ExperimentZipFile(Experiment): @@ -662,43 +624,6 @@ def _fix_json_version_3_0_0(self, jsonFile): return oldFormat -class webExperiment(Experiment): - """ - Experiment loaded from a remote ArgosWEB server. - - Extends :class:`Experiment` to fetch images via HTTP rather than - from the local filesystem. - """ - - def getImage(self, imageName: str): - """ - Fetch an experiment image from the remote server via HTTP. - - Parameters - ---------- - imageName : str - The name of the image. - - Returns - ------- - numpy.ndarray - The image data as a NumPy array. - - Raises - ------ - ValueError - If the image is not found on the server (HTTP status != 200). - """ - imgUrl = self.getImageURL(imageName) - response = requests.get(imgUrl) - - if response.status_code != 200: - raise ValueError(f"Image {imageName} not found on the server.") - - imageFile = BytesIO(response.content) - return plt.imread(imageFile) - - class TrialSet(dict): """ A collection of trials belonging to a single trial set. @@ -724,18 +649,6 @@ class TrialSet(dict): _trialsDict = None - @property - def client(self): - """ - The GraphQL client from the parent experiment. - - Returns - ------- - Client or None - The GQL client instance, or None for file-based experiments. - """ - return self.experiment.client - @property def experiment(self): """ @@ -914,18 +827,6 @@ def experiment(self): """ return self._trialSet.experiment - @property - def client(self): - """ - The GraphQL client from the root experiment. - - Returns - ------- - Client or None - The GQL client instance, or None for file-based experiments. - """ - return self._trialSet.experiment.client - @property def experiment(self): """ @@ -1547,18 +1448,6 @@ def experiment(self): """ return self._experiment - @property - def client(self): - """ - The GraphQL client from the parent experiment. - - Returns - ------- - Client or None - The GQL client instance, or None for file-based experiments. - """ - return self.experiment.client - @property def name(self): """ diff --git a/argos/experimentSetup/dataObjectsFactory.py b/argos/experimentSetup/dataObjectsFactory.py index 21baae1..e33f748 100644 --- a/argos/experimentSetup/dataObjectsFactory.py +++ b/argos/experimentSetup/dataObjectsFactory.py @@ -1,24 +1,13 @@ """ -Factory classes for loading experiments from different data sources. +Factory for loading experiments from local files. -This module provides two factories: - -- ``fileExperimentFactory`` -- loads experiments from local files (JSON or ZIP). -- ``webExperimentFactory`` -- fetches experiments from an ArgosWEB server via GraphQL. - -Both factories return ``Experiment`` (or subclass) objects that provide a -unified interface to the experiment data. +Provides ``fileExperimentFactory`` which loads experiments from local +files (JSON or ZIP) and returns ``Experiment`` (or ``ExperimentZipFile``) +objects. """ import glob -try: - from gql import gql, Client - from gql.transport.aiohttp import AIOHTTPTransport -except: - print("qgl is not installed") - -import pandas -from argos.experimentSetup.dataObjects import webExperiment,Experiment,ExperimentZipFile +from argos.experimentSetup.dataObjects import Experiment, ExperimentZipFile import os from argos.utils.jsonutils import loadJSON from argos.utils.logging import get_logger as argos_get_logger @@ -121,350 +110,3 @@ def __getitem__(self, item): The loaded experiment object. """ return self.getExperiment(experimentPath=item) - - -class webExperimentFactory: - """ - Factory that fetches experiment data from an ArgosWEB server via GraphQL. - - Connects to the server's GraphQL endpoint and provides methods to list, - describe, and fully load experiments. - - Parameters - ---------- - url : str - The base URL of the ArgosWEB server (e.g., ``"http://localhost:3000"``). - token : str - The authorization token for the server. Use an empty string for - unauthenticated access. - - Examples - -------- - >>> factory = webExperimentFactory("http://argos-server:3000", "my-token") - >>> experiment = factory.getExperiment("MyExperiment") - >>> print(factory.listExperimentsNames()) - """ - - _client = None - _url = None - - @property - def url(self): - """ - The base URL of the ArgosWEB server. - - Returns - ------- - str - The server URL. - """ - return self._url - - @property - def client(self): - """ - The GraphQL client used for server communication. - - Returns - ------- - Client - The GQL client instance. - """ - return self._client - - def __init__(self, url: str, token: str): - """ - Initialize the web experiment factory. - - Parameters - ---------- - url : str - The base URL of the ArgosWEB server. - token : str - The authorization token. Use an empty string for unauthenticated access. - """ - - graphqlUrl = f"{url}/graphql" - - headers = None if token == '' else dict(authorization=token) - transport = AIOHTTPTransport(url=graphqlUrl, headers=headers) - self._client = Client(transport=transport, fetch_schema_from_transport=True) - - self._url = url - - - def getExperimentMetadata(self,experimentName): - """ - Fetch the full metadata for an experiment from the server. - - Queries the GraphQL API to retrieve entity types, entities, trial sets, - and trials with all their properties and relationships. - - Parameters - ---------- - experimentName : str - The name of the experiment to fetch. - - Returns - ------- - dict - A dictionary containing the full experiment metadata with keys - ``experimentsWithData``, ``entitiesTypes``, and ``trialSets``. - """ - experimentDesc = self.getExperimentDescriptor(experimentName) - - ## Get the entities types - query = ''' - { - entitiesTypes(experimentId: "%s"){ - key - name - numberOfEntities - properties{ - key - type - label - description - required - trialField - value - } - } - } - ''' % experimentDesc['project']['id'] - entitiesTypes = self._client.execute(gql(query)) - - - - for entityType in entitiesTypes['entitiesTypes']: - query = ''' - { - entities(experimentId: "%s", entitiesTypeKey: "%s"){ - key - name - entitiesTypeKey - state - properties{ - val - key - } - } - } - ''' % (experimentDesc['project']['id'], entityType['key']) - result = self.client.execute(gql(query)) - - entityType.update(result) - - ## Get the trialsets - query = """{ - trialSets(experimentId: "%s"){ - key - name - description - numberOfTrials - properties{ - key - type - label - description - required - trialField - value - } - state - } - } - """ % experimentDesc['project']['id'] - trialsets = self._client.execute(gql(query)) - - - for trialset in trialsets['trialSets']: - query = ''' - { - trials(experimentId: "%s", trialSetKey: "%s"){ - key - name - status - created - cloneFrom - numberOfEntities - state - properties{ - val - key - } - entities{ - entitiesTypeKey - properties{ - val - key - } - key - containsEntities - } - deployedEntities{ - entitiesTypeKey - properties{ - val - key - } - key - containsEntities - } - } - } - ''' % (experimentDesc['project']['id'], trialset['key']) - - result = self.client.execute(gql(query)) - trialset.update(result) - - - ret = dict(experimentsWithData=experimentDesc) - ret.update(trialsets) - ret.update(entitiesTypes) - return ret - - def getExperiment(self,experimentName): - """ - Load a full experiment from the server. - - Fetches the experiment metadata via GraphQL and returns a - ``webExperiment`` object with the data populated. - - Parameters - ---------- - experimentName : str - The name of the experiment to load. - - Returns - ------- - webExperiment - The loaded experiment object. - """ - experimentDict = self.getExperimentMetadata(experimentName) - experimentDict['experimentsWithData']['url'] = self.url - return webExperiment(setupFileOrData=experimentDict) - - - - def getExperimentsDescriptionsList(self): - """ - List all experiments on the server with their metadata. - - Queries the GraphQL API for all experiments and returns their - descriptors including name, description, dates, and map definitions. - - Returns - ------- - list[dict] - A list of experiment descriptor dicts, each containing: - - - ``name`` : experiment name - - ``description`` : experiment description - - ``begin``, ``end`` : experiment date range - - ``numberOfTrials`` : trial count - - ``project.id`` : internal project ID - - ``maps`` : list of image map definitions with coordinates - """ - query = ''' - { - experimentsWithData { - name - description - begin - end - numberOfTrials - project { - id - } - maps { - imageUrl - imageName - lower - upper - left - right - width - height - embedded - } - } - } - ''' - return self._client.execute(gql(query))['experimentsWithData'] - - def getExperimentsDescriptionsTable(self): - """ - Get all experiment descriptions as a Pandas DataFrame. - - Returns - ------- - pandas.DataFrame - A flattened DataFrame of all experiments on the server with - their metadata. - """ - return pandas.json_normalize(self.getExperimentsDescriptionsList()) - - - def getExperimentDescriptor(self,experimentName): - """ - Get the descriptor for a specific experiment by name. - - Parameters - ---------- - experimentName : str - The name of the experiment. - - Returns - ------- - dict - The experiment descriptor containing ``name``, ``description``, - ``begin``, ``end``, ``numberOfTrials``, ``project.id``, and ``maps``. - - Raises - ------ - IndexError - If no experiment with the given name is found on the server. - """ - descs = self.getExperimentsDescriptionsList() - - return [x for x in descs if x['name']==experimentName][0] - - - def listExperimentsNames(self): - """ - List the names of all experiments on the server. - - Returns - ------- - list[str] - A list of experiment name strings. - """ - return [x['name'] for x in self.listExperimentsDescriptions()] - - def __getitem__(self, item): - """ - Load an experiment by name using dictionary-style access. - - Parameters - ---------- - item : str - The experiment name. - - Returns - ------- - webExperiment - The loaded experiment object. - """ - return self.getExperiment(experimentName=item) - - def keys(self): - """ - Return the names of all experiments on the server. - - Returns - ------- - list[str] - A list of experiment name strings. - """ - return self.listExperiments()['name'] diff --git a/docs/developer_guide/api/experiment_setup.md b/docs/developer_guide/api/experiment_setup.md index 6b352b6..e050b08 100644 --- a/docs/developer_guide/api/experiment_setup.md +++ b/docs/developer_guide/api/experiment_setup.md @@ -13,10 +13,8 @@ Each class in this module has a distinct responsibility in the experiment lifecy | Class | Role | Analogy | |-------|------|---------| | `fileExperimentFactory` | Locates and loads experiment data from local disk | File browser | -| `webExperimentFactory` | Fetches experiment data from ArgosWEB via GraphQL | API client | | `Experiment` | Root container holding all entity types, trial sets, and image maps | Project folder | | `ExperimentZipFile` | Variant of Experiment that reads from a `.zip` archive (with version migration) | ZIP reader | -| `webExperiment` | Variant of Experiment that fetches images over HTTP | Remote reader | | `TrialSet` | Named collection of trials (e.g., "design", "deploy") | Folder of trials | | `Trial` | A single experimental configuration with per-entity attributes | Configuration snapshot | | `EntityType` | Schema definition + collection of entities of one type (e.g., "Sensor") | Device class | @@ -28,7 +26,7 @@ Each class in this module has a distinct responsibility in the experiment lifecy ### Experiment Hierarchy -`Experiment` is the base class. Two subclasses override only the data-loading and image-fetching behavior, keeping the rest of the interface identical: +`Experiment` is the base class. `ExperimentZipFile` overrides the data-loading and image-fetching behavior, keeping the rest of the interface identical: ![Diagram](../../images/diagrams/developer_guide_api_experiment_setup_0_a46b01b4.svg) @@ -56,19 +54,13 @@ classDiagram -_fix_json_version_3_0_0() } - class webExperiment { - +getImage() ⟵ overrides - } - Experiment <|-- ExperimentZipFile : "reads from ZIP archive" - Experiment <|-- webExperiment : "fetches images via HTTP" ``` --> **What each subclass changes:** - **ExperimentZipFile** -- `refresh()` extracts `data.json` from the ZIP, detects the schema version, and applies the appropriate migration. `getImage()` reads images from inside the archive. `_init_ImageMaps()` parses the ZIP's internal map format. -- **webExperiment** -- `getImage()` does an HTTP GET to fetch the image from the ArgosWEB server. Everything else is inherited. ### Container Hierarchy (dict-based) @@ -217,58 +209,6 @@ sequenceDiagram --- -## Call Workflow: Loading an Experiment from Web - -![Diagram](../../images/diagrams/developer_guide_api_experiment_setup_3_fde07684.svg) - ->Factory: Experiment descriptor - - Factory->>GQL: Query entitiesTypes (key, name, properties) - GQL-->>Factory: Entity type list - loop For each entityType - Factory->>GQL: Query entities (key, name, properties) - GQL-->>Factory: Entity list for type - end - - Factory->>GQL: Query trialSets (key, name, properties) - GQL-->>Factory: Trial set list - loop For each trialSet - Factory->>GQL: Query trials (name, properties, entities) - GQL-->>Factory: Trial list for set - end - - Factory->>Factory: Assemble metadata dict - Factory->>Exp: webExperiment(metadata) - Note over Exp: Same __init__ as Experiment:
refresh → _initTrialSets → _initEntitiesTypes - - Exp-->>Factory: return webExperiment - Factory-->>User: experiment object ready - - Note over User: Images fetched on demand - User->>Exp: experiment.getImage("field_map") - Exp->>GQL: HTTP GET image URL - GQL-->>Exp: Image bytes - Exp-->>User: numpy.ndarray -``` ---> - ---- - ## Call Workflow: Accessing Trial Entity Data When you access `trial.entities` or `trial.entitiesTable`, the containment hierarchy is resolved on-the-fly: @@ -317,15 +257,6 @@ sequenceDiagram --- -## Module Entry Point - -::: argos.experimentSetup.getExperimentSetup - options: - show_root_heading: true - heading_level: 3 - ---- - ## Factories ### fileExperimentFactory @@ -348,34 +279,6 @@ sequenceDiagram --- -### webExperimentFactory - -**Role:** Connects to an ArgosWEB server via GraphQL and fetches experiment data remotely. Supports listing experiments, fetching metadata, and loading complete experiments. - -**Key behavior:** - -- Authenticates with a token-based authorization header -- Issues multiple GraphQL queries to assemble the full experiment: entity types, entities, trial sets, trials -- Returns a `webExperiment` object whose `getImage()` fetches images over HTTP - -::: argos.experimentSetup.dataObjectsFactory.webExperimentFactory - options: - show_root_heading: true - heading_level: 4 - members: - - __init__ - - getExperiment - - getExperimentMetadata - - getExperimentsDescriptionsList - - getExperimentsDescriptionsTable - - getExperimentDescriptor - - listExperimentsNames - - url - - client - - keys - ---- - ## Data Objects ### Experiment @@ -430,19 +333,6 @@ sequenceDiagram --- -### webExperiment - -**Role:** Handles experiments fetched from a remote ArgosWEB server. The only override is `getImage()`, which fetches images via HTTP GET instead of reading from disk. - -::: argos.experimentSetup.dataObjects.webExperiment - options: - show_root_heading: true - heading_level: 4 - members: - - getImage - ---- - ### TrialSet **Role:** A named collection of trials (e.g., `"design"`, `"deploy"`). Extends `dict` so trials can be accessed by name: `trial_set["myTrial"]`. Defines the attribute type schema that all trials in the set share. diff --git a/docs/developer_guide/api/index.md b/docs/developer_guide/api/index.md index 9262f86..923be5c 100644 --- a/docs/developer_guide/api/index.md +++ b/docs/developer_guide/api/index.md @@ -12,8 +12,8 @@ argos/ CLI.py # CLI command handlers manager.py # experimentManager class experimentSetup/ - __init__.py # getExperimentSetup(), WEB/FILE constants - dataObjectsFactory.py # fileExperimentFactory, webExperimentFactory + __init__.py # FILE constant + dataObjectsFactory.py # fileExperimentFactory dataObjects.py # Experiment, TrialSet, Trial, EntityType, Entity fillContained.py # fill_properties_by_contained() kafka/ @@ -44,10 +44,6 @@ argos/ from argos.experimentSetup import fileExperimentFactory experiment = fileExperimentFactory("/path").getExperiment() -# Load an experiment from web -from argos.experimentSetup import webExperimentFactory -experiment = webExperimentFactory(url, token).getExperiment("name") - # Use the experiment manager (with ThingsBoard) from argos.manager import experimentManager manager = experimentManager("/path") diff --git a/docs/developer_guide/architecture/core_concepts.md b/docs/developer_guide/architecture/core_concepts.md index dcc6b84..16ea522 100644 --- a/docs/developer_guide/architecture/core_concepts.md +++ b/docs/developer_guide/architecture/core_concepts.md @@ -59,7 +59,7 @@ graph TD | Module | Depends on (internal) | Depends on (external) | |--------|----------------------|----------------------| -| `experimentSetup` | `utils.jsonutils`, `utils.logging` | `gql` (optional, for web), `matplotlib`, `requests` | +| `experimentSetup` | `utils.jsonutils`, `utils.logging` | `matplotlib`, `requests` | | `manager` | `experimentSetup`, `utils.jsonutils`, `utils.logging` | `tb_rest_client` (optional) | | `kafka.consumer` | `utils.parquetUtils` | `kafka-python`, `numpy`, `pandas` | | `CLI` | `experimentSetup`, `kafka.consumer`, `manager` | `kafka-python` | @@ -101,10 +101,6 @@ classDiagram +getImage() } - class webExperiment { - +getImage() - } - class TrialSet { +name +experiment: Experiment @@ -141,7 +137,6 @@ classDiagram } Experiment <|-- ExperimentZipFile - Experiment <|-- webExperiment Experiment "1" --> "*" TrialSet Experiment "1" --> "*" EntityType TrialSet "1" --> "*" Trial @@ -163,13 +158,9 @@ pyArgos uses the Factory pattern to abstract experiment loading from different s B{Source Type?} - B -->|FILE| C[fileExperimentFactory] - B -->|WEB| D[webExperimentFactory] + A[Client Code] --> C[fileExperimentFactory] C --> E[Experiment / ExperimentZipFile] - D --> F[webExperiment] E --> G[Unified Experiment Interface] - F --> G ``` --> @@ -180,13 +171,6 @@ graph TD - Handles version migrations (1.0.0 through 3.0.0) - Default: uses current working directory -### `webExperimentFactory` - -- Fetches from ArgosWEB via GraphQL -- Requires server URL and authentication token -- Retrieves entity types, entities, trial sets, and trials -- Images are fetched via HTTP on demand - --- ## Swimlane: Full Experiment Deployment @@ -333,7 +317,7 @@ This makes it easy to filter, join, and analyze experiment metadata using standa ### Why optional external dependencies? -ThingsBoard (`tb_rest_client`), GraphQL (`gql`), Cassandra (`cassandra-driver`), and MongoDB (`pymongo`) are all optional. pyArgos gracefully handles their absence with try/except imports. This allows users to install only what they need — a researcher doing offline analysis doesn't need ThingsBoard, and a deployment engineer doesn't need Cassandra. +ThingsBoard (`tb_rest_client`), Cassandra (`cassandra-driver`), and MongoDB (`pymongo`) are all optional. pyArgos gracefully handles their absence with try/except imports. This allows users to install only what they need — a researcher doing offline analysis doesn't need ThingsBoard, and a deployment engineer doesn't need Cassandra. ### Why threads for Kafka consumers? diff --git a/docs/developer_guide/architecture/data_flow.md b/docs/developer_guide/architecture/data_flow.md index 3f0ce88..73db620 100644 --- a/docs/developer_guide/architecture/data_flow.md +++ b/docs/developer_guide/architecture/data_flow.md @@ -36,32 +36,6 @@ sequenceDiagram --- -## Remote Experiment Loading - -![Diagram](../../images/diagrams/developer_guide_architecture_data_flow_1_ff8aacf7.svg) - ->Factory: Experiment metadata - Factory->>Exp: Create webExperiment - Factory-->>Client: Return webExperiment -``` ---> - ---- - ## Kafka to Parquet Pipeline ![Diagram](../../images/diagrams/developer_guide_architecture_data_flow_2_bfa05a42.svg) diff --git a/docs/developer_guide/architecture/experiment_setup.md b/docs/developer_guide/architecture/experiment_setup.md index 9bac4ed..a8fed6e 100644 --- a/docs/developer_guide/architecture/experiment_setup.md +++ b/docs/developer_guide/architecture/experiment_setup.md @@ -8,7 +8,7 @@ This page provides a deep architectural reference for the `argos.experimentSetup The `experimentSetup` module is responsible for: -1. **Loading** experiment definitions from files (JSON/ZIP) or the ArgosWEB server (GraphQL) +1. **Loading** experiment definitions from files (JSON/ZIP) 2. **Parsing** the raw JSON into a typed object hierarchy 3. **Exposing** experiment metadata as Pandas DataFrames for analysis 4. **Resolving** entity containment hierarchies (parent-child property inheritance) @@ -18,8 +18,8 @@ The `experimentSetup` module is responsible for: ``` argos/experimentSetup/ - __init__.py # Module entry point, WEB/FILE constants - dataObjectsFactory.py # Factory classes (file, web) + __init__.py # Module entry point, FILE constant + dataObjectsFactory.py # Factory classes (file) dataObjects.py # Data object hierarchy (Experiment, Trial, Entity, ...) fillContained.py # Containment hierarchy resolution runner.py # Development/test runner script @@ -32,7 +32,7 @@ argos/experimentSetup/ ### Experiment Classes -The `Experiment` base class defines the full interface for accessing experiment data. Two subclasses override only the data-loading and image-fetching behavior: +The `Experiment` base class defines the full interface for accessing experiment data. `ExperimentZipFile` overrides the data-loading and image-fetching behavior: ![Diagram](../../images/diagrams/developer_guide_architecture_experiment_setup_0_f2b553c6.svg) @@ -84,26 +84,19 @@ classDiagram #_fix_json_version_3_0_0(jsonFile) } - class webExperiment { - <> - +getImage(imageName)◄ - } - Experiment <|-- ExperimentZipFile : inherits - Experiment <|-- webExperiment : inherits note for ExperimentZipFile "◄ = overridden method" - note for webExperiment "◄ = overridden method" ``` --> **Key override points:** -| Method | Experiment (base) | ExperimentZipFile | webExperiment | -|--------|-------------------|-------------------|---------------| -| `refresh()` | Loads JSON from file/dict | Extracts `data.json` from ZIP, applies version migration | Inherited (uses base) | -| `getImage()` | Reads from local filesystem path | Reads from inside the ZIP archive | Fetches via HTTP GET | -| `_init_ImageMaps()` | Parses `experimentsWithData.maps` with full URLs | Parses `maps` from ZIP metadata (no URLs) | Inherited (uses base) | +| Method | Experiment (base) | ExperimentZipFile | +|--------|-------------------|-------------------| +| `refresh()` | Loads JSON from file/dict | Extracts `data.json` from ZIP, applies version migration | +| `getImage()` | Reads from local filesystem path | Reads from inside the ZIP archive | +| `_init_ImageMaps()` | Parses `maps` with full URLs | Parses `maps` from ZIP metadata (no URLs) | ### Container Classes (dict-based) @@ -226,10 +219,7 @@ graph TD B{Source type?} - - B -->|FILE| C[fileExperimentFactory] - B -->|WEB| D[webExperimentFactory] + A[Client calls factory directly] --> C[fileExperimentFactory] C --> E[Scan runtimeExperimentData/] E --> F{Found .zip files?} @@ -252,14 +242,6 @@ flowchart TD L --> R - D --> S[GraphQL queries] - S --> T[Query entitiesTypes + entities] - S --> U[Query trialSets + trials] - T --> V[Build metadata dict] - U --> V - V --> W[webExperiment.__init__] - W --> R - R --> X[Return Experiment object] ``` --> @@ -683,8 +665,8 @@ Experiment metadata is inherently tabular (entities have rows of properties, tri ### Why factory pattern? -The same `Experiment` interface is needed regardless of whether data comes from a local file, a ZIP archive, or a remote server. The factory pattern: +The same `Experiment` interface is needed regardless of whether data comes from a local JSON file or a ZIP archive. The factory pattern: -- Hides the complexity of source detection (ZIP vs JSON vs GraphQL) +- Hides the complexity of source detection (ZIP vs JSON) - Handles version migration transparently - Returns a uniform interface that client code can rely on diff --git a/docs/developer_guide/data_model.md b/docs/developer_guide/data_model.md index aa5088a..2c60ddd 100644 --- a/docs/developer_guide/data_model.md +++ b/docs/developer_guide/data_model.md @@ -15,7 +15,7 @@ An **Experiment** is a complete scientific or engineering study conducted with I - **Where things are** (image maps with geo-coordinates) - **When it runs** (start and end dates) -An experiment is created in [ArgosWEB](https://github.com/argosp) and exported as a ZIP file, or loaded directly from the server via GraphQL. +An experiment is created in [ArgosWEB](https://github.com/argosp) and exported as a ZIP file for use with pyArgos. ### What is an Entity Type? diff --git a/docs/developer_guide/experiment_setup_index.md b/docs/developer_guide/experiment_setup_index.md index f33dbae..12bb6b0 100644 --- a/docs/developer_guide/experiment_setup_index.md +++ b/docs/developer_guide/experiment_setup_index.md @@ -27,7 +27,7 @@ The architecture page covers the **internal design** of the module: how classes - [**Experiment Setup Architecture**](architecture/experiment_setup.md) - Module overview and file layout - - Inheritance hierarchy (Experiment → ExperimentZipFile / webExperiment) + - Inheritance hierarchy (Experiment → ExperimentZipFile) - Container classes (dict-based TrialSet, EntityType) - Full object composition diagram - Factory pattern decision flow @@ -51,12 +51,10 @@ The API page provides the **class-level documentation** with role descriptions, - Class roles summary table - Inheritance and container hierarchy diagrams - Swimlane: loading experiment from file - - Swimlane: loading experiment from web (GraphQL) - Swimlane: accessing trial entity data (containment resolution) - Auto-generated docs for every class and method: - - `getExperimentSetup` (module entry point) - - `fileExperimentFactory`, `webExperimentFactory` - - `Experiment`, `ExperimentZipFile`, `webExperiment` + - `fileExperimentFactory` + - `Experiment`, `ExperimentZipFile` - `TrialSet`, `Trial` - `EntityType`, `Entity` - `fill_properties_by_contained`, `spread_attributes`, `get_parent`, `key_from_name` @@ -72,7 +70,7 @@ The user guide covers the **practical usage** -- how to create experiments, defi - Configuring datasources - Defining entities (DEVICE / ASSET) - Setting up trials (template → design → upload) - - Loading experiments in Python (file and web) + - Loading experiments in Python (from file) --- diff --git a/docs/user_guide/concepts.md b/docs/user_guide/concepts.md index 5a0bacb..c17709c 100644 --- a/docs/user_guide/concepts.md +++ b/docs/user_guide/concepts.md @@ -51,9 +51,8 @@ All of this is done through forms, tables, and map interfaces in the browser ### How pyArgos fits in pyArgos is the **Python programmatic interface** to the data that ArgosWEB -creates. It reads the exported ZIP file (or connects to the ArgosWEB server -via GraphQL) and makes the experiment data available as Python objects and -Pandas DataFrames. +creates. It reads the exported ZIP file and makes the experiment data +available as Python objects and Pandas DataFrames. ``` ┌──────────────────────┐ diff --git a/docs/user_guide/experiment_setup.md b/docs/user_guide/experiment_setup.md index 514dcd5..8679b61 100644 --- a/docs/user_guide/experiment_setup.md +++ b/docs/user_guide/experiment_setup.md @@ -171,18 +171,6 @@ trial = experiment.trialSet["design"]["morningTrial"] print(trial.entitiesTable()) ``` -### From ArgosWEB - -```python -from argos.experimentSetup import webExperimentFactory - -factory = webExperimentFactory(url="http://argos-server/graphql", token="your-token") -experiment = factory.getExperiment("MyExperiment") - -# List all experiments -print(factory.listExperimentsNames()) -``` - --- ## Using the Generated Starter Script