diff --git a/.coverage b/.coverage
new file mode 100644
index 0000000..b5d30d2
Binary files /dev/null and b/.coverage differ
diff --git a/argos/__init__.py b/argos/__init__.py
index d82a615..a5c32d5 100755
--- a/argos/__init__.py
+++ b/argos/__init__.py
@@ -1,5 +1,5 @@
__version__ = '1.3.0'
-from .experimentSetup import WEB,FILE
+from .experimentSetup import FILE
from .manager import experimentManager
diff --git a/argos/examples/web/expConf.json b/argos/examples/web/expConf.json
deleted file mode 100644
index 756168e..0000000
--- a/argos/examples/web/expConf.json
+++ /dev/null
@@ -1,50 +0,0 @@
-{
- "name": "edenTest",
- "graphql": {
- "url": "XXXX",
- "token": "YYYY"
- },
- "thingsboard": {
- "login": {
- "username": "tenant@thingsboard.org",
- "password": "tenant"
- },
- "server": {
- "ip": "127.0.0.1",
- "port": "8080"
- }
- },
- "kafka": {
- "consumers": {
- "deviceType1": {
- "slide": "60",
- "toParquet": ["argos.kafka.processes.to_parquet_type1", {}],
- "processes": {
- "180": {
- "argos.kafka.processes.function1": {
- "arg1": "value1",
- "arg2": "value2"
- }
- },
- "300": {
- "argos.kafka.processes.function2": {
- "arg3": "value3"
- }
- }
- }
- },
- "deviceType2": {
- "slide": 10,
- "toParquet": ["argos.kafka.processes.to_parquet_type2", {}],
- "processes": {
- "60": {
- "argos.kafka.processes.function3": {
- "arg4": "value4"
- }
- }
- }
- }
- },
- "ip": "127.0.0.1"
- }
-}
diff --git a/argos/examples/web/runExperiment.sh b/argos/examples/web/runExperiment.sh
deleted file mode 100644
index 7c8f5f5..0000000
--- a/argos/examples/web/runExperiment.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-# Experiment configuration example can be found in this directory as expConf.json
-
-# Experiment setup
-python argos-experiment-web.py --expConf {The experiment configuration} --setup
-
-# Trial load
-python argos-experiment-web.py --expConf {The experiment configuration} --load {trial set name} {trial name}
-
-# Run consumers
-python argos-experiment-web.py --expConf {The experiment configuration} --runConsumers {default folder for saving data}
-
-# Finalize (Updating the database documents desc with the information from the trials)
-python argos-exeperiment-web.py --expConf {The experiments configuration} --finalize
\ No newline at end of file
diff --git a/argos/experimentSetup/.ipynb_checkpoints/dataObjects-checkpoint.py b/argos/experimentSetup/.ipynb_checkpoints/dataObjects-checkpoint.py
deleted file mode 100644
index a45cbce..0000000
--- a/argos/experimentSetup/.ipynb_checkpoints/dataObjects-checkpoint.py
+++ /dev/null
@@ -1,1126 +0,0 @@
-import zipfile
-import os
-import json
-import pandas
-import requests
-from io import BytesIO
-import matplotlib.pyplot as plt
-from ..utils.jsonutils import loadJSON
-from ..utils.logging import get_logger as argos_get_logger
-
-class Experiment:
- """
- Interface to the WEB experiment object.
-
- """
-
- _setupFileNameOrData = None # experiment description holds its name, descrition and ect.
- _experimentSetup = None
-
- _trialSetsDict = None # A dictionary of the trial sets.
- _entitiesTypesDict = None # A dictionary of the devices types.
-
- _client = None # The client of the connection to the WEB.
-
- _imagesMap = None
-
- def refresh(self):
- """
- Loads the experiment setup and rebuilds all the trial sets and entity types.
- """
- self._experimentSetup = loadJSON(self._setupFileNameOrData)
- self._initTrialSets()
- self._initEntitiesTypes()
-
- @property
- def setup(self):
- return self._experimentSetup
-
- @property
- def url(self):
- return self.setup['experimentsWithData']['url']
-
- @property
- def client(self):
- return self._client
-
- @property
- def id(self):
- ## DO NOT USE the self._experimentDescription['id']. It is not useful here.
- return self.setup['project']['id']
-
- @property
- def name(self):
- return self.setup['name']
-
- @property
- def description(self):
- return self.setup['description']
-
- @property
- def trialSet(self):
- return self._trialSetsDict
-
- @property
- def entityType(self):
- return self._entitiesTypesDict
-
- @property
- def entityTypeTable(self):
- entityTypeList = []
- for entityTypeName, entityTypeData in self.entityType.items():
- entityTypeList.append(entityTypeData.propertiesTable.assign(entityType=entityTypeName))
-
- return pandas.concat(entityTypeList, ignore_index=True)
-
- @property
- def entitiesTable(self):
- return self.entitiesTableFull.drop(columns=["key","entitiesTypeKey"])
-
- def trialsTable(self,trialsetName):
- return self.trialSet[trialsetName].trialsTable
-
- @property
- def entitiesTableFull(self):
- entityTypeList = []
- for entityTypeName, entityTypeData in self.entityType.items():
- for entityTypeDataName, entityData in entityTypeData.items():
- entityTypeList.append(
- pandas.DataFrame(entityData.propertiesList, index=[0]).assign(entityType=entityTypeName))
-
- return pandas.concat(entityTypeList, ignore_index=True)
-
- def __init__(self, setupFileOrData):
- """
- Experiment object contains information on a specific experiment
-
- Parameters
- -----------
-
- setupFileOrData: str,dict
-
- - A file name that contains a dictionary with all the information on the experiment, that was downloaded from
- argosWEB (using the download metadata).
- - dict that includes all the data.
-
-
- """
- self.logger = argos_get_logger(self)
- self._trialSetsDict = dict()
- self._entitiesTypesDict = dict()
-
- self._setupFileNameOrData = setupFileOrData
- self.refresh()
-
- self.logger.execution("Loading images")
- self._init_ImageMaps()
-
- def _init_ImageMaps(self):
- ## Initializing the images map
- self._imagesMap = dict()
- if 'experimentsWithData' in self.setup:
- for imgs in self.setup['experimentsWithData']['maps']:
- imgName = imgs['imageName']
- imageFullURL = f"{self.url}/{imgs['imageUrl']}"
- imgs['imageURL'] = imageFullURL
-
- self._imagesMap[imgName] = imgs
-
-
- @property
- def imageMap(self):
- return self._imagesMap
-
- def getImageURL(self, imageName: str):
- return self._imagesMap[imageName]['imageURL']
-
- def getImageJSMappingFunction(self, imageName: str):
- """
- Return the javascript mapping function that maps the
- coordindates of the image to from the coordinates to 0..1 coordinats
-
- Used in thingsboard dashboards. Therefore the inputs to the function are
- (origXpos,origYpos).
-
-
- Parameters
- ----------
- imageName: str
- The name of the image
-
-
- Returns
- -------
- A string with the mapping function (in javascript).
-
- """
- metadata = self.getImageMetadata(imageName)
-
- xdim = metadata['right'] - metadata['left']
- ydim = metadata['upper'] - metadata['lower']
-
- Xconvert = f"image_x = (origXPos - ({metadata['left']}))/{xdim};"
- Yconvert = f"image_y = (({metadata['upper']})-origYPos)/{ydim};"
-
- return_string = "return {x: image_x, y: image_y};"
-
- return "\n".join([Xconvert, Yconvert, return_string])
-
- def getImageMetadata(self, imageName: str):
- return self._imagesMap[imageName]
-
- def _initTrialSets(self):
-
- for trialset in self.setup['trialSets']:
- self._trialSetsDict[trialset['name']] = TrialSet(experiment=self, metadata=trialset)
-
- def _initEntitiesTypes(self):
-
- for entityType in self.setup['entityTypes']:
- self._entitiesTypesDict[entityType['name']] = EntityType(experiment=self, metadata=entityType)
-
- def toJSON(self):
- """
- Create a single JSON with the data of the experiment.
-
-
- :return:
- """
- ret = dict()
-
- entityTypeMap = dict()
- for entityName, entityType in self.entityType.items():
- entityTypeMap[entityName] = entityType.toJSON()
-
- trialMap = dict()
- for trialName, trialData in self.trialSet.items():
- trialMap[trialName] = trialData.toJSON()
-
- ret['entityType'] = entityTypeMap
- ret['trialSet'] = trialMap
-
- expr = dict()
-
- for field in ['maps', 'begin', 'end', 'description']:
- expr[field] = self._setupFileNameOrData['experimentsWithData'][field]
-
- ret['experiment'] = expr
- return ret
-
- def getExperimentEntities(self):
- """
- Returns the list of all the entities
-
- :return: dict
- Return a list of the entities.
- """
- retList = []
-
- for entitytypeName, entityTypeObj in self._entitiesTypesDict.items():
- for entityName, entityData in entityTypeObj.items():
- retList.append(dict(entityName=entityName, entityTypeName=entityData.entityType.name))
-
- return retList
-
- def getEntitiesTypeByID(self, entityTypeID):
-
- ret = None
- for entityTypeName, entityTypeData in self.entityType.items():
- if entityTypeID == entityTypeData.keyID:
- ret = entityTypeData
- break
-
- return ret
-
- def getImage(self, imageName: str):
-
- imgUrl = os.path.join(self.setup['experimentsWithData']['url'], "images", f"{imageName}.png")
-
- # maybe we can skip the open(...), didn't want to risk it
- try:
- with open(imgUrl) as imageFile:
- img = plt.imread(imageFile)
- except UnicodeDecodeError:
- img = plt.imread(imgUrl)
- return img
-
-class ExperimentZipFile(Experiment):
-
- def __init__(self, setupFileOrData):
- super().__init__(setupFileOrData=setupFileOrData)
-
- def getImage(self, imageName: str):
-
- imageurl = self._imagesMap[imageName]['imageURL']
-
- with zipfile.ZipFile(self._setupFileNameOrData) as archive:
- imageFile = archive.open(os.path.join("images",imageurl))
-
- return plt.imread(imageFile)
-
-
- def refresh(self):
- """
- Loads the experiment setup and rebuilds all the trial sets and entity types.
- """
- self.logger.execution("------- Start ----")
- self.logger.debug(f"Loading file {self._setupFileNameOrData}")
- with zipfile.ZipFile(self._setupFileNameOrData) as archive:
- experimentDict =loadJSON("\n".join([x.decode() for x in archive.open("data.json").readlines()]))
-
-
- fileVersion = experimentDict.get("version","1.0.0.").replace(".","_")
- self.logger.debug(f"Got file version {fileVersion}")
-
-
- experimentDict = getattr(self,f"_fix_json_version_{fileVersion}")(experimentDict)
-
- self.logger.execution("Experiemnt dict")
- self._experimentSetup = experimentDict
-
- self.logger.execution("Init trial sets")
- self._initTrialSets()
-
- self.logger.execution("Init entity type")
- self._initEntitiesTypes()
-
- def _init_ImageMaps(self):
- ## Initializing the images map
- with zipfile.ZipFile(self._setupFileNameOrData) as archive:
- experimentDict =loadJSON("\n".join([x.decode() for x in archive.open("data.json").readlines()]))
-
- self._imagesMap = dict()
-
- experimentDatakey = 'experiment' if 'experiment' in experimentDict else 'experimentWithData'
-
- if 'experiment' in experimentDict:
- for imgs in experimentDict[experimentDatakey]['maps']:
- imgName = imgs['imageName']
- self._imagesMap[imgName] = imgs
-
- def _fix_json_version_1_0_0_(self,jsonFile):
- return jsonFile
-
- def _fix_json_version_2_0_0_(self,jsonFile):
-
- oldFormat = dict(experiment=jsonFile['experiment'],
- entityTypes=jsonFile['entityTypes'],
- trialSets=jsonFile['trialSets'])
-
-
- for trialSet in oldFormat['trialSets']:
- trialSet['trials'] = []
- currentKey = trialSet['key']
- for trial in jsonFile['trials']:
- if trial['trialSetKey'] == currentKey:
- trialSet['trials'].append(trial)
-
-
- for entityType in oldFormat['entityTypes']:
- entityType['entities'] = []
- currentKey = entityType['key']
- for entity in jsonFile['entities']:
- if entity['entitiesTypeKey'] == currentKey:
- entityType['entities'].append(entity)
-
- return oldFormat
-
-
-
-class webExperiment(Experiment):
-
- def getImage(self, imageName: str):
- imgUrl = self.getImageURL(imageName)
- response = requests.get(imgUrl)
-
- if response.status_code != 200:
- raise ValueError(f"Image {imageName} not found on the server.")
-
- imageFile = BytesIO(response.content)
- return plt.imread(imageFile)
-
-
-class TrialSet(dict):
- """
- Interface to the web trial set object.
-
- """
- _experiment = None
- _metadata = None
-
- _trialsDict = None
-
- @property
- def client(self):
- return self.experiment.client
-
- @property
- def experiment(self):
- return self._experiment
-
- @property
- def keyID(self):
- return self._metadata['key']
-
- @property
- def id(self):
- return self._metadata['id']
-
- @property
- def name(self):
- return self._metadata['name']
-
- @property
- def description(self):
- return self._metadata['description']
-
- @property
- def numberOfTrials(self):
- return self._metadata['numberOfTrials']
-
- @property
- def propertiesTable(self):
- if 'properties' in self._metadata:
- ret = pandas.DataFrame(self._metadata['properties']) .set_index('key')
- else:
- ret = pandas.DataFrame()
-
- return ret
-
- @property
- def properties(self):
- ret = dict()
- for prop in self._metadata['properties']:
- ret[prop['label']] = prop
-
- return ret
-
- def toJSON(self):
- ret = dict()
- ret['name'] = self.name
- ret['properties'] = self.properties
-
- trialsJSON = {}
- for trialName, trialData in self.items():
- trialsJSON[trialName] = trialData.toJSON()
-
- ret['trials'] = trialsJSON
-
- return ret
-
- @property
- def trials(self):
- retList = []
- for trialName, trialData in self.items():
- trialProps = trialData.propertiesTable.assign(trialName=trialName, key=trialData.key)
- retList.append(trialProps)
-
- return retList
-
- @property
- def trialsTable(self):
- return pandas.DataFrame(self.toJSON()['trials']).T
-
- def __init__(self, experiment: Experiment, metadata: dict):
- """
- Trial set object contains information on a specific trial set
-
- :param experimentId: The experiment id
- :param desc: A dictionary with information on the trial set
- :param client: GraphQL client
- """
- self._experiment = experiment
- self._metadata = metadata
-
- self._initTrials()
-
- def _initTrials(self):
-
- for trial in self._metadata['trials']:
- self[trial['name']] = Trial(trialSet=self, metadata=trial)
-
-
-class Trial:
- """
- Interface to the WEB trials.
-
- """
-
- _trialSet = None
- _metadata = None
-
- @property
- def experiment(self):
- return self._trialSet.experiment
-
- @property
- def client(self):
- return self._trialSet.experiment.client
-
- @property
- def experiment(self):
- return self._trialSet.experiment
-
- @property
- def key(self):
- return self._metadata['key']
-
- @property
- def id(self):
- return self._metadata['id']
-
- @property
- def name(self):
- return self._metadata['name']
-
- @property
- def trialSetKey(self):
- return self._metadata['trialSetKey']
-
- @property
- def created(self):
- return self._metadata['created']
-
- @property
- def status(self):
- return self._metadata['status']
-
- @property
- def cloneFrom(self):
- return self._metadata['cloneFrom']
-
- @property
- def numberOfEntities(self):
- return self._metadata['numberOfEntities']
-
- @property
- def state(self):
- return self._metadata['state']
-
- @property
- def properties(self):
- return self._properties
-
- @property
- def trialSet(self):
- return self._trialSet
-
- @property
- def propertiesTable(self):
- return pandas.DataFrame(self.properties, index=[0])
-
- def __init__(self, trialSet: TrialSet, metadata: dict):
- """
- Trial object contains information on a specific trial
-
- Parameters
- ----------
-
- experiment: The experiment object
- trialSet: The trial set object
- desc: A dictionary with information on the trial
- client: GraphQL client
- """
- self._trialSet = trialSet
- self._metadata = metadata
- if metadata['properties']:
- propertiesPandas = pandas.DataFrame(metadata['properties']).set_index('key')
-
- properties = propertiesPandas.merge(trialSet.propertiesTable, left_index=True, right_index=True)[
- ['val', 'type', 'label', 'description']]
- getParser = lambda x: getattr(self, f"_parseProperty_{x.replace('-', '_')}")
-
-
- # this wont work well for location property in the trial because it has 2 fields.
- # we have to get the list, and the change all the lists with size 1 to the object itself (like we do now)
- # and leave all the lists with size 2 as is.
- parsedValuesList = []
- for key, data in properties.T.to_dict().items():
- parsed_data = getParser(data['type'])(data, data)
- if len(parsed_data[1]) > 0:
- val = parsed_data[1][0]
- else:
- val = None
- parsedValuesList.append((data['label'], val))
-
- self._properties = dict(parsedValuesList)
- else:
-
- self._properties = metadata['properties']
-
- def toJSON(self):
- val = self.properties
- val['name'] = self.name
- val['status'] = self.status
- val['state'] = self.status
- return val
-
- def __str__(self):
- return json.dumps(self.toJSON())
-
- def __repr__(self):
- return json.dumps(dict(name=self.name, status=self.status, state=self.state))
-
- def _parseProperty_location(self, property, propertyMetadata):
- """
- Parse the location property.
-
- Returns 3 values: location name, latitude and longitude.
- The column names are at this order.
-
- :param property:
- property
- :param propertyMetadata:
- The data that describes the property.
-
- :return: list,list
- Returns list of column names and list of values.
- """
- try:
- if isinstance(property['val'],dict):
- locationDict = property['val']
- else:
- locationDict = json.loads(property['val'])
-
- locationName = locationDict['name']
- coords = locationDict['coordinates']
- if isinstance(coords,str):
- coords = eval(coords)
-
- latitude = float(coords[0])
- longitude = float(coords[1])
- data = [locationName, latitude, longitude]
- columns = ['locationName', 'latitude', 'longitude']
- except KeyError:
- # data = [None] * 3
- data = []
- columns = []
- except TypeError:
- # data = [None] * 3
- data = []
- columns = []
-
- return columns, data
-
- def _parseProperty_text(self, property, propertyMetadata):
- """
- Parse the text property.
-
- Returns 2 values: location name, latitude and longitude.
- :param property:
- :param propertyMetadata:
-
- :return: list,list
- Returns list of column names and list of values.
- """
- propertyLabel = propertyMetadata['label']
- data = [property['val']]
- columns = [propertyLabel]
-
- return columns, data
-
- def _parseProperty_textArea(self, property, propertyMetadata):
- """
- Parse the text property.
-
- Returns 2 values: location name, latitude and longitude.
- :param property:
- :param propertyMetadata:
-
- :return: list,list
- Returns list of column names and list of values.
- """
- propertyLabel = propertyMetadata['label']
-
- data = [property['val']]
- columns = [propertyLabel]
-
- return columns, data
-
- def _parseProperty_boolean(self, property, propertyMetadata):
- """
- Parse the text property.
-
- Returns 2 values: location name, latitude and longitude.
- :param property:
- :param propertyMetadata:
-
- :return: list,list
- Returns list of column names and list of values.
- """
- propertyLabel = propertyMetadata['label']
- data = [bool(property['val'])]
- columns = [propertyLabel]
-
- return columns, data
-
- def _parseProperty_number(self, property, propertyMetadata):
- """
- Parse the text property.
-
- Returns 2 values: location name, latitude and longitude.
- :param property:
- :param propertyMetadata:
-
- :return: list,list
- Returns list of column names and list of values.
- """
- try:
- propertyLabel = propertyMetadata['label']
- flt = property['val']
- if flt is None:
- data = [None]
- else:
- data = [float(property['val'])]
- columns = [propertyLabel]
- except ValueError:
- #print(f"\tCannot convert to float property {propertyLabel}. Got value '{property['val']}'")
- data = []
- columns = []
-
- return columns, data
-
- def _parseProperty_datetime_local(self, property, propertyMetadata):
- """
- Parse the text property.
-
- Returns 2 values: location name, latitude and longitude.
- :param property:
- :param propertyMetadata:
-
- :return: list,list
- Returns list of column names and list of values.
- """
- propertyLabel = propertyMetadata['label']
- localdata = pandas.to_datetime(property['val'], utc=False)
- if localdata is None:
- data = [None]
- else:
- data = [localdata.tz_localize("israel")]
- columns = [propertyLabel]
-
- return columns, data
-
- def _parseProperty_selectList(self, property, propertyMetadata):
- propertyLabel = propertyMetadata['label']
- data = [property['val']]
- columns = [propertyLabel]
-
- return columns, data
-
- def _composeEntityProperties(self, entityType, properties):
- """
- Just resolves the properties names.
- If it is location, split into 3 coordinates.
- :param properties:
- :return:
- """
- data = []
- columns = []
- for property in properties:
- propertyKey = property['key']
- propertyMetadata = entityType.propertiesTable.loc[propertyKey]
- propertyType = propertyMetadata['type']
-
- prop_type_handler = getattr(self, f"_parseProperty_{propertyType}")
-
- pcolumns, pdata = prop_type_handler(property, propertyMetadata)
- columns += pcolumns
- data += pdata
-
- entity_trial_properties = pandas.DataFrame(data=[data], columns=columns, index=[0])
- return entity_trial_properties
-
- def _composeProperties(self, entities):
-
- fullData = self.experiment.entitiesTableFull.set_index("key").join(entities, rsuffix="_r", how="inner").reset_index()
- dfList = []
- for indx, (entitykey, entitydata) in enumerate(fullData.iterrows()):
-
- properties = entitydata['properties']
- entityType = self.experiment.getEntitiesTypeByID(entityTypeID=entitydata.entitiesTypeKey)
-
- entity_trial_properties = self._composeEntityProperties(entityType, properties)
-
- entityProperties = entityType[entitydata['name']].propertiesTable.copy()
- entity_total_properties = entity_trial_properties.join(entityProperties,
- how='left',rsuffix='_prop') # .assign(trialSet = self.trialSet.name,
-
- dfList.append(entity_total_properties)
- new_df = pandas.concat(dfList, sort=False, ignore_index=True).drop(columns=["key","entitiesTypeKey"])
-
- return new_df
-
- @property
- def designEntities(self):
- ret = self.designEntitiesTable
- if ret.empty:
- return dict()
- else:
- return ret.set_index("entityName").T.to_dict()
-
- @property
- def deployEntities(self):
- ret = self.deployEntitiesTable
- if ret.empty:
- return dict()
- else:
- return ret.set_index("entityName").T.to_dict()
-
- def entities(self, status):
- """
-
- :param status: str
- design or deploy
- :return:
- """
- return getattr(self, f"{status}Entities")
-
- def entitiesTable(self, status):
- """
-
- :param status: str
- design or deploy
- :return:
- """
- return getattr(self, f"{status}EntitiesTable")
-
- def getDesignPropertiesTableByEntityID(self, entityID):
- try:
- entity = pandas.DataFrame(self._metadata['entities']).set_index('key').loc[entityID]
- entityType = self.experiment.getEntitiesTypeByID(entityTypeID=entity.entitiesTypeKey)
- ret = self._composeEntityProperties(entityType, entity.propertiesList)
- except KeyError:
- ret = pandas.DataFrame()
- return ret
-
- def getDesignPropertiesByEntityID(self, entityID):
- ret = self.getDesignPropertiesTableByEntityID(entityID)
- if ret.empty:
- return dict()
- else:
- return ret.loc[0].T.to_dict()
-
- def _prepareEntitiesMetadata(self,metadata):
-
- retList = []
- for entityData in metadata:
- for propData in entityData['properties']:
- properties = pandas.DataFrame(propData,index=[0])
- itm = pandas.DataFrame(properties).assign(entitiesTypeKey=entityData['entitiesTypeKey'],containsEntities=entityData['containsEntities'])
- retList.append(itm)
-
-
- return pandas.concat(retList,ignore_index=True)
-
- @property
- def designEntitiesTable(self):
- entities = pandas.DataFrame(self._metadata['entities']).set_index('key')
- return self._composeProperties(entities)
-
- @property
- def deployEntitiesTable(self):
- if not self._metadata['deployedEntities']:
- return pandas.DataFrame()
- else:
- entities = pandas.DataFrame(self._metadata['deployedEntities']).set_index('key')
- return self._composeProperties(entities)
-
- def getDeployPropertiesTableByEntityID(self, entityID):
- try:
- entity = pandas.DataFrame(self._metadata['deployedEntities']).set_index('key').loc[entityID]
- entityType = self.experiment.getEntitiesTypeByID(entityTypeID=entity.entitiesTypeKey)
- ret = self._composeEntityProperties(entityType, entity.propertiesList)
- except KeyError:
- ret = pandas.DataFrame()
- return ret
-
- def getDeployPropertiesByEntityID(self, entityID):
- ret = self.getDeployPropertiesTableByEntityID(entityID)
- if ret.empty:
- return dict()
- else:
- return ret.loc[0].T.to_dict()
-
- def __getitem__(self, item):
- return self._properties.loc[item].val
-
-
-class EntityType(dict):
- _experiment = None
- _metadata = None
-
- @property
- def experiment(self):
- return self._experiment
-
- @property
- def client(self):
- return self.experiment.client
-
- @property
- def keyID(self):
- return self._metadata['key']
-
- @property
- def id(self):
- return self._metadata['id']
-
- @property
- def name(self):
- return self._metadata['name']
-
- @property
- def numberOfEntities(self):
- return self._metadata['numberOfEntities']
-
- @property
- def state(self):
- return self._metadata['state']
-
- @property
- def propertiesTable(self):
- if 'properties' in self._metadata:
- ret = pandas.DataFrame(self._metadata['properties']).set_index('key')
- else:
- ret = pandas.DataFrame()
-
- return ret
-
- @property
- def properties(self):
- ret = dict()
- for prop in self._metadata['properties']:
- ret[prop['label']] = prop
-
- return ret
-
- def toJSON(self):
- ret = dict()
- ret['name'] = self.name
- ret['properties'] = self.properties
-
- entityJSON = {}
- for entityName, entityData in self.items():
- entityJSON[entityName] = entityData.toJSON()
-
- ret['entities'] = entityJSON
-
- return ret
-
- def __init__(self, experiment: Experiment, metadata: dict):
- """
- EntityType object contains information on a specific entity type
-
- :param experiment: The experiment object
- :param metadata: dict
- The metadata of the entity type (the properties and ect).
- """
- self._experiment = experiment
- self._metadata = metadata
-
- self._initEntities()
-
- def _initEntities(self):
- for entity in self._metadata['entities']:
- self[entity['name']] = Entity(entityType=self, metadata=entity)
-
- @property
- def entities(self):
- retList = []
- for entityName, entityData in self.items():
- trialProps = entityData.propertiesTable.assign(entityName=entityName, key=entityData.key)
- retList.append(trialProps)
- return pandas.concat(retList, ignore_index=True).drop(columns=["key","entitiesTypeKey"])
-
-
-class Entity:
- _entityType = None
- _metadata = None
-
- @property
- def entityType(self):
- return self._entityType
-
- @property
- def experiment(self):
- return self.entityType.experiment
-
- @property
- def client(self):
- return self.experiment.client
-
- @property
- def key(self):
- return self._metadata['key']
-
- @property
- def id(self):
- return self._metadata['id']
-
- @property
- def name(self):
- return self._metadata['name']
-
- @property
- def entityTypeKey(self):
- return self._metadata['entitiesTypeKey']
-
- @property
- def properties(self):
-
- ret = dict(self._properties)
-
- ret['key'] = self.key
- ret['entitiesTypeKey'] = self._metadata['entitiesTypeKey']
- ret['name'] = self.name
- ret['entityType'] = self.entityType.name
-
- return ret
-
- @property
- def allProperties(self):
- trialsetdict = dict()
-
- for trialsSetsName in self.experiment.trialSet.keys():
- trialsetdict[trialsSetsName] = dict()
- for trialName in self.experiment.trialSet[trialsSetsName].keys():
- ddp = trialsetdict[trialsSetsName].setdefault(trialName, dict())
-
- ddp['design'] = self.trialDesign(trialsSetsName, trialName)
- ddp['deploy'] = self.trialDeploy(trialsSetsName, trialName)
-
- return trialsetdict
-
- @property
- def allPropertiesList(self):
-
- trialsetlist = []
-
- for trialsSetsName in self.experiment.trialSet.keys():
- for trialName in self.experiment.trialSet[trialsSetsName].keys():
- design = self.trialDesign(trialsSetsName, trialName)
- deploy = self.trialDeploy(trialsSetsName, trialName)
-
- design['trialSetName'] = trialsSetsName
- design['trialName'] = trialName
- design['state'] = 'design'
-
- deploy['trialSetName'] = trialsSetsName
- deploy['trialName'] = trialName
- deploy['state'] = 'deploy'
-
- trialsetlist.append(design)
- trialsetlist.append(deploy)
-
- return trialsetlist
-
- @property
- def allPropertiesTable(self):
- return pandas.DataFrame(self.allPropertiesList)
-
- @property
- def propertiesTable(self):
- val = pandas.DataFrame(self.properties, index=[0])
- val = val.assign(entityName=self.name).drop("name", axis=1)
- return val
-
- def toJSON(self):
- ret = dict()
- ret['properties'] = dict(self._properties)
- ret['name'] = self.name
- ret['entityType'] = self.entityType.name
- ret['trialProperties'] = self.allPropertiesList
- return ret
-
- def __str__(self):
- return json.dumps(self.toJSON())
-
- def __repr__(self):
- props = self.properties
- props['name'] = self.name
- return json.dumps(props)
-
- @property
- def designProperties(self):
-
- trialsetdict = dict()
-
- for trialsSetsName in self.experiment.trialSet.keys():
- trialsetdict[trialsSetsName] = dict()
- for trialName in self.experiment.trialSet[trialsSetsName].keys():
- trialsetdict[trialsSetsName][trialName] = self.trialDesign(trialsSetsName, trialName)
-
- return trialsetdict
-
- @property
- def deployProperties(self):
- trialsetdict = dict()
-
- for trialsSetsName in self.experiment.trialSet.keys():
- trialsetdict[trialsSetsName] = dict()
- for trialName in self.experiment.trialSet[trialsSetsName].keys():
- trialsetdict[trialsSetsName][trialName] = self.trialDeploy(trialsSetsName, trialName)
-
- return trialsetdict
-
- def trialDesign(self, trialSet, trialName):
- ret = self.experiment.trialSet[trialSet][trialName].getDesignPropertiesByEntityID(self.key)
- for fld in ['key', 'name', 'entityType', 'entitiesTypeKey']:
- if fld in ret:
- del ret[fld]
-
- return ret
-
- def trialDeploy(self, trialSet, trialName):
- properties = self.experiment.trialSet[trialSet][trialName].deployEntities
- ret = properties.get(self.name, dict())
-
- for fld in ['key', 'name', 'entityType', 'entitiesTypeKey']:
- if fld in ret:
- del ret[fld]
-
- return ret
-
- def trial(self, trialSet, trialName, state):
- """
- Gets the properties of the trial useng the state
-
- Parameters
- -----------
- trialSet: str
- The trialset name
- trialName: str
- The trial name
- state: str
- 'design' or 'deploy'
-
- Returns
- -------
- dict
- """
-
- return getattr(self, f"trial{state.title()}")(trialSet, trialName)
-
- def __init__(self, entityType: EntityType, metadata: dict):
- """
-
- :param entityType: EntityType
- The entity type
-
- :param metadata : dict
- The metadata of the
- """
- self._entityType = entityType
- self._metadata = metadata
- if 'properties' in metadata:
- propertiesPandas = pandas.DataFrame(metadata['properties']).set_index('key')
-
- properties = propertiesPandas.merge(entityType.propertiesTable.query("trialField==False"), left_index=True,
- right_index=True)[['val', 'type', 'label', 'description']] \
- .set_index("label")
- else:
- properties = pandas.DataFrame()
-
- self._properties = dict([(key, data['val']) for key, data in properties.T.to_dict().items()])
-
diff --git a/argos/experimentSetup/__init__.py b/argos/experimentSetup/__init__.py
index 525bc1c..1d636f3 100644
--- a/argos/experimentSetup/__init__.py
+++ b/argos/experimentSetup/__init__.py
@@ -2,65 +2,9 @@
Experiment setup module for pyArgos.
Provides factory classes and data objects for loading and working with
-Argos experiment configurations from local files or the ArgosWEB server.
-
-Constants
----------
-WEB : str
- Source type for web-based experiments (``"web"``).
-FILE : str
- Source type for file-based experiments (``"json"``).
+Argos experiment configurations from local files.
"""
-from .dataObjectsFactory import fileExperimentFactory,webExperimentFactory
+from .dataObjectsFactory import fileExperimentFactory
-WEB = "web"
FILE = "json"
-
-
-
-def getExperimentSetup(experimentType,experimentName, **kwargs):
- """
- Load an experiment using the appropriate factory based on source type.
-
- A convenience function that routes to either ``fileExperimentFactory``
- or ``webExperimentFactory`` depending on the ``experimentType`` parameter.
-
- Parameters
- ----------
- experimentType : str
- The source type: ``"web"`` for ArgosWEB or ``"json"`` for local files.
- Use the module constants ``WEB`` and ``FILE``.
- experimentName : str
- The name of the experiment to load.
- **kwargs : dict
- Additional keyword arguments passed to the factory:
-
- - For ``WEB``: ``url`` (str) and ``token`` (str) are required.
- - For ``FILE``: ``experimentPath`` (str, optional) specifies the directory.
-
- Returns
- -------
- Experiment
- The loaded experiment object.
-
- Raises
- ------
- ValueError
- If ``experimentType`` is not ``WEB`` or ``FILE``.
-
- Examples
- --------
- >>> from argos.experimentSetup import getExperimentSetup, FILE, WEB
- >>> experiment = getExperimentSetup(FILE, "MyExp", experimentPath="/path/to/exp")
- >>> experiment = getExperimentSetup(WEB, "MyExp", url="http://server", token="abc")
- """
- if experimentType not in [WEB, FILE]:
- raise ValueError(f"experimentType must be {FILE} or {WEB}. Got {experimentType}")
-
- if experimentType==WEB:
- experiment = webExperimentFactory(url=kwargs['url'],token=kwargs['token'])
- else:
- experiment= fileExperimentFactory(**kwargs)
-
- return experiment.getExperiment(experimentName)
diff --git a/argos/experimentSetup/dataObjects.py b/argos/experimentSetup/dataObjects.py
index 69b72e4..6230afc 100644
--- a/argos/experimentSetup/dataObjects.py
+++ b/argos/experimentSetup/dataObjects.py
@@ -11,8 +11,6 @@
import os
import json
import pandas
-import requests
-from io import BytesIO
import matplotlib.pyplot as plt
import warnings
@@ -75,8 +73,6 @@ class Experiment:
_trialSetsDict = None # A dictionary of the trial sets.
_entitiesTypesDict = None # A dictionary of the devices types.
- _client = None # The client of the connection to the WEB.
-
_imagesMap = None
def refresh(self):
@@ -103,30 +99,6 @@ def setup(self):
"""
return self._experimentSetup
- @property
- def url(self):
- """
- The base URL of the experiment on the ArgosWEB server.
-
- Returns
- -------
- str
- The URL string from the experiment's ``experimentsWithData`` section.
- """
- return self.setup['experimentsWithData']['url']
-
- @property
- def client(self):
- """
- The GraphQL client used for web-based experiments.
-
- Returns
- -------
- Client or None
- The GQL client instance, or None for file-based experiments.
- """
- return self._client
-
@property
def name(self):
"""
@@ -136,7 +108,6 @@ def name(self):
- v3.0.0 ZIP (after migration): ``setup['experiment']['name']``
- v2.0.0 ZIP (after migration): ``setup['experiment']['name']``
- - Web (GraphQL): ``setup['experimentsWithData']['name']``
- Legacy/direct: ``setup['name']``
Returns
@@ -169,7 +140,6 @@ def _get_experiment_field(self, field):
1. ``setup[field]`` (legacy/direct)
2. ``setup['experiment'][field]`` (v2/v3 ZIP after migration)
- 3. ``setup['experimentsWithData'][field]`` (web/GraphQL)
Parameters
----------
@@ -190,8 +160,6 @@ def _get_experiment_field(self, field):
return self.setup[field]
if 'experiment' in self.setup and field in self.setup['experiment']:
return self.setup['experiment'][field]
- if 'experimentsWithData' in self.setup and field in self.setup['experimentsWithData']:
- return self.setup['experimentsWithData'][field]
raise KeyError(f"Field '{field}' not found in experiment setup")
@property
@@ -310,15 +278,8 @@ def __init__(self, setupFileOrData):
self._init_ImageMaps()
def _init_ImageMaps(self):
- ## Initializing the images map
+ """Initialize the images map. Overridden by ExperimentZipFile."""
self._imagesMap = dict()
- if 'experimentsWithData' in self.setup:
- for imgs in self.setup['experimentsWithData']['maps']:
- imgName = imgs['imageName']
- imageFullURL = f"{self.url}/{imgs['imageUrl']}"
- imgs['imageURL'] = imageFullURL
-
- self._imagesMap[imgName] = imgs
@property
def imageMap(self):
@@ -486,7 +447,10 @@ def getEntitiesTypeByID(self, entityTypeID):
def getImage(self, imageName: str):
"""
- Load an experiment image from the local filesystem.
+ Load an experiment image.
+
+ Override in subclasses to provide image loading from the
+ appropriate source (e.g., ZIP archive).
Parameters
----------
@@ -496,16 +460,14 @@ def getImage(self, imageName: str):
Returns
-------
numpy.ndarray
- The image data as a NumPy array (via matplotlib.pyplot.imread).
+ The image data as a NumPy array.
+
+ Raises
+ ------
+ NotImplementedError
+ If not overridden by a subclass.
"""
- imgUrl = os.path.join(self.setup['experimentsWithData']['url'], "images", f"{imageName}.png")
- # maybe we can skip the open(...), didn't want to risk it
- try:
- with open(imgUrl) as imageFile:
- img = plt.imread(imageFile)
- except UnicodeDecodeError:
- img = plt.imread(imgUrl)
- return img
+ raise NotImplementedError("getImage must be called on ExperimentZipFile, not base Experiment")
class ExperimentZipFile(Experiment):
@@ -662,43 +624,6 @@ def _fix_json_version_3_0_0(self, jsonFile):
return oldFormat
-class webExperiment(Experiment):
- """
- Experiment loaded from a remote ArgosWEB server.
-
- Extends :class:`Experiment` to fetch images via HTTP rather than
- from the local filesystem.
- """
-
- def getImage(self, imageName: str):
- """
- Fetch an experiment image from the remote server via HTTP.
-
- Parameters
- ----------
- imageName : str
- The name of the image.
-
- Returns
- -------
- numpy.ndarray
- The image data as a NumPy array.
-
- Raises
- ------
- ValueError
- If the image is not found on the server (HTTP status != 200).
- """
- imgUrl = self.getImageURL(imageName)
- response = requests.get(imgUrl)
-
- if response.status_code != 200:
- raise ValueError(f"Image {imageName} not found on the server.")
-
- imageFile = BytesIO(response.content)
- return plt.imread(imageFile)
-
-
class TrialSet(dict):
"""
A collection of trials belonging to a single trial set.
@@ -724,18 +649,6 @@ class TrialSet(dict):
_trialsDict = None
- @property
- def client(self):
- """
- The GraphQL client from the parent experiment.
-
- Returns
- -------
- Client or None
- The GQL client instance, or None for file-based experiments.
- """
- return self.experiment.client
-
@property
def experiment(self):
"""
@@ -914,18 +827,6 @@ def experiment(self):
"""
return self._trialSet.experiment
- @property
- def client(self):
- """
- The GraphQL client from the root experiment.
-
- Returns
- -------
- Client or None
- The GQL client instance, or None for file-based experiments.
- """
- return self._trialSet.experiment.client
-
@property
def experiment(self):
"""
@@ -1547,18 +1448,6 @@ def experiment(self):
"""
return self._experiment
- @property
- def client(self):
- """
- The GraphQL client from the parent experiment.
-
- Returns
- -------
- Client or None
- The GQL client instance, or None for file-based experiments.
- """
- return self.experiment.client
-
@property
def name(self):
"""
diff --git a/argos/experimentSetup/dataObjectsFactory.py b/argos/experimentSetup/dataObjectsFactory.py
index 21baae1..e33f748 100644
--- a/argos/experimentSetup/dataObjectsFactory.py
+++ b/argos/experimentSetup/dataObjectsFactory.py
@@ -1,24 +1,13 @@
"""
-Factory classes for loading experiments from different data sources.
+Factory for loading experiments from local files.
-This module provides two factories:
-
-- ``fileExperimentFactory`` -- loads experiments from local files (JSON or ZIP).
-- ``webExperimentFactory`` -- fetches experiments from an ArgosWEB server via GraphQL.
-
-Both factories return ``Experiment`` (or subclass) objects that provide a
-unified interface to the experiment data.
+Provides ``fileExperimentFactory`` which loads experiments from local
+files (JSON or ZIP) and returns ``Experiment`` (or ``ExperimentZipFile``)
+objects.
"""
import glob
-try:
- from gql import gql, Client
- from gql.transport.aiohttp import AIOHTTPTransport
-except:
- print("qgl is not installed")
-
-import pandas
-from argos.experimentSetup.dataObjects import webExperiment,Experiment,ExperimentZipFile
+from argos.experimentSetup.dataObjects import Experiment, ExperimentZipFile
import os
from argos.utils.jsonutils import loadJSON
from argos.utils.logging import get_logger as argos_get_logger
@@ -121,350 +110,3 @@ def __getitem__(self, item):
The loaded experiment object.
"""
return self.getExperiment(experimentPath=item)
-
-
-class webExperimentFactory:
- """
- Factory that fetches experiment data from an ArgosWEB server via GraphQL.
-
- Connects to the server's GraphQL endpoint and provides methods to list,
- describe, and fully load experiments.
-
- Parameters
- ----------
- url : str
- The base URL of the ArgosWEB server (e.g., ``"http://localhost:3000"``).
- token : str
- The authorization token for the server. Use an empty string for
- unauthenticated access.
-
- Examples
- --------
- >>> factory = webExperimentFactory("http://argos-server:3000", "my-token")
- >>> experiment = factory.getExperiment("MyExperiment")
- >>> print(factory.listExperimentsNames())
- """
-
- _client = None
- _url = None
-
- @property
- def url(self):
- """
- The base URL of the ArgosWEB server.
-
- Returns
- -------
- str
- The server URL.
- """
- return self._url
-
- @property
- def client(self):
- """
- The GraphQL client used for server communication.
-
- Returns
- -------
- Client
- The GQL client instance.
- """
- return self._client
-
- def __init__(self, url: str, token: str):
- """
- Initialize the web experiment factory.
-
- Parameters
- ----------
- url : str
- The base URL of the ArgosWEB server.
- token : str
- The authorization token. Use an empty string for unauthenticated access.
- """
-
- graphqlUrl = f"{url}/graphql"
-
- headers = None if token == '' else dict(authorization=token)
- transport = AIOHTTPTransport(url=graphqlUrl, headers=headers)
- self._client = Client(transport=transport, fetch_schema_from_transport=True)
-
- self._url = url
-
-
- def getExperimentMetadata(self,experimentName):
- """
- Fetch the full metadata for an experiment from the server.
-
- Queries the GraphQL API to retrieve entity types, entities, trial sets,
- and trials with all their properties and relationships.
-
- Parameters
- ----------
- experimentName : str
- The name of the experiment to fetch.
-
- Returns
- -------
- dict
- A dictionary containing the full experiment metadata with keys
- ``experimentsWithData``, ``entitiesTypes``, and ``trialSets``.
- """
- experimentDesc = self.getExperimentDescriptor(experimentName)
-
- ## Get the entities types
- query = '''
- {
- entitiesTypes(experimentId: "%s"){
- key
- name
- numberOfEntities
- properties{
- key
- type
- label
- description
- required
- trialField
- value
- }
- }
- }
- ''' % experimentDesc['project']['id']
- entitiesTypes = self._client.execute(gql(query))
-
-
-
- for entityType in entitiesTypes['entitiesTypes']:
- query = '''
- {
- entities(experimentId: "%s", entitiesTypeKey: "%s"){
- key
- name
- entitiesTypeKey
- state
- properties{
- val
- key
- }
- }
- }
- ''' % (experimentDesc['project']['id'], entityType['key'])
- result = self.client.execute(gql(query))
-
- entityType.update(result)
-
- ## Get the trialsets
- query = """{
- trialSets(experimentId: "%s"){
- key
- name
- description
- numberOfTrials
- properties{
- key
- type
- label
- description
- required
- trialField
- value
- }
- state
- }
- }
- """ % experimentDesc['project']['id']
- trialsets = self._client.execute(gql(query))
-
-
- for trialset in trialsets['trialSets']:
- query = '''
- {
- trials(experimentId: "%s", trialSetKey: "%s"){
- key
- name
- status
- created
- cloneFrom
- numberOfEntities
- state
- properties{
- val
- key
- }
- entities{
- entitiesTypeKey
- properties{
- val
- key
- }
- key
- containsEntities
- }
- deployedEntities{
- entitiesTypeKey
- properties{
- val
- key
- }
- key
- containsEntities
- }
- }
- }
- ''' % (experimentDesc['project']['id'], trialset['key'])
-
- result = self.client.execute(gql(query))
- trialset.update(result)
-
-
- ret = dict(experimentsWithData=experimentDesc)
- ret.update(trialsets)
- ret.update(entitiesTypes)
- return ret
-
- def getExperiment(self,experimentName):
- """
- Load a full experiment from the server.
-
- Fetches the experiment metadata via GraphQL and returns a
- ``webExperiment`` object with the data populated.
-
- Parameters
- ----------
- experimentName : str
- The name of the experiment to load.
-
- Returns
- -------
- webExperiment
- The loaded experiment object.
- """
- experimentDict = self.getExperimentMetadata(experimentName)
- experimentDict['experimentsWithData']['url'] = self.url
- return webExperiment(setupFileOrData=experimentDict)
-
-
-
- def getExperimentsDescriptionsList(self):
- """
- List all experiments on the server with their metadata.
-
- Queries the GraphQL API for all experiments and returns their
- descriptors including name, description, dates, and map definitions.
-
- Returns
- -------
- list[dict]
- A list of experiment descriptor dicts, each containing:
-
- - ``name`` : experiment name
- - ``description`` : experiment description
- - ``begin``, ``end`` : experiment date range
- - ``numberOfTrials`` : trial count
- - ``project.id`` : internal project ID
- - ``maps`` : list of image map definitions with coordinates
- """
- query = '''
- {
- experimentsWithData {
- name
- description
- begin
- end
- numberOfTrials
- project {
- id
- }
- maps {
- imageUrl
- imageName
- lower
- upper
- left
- right
- width
- height
- embedded
- }
- }
- }
- '''
- return self._client.execute(gql(query))['experimentsWithData']
-
- def getExperimentsDescriptionsTable(self):
- """
- Get all experiment descriptions as a Pandas DataFrame.
-
- Returns
- -------
- pandas.DataFrame
- A flattened DataFrame of all experiments on the server with
- their metadata.
- """
- return pandas.json_normalize(self.getExperimentsDescriptionsList())
-
-
- def getExperimentDescriptor(self,experimentName):
- """
- Get the descriptor for a specific experiment by name.
-
- Parameters
- ----------
- experimentName : str
- The name of the experiment.
-
- Returns
- -------
- dict
- The experiment descriptor containing ``name``, ``description``,
- ``begin``, ``end``, ``numberOfTrials``, ``project.id``, and ``maps``.
-
- Raises
- ------
- IndexError
- If no experiment with the given name is found on the server.
- """
- descs = self.getExperimentsDescriptionsList()
-
- return [x for x in descs if x['name']==experimentName][0]
-
-
- def listExperimentsNames(self):
- """
- List the names of all experiments on the server.
-
- Returns
- -------
- list[str]
- A list of experiment name strings.
- """
- return [x['name'] for x in self.listExperimentsDescriptions()]
-
- def __getitem__(self, item):
- """
- Load an experiment by name using dictionary-style access.
-
- Parameters
- ----------
- item : str
- The experiment name.
-
- Returns
- -------
- webExperiment
- The loaded experiment object.
- """
- return self.getExperiment(experimentName=item)
-
- def keys(self):
- """
- Return the names of all experiments on the server.
-
- Returns
- -------
- list[str]
- A list of experiment name strings.
- """
- return self.listExperiments()['name']
diff --git a/docs/developer_guide/api/experiment_setup.md b/docs/developer_guide/api/experiment_setup.md
index 6b352b6..e050b08 100644
--- a/docs/developer_guide/api/experiment_setup.md
+++ b/docs/developer_guide/api/experiment_setup.md
@@ -13,10 +13,8 @@ Each class in this module has a distinct responsibility in the experiment lifecy
| Class | Role | Analogy |
|-------|------|---------|
| `fileExperimentFactory` | Locates and loads experiment data from local disk | File browser |
-| `webExperimentFactory` | Fetches experiment data from ArgosWEB via GraphQL | API client |
| `Experiment` | Root container holding all entity types, trial sets, and image maps | Project folder |
| `ExperimentZipFile` | Variant of Experiment that reads from a `.zip` archive (with version migration) | ZIP reader |
-| `webExperiment` | Variant of Experiment that fetches images over HTTP | Remote reader |
| `TrialSet` | Named collection of trials (e.g., "design", "deploy") | Folder of trials |
| `Trial` | A single experimental configuration with per-entity attributes | Configuration snapshot |
| `EntityType` | Schema definition + collection of entities of one type (e.g., "Sensor") | Device class |
@@ -28,7 +26,7 @@ Each class in this module has a distinct responsibility in the experiment lifecy
### Experiment Hierarchy
-`Experiment` is the base class. Two subclasses override only the data-loading and image-fetching behavior, keeping the rest of the interface identical:
+`Experiment` is the base class. `ExperimentZipFile` overrides the data-loading and image-fetching behavior, keeping the rest of the interface identical:
@@ -56,19 +54,13 @@ classDiagram
-_fix_json_version_3_0_0()
}
- class webExperiment {
- +getImage() ⟵ overrides
- }
-
Experiment <|-- ExperimentZipFile : "reads from ZIP archive"
- Experiment <|-- webExperiment : "fetches images via HTTP"
```
-->
**What each subclass changes:**
- **ExperimentZipFile** -- `refresh()` extracts `data.json` from the ZIP, detects the schema version, and applies the appropriate migration. `getImage()` reads images from inside the archive. `_init_ImageMaps()` parses the ZIP's internal map format.
-- **webExperiment** -- `getImage()` does an HTTP GET to fetch the image from the ArgosWEB server. Everything else is inherited.
### Container Hierarchy (dict-based)
@@ -217,58 +209,6 @@ sequenceDiagram
---
-## Call Workflow: Loading an Experiment from Web
-
-
-
->Factory: Experiment descriptor
-
- Factory->>GQL: Query entitiesTypes (key, name, properties)
- GQL-->>Factory: Entity type list
- loop For each entityType
- Factory->>GQL: Query entities (key, name, properties)
- GQL-->>Factory: Entity list for type
- end
-
- Factory->>GQL: Query trialSets (key, name, properties)
- GQL-->>Factory: Trial set list
- loop For each trialSet
- Factory->>GQL: Query trials (name, properties, entities)
- GQL-->>Factory: Trial list for set
- end
-
- Factory->>Factory: Assemble metadata dict
- Factory->>Exp: webExperiment(metadata)
- Note over Exp: Same __init__ as Experiment:
refresh → _initTrialSets → _initEntitiesTypes
-
- Exp-->>Factory: return webExperiment
- Factory-->>User: experiment object ready
-
- Note over User: Images fetched on demand
- User->>Exp: experiment.getImage("field_map")
- Exp->>GQL: HTTP GET image URL
- GQL-->>Exp: Image bytes
- Exp-->>User: numpy.ndarray
-```
--->
-
----
-
## Call Workflow: Accessing Trial Entity Data
When you access `trial.entities` or `trial.entitiesTable`, the containment hierarchy is resolved on-the-fly:
@@ -317,15 +257,6 @@ sequenceDiagram
---
-## Module Entry Point
-
-::: argos.experimentSetup.getExperimentSetup
- options:
- show_root_heading: true
- heading_level: 3
-
----
-
## Factories
### fileExperimentFactory
@@ -348,34 +279,6 @@ sequenceDiagram
---
-### webExperimentFactory
-
-**Role:** Connects to an ArgosWEB server via GraphQL and fetches experiment data remotely. Supports listing experiments, fetching metadata, and loading complete experiments.
-
-**Key behavior:**
-
-- Authenticates with a token-based authorization header
-- Issues multiple GraphQL queries to assemble the full experiment: entity types, entities, trial sets, trials
-- Returns a `webExperiment` object whose `getImage()` fetches images over HTTP
-
-::: argos.experimentSetup.dataObjectsFactory.webExperimentFactory
- options:
- show_root_heading: true
- heading_level: 4
- members:
- - __init__
- - getExperiment
- - getExperimentMetadata
- - getExperimentsDescriptionsList
- - getExperimentsDescriptionsTable
- - getExperimentDescriptor
- - listExperimentsNames
- - url
- - client
- - keys
-
----
-
## Data Objects
### Experiment
@@ -430,19 +333,6 @@ sequenceDiagram
---
-### webExperiment
-
-**Role:** Handles experiments fetched from a remote ArgosWEB server. The only override is `getImage()`, which fetches images via HTTP GET instead of reading from disk.
-
-::: argos.experimentSetup.dataObjects.webExperiment
- options:
- show_root_heading: true
- heading_level: 4
- members:
- - getImage
-
----
-
### TrialSet
**Role:** A named collection of trials (e.g., `"design"`, `"deploy"`). Extends `dict` so trials can be accessed by name: `trial_set["myTrial"]`. Defines the attribute type schema that all trials in the set share.
diff --git a/docs/developer_guide/api/index.md b/docs/developer_guide/api/index.md
index 9262f86..923be5c 100644
--- a/docs/developer_guide/api/index.md
+++ b/docs/developer_guide/api/index.md
@@ -12,8 +12,8 @@ argos/
CLI.py # CLI command handlers
manager.py # experimentManager class
experimentSetup/
- __init__.py # getExperimentSetup(), WEB/FILE constants
- dataObjectsFactory.py # fileExperimentFactory, webExperimentFactory
+ __init__.py # FILE constant
+ dataObjectsFactory.py # fileExperimentFactory
dataObjects.py # Experiment, TrialSet, Trial, EntityType, Entity
fillContained.py # fill_properties_by_contained()
kafka/
@@ -44,10 +44,6 @@ argos/
from argos.experimentSetup import fileExperimentFactory
experiment = fileExperimentFactory("/path").getExperiment()
-# Load an experiment from web
-from argos.experimentSetup import webExperimentFactory
-experiment = webExperimentFactory(url, token).getExperiment("name")
-
# Use the experiment manager (with ThingsBoard)
from argos.manager import experimentManager
manager = experimentManager("/path")
diff --git a/docs/developer_guide/architecture/core_concepts.md b/docs/developer_guide/architecture/core_concepts.md
index dcc6b84..16ea522 100644
--- a/docs/developer_guide/architecture/core_concepts.md
+++ b/docs/developer_guide/architecture/core_concepts.md
@@ -59,7 +59,7 @@ graph TD
| Module | Depends on (internal) | Depends on (external) |
|--------|----------------------|----------------------|
-| `experimentSetup` | `utils.jsonutils`, `utils.logging` | `gql` (optional, for web), `matplotlib`, `requests` |
+| `experimentSetup` | `utils.jsonutils`, `utils.logging` | `matplotlib`, `requests` |
| `manager` | `experimentSetup`, `utils.jsonutils`, `utils.logging` | `tb_rest_client` (optional) |
| `kafka.consumer` | `utils.parquetUtils` | `kafka-python`, `numpy`, `pandas` |
| `CLI` | `experimentSetup`, `kafka.consumer`, `manager` | `kafka-python` |
@@ -101,10 +101,6 @@ classDiagram
+getImage()
}
- class webExperiment {
- +getImage()
- }
-
class TrialSet {
+name
+experiment: Experiment
@@ -141,7 +137,6 @@ classDiagram
}
Experiment <|-- ExperimentZipFile
- Experiment <|-- webExperiment
Experiment "1" --> "*" TrialSet
Experiment "1" --> "*" EntityType
TrialSet "1" --> "*" Trial
@@ -163,13 +158,9 @@ pyArgos uses the Factory pattern to abstract experiment loading from different s
B{Source Type?}
- B -->|FILE| C[fileExperimentFactory]
- B -->|WEB| D[webExperimentFactory]
+ A[Client Code] --> C[fileExperimentFactory]
C --> E[Experiment / ExperimentZipFile]
- D --> F[webExperiment]
E --> G[Unified Experiment Interface]
- F --> G
```
-->
@@ -180,13 +171,6 @@ graph TD
- Handles version migrations (1.0.0 through 3.0.0)
- Default: uses current working directory
-### `webExperimentFactory`
-
-- Fetches from ArgosWEB via GraphQL
-- Requires server URL and authentication token
-- Retrieves entity types, entities, trial sets, and trials
-- Images are fetched via HTTP on demand
-
---
## Swimlane: Full Experiment Deployment
@@ -333,7 +317,7 @@ This makes it easy to filter, join, and analyze experiment metadata using standa
### Why optional external dependencies?
-ThingsBoard (`tb_rest_client`), GraphQL (`gql`), Cassandra (`cassandra-driver`), and MongoDB (`pymongo`) are all optional. pyArgos gracefully handles their absence with try/except imports. This allows users to install only what they need — a researcher doing offline analysis doesn't need ThingsBoard, and a deployment engineer doesn't need Cassandra.
+ThingsBoard (`tb_rest_client`), Cassandra (`cassandra-driver`), and MongoDB (`pymongo`) are all optional. pyArgos gracefully handles their absence with try/except imports. This allows users to install only what they need — a researcher doing offline analysis doesn't need ThingsBoard, and a deployment engineer doesn't need Cassandra.
### Why threads for Kafka consumers?
diff --git a/docs/developer_guide/architecture/data_flow.md b/docs/developer_guide/architecture/data_flow.md
index 3f0ce88..73db620 100644
--- a/docs/developer_guide/architecture/data_flow.md
+++ b/docs/developer_guide/architecture/data_flow.md
@@ -36,32 +36,6 @@ sequenceDiagram
---
-## Remote Experiment Loading
-
-
-
->Factory: Experiment metadata
- Factory->>Exp: Create webExperiment
- Factory-->>Client: Return webExperiment
-```
--->
-
----
-
## Kafka to Parquet Pipeline
diff --git a/docs/developer_guide/architecture/experiment_setup.md b/docs/developer_guide/architecture/experiment_setup.md
index 9bac4ed..a8fed6e 100644
--- a/docs/developer_guide/architecture/experiment_setup.md
+++ b/docs/developer_guide/architecture/experiment_setup.md
@@ -8,7 +8,7 @@ This page provides a deep architectural reference for the `argos.experimentSetup
The `experimentSetup` module is responsible for:
-1. **Loading** experiment definitions from files (JSON/ZIP) or the ArgosWEB server (GraphQL)
+1. **Loading** experiment definitions from files (JSON/ZIP)
2. **Parsing** the raw JSON into a typed object hierarchy
3. **Exposing** experiment metadata as Pandas DataFrames for analysis
4. **Resolving** entity containment hierarchies (parent-child property inheritance)
@@ -18,8 +18,8 @@ The `experimentSetup` module is responsible for:
```
argos/experimentSetup/
- __init__.py # Module entry point, WEB/FILE constants
- dataObjectsFactory.py # Factory classes (file, web)
+ __init__.py # Module entry point, FILE constant
+ dataObjectsFactory.py # Factory classes (file)
dataObjects.py # Data object hierarchy (Experiment, Trial, Entity, ...)
fillContained.py # Containment hierarchy resolution
runner.py # Development/test runner script
@@ -32,7 +32,7 @@ argos/experimentSetup/
### Experiment Classes
-The `Experiment` base class defines the full interface for accessing experiment data. Two subclasses override only the data-loading and image-fetching behavior:
+The `Experiment` base class defines the full interface for accessing experiment data. `ExperimentZipFile` overrides the data-loading and image-fetching behavior:
@@ -84,26 +84,19 @@ classDiagram
#_fix_json_version_3_0_0(jsonFile)
}
- class webExperiment {
- <>
- +getImage(imageName)◄
- }
-
Experiment <|-- ExperimentZipFile : inherits
- Experiment <|-- webExperiment : inherits
note for ExperimentZipFile "◄ = overridden method"
- note for webExperiment "◄ = overridden method"
```
-->
**Key override points:**
-| Method | Experiment (base) | ExperimentZipFile | webExperiment |
-|--------|-------------------|-------------------|---------------|
-| `refresh()` | Loads JSON from file/dict | Extracts `data.json` from ZIP, applies version migration | Inherited (uses base) |
-| `getImage()` | Reads from local filesystem path | Reads from inside the ZIP archive | Fetches via HTTP GET |
-| `_init_ImageMaps()` | Parses `experimentsWithData.maps` with full URLs | Parses `maps` from ZIP metadata (no URLs) | Inherited (uses base) |
+| Method | Experiment (base) | ExperimentZipFile |
+|--------|-------------------|-------------------|
+| `refresh()` | Loads JSON from file/dict | Extracts `data.json` from ZIP, applies version migration |
+| `getImage()` | Reads from local filesystem path | Reads from inside the ZIP archive |
+| `_init_ImageMaps()` | Parses `maps` with full URLs | Parses `maps` from ZIP metadata (no URLs) |
### Container Classes (dict-based)
@@ -226,10 +219,7 @@ graph TD
B{Source type?}
-
- B -->|FILE| C[fileExperimentFactory]
- B -->|WEB| D[webExperimentFactory]
+ A[Client calls factory directly] --> C[fileExperimentFactory]
C --> E[Scan runtimeExperimentData/]
E --> F{Found .zip files?}
@@ -252,14 +242,6 @@ flowchart TD
L --> R
- D --> S[GraphQL queries]
- S --> T[Query entitiesTypes + entities]
- S --> U[Query trialSets + trials]
- T --> V[Build metadata dict]
- U --> V
- V --> W[webExperiment.__init__]
- W --> R
-
R --> X[Return Experiment object]
```
-->
@@ -683,8 +665,8 @@ Experiment metadata is inherently tabular (entities have rows of properties, tri
### Why factory pattern?
-The same `Experiment` interface is needed regardless of whether data comes from a local file, a ZIP archive, or a remote server. The factory pattern:
+The same `Experiment` interface is needed regardless of whether data comes from a local JSON file or a ZIP archive. The factory pattern:
-- Hides the complexity of source detection (ZIP vs JSON vs GraphQL)
+- Hides the complexity of source detection (ZIP vs JSON)
- Handles version migration transparently
- Returns a uniform interface that client code can rely on
diff --git a/docs/developer_guide/data_model.md b/docs/developer_guide/data_model.md
index aa5088a..2c60ddd 100644
--- a/docs/developer_guide/data_model.md
+++ b/docs/developer_guide/data_model.md
@@ -15,7 +15,7 @@ An **Experiment** is a complete scientific or engineering study conducted with I
- **Where things are** (image maps with geo-coordinates)
- **When it runs** (start and end dates)
-An experiment is created in [ArgosWEB](https://github.com/argosp) and exported as a ZIP file, or loaded directly from the server via GraphQL.
+An experiment is created in [ArgosWEB](https://github.com/argosp) and exported as a ZIP file for use with pyArgos.
### What is an Entity Type?
diff --git a/docs/developer_guide/experiment_setup_index.md b/docs/developer_guide/experiment_setup_index.md
index f33dbae..12bb6b0 100644
--- a/docs/developer_guide/experiment_setup_index.md
+++ b/docs/developer_guide/experiment_setup_index.md
@@ -27,7 +27,7 @@ The architecture page covers the **internal design** of the module: how classes
- [**Experiment Setup Architecture**](architecture/experiment_setup.md)
- Module overview and file layout
- - Inheritance hierarchy (Experiment → ExperimentZipFile / webExperiment)
+ - Inheritance hierarchy (Experiment → ExperimentZipFile)
- Container classes (dict-based TrialSet, EntityType)
- Full object composition diagram
- Factory pattern decision flow
@@ -51,12 +51,10 @@ The API page provides the **class-level documentation** with role descriptions,
- Class roles summary table
- Inheritance and container hierarchy diagrams
- Swimlane: loading experiment from file
- - Swimlane: loading experiment from web (GraphQL)
- Swimlane: accessing trial entity data (containment resolution)
- Auto-generated docs for every class and method:
- - `getExperimentSetup` (module entry point)
- - `fileExperimentFactory`, `webExperimentFactory`
- - `Experiment`, `ExperimentZipFile`, `webExperiment`
+ - `fileExperimentFactory`
+ - `Experiment`, `ExperimentZipFile`
- `TrialSet`, `Trial`
- `EntityType`, `Entity`
- `fill_properties_by_contained`, `spread_attributes`, `get_parent`, `key_from_name`
@@ -72,7 +70,7 @@ The user guide covers the **practical usage** -- how to create experiments, defi
- Configuring datasources
- Defining entities (DEVICE / ASSET)
- Setting up trials (template → design → upload)
- - Loading experiments in Python (file and web)
+ - Loading experiments in Python (from file)
---
diff --git a/docs/user_guide/concepts.md b/docs/user_guide/concepts.md
index 5a0bacb..c17709c 100644
--- a/docs/user_guide/concepts.md
+++ b/docs/user_guide/concepts.md
@@ -51,9 +51,8 @@ All of this is done through forms, tables, and map interfaces in the browser
### How pyArgos fits in
pyArgos is the **Python programmatic interface** to the data that ArgosWEB
-creates. It reads the exported ZIP file (or connects to the ArgosWEB server
-via GraphQL) and makes the experiment data available as Python objects and
-Pandas DataFrames.
+creates. It reads the exported ZIP file and makes the experiment data
+available as Python objects and Pandas DataFrames.
```
┌──────────────────────┐
diff --git a/docs/user_guide/experiment_setup.md b/docs/user_guide/experiment_setup.md
index 514dcd5..8679b61 100644
--- a/docs/user_guide/experiment_setup.md
+++ b/docs/user_guide/experiment_setup.md
@@ -171,18 +171,6 @@ trial = experiment.trialSet["design"]["morningTrial"]
print(trial.entitiesTable())
```
-### From ArgosWEB
-
-```python
-from argos.experimentSetup import webExperimentFactory
-
-factory = webExperimentFactory(url="http://argos-server/graphql", token="your-token")
-experiment = factory.getExperiment("MyExperiment")
-
-# List all experiments
-print(factory.listExperimentsNames())
-```
-
---
## Using the Generated Starter Script