diff --git a/README.md b/README.md index 836669e..5191f83 100644 --- a/README.md +++ b/README.md @@ -162,3 +162,75 @@ Password: ${HTTP_GUEST_PASSWORD_RABBIT} 1. Start the queue (see [above](#activationdeactivation-of-queue)). 2. Use the CLI commands or the API with/without docker (see [above](#usage-of-main-application)). + +## Testing ## + +### Mock data ### + +To generate mock data one may call for examples + +```bash +rm -rf "data/example" +just create-mocks \ + --path "data/example" \ + --max-depth 10 \ + --max-folders 100 \ + --max-files 1000 +``` + +which creates a fresh relative directory `data/examples`, +consisting of at most roughly 1000 files and 100 folders, +and not exceeding a depth of 10. + +### Request ### + +Fill in `setup/requests.yaml` as follows: + +```yaml +label: 'Mock example' + +# apply some generous limits +options: + max-depth: 100 + max-count: 10_000_000 + max-time: 00:05:00 + +data: + # the locaiton of the mock directory + inputs: + location: OS + path: 'data/example' +``` + +### Execution ### + +1. Start the queue (see [above](#activationdeactivation-of-queue)). + +2. Ensure that the queue-users are registered (see [above](#set-up-users)). + +3. Use the CLI commands or the API with/without docker (see [above](#usage-of-main-application)). + +4. Run the feature: + + - For the CLI option, call + + ```bash + just run-cli SEARCH-FS + ``` + + - For the FastApi options (with or without docker), + make a POST-call (e.g. in [Postman](https://www.postman.com)) + against the endpoint `/feature/search-fs` + using the JSON-body + + ```json + { + "ref": { + "location": "OS", + "path": "setup/requests.yaml" + } + } + ``` + + The file reference in this body can of course be a json + and located anywhere on your system. diff --git a/docker-compose.yaml b/docker-compose.yaml index 128c82b..99d3d67 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -186,7 +186,7 @@ services: echo "success" ' - queue-server: + queue-server: &ref_queue-server image: local/examplerabbitmq:queue # <- i.e. use this image hostname: ${HTTP_HOST_NAME_RABBIT} @@ -198,8 +198,12 @@ services: - credentials volumes: - - ${PATH_LOGS_QUEUE}://var/log/rabbitmq:rw - - ${PATH_LOGS_QUEUE_STATE}://var/lib/rabbitmq:rw + - type: bind + source: ${PATH_LOGS_QUEUE} + target: //var/log/rabbitmq + - type: bind + source: ${PATH_LOGS_QUEUE_STATE} + target: //var/lib/rabbitmq ports: # HOST-IP:HOST-PORT:CONTAINER (note: uses values in .env) @@ -213,34 +217,26 @@ services: rabbitmq-server queue-admin: - image: local/examplerabbitmq:queue # <- i.e. use this image - - env_file: - - .env - - .env.docker-vars - - secrets: - - credentials - - volumes: - - ${PATH_LOGS_QUEUE}://var/log/rabbitmq:rw - - ${PATH_LOGS_QUEUE_STATE}://var/lib/rabbitmq:rw - + <<: *ref_queue-server + hostname: "admin" + depends_on: + - queue-server + ports: [] networks: - default - tty: true stdin_open: true - depends_on: - - queue-server command: |- bash --login -c ' source /run/secrets/credentials rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} list_users + # create admin rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} delete_user $${HTTP_ADMIN_USER_RABBIT} 2> /dev/null rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} add_user $${HTTP_ADMIN_USER_RABBIT} $${HTTP_ADMIN_PASSWORD_RABBIT} + rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} set_permissions -p / $${HTTP_ADMIN_USER_RABBIT} ".*" ".*" ".*" rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} set_user_tags $${HTTP_ADMIN_USER_RABBIT} administrator + # create guest rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} delete_user $${HTTP_GUEST_USER_RABBIT} 2> /dev/null rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} add_user $${HTTP_GUEST_USER_RABBIT} $${HTTP_GUEST_PASSWORD_RABBIT} diff --git a/docs/models/application/Models/RequestTask.md b/docs/models/application/Models/RequestTask.md index 5518f09..11216a7 100644 --- a/docs/models/application/Models/RequestTask.md +++ b/docs/models/application/Models/RequestTask.md @@ -4,7 +4,8 @@ | Name | Type | Description | Notes | |------------ | ------------- | ------------- | -------------| | **label** | **String** | Label of task | [default to null] | -| **options** | [**Map**](AnyType.md) | Structure of requests payload > options NOTE: not yet implemented | [default to null] | +| **ignore** | **Boolean** | | [optional] [default to false] | +| **options** | [**RequestTaskOptions**](RequestTaskOptions.md) | | [default to null] | | **data** | [**RequestTaskData**](RequestTaskData.md) | | [default to null] | [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/docs/models/application/Models/RequestTaskOptions.md b/docs/models/application/Models/RequestTaskOptions.md new file mode 100644 index 0000000..b19051b --- /dev/null +++ b/docs/models/application/Models/RequestTaskOptions.md @@ -0,0 +1,11 @@ +# RequestTaskOptions +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **max-depth** | **Integer** | Limits the search depth | [optional] [default to 50] | +| **max-items** | **Integer** | Limits the amount of items that can be found | [optional] [default to 1000000] | +| **max-duration** | **String** | Limits the amount of time spent for a search | [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/models/application/Models/RequestsPayload.md b/docs/models/application/Models/RequestsPayload.md index f04976a..698e92d 100644 --- a/docs/models/application/Models/RequestsPayload.md +++ b/docs/models/application/Models/RequestsPayload.md @@ -4,7 +4,8 @@ | Name | Type | Description | Notes | |------------ | ------------- | ------------- | -------------| | **label** | **String** | Label of task | [default to null] | -| **options** | [**Map**](AnyType.md) | Structure of requests payload > options NOTE: not yet implemented | [default to null] | +| **ignore** | **Boolean** | | [optional] [default to false] | +| **options** | [**RequestTaskOptions**](RequestTaskOptions.md) | | [default to null] | | **data** | [**RequestTaskData**](RequestTaskData.md) | | [default to null] | [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/docs/models/application/README.md b/docs/models/application/README.md index 8b0298d..93ea15f 100644 --- a/docs/models/application/README.md +++ b/docs/models/application/README.md @@ -23,6 +23,7 @@ All URIs are relative to *https://acme.org* - [RepoInfo_urls](./Models/RepoInfo_urls.md) - [RequestTask](./Models/RequestTask.md) - [RequestTaskData](./Models/RequestTaskData.md) + - [RequestTaskOptions](./Models/RequestTaskOptions.md) - [RequestsPayload](./Models/RequestsPayload.md) diff --git a/justfile b/justfile index 6ef1341..af4ea97 100644 --- a/justfile +++ b/justfile @@ -294,8 +294,8 @@ check-time-matches-cron cron_expr time="*": @{{PYVENV_ON}} && {{PYVENV}} -m scripts.cron "{{cron_expr}}" --time "{{time}}" # Recipe only works if local file scripts/mocks.py exists -mocks *args: - @{{PYVENV_ON}} && {{PYVENV}} -m scripts.mocks +create-mocks *args: + @{{PYVENV_ON}} && {{PYVENV}} -m scripts.mocks {{args}} # -------------------------------- # TARGETS: terminate execution diff --git a/models/schema-application.yaml b/models/schema-application.yaml index 7797243..960b6ec 100644 --- a/models/schema-application.yaml +++ b/models/schema-application.yaml @@ -133,6 +133,9 @@ components: description: |- Label of task type: string + ignore: + type: boolean + default: false options: $ref: "#/components/schemas/RequestTaskOptions" data: @@ -144,7 +147,25 @@ components: NOTE: not yet implemented type: object + required: + - max-duration additionalProperties: true + properties: + max-depth: + description: |- + Limits the search depth + type: integer + default: 50 + max-items: + description: |- + Limits the amount of items that can be found + type: integer + default: 1_000_000 + max-duration: + description: |- + Limits the amount of time spent for a search + type: string + format: duration RequestTaskData: description: |- diff --git a/scripts/mocks.py b/scripts/mocks.py index f903df9..301608b 100644 --- a/scripts/mocks.py +++ b/scripts/mocks.py @@ -2,20 +2,134 @@ # -*- coding: utf-8 -*- """ -Script to check if current time matches a given CRON expression +Script to generate a local mock directory """ # ---------------------------------------------------------------- # IMPORTS # ---------------------------------------------------------------- +import os import sys +from pathlib import Path + +os.chdir(Path(__file__).parent.parent) +sys.path.insert(0, os.getcwd()) + +import logging +import math +import random +from argparse import ArgumentParser +from argparse import RawTextHelpFormatter + +from lorem_text import lorem +from mimesis import Generic + +from src._core.constants import * +from src._core.logging import * + +# ---------------------------------------------------------------- +# SETTINGS +# ---------------------------------------------------------------- + +gen = Generic() # ---------------------------------------------------------------- # METHODS # ---------------------------------------------------------------- -# + +def parse_args(*args: str): + parser = ArgumentParser( + prog="mocks", + description="creates a local mock directory for the feature SEARCH-FS", + formatter_class=RawTextHelpFormatter, + ) + parser.add_argument( + "--path", + type=str, + help="absolute or relative path to directory to be created", + # nargs="?", + # default="data/example", + ) + parser.add_argument( + "--max-depth", + type=int, + help="maximum depth of folders", + # nargs="?", + # default=4, + ) + parser.add_argument( + "--max-folders", + type=int, + help="maximum count files count", + # nargs="?", + # default=100, + ) + parser.add_argument( + "--max-files", + type=int, + help="maximum count files count", + # nargs="?", + # default=1000, + ) + logging.info(args) + args_parsed = parser.parse_args(args) + return args_parsed + + +def create_folder( + path: str, + /, + *, + depth: int, + k_folders: int, + k_files: int, +): + """ + Recursive depth-first method to create folders and files + """ + # ensure folder exists + logging.info(f"- create folder '{path}'") + p = Path(path) + p.mkdir(parents=True, exist_ok=True) + + # generate random file names + k = random.randint(1, k_files) + basenames = [gen.food.dish() for _ in range(k)] + extension = random.choices([".txt", ".csv", ".md"], k=k) + filenames = [f"{x}{ext}" for x, ext in zip(basenames, extension)] + + # create files and contents + for filename in filenames: + path_ = os.path.join(path, filename) + content = lorem.paragraph().encode() + logging.info(f"- create file '{filename}' with {len(content)/SIZE_1_KB:.4f} kb of data") # fmt: skip + + p = Path(path_) + p.touch(exist_ok=True) + with open(path_, "wb") as fp: + fp.write(content) + + # if depth remaining is 0 stop + if depth == 0: + return + + # otherwise proceed + k = random.randint(1, k_folders) + foldernames = [gen.address.city() for _ in range(k)] + + for foldername in foldernames: + path_ = os.path.join(path, foldername) + create_folder( + path_, + depth=depth - 1, + k_folders=k_folders, + k_files=k_files, + ) + + return + # ---------------------------------------------------------------- # EXECUTION @@ -23,4 +137,30 @@ if __name__ == "__main__": sys.tracebacklimit = 0 - raise NotImplementedError("generation of mocks not yet implemented") + + # initialise logging + configure_logging(name="root", level="INFO", path=None, serialise=False) + + # parse the args + args = parse_args(*sys.argv[1:]) + + path = args.path + depth = args.max_depth + count_files = args.max_files + count_folder = args.max_folders + + # derive how many files/subfolders per folder using heuristics + k_folders = 0 + if depth > 1: + k_folders = math.ceil(math.pow(count_folder, 1 / depth)) + + k_files = math.ceil(count_files / count_folder) + k_files = max(1, k_files) + + # call the recursive method + create_folder( + path, + depth=depth, + k_folders=k_folders, + k_files=k_files, + ) diff --git a/src/algorithms/__init__.py b/src/algorithms/__init__.py new file mode 100644 index 0000000..f4f4e92 --- /dev/null +++ b/src/algorithms/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Module for algorithms independent of application +""" diff --git a/src/algorithms/filesmanager/__init__.py b/src/algorithms/filesmanager/__init__.py new file mode 100644 index 0000000..f68fd0f --- /dev/null +++ b/src/algorithms/filesmanager/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Algorithms to work with file systems +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from .search import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "recursive_file_search", +] diff --git a/src/algorithms/filesmanager/search.py b/src/algorithms/filesmanager/search.py new file mode 100644 index 0000000..7bb545d --- /dev/null +++ b/src/algorithms/filesmanager/search.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Recursive search algorithms +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from collections import deque +from typing import Generator + +from ...models.filesmanager import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "recursive_file_search", +] + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def recursive_file_search( + manager: FilesManager, + /, + *, + path: str, +) -> Generator[tuple[int, str, str], None, None]: + """ + Uses a FIFO-queue to search for all files in a given directory + """ + # create and initialise queue + q = deque([(0, path)]) + + # keep alive as long as queue not empty + while len(q) > 0: + # handle next task + d, path = q.pop() + + # obtain folder handler + folder = manager.get_folder(path) + + # obtain filenames + filenames = folder.get_filenames() + for filename in filenames: + yield d, path, filename + + # add subfolders to queue + subpaths = folder.get_subfolder_paths() + for subpath in subpaths: + q.append((d + 1, subpath)) diff --git a/src/features/feat_searchfs/__init__.py b/src/features/feat_searchfs/__init__.py index 00ff6fe..7fcd49c 100644 --- a/src/features/feat_searchfs/__init__.py +++ b/src/features/feat_searchfs/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ -Submodule for the IFC-TO-{FILE} feature +Submodule for the SEARCH-FS feature """ # ---------------------------------------------------------------- @@ -16,6 +16,5 @@ # ---------------------------------------------------------------- __all__ = [ - "superfeature_direct", - "superfeature_queue", + "superfeature", ] diff --git a/src/features/feat_searchfs/feature.py b/src/features/feat_searchfs/feature.py index 7da09ff..18abcc7 100644 --- a/src/features/feat_searchfs/feature.py +++ b/src/features/feat_searchfs/feature.py @@ -5,13 +5,20 @@ # IMPORTS # ---------------------------------------------------------------- +import json import logging +from datetime import datetime +from datetime import timedelta +from functools import partial +from pika import BasicProperties +from pika import BlockingConnection from safetywrap import Err from safetywrap import Ok from safetywrap import Result from ..._core.logging import * +from ...algorithms.filesmanager import * from ...models.application import * from ...models.filesmanager import * from ...models.internal.errors import * @@ -38,17 +45,68 @@ def feature( *, label: str, - ref_inputs: FileRef, + ref: FileRef, options: RequestTaskOptions, ) -> Result[str, str]: """ Feature `SEARCH-FS` """ + feat = EnumFeatures.SEARCH_FS + # NOTE: currently unused + # cfg_general = config.parser_config().parse() + managers = config.get_managers() + + # create guard to safeguard against computational limits + guard = partial( + guard_limits, + max_depth=options.max_depth, + max_items=options.max_items, + max_duration=options.max_duration, + t_max=datetime.now() + options.max_duration, + ) + try: - managers = config.get_managers() - cfg_general = config.parser_config().parse() + """ + connect to message queue and perform task + """ + # FIXME: publication to exchages fails + # msg_exchange = feat.value + msg_exchange = "" + msg_route = label + msg_properties = BasicProperties(type="info") + + settings = config.get_queue_parameters() + with BlockingConnection(settings) as connection: + chan = connection.channel() + # FIXME: publication to exchages fails + # chan.exchange_declare(exchange=msg_exchange, exchange_type="direct") + chan.queue_declare(queue=msg_route) + + # locate directory in file system + root = ref.path + loc = ref.location + manager = managers[loc] - raise NotImplementedError("feature SEARCH-FS not yet implemented") + """ + run search algorithm and apply guards to prevent unlimited search duration + """ + for count, (d, subpath, filename) in enumerate( + # NOTE: algorithm returns a generator + recursive_file_search(manager, path=root), + # keep track of number of items found + start=1, + ): + # apply guard + guard(d=d, count=count) + # if not blocked by guard log to queue + body = {"path": subpath, "filename": filename} + contents = json.dumps(body).encode() + chan.basic_publish( + exchange=msg_exchange, + routing_key=msg_route, + body=contents, + properties=msg_properties, + ) return Ok("success") @@ -65,3 +123,34 @@ def feature( except BaseException as err: # DEV-NOTE: pass on all other kinds of exceptions raise err + + +# ---------------------------------------------------------------- +# AUXILIARY METHODS +# ---------------------------------------------------------------- + + +def guard_limits( + *, + d: int, + count: int, + max_depth: int, + max_items: int, + max_duration: timedelta, + t_max: datetime, +): + """ + Applies guard clauses to terminate search algorithm if limits are breached + """ + # terminate if search takes too long + if datetime.now() >= t_max: + raise TimeoutError(f"search algorithm terminated - exceeded maximum tolerated duration of {max_duration}") # fmt: skip + + # terminate if depth exceeds limits + if d > max_depth: + raise Exception(f"search algorithm terminated - directory depth exceeeded maximum tolerated depth of {max_depth}") # fmt: skip + + # terminate if number of items exceeds limits + count += 1 + if count > max_items: + raise Exception(f"search algorithm terminated - item count exceeededs maximum tolerated value of {max_items}") # fmt: skip diff --git a/src/features/feat_searchfs/superfeature.py b/src/features/feat_searchfs/superfeature.py index 9dbffc5..b55470b 100644 --- a/src/features/feat_searchfs/superfeature.py +++ b/src/features/feat_searchfs/superfeature.py @@ -37,10 +37,11 @@ def superfeature( """ errors = list[str]() n_tot = len(tasks) + for task in tasks: result = feature( label=task.label, - ref_inputs=task.data.inputs, + ref=task.data.inputs, options=task.options, ) diff --git a/src/models/application/__init__.py b/src/models/application/__init__.py index ec0afc1..1e0de29 100644 --- a/src/models/application/__init__.py +++ b/src/models/application/__init__.py @@ -38,9 +38,15 @@ def parse_tasks(payload: RequestsPayload, /) -> list[RequestTask]: """ Given a payload retuns the list of tasks it encodes """ + # parse either list or single item as list match payload.root: - case list() as tasks: - return tasks + case list() as x: + tasks = x case _ as task: - return [task] + tasks = [task] + + # filter out tasks to be ignored + tasks = [task for task in tasks if not task.ignore] + + return tasks diff --git a/src/models/filesmanager/os/classes.py b/src/models/filesmanager/os/classes.py index d8cde41..35b166e 100644 --- a/src/models/filesmanager/os/classes.py +++ b/src/models/filesmanager/os/classes.py @@ -15,7 +15,6 @@ from pydantic import AwareDatetime from ...._core.constants import * -from ...._core.utils.code import * from ...._core.utils.time import * from ...generated.application import MetaData @@ -244,7 +243,6 @@ def date_modified(self) -> AwareDatetime | None: t = datetime.fromtimestamp(meta.st_mtime) return add_timezone(t, tz=self._timezone) - @make_lazy def get_meta_data(self) -> MetaData: """ Gets bundled meta data associated to file. @@ -291,7 +289,6 @@ class OSFilesManagerFolder: _manager: OSFilesManager _path: str _timezone: timezone | None - _filenames: list[str] | None def __init__( self, @@ -305,7 +302,6 @@ def __init__( self._manager = manager self._path = path self._timezone = tz - self._filenames = None return @property @@ -326,44 +322,72 @@ def path(self) -> str: def name(self) -> str: return os.path.basename(self._path) - @property - def subfolders(self) -> list[OSFilesManagerFolder]: - tz = self._timezone + def get_file(self, name: str, /) -> OSFilesManagerFile: + """ + Gets file object by name within folder + """ + path = Path(self._path, name).as_posix() + file = self._manager.get_file(path) + return file + + def get_filenames(self) -> list[str]: + """ + Get all filenames in folder + """ names = os.listdir(self._path) - paths = [Path(self._path, name).as_posix() for name in names] - paths = [path for path in paths if Path(path).is_dir()] - return [OSFilesManagerFolder(self._manager, path=path, tz=tz) for path in paths] + paths = [Path(self.path, name).as_posix() for name in names] + filenames = [name for name, path in zip(names, paths) if Path(path).is_file()] + return filenames - def get_subfolder(self, name: str) -> OSFilesManagerFolder: - return self._manager.get_folder(Path(self._path, name).as_posix()) + def get_files(self) -> list[OSFilesManagerFile]: + """ + Gets all file objects in folder + """ + filenames = self.get_filenames() + files = [self.get_file(filename) for filename in filenames] + return files - @property - def files(self) -> list[OSFilesManagerFile]: + def get_subfolder(self, name: str, /) -> OSFilesManagerFolder: + """ + Gets subfolder object by name within folder + """ tz = self._timezone - names = os.listdir(self._path) - paths = [Path(self._path, name).as_posix() for name in names] - paths = [path for path in paths if Path(path).is_file()] - return [OSFilesManagerFile(path=path, tz=tz) for path in paths] + path = Path(self._path, name).as_posix() + folder = OSFilesManagerFolder(self._manager, path=path, tz=tz) + return folder - @property - def filenames(self) -> list[str]: + def get_subfolder_paths(self) -> list[str]: + """ + Gets all paths to subfolders within folder + """ names = os.listdir(self._path) - paths = [Path(self.path, name).as_posix() for name in names] - self._filenames = [name for name, path in zip(names, paths) if Path(path).is_file()] - return self._filenames + paths = [Path(self._path, name).as_posix() for name in names] + paths = [path for path in paths if Path(path).is_dir()] + return paths - def has_file(self, file: OSFilesManagerFile) -> bool: - return file.filename in self.filenames + def get_subfolders(self) -> list[str]: + """ + Gets all subfolder objects within folder + """ + tz = self._timezone + get_folder = lambda path: OSFilesManagerFolder(self._manager, path=path, tz=tz) + paths = self.get_subfolder_paths() + folders = list(map(get_folder, paths)) + return folders - def get_file(self, name: str) -> OSFilesManagerFile: - return self._manager.get_file(Path(self._path, name).as_posix()) + def has_file(self, file: OSFilesManagerFile, /) -> bool: + """ + Checks if file of given name exists in folder + """ + filenames = self.get_filenames() + return file.filename in filenames - @make_lazy def get_files_meta_data(self) -> list[MetaData]: """ Gets a list of metadata associated to files """ - return [file.get_meta_data() for file in self.files] + files = self.get_files() + return [file.get_meta_data() for file in files] def write_bytes( self, @@ -398,9 +422,9 @@ def clear_folder(self) -> bool: Removes all contents of current folder """ success = True - for file in self.files: + for file in self.get_files(): success = success and file.delete_self() - for subfolder in self.subfolders: + for subfolder in self.get_subfolders(): success = success and subfolder.delete_self() return success diff --git a/src/models/filesmanager/traits.py b/src/models/filesmanager/traits.py index 8ccf0c3..71e274a 100644 --- a/src/models/filesmanager/traits.py +++ b/src/models/filesmanager/traits.py @@ -240,42 +240,45 @@ def name(self) -> str: """ ... - @property - def subfolders(self) -> list[FilesManagerFolder]: + def get_file(self, name: str, /) -> FilesManagerFile: """ - Gets list of subfolder within folder + Gets file object by name within folder """ ... - def get_subfolder(self, name: str) -> FilesManagerFolder: + def get_filenames(self) -> list[str]: """ - Gets subfolder by name within folder + Get all filenames in folder """ ... - @property - def files(self) -> list[FilesManagerFile]: + def get_files(self) -> list[FilesManagerFile]: """ - Gets list of files within folder + Gets all file objects in folder """ ... - @property - def filenames(self) -> list[str]: + def get_subfolder(self, name: str, /) -> FilesManagerFolder: + """ + Gets subfolder object by name within folder + """ + ... + + def get_subfolder_paths(self) -> list[str]: """ - Returns names of all files in folder. + Gets all paths to subfolders within folder """ ... - def has_file(self, file: FilesManagerFile) -> bool: + def get_subfolders(self) -> list[str]: """ - Checks if file of given name exists in folder. + Gets all subfolder objects within folder """ ... - def get_file(self, name: str) -> FilesManagerFile: + def has_file(self, file: FilesManagerFile, /) -> bool: """ - Gets file by name within folder + Checks if file of given name exists in folder """ ... diff --git a/src/models/generated/application.py b/src/models/generated/application.py index 94026b1..5259e73 100644 --- a/src/models/generated/application.py +++ b/src/models/generated/application.py @@ -3,6 +3,7 @@ from __future__ import annotations +from datetime import timedelta from enum import Enum from typing import Any @@ -59,6 +60,19 @@ class RequestTaskOptions(BaseModel): extra="allow", populate_by_name=True, ) + max_depth: int = Field( + default=50, alias="max-depth", description="Limits the search depth" + ) + max_items: int = Field( + default=1000000, + alias="max-items", + description="Limits the amount of items that can be found", + ) + max_duration: timedelta = Field( + ..., + alias="max-duration", + description="Limits the amount of time spent for a search", + ) class MetaData(BaseModel): @@ -171,6 +185,7 @@ class RequestTask(BaseModel): populate_by_name=True, ) label: str = Field(..., description="Label of task") + ignore: bool = False options: RequestTaskOptions data: RequestTaskData diff --git a/src/setup/config.py b/src/setup/config.py index e0194a8..7701e23 100644 --- a/src/setup/config.py +++ b/src/setup/config.py @@ -10,6 +10,8 @@ from pathlib import Path import toml +from pika import ConnectionParameters +from pika import PlainCredentials from pydantic import SecretStr from ..__paths__ import * @@ -116,6 +118,22 @@ def get_managers() -> dict[EnumFilesSystem, FilesManager]: } +@compute_once +def get_queue_parameters() -> ConnectionParameters: + """ + Returns connection parameters for queue + """ + # use guest credentials + user = http_user_rabbit_guest() + pw = http_password_rabbit_guest().get_secret_value() + creds = PlainCredentials(username=user, password=pw) + # apply to queue + host = http_ip() + port = http_port_rabbit_queue() + settings = ConnectionParameters(host=host, port=port, credentials=creds) # fmt: skip + return settings + + # ---------------------------------------------------------------- # LAZY LOADED RESOURCES / PROPERTIES # ---------------------------------------------------------------- diff --git a/templates/template-requests-multiple.yaml b/templates/template-requests-multiple.yaml index a7e4ac8..0c1480d 100644 --- a/templates/template-requests-multiple.yaml +++ b/templates/template-requests-multiple.yaml @@ -1,19 +1,50 @@ +# -------------------------------- +# TASK 1 +# -------------------------------- + - label: 'First task' - options: {} + options: &ref_options + max-depth: 100 + max-items: 10_000_000 + max-duration: 00:30:00 data: inputs: location: OS path: 'path/to/directory1' +# -------------------------------- +# TASK 2 +# -------------------------------- + - label: 'Second task' - options: {} + options: + <<: *ref_options data: inputs: location: OS path: 'path/to/directory2' -- label: 'Third task' - options: {} +# -------------------------------- +# TASK 3 - use ignore: true to skip +# -------------------------------- + +- label: 'Bad third task' + ignore: true + options: + <<: *ref_options + data: + inputs: + location: OS + path: 'path/to/bad directory3' + +# -------------------------------- +# TASK 4 +# -------------------------------- + +- label: 'Fourth task' + options: + <<: *ref_options + data: inputs: location: OS diff --git a/templates/template-requests.yaml b/templates/template-requests.yaml index 31243e3..4f8593b 100644 --- a/templates/template-requests.yaml +++ b/templates/template-requests.yaml @@ -2,7 +2,10 @@ label: 'Some label' # NOTE: not yet implemented -options: {} +options: + max-depth: 100 + max-items: 10_000_000 + max-duration: 00:30:00 # The main request data: