Feat-SearchFS ---> Staging

README.md

@@ -162,3 +162,75 @@ Password: ${HTTP_GUEST_PASSWORD_RABBIT}
 1. Start the queue (see [above](#activationdeactivation-of-queue)).
 
 2. Use the CLI commands or the API with/without docker (see [above](#usage-of-main-application)).
+
+## Testing ##
+
+### Mock data ###
+
+To generate mock data one may call for examples
+
+```bash
+rm -rf "data/example"
+just create-mocks \
+    --path "data/example" \
+    --max-depth 10 \
+    --max-folders 100 \
+    --max-files 1000
+```
+
+which creates a fresh relative directory `data/examples`,
+consisting of at most roughly 1000 files and 100 folders,
+and not exceeding a depth of 10.
+
+### Request ###
+
+Fill in `setup/requests.yaml` as follows:
+
+```yaml
+label: 'Mock example'
+
+# apply some generous limits
+options:
+  max-depth: 100
+  max-count: 10_000_000
+  max-time: 00:05:00
+
+data:
+  # the locaiton of the mock directory
+  inputs:
+    location: OS
+    path: 'data/example'
+```
+
+### Execution ###
+
+1. Start the queue (see [above](#activationdeactivation-of-queue)).
+
+2. Ensure that the queue-users are registered (see [above](#set-up-users)).
+
+3. Use the CLI commands or the API with/without docker (see [above](#usage-of-main-application)).
+
+4. Run the feature:
+
+    - For the CLI option, call
+
+        ```bash
+        just run-cli SEARCH-FS
+        ```
+
+    - For the FastApi options (with or without docker),
+        make a POST-call (e.g. in [Postman](https://www.postman.com))
+        against the endpoint `/feature/search-fs`
+        using the JSON-body
+
+        ```json
+        {
+            "ref": {
+                "location": "OS",
+                "path": "setup/requests.yaml"
+            }
+        }
+        ```
+
+        The file reference in this body can of course be a json
+        and located anywhere on your system.

docker-compose.yaml

@@ -186,7 +186,7 @@ services:
         echo "success"
       '
 
-  queue-server:
+  queue-server: &ref_queue-server
     image: local/examplerabbitmq:queue # <- i.e. use this image
     hostname: ${HTTP_HOST_NAME_RABBIT}
 
@@ -198,8 +198,12 @@ services:
       - credentials
 
     volumes:
-      - ${PATH_LOGS_QUEUE}://var/log/rabbitmq:rw
-      - ${PATH_LOGS_QUEUE_STATE}://var/lib/rabbitmq:rw
+      - type: bind
+        source: ${PATH_LOGS_QUEUE}
+        target: //var/log/rabbitmq
+      - type: bind
+        source: ${PATH_LOGS_QUEUE_STATE}
+        target: //var/lib/rabbitmq
 
     ports:
       # HOST-IP:HOST-PORT:CONTAINER (note: uses values in .env)
@@ -213,34 +217,26 @@ services:
       rabbitmq-server
 
   queue-admin:
-    image: local/examplerabbitmq:queue # <- i.e. use this image
-
-    env_file:
-      - .env
-      - .env.docker-vars
-
-    secrets:
-      - credentials
-
-    volumes:
-      - ${PATH_LOGS_QUEUE}://var/log/rabbitmq:rw
-      - ${PATH_LOGS_QUEUE_STATE}://var/lib/rabbitmq:rw
-
+    <<: *ref_queue-server
+    hostname: "admin"
+    depends_on:
+      - queue-server
+    ports: []
     networks:
       - default
-
     tty: true
     stdin_open: true
-    depends_on:
-      - queue-server
     command: |-
       bash --login -c '
         source /run/secrets/credentials
         rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} list_users
+
         # create admin
         rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} delete_user $${HTTP_ADMIN_USER_RABBIT} 2> /dev/null
         rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} add_user $${HTTP_ADMIN_USER_RABBIT} $${HTTP_ADMIN_PASSWORD_RABBIT}
+        rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} set_permissions -p / $${HTTP_ADMIN_USER_RABBIT} ".*" ".*" ".*"
         rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} set_user_tags $${HTTP_ADMIN_USER_RABBIT} administrator
+
         # create guest
         rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} delete_user $${HTTP_GUEST_USER_RABBIT} 2> /dev/null
         rabbitmqctl --node rabbit@${HTTP_HOST_NAME_RABBIT} add_user $${HTTP_GUEST_USER_RABBIT} $${HTTP_GUEST_PASSWORD_RABBIT}

docs/models/application/Models/RequestTask.md

@@ -4,7 +4,8 @@
 | Name | Type | Description | Notes |
 |------------ | ------------- | ------------- | -------------|
 | **label** | **String** | Label of task | [default to null] |
-| **options** | [**Map**](AnyType.md) | Structure of requests payload > options  NOTE: not yet implemented | [default to null] |
+| **ignore** | **Boolean** |  | [optional] [default to false] |
+| **options** | [**RequestTaskOptions**](RequestTaskOptions.md) |  | [default to null] |
 | **data** | [**RequestTaskData**](RequestTaskData.md) |  | [default to null] |
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

docs/models/application/Models/RequestTaskOptions.md

@@ -0,0 +1,11 @@
+# RequestTaskOptions
+## Properties
+
+| Name | Type | Description | Notes |
+|------------ | ------------- | ------------- | -------------|
+| **max-depth** | **Integer** | Limits the search depth | [optional] [default to 50] |
+| **max-items** | **Integer** | Limits the amount of items that can be found | [optional] [default to 1000000] |
+| **max-duration** | **String** | Limits the amount of time spent for a search | [default to null] |
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+

docs/models/application/Models/RequestsPayload.md

@@ -4,7 +4,8 @@
 | Name | Type | Description | Notes |
 |------------ | ------------- | ------------- | -------------|
 | **label** | **String** | Label of task | [default to null] |
-| **options** | [**Map**](AnyType.md) | Structure of requests payload > options  NOTE: not yet implemented | [default to null] |
+| **ignore** | **Boolean** |  | [optional] [default to false] |
+| **options** | [**RequestTaskOptions**](RequestTaskOptions.md) |  | [default to null] |
 | **data** | [**RequestTaskData**](RequestTaskData.md) |  | [default to null] |
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

docs/models/application/README.md

@@ -23,6 +23,7 @@ All URIs are relative to *https://acme.org*
  - [RepoInfo_urls](./Models/RepoInfo_urls.md)
  - [RequestTask](./Models/RequestTask.md)
  - [RequestTaskData](./Models/RequestTaskData.md)
+ - [RequestTaskOptions](./Models/RequestTaskOptions.md)
  - [RequestsPayload](./Models/RequestsPayload.md)
 
 

justfile

@@ -294,8 +294,8 @@ check-time-matches-cron cron_expr time="*":
     @{{PYVENV_ON}} && {{PYVENV}} -m scripts.cron "{{cron_expr}}" --time "{{time}}"
 
 # Recipe only works if local file scripts/mocks.py exists
-mocks *args:
-    @{{PYVENV_ON}} && {{PYVENV}} -m scripts.mocks
+create-mocks *args:
+    @{{PYVENV_ON}} && {{PYVENV}} -m scripts.mocks {{args}}
 
 # --------------------------------
 # TARGETS: terminate execution

models/schema-application.yaml

@@ -133,6 +133,9 @@ components:
           description: |-
             Label of task
           type: string
+        ignore:
+          type: boolean
+          default: false
         options:
           $ref: "#/components/schemas/RequestTaskOptions"
         data:
@@ -144,7 +147,25 @@ components:
 
         NOTE: not yet implemented
       type: object
+      required:
+        - max-duration
       additionalProperties: true
+      properties:
+        max-depth:
+          description: |-
+            Limits the search depth
+          type: integer
+          default: 50
+        max-items:
+          description: |-
+            Limits the amount of items that can be found
+          type: integer
+          default: 1_000_000
+        max-duration:
+          description: |-
+            Limits the amount of time spent for a search
+          type: string
+          format: duration
 
     RequestTaskData:
       description: |-

scripts/mocks.py

@@ -2,25 +2,165 @@
 # -*- coding: utf-8 -*-
 
 """
-Script to check if current time matches a given CRON expression
+Script to generate a local mock directory
 """
 
 # ----------------------------------------------------------------
 # IMPORTS
 # ----------------------------------------------------------------
 
+import os
 import sys
+from pathlib import Path
+
+os.chdir(Path(__file__).parent.parent)
+sys.path.insert(0, os.getcwd())
+
+import logging
+import math
+import random
+from argparse import ArgumentParser
+from argparse import RawTextHelpFormatter
+
+from lorem_text import lorem
+from mimesis import Generic
+
+from src._core.constants import *
+from src._core.logging import *
+
+# ----------------------------------------------------------------
+# SETTINGS
+# ----------------------------------------------------------------
+
+gen = Generic()
 
 # ----------------------------------------------------------------
 # METHODS
 # ----------------------------------------------------------------
 
-#
+
+def parse_args(*args: str):
+    parser = ArgumentParser(
+        prog="mocks",
+        description="creates a local mock directory for the feature SEARCH-FS",
+        formatter_class=RawTextHelpFormatter,
+    )
+    parser.add_argument(
+        "--path",
+        type=str,
+        help="absolute or relative path to directory to be created",
+        # nargs="?",
+        # default="data/example",
+    )
+    parser.add_argument(
+        "--max-depth",
+        type=int,
+        help="maximum depth of folders",
+        # nargs="?",
+        # default=4,
+    )
+    parser.add_argument(
+        "--max-folders",
+        type=int,
+        help="maximum count files count",
+        # nargs="?",
+        # default=100,
+    )
+    parser.add_argument(
+        "--max-files",
+        type=int,
+        help="maximum count files count",
+        # nargs="?",
+        # default=1000,
+    )
+    logging.info(args)
+    args_parsed = parser.parse_args(args)
+    return args_parsed
+
+
+def create_folder(
+    path: str,
+    /,
+    *,
+    depth: int,
+    k_folders: int,
+    k_files: int,
+):
+    """
+    Recursive depth-first method to create folders and files
+    """
+    # ensure folder exists
+    logging.info(f"- create folder '{path}'")
+    p = Path(path)
+    p.mkdir(parents=True, exist_ok=True)
+
+    # generate random file names
+    k = random.randint(1, k_files)
+    basenames = [gen.food.dish() for _ in range(k)]
+    extension = random.choices([".txt", ".csv", ".md"], k=k)
+    filenames = [f"{x}{ext}" for x, ext in zip(basenames, extension)]
+
+    # create files and contents
+    for filename in filenames:
+        path_ = os.path.join(path, filename)
+        content = lorem.paragraph().encode()
+        logging.info(f"- create file '{filename}' with {len(content)/SIZE_1_KB:.4f} kb of data")  # fmt: skip
+
+        p = Path(path_)
+        p.touch(exist_ok=True)
+        with open(path_, "wb") as fp:
+            fp.write(content)
+
+    # if depth remaining is 0 stop
+    if depth == 0:
+        return
+
+    # otherwise proceed
+    k = random.randint(1, k_folders)
+    foldernames = [gen.address.city() for _ in range(k)]
+
+    for foldername in foldernames:
+        path_ = os.path.join(path, foldername)
+        create_folder(
+            path_,
+            depth=depth - 1,
+            k_folders=k_folders,
+            k_files=k_files,
+        )
+
+    return
+
 
 # ----------------------------------------------------------------
 # EXECUTION
 # ----------------------------------------------------------------
 
 if __name__ == "__main__":
     sys.tracebacklimit = 0
-    raise NotImplementedError("generation of mocks not yet implemented")
+
+    # initialise logging
+    configure_logging(name="root", level="INFO", path=None, serialise=False)
+
+    # parse the args
+    args = parse_args(*sys.argv[1:])
+
+    path = args.path
+    depth = args.max_depth
+    count_files = args.max_files
+    count_folder = args.max_folders
+
+    # derive how many files/subfolders per folder using heuristics
+    k_folders = 0
+    if depth > 1:
+        k_folders = math.ceil(math.pow(count_folder, 1 / depth))
+
+    k_files = math.ceil(count_files / count_folder)
+    k_files = max(1, k_files)
+
+    # call the recursive method
+    create_folder(
+        path,
+        depth=depth,
+        k_folders=k_folders,
+        k_files=k_files,
+    )