diff --git a/.dockerignore b/.dockerignore index 780e06a..7497d22 100644 --- a/.dockerignore +++ b/.dockerignore @@ -34,8 +34,6 @@ !/src !/src/**/ !/src/**/*.py -# the app-config (used internally for code only) -!/src/setup/*.yaml !/tests !/tests/**/ diff --git a/.gitignore b/.gitignore index a68ea92..580b664 100644 --- a/.gitignore +++ b/.gitignore @@ -50,8 +50,6 @@ !/src !/src/**/ !/src/**/*.py -# the app-config (used internally for code only) -!/src/setup/*.yaml !/tests !/tests/**/ diff --git a/README.md b/README.md index eeb2d32..836669e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ # Example Rabbit MQ # -This repository provides an example implementation of a tool with a single feature `SEARCH-FS`, +This repository provides an example implementation of a tool with a single feature `SEARCH-FILESYSTEM`, which upon performs the following: - given a request payload; @@ -61,8 +61,6 @@ The main code base can be run in three modes: - via the API - via the API within docker -The cli-usage is limited - ### Docker-free usage ### For all sakes and purposes, so that local docker-less execution is possible, @@ -73,6 +71,21 @@ just build ``` which assumes that the .env file has been correctly set up. +One can then either call + +```bash +just run-server +``` + +to start the server +(which can be interacted with via Postman and/or cURL commands) +or else use the CLI: + +```bash +just run-cli --help # displays usage +just run-cli version # displays version +just run-cli SEARCH-FS # runs the main feature +``` ### Usage with docker ### @@ -84,7 +97,13 @@ just docker-build # builds the application just docker-qa # performs qa on the docker image of the main code base ``` -To start the ap +to build the application (once), +then use the following commands to start/stop the server within docker: + +```bash +just docker-start-server +just docker-stop-server +``` ## Usage of Rabbit Message Queue ## @@ -140,5 +159,6 @@ Password: ${HTTP_GUEST_PASSWORD_RABBIT} ## Execution ## -Start the queue (see [above](#activationdeactivation-of-queue)). -Start the server +1. Start the queue (see [above](#activationdeactivation-of-queue)). + +2. Use the CLI commands or the API with/without docker (see [above](#usage-of-main-application)). diff --git a/dist/VERSION b/dist/VERSION index 77d6f4c..6e8bf73 100644 --- a/dist/VERSION +++ b/dist/VERSION @@ -1 +1 @@ -0.0.0 +0.1.0 diff --git a/docker-compose.yaml b/docker-compose.yaml index bd10ddc..128c82b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -87,6 +87,7 @@ services: user: basicuser command: |- bash --login -c ' + source /run/secrets/credentials just prettify || exit 1 just tests-unit || exit 1 ' @@ -115,7 +116,9 @@ services: # # for debugging # volumes: - # - ${PATH_LOGS}://home/basicuser/app/logs:rw + # - type: bind + # source: ${PATH_LOGS} + # target: //home/basicuser/app/logs # - ./setup://home/basicuser/app/setup:ro # - ./data://home/basicuser/app/data:rw # - ./scripts://home/basicuser/app/scripts:ro @@ -143,14 +146,18 @@ services: # the healthcheck test: |- bash --login -c ' - curl -f "${HTTP_IP}:${HTTP_PORT}/api/ping" || exit 1 + curl -f "${HTTP_IP}:${HTTP_PORT}/ping" || exit 1 ' - user: root + user: basicuser # restart: unless-stopped restart: no # entrypoint: "//home/basicuser/app/scripts/entrypoint.sh" - command: just serve-fastapi + command: |- + bash --login -c ' + source /run/secrets/credentials + just run-server + ' # -------------------------------- # SERVICE: queue diff --git a/docs/models/application/Models/EnumDataFileFormat.md b/docs/models/application/Models/EnumDataFileFormat.md new file mode 100644 index 0000000..3fde64a --- /dev/null +++ b/docs/models/application/Models/EnumDataFileFormat.md @@ -0,0 +1,8 @@ +# EnumDataFileFormat +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/models/application/Models/EnumFeatures.md b/docs/models/application/Models/EnumFeatures.md new file mode 100644 index 0000000..78dd4e0 --- /dev/null +++ b/docs/models/application/Models/EnumFeatures.md @@ -0,0 +1,8 @@ +# EnumFeatures +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/models/application/Models/FileRef.md b/docs/models/application/Models/FileRef.md new file mode 100644 index 0000000..216fe03 --- /dev/null +++ b/docs/models/application/Models/FileRef.md @@ -0,0 +1,11 @@ +# FileRef +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **location** | [**EnumFilesSystem**](EnumFilesSystem.md) | | [optional] [default to null] | +| **path** | **String** | Absolute path to file. | [optional] [default to .] | +| **format** | [**EnumDataFileFormat**](EnumDataFileFormat.md) | | [optional] [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/models/application/Models/GeneralConfig.md b/docs/models/application/Models/GeneralConfig.md new file mode 100644 index 0000000..3971dac --- /dev/null +++ b/docs/models/application/Models/GeneralConfig.md @@ -0,0 +1,9 @@ +# GeneralConfig +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **version** | **String** | User defined version. Bump this value with every change to the config. | [optional] [default to X.Y.Z] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/models/application/Models/MetaData.md b/docs/models/application/Models/MetaData.md new file mode 100644 index 0000000..193809a --- /dev/null +++ b/docs/models/application/Models/MetaData.md @@ -0,0 +1,16 @@ +# MetaData +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **filename** | **String** | Filename (without path, but with extension) | [default to null] | +| **basename** | **String** | Filename without path and without extension | [default to null] | +| **ext** | **String** | Extension of file | [default to null] | +| **size** | **Integer** | Size of file in bytes | [default to null] | +| **author** | **String** | Author of file | [optional] [default to null] | +| **author\_id** | **String** | Id of author of file | [optional] [default to null] | +| **time-created** | [**datetime**](datetime.md) | | [optional] [default to null] | +| **time-updated** | [**datetime**](datetime.md) | | [optional] [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/models/application/Models/ProxyConfig.md b/docs/models/application/Models/ProxyConfig.md new file mode 100644 index 0000000..b4d5e03 --- /dev/null +++ b/docs/models/application/Models/ProxyConfig.md @@ -0,0 +1,9 @@ +# ProxyConfig +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **ref** | [**FileRef**](FileRef.md) | | [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/models/application/Models/RequestTask.md b/docs/models/application/Models/RequestTask.md new file mode 100644 index 0000000..5518f09 --- /dev/null +++ b/docs/models/application/Models/RequestTask.md @@ -0,0 +1,11 @@ +# RequestTask +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **label** | **String** | Label of task | [default to null] | +| **options** | [**Map**](AnyType.md) | Structure of requests payload > options NOTE: not yet implemented | [default to null] | +| **data** | [**RequestTaskData**](RequestTaskData.md) | | [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/models/application/Models/RequestTaskData.md b/docs/models/application/Models/RequestTaskData.md new file mode 100644 index 0000000..4affb3a --- /dev/null +++ b/docs/models/application/Models/RequestTaskData.md @@ -0,0 +1,9 @@ +# RequestTaskData +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **inputs** | [**FileRef**](FileRef.md) | | [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/models/application/Models/RequestsPayload.md b/docs/models/application/Models/RequestsPayload.md new file mode 100644 index 0000000..f04976a --- /dev/null +++ b/docs/models/application/Models/RequestsPayload.md @@ -0,0 +1,11 @@ +# RequestsPayload +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **label** | **String** | Label of task | [default to null] | +| **options** | [**Map**](AnyType.md) | Structure of requests payload > options NOTE: not yet implemented | [default to null] | +| **data** | [**RequestTaskData**](RequestTaskData.md) | | [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/models/application/README.md b/docs/models/application/README.md index 241eea0..8b0298d 100644 --- a/docs/models/application/README.md +++ b/docs/models/application/README.md @@ -1,4 +1,4 @@ -# Documentation for Schemata for config files. +# Documentation for General models for application ## Documentation for API Endpoints @@ -12,9 +12,18 @@ All URIs are relative to *https://acme.org* ## Documentation for Models + - [EnumDataFileFormat](./Models/EnumDataFileFormat.md) + - [EnumFeatures](./Models/EnumFeatures.md) - [EnumFilesSystem](./Models/EnumFilesSystem.md) + - [FileRef](./Models/FileRef.md) + - [GeneralConfig](./Models/GeneralConfig.md) + - [MetaData](./Models/MetaData.md) + - [ProxyConfig](./Models/ProxyConfig.md) - [RepoInfo](./Models/RepoInfo.md) - [RepoInfo_urls](./Models/RepoInfo_urls.md) + - [RequestTask](./Models/RequestTask.md) + - [RequestTaskData](./Models/RequestTaskData.md) + - [RequestsPayload](./Models/RequestsPayload.md) diff --git a/justfile b/justfile index ab2863c..6ef1341 100644 --- a/justfile +++ b/justfile @@ -434,9 +434,9 @@ clear-logs log_path="${PATH_LOGS}": @rm -rf "{{log_path}}" 2> /dev/null create-logs log_path="${PATH_LOGS}": - @just create-logs-part "debug" "{{log_path}}" - @just create-logs-part "out" "{{log_path}}" - @just create-logs-part "err" "{{log_path}}" + @just _create-logs-part "debug" "{{log_path}}" + @just _create-logs-part "out" "{{log_path}}" + @just _create-logs-part "err" "{{log_path}}" _create-logs-part part log_path="${PATH_LOGS}": @mkdir -p "{{log_path}}" diff --git a/models/schema-application.yaml b/models/schema-application.yaml index 6da5f71..7797243 100644 --- a/models/schema-application.yaml +++ b/models/schema-application.yaml @@ -1,7 +1,10 @@ openapi: 3.0.3 info: - version: "0.0.0" - title: Schemata for config files. + version: "0.1.0" + title: General models for application + description: |- + Schemata for data models in application - configs, enums, etc. + servers: - url: "https://acme.org" paths: {} @@ -44,10 +47,192 @@ components: type: string format: uri + # -------------------------------- + # Proxy config + # -------------------------------- + + ProxyConfig: + description: |- + A proxy config which simply links to another config file. + type: object + required: + - ref + # since this can be set externally, allow superfluous properties (which will be ignored) + additionalProperties: true + properties: + ref: + $ref: "#/components/schemas/FileRef" + + FileRef: + description: |- + Structured reference to a file + type: object + required: [] + additionalProperties: false + properties: + location: + description: |- + Which files management system is used to locate the file + $ref: "#/components/schemas/EnumFilesSystem" + default: "OS" + path: + description: |- + Absolute path to file. + type: string + default: "." + format: + description: |- + Optional format of file to be loaded (e.g. `".json"` or `".yaml"`). + $ref: "#/components/schemas/EnumDataFileFormat" + + # -------------------------------- + # General Config of Application + # -------------------------------- + + GeneralConfig: + description: |- + Structure of configuration of application for use with features. + + NOTE: not yet implemented + + type: object + required: [] + # since this can be set externally, allow superfluous properties (which will be ignored) + additionalProperties: true + properties: + version: + description: |- + User defined version. Bump this value with every change to the config. + type: string + default: "X.Y.Z" + + # -------------------------------- + # User Request + # -------------------------------- + + RequestsPayload: + description: |- + Structure of requests payload + oneOf: + - $ref: "#/components/schemas/RequestTask" + - type: array + items: + $ref: "#/components/schemas/RequestTask" + + RequestTask: + description: |- + Structure of requests payload + type: object + required: + - label + - options + - data + additionalProperties: false + properties: + label: + description: |- + Label of task + type: string + options: + $ref: "#/components/schemas/RequestTaskOptions" + data: + $ref: "#/components/schemas/RequestTaskData" + + RequestTaskOptions: + description: |- + Structure of requests payload > options + + NOTE: not yet implemented + type: object + additionalProperties: true + + RequestTaskData: + description: |- + Structure of requests payload > data + required: + - inputs + additionalProperties: false + properties: + inputs: + $ref: "#/components/schemas/FileRef" + + + # -------------------------------- + # User Request + # -------------------------------- + + MetaData: + description: |- + Struct containing information about an object in a filesystem + required: + - filename + - basename + - ext + - size + additionalProperties: false + properties: + filename: + description: |- + Filename (without path, but with extension) + type: string + basename: + description: |- + Filename without path and without extension + type: string + ext: + description: |- + Extension of file + type: string + size: + description: |- + Size of file in bytes + type: int + author: + description: |- + Author of file + type: string + author_id: + description: |- + Id of author of file + type: string + time-created: + type: datetime + time-updated: + type: datetime + # ---------------------------------------------------------------- # ENUMS # ---------------------------------------------------------------- + # -------------------------------- + # ENUM: file formats + # -------------------------------- + + EnumFeatures: + description: |- + Enumeration of features + type: string + enum: + - version + - SEARCH-FS + + # -------------------------------- + # ENUM: file formats + # -------------------------------- + + EnumDataFileFormat: + description: |- + Enumeration of data file formats. + type: string + enum: + - .json + - .yaml + - .toml + - .xml + - .parquet + - .csv + - .xlsx + # -------------------------------- # ENUM: for file system # -------------------------------- @@ -58,5 +243,5 @@ components: type: string enum: - OS - - BLOB + - BLOB-STORAGE - SHAREPOINT diff --git a/pyproject.toml b/pyproject.toml index e4e0ec3..41b7190 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "example-rabbit-mq" -version = "0.0.0" +version = "0.1.0" description = 'Example tool to recursively search a file system from a given point and message rabbit mq' authors = [ {name="raj-open", email="raj-open@users.noreply.github.com"}, @@ -89,6 +89,7 @@ dependencies = [ "datetime>=5.5", "python-dateutil>=2.9.0.post0", "tzdata>=2025.2", # NOTE: necessary to ensure that OS has access to timezones + "tzlocal>=5.3.1", "pytz>=2025.2", "lorem-text>=3.0", "flatDict>=4.0.1", diff --git a/src/_core/__init__.py b/src/_core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/_core/constants.py b/src/_core/constants.py new file mode 100644 index 0000000..e570f3b --- /dev/null +++ b/src/_core/constants.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from enum import StrEnum +from typing import Literal + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "BASIC_FILETYPES", + "ENCODING", + "MAP_MIME_TYPE_TO_FILETYPE", + "MIME_TYPES", + "SIZE_1_KB", + "SIZE_1_MB", + "EnumMimeTypes", +] + +# ---------------------------------------------------------------- +# CONSTANTS +# ---------------------------------------------------------------- + +SIZE_1_KB = 2**10 +SIZE_1_MB = 2**20 + +ENCODING = Literal[ + "ascii", + "utf-8", + "utf-8-sig", + "unicode_escape", +] + +BASIC_FILETYPES = Literal[ + ".json", + ".yaml", + ".toml", + ".xml", + ".parquet", + ".csv", + ".xlsx", +] + + +class EnumMimeTypes(StrEnum): + BYTES = "application/octet-stream" + TEXT = "text/plain" + JSON = "application/json" + # see https://learn.microsoft.com/previous-versions/office/office-2007-resource-kit/ee309278(v=office.12) + XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + YAML = "application/x-yaml" + + +MIME_TYPES = Literal[ + "application/octet-stream", + "text/plain", + "application/json", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/x-yaml", +] + + +MAP_MIME_TYPE_TO_FILETYPE: dict[MIME_TYPES, BASIC_FILETYPES] = { + "application/x-yaml": ".yaml", + "application/json": ".json", +} diff --git a/src/_core/logging/__init__.py b/src/_core/logging/__init__.py new file mode 100644 index 0000000..d541ff1 --- /dev/null +++ b/src/_core/logging/__init__.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from .basic import * +from .constants import * +from .decorators import * +from .errors import * +from .special import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "LOG_LEVELS", + "configure_logging", + "echo_async_function", + "echo_function", + "echo_generator", + "error_with_trace", + "error_with_trace_multiline", + "log", + "log_console", + "log_debug_wrapped", + "log_debug_wrapped_args", + "log_dev", +] diff --git a/src/_core/logging/basic.py b/src/_core/logging/basic.py new file mode 100644 index 0000000..e73df5b --- /dev/null +++ b/src/_core/logging/basic.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from __future__ import annotations + +import json +import logging +import os +from logging import CRITICAL +from logging import DEBUG +from logging import ERROR +from logging import INFO +from logging import WARNING +from logging import FileHandler +from logging import Formatter +from logging import LogRecord +from logging import Logger +from logging import getLogger +from pathlib import Path +from typing import Any + +from strip_ansi import strip_ansi + +from .constants import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "configure_logging", +] + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def configure_logging( + *, + level: LOG_LEVELS | int = INFO, + name: str | None = None, + path: str | None = None, + format_date: str = r"%Y-%m-%d %H:%M:%S", + serialise: bool = True, +) -> Logger: + """ + Establishes logging for console and files. + """ + name = name or "root" + fmt_console = "%(asctime)s $\x1b[92;1m%(name)s\x1b[0m [\x1b[1m%(levelname)s\x1b[0m] %(message)s" # fmt: skip + logging.basicConfig(format=fmt_console, datefmt=format_date, encoding="utf-8") + logger = getLogger(name=name) + if isinstance(level, str): + level = logging.getLevelNamesMapping().get(level, INFO) + logger.setLevel(level) + + if not isinstance(path, str): + return + + if serialise: + fmt = JsonFormatter(r"%(message)s") + + else: + fmt = Formatter(fmt=strip_ansi(fmt_console), datefmt=format_date) + + for path_file, level in [ + (f"{path}/out.log", INFO), + (f"{path}/out.log", WARNING), + (f"{path}/err.log", ERROR), + (f"{path}/err.log", CRITICAL), + (f"{path}/debug.log", DEBUG), + ]: + create_file_if_not_exists(path_file) + handler = FileHandler(path_file, encoding="utf-8") + handler.setFormatter(fmt) + handler.setLevel(level) + handler.addFilter(LoggingLevelFilter(level)) + logger.addHandler(handler) + + return logger + + +# ---------------------------------------------------------------- +# MAIN CLASSES + METHODS +# ---------------------------------------------------------------- + + +class LoggingLevelFilter(logging.Filter): + def __init__(self, logging_level: int): + super().__init__() + self.logging_level = logging_level + + def filter(self, record: LogRecord) -> bool: + return record.levelno == self.logging_level + + +class JsonFormatter(Formatter): + def format(self, record: LogRecord, /): + """ + intercepts logging: + + - replaces message by entire record + - filters to desired keys in the given order + - serialises to a valid JSON if possible + """ + # force universal path standard + record.pathname = Path(record.pathname).as_posix() + + # make the record the entire message: + parts = record.__dict__ + parts["message"] = parts.get("msg", None) + parts = {key: parts.get(key, None) for key in REPORT_KEYS} + + # ensure proper JSON'ised message + record.msg = serialise(parts) + + return super().format(record) + + +# ---------------------------------------------------------------- +# AUXILIARY CLASSES + METHODS +# ---------------------------------------------------------------- + + +def create_dir_if_not_exists( + path: str, + /, +): + p = Path(path) + if p.exists(): + return + p.mkdir(parents=True, exist_ok=True) + + +def create_file_if_not_exists( + path: str, + /, + *, + rights: int = 0o664, +): + """ + Creates a file if it does not already exist + + NOTE: Digits of `rights` define + + - digit 1: rights for user + - digit 2: rights for group + - digit 3: rights for others + + Each digit is an octal number 0-7 (think binary) + + | digit | binary | rights | + | ----: | :----: | :----: | + | 0 | 000 | - - - | + | 1 | 001 | - - x | + | 2 | 010 | - w - | + | 3 | 011 | - w x | + | 4 | 100 | r - - | + | 5 | 101 | r - x | + | 6 | 110 | r w - | + | 7 | 111 | r w x | + + where + + - `r` = read access + - `w` = write access + - `x` = execution rights + + e.g. `0o664` means read+write for user and group, + and read only for others. + """ + create_dir_if_not_exists(os.path.dirname(path)) + p = Path(path) + if p.exists(): + return + p.touch(mode=rights, exist_ok=True) + + +def serialise(value: Any, /) -> str: + """ + Safe jsonisation of a value if possible. + Otherwise resorts to mere stringification. + """ + try: + return json.dumps( + value, + skipkeys=False, + ensure_ascii=False, + allow_nan=True, + sort_keys=False, + ) + + except Exception as _: + pass + + try: + return str(value) + + except Exception as _: + pass + + return None diff --git a/src/_core/logging/constants.py b/src/_core/logging/constants.py new file mode 100644 index 0000000..88f5b99 --- /dev/null +++ b/src/_core/logging/constants.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from typing import Literal + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "LOG_LEVELS", + "REPORT_KEYS", +] + +# ---------------------------------------------------------------- +# CONSTANTS/VARIABLES +# ---------------------------------------------------------------- + +REPORT_KEYS = [ + "asctime", + "levelname", + # "levelno", + "name", + # "msg", + "message", + # "args", + "pathname", + # "filename", + "module", + "lineno", + "funcName", + # "exc_info", + # "exc_text", + # "stack_info", + "created", + "relativeCreated", + "msecs", + "thread", + "threadName", + "processName", + "process", +] + +LOG_LEVELS = Literal[ + "DEBUG", + "INFO", + # "WARN", + "WARNING", + "ERROR", + # "CRITICAL", + "FATAL", +] diff --git a/src/_core/logging/decorators.py b/src/_core/logging/decorators.py new file mode 100644 index 0000000..f0e2f80 --- /dev/null +++ b/src/_core/logging/decorators.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from contextvars import ContextVar +from functools import wraps +from typing import Awaitable +from typing import Callable +from typing import Generator +from typing import ParamSpec +from typing import TypeVar + +from safetywrap import Err + +from ..utils.basic import * +from ..utils.time import * +from .constants import * +from .special import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "echo_async_function", + "echo_function", + "echo_generator", +] + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS/VARIABLES +# ---------------------------------------------------------------- + +PARAMS = ParamSpec("PARAMS") +TX = TypeVar("TX") +RX = TypeVar("RX") +RETURN = TypeVar("RETURN") +_DEPTH = ContextVar("depth", default=[0]) + +# ---------------------------------------------------------------- +# DECORATORS +# ---------------------------------------------------------------- + + +def echo_function( + *, + tag: str | None = None, + message: str | None = None, + level: LOG_LEVELS | int | None = None, + close: bool = True, + depth: int | None = None, +): + """ + Decorates a method via logging before and after (including in the case of errors). + """ + + def dec( + action: Callable[PARAMS, RETURN], + /, + ) -> Callable[PARAMS, RETURN]: + # prepare the message + tag_ = tag or f"fct:{action.__name__}" + message_ = message or tag_ + + # modify function + @wraps(action) + def wrapped_action(*_: PARAMS.args, **__: PARAMS.kwargs) -> RETURN: + timer = Timer(logger=None) + message_end, message_error = echo_beginning(_, __, timer=timer, depth=depth, close=close, message=message_, level=level) # fmt: skip + + try: + output = action(*_, **__) + match output: + case Err(): + echo_end(timer=timer, close=close, message=message_error, level=level) # fmt: skip + return output + + # case Ok() | _: + case _: + echo_end(timer=timer, close=close, message=message_end, level=level) # fmt: skip + + return output + + except BaseException as err: + echo_end(timer=timer, close=close, message=message_error, level=level) # fmt: skip + raise err + + return wrapped_action + + return dec + + +def echo_generator( + *, + tag: str | None = None, + message: str | None = None, + level: LOG_LEVELS | int | None = None, + close: bool = True, + depth: int | None = None, +): + """ + Decorates a method with Generator return type, + via logging before and after (including in the case of errors). + """ + + def dec( + action: Callable[PARAMS, Generator[TX, RX, RETURN]], + /, + ) -> Callable[PARAMS, Generator[TX, RX, RETURN]]: + # prepare the message + tag_ = tag or f"fct:{action.__name__}" + message_ = message or tag_ + + # modify function + @wraps(action) + def wrapped_action(*_: PARAMS.args, **__: PARAMS.kwargs) -> Generator[TX, RX, RETURN]: + timer = Timer(logger=None) + message_end, message_error = echo_beginning(_, __, timer=timer, depth=depth, close=close, message=message_, level=level) # fmt: skip + + try: + output = yield from action(*_, **__) + match output: + case Err(): + echo_end(timer=timer, close=close, message=message_error, level=level) # fmt: skip + + # case Ok() | _: + case _: + echo_end(timer=timer, close=close, message=message_end, level=level) # fmt: skip + + return output + + except BaseException as err: + echo_end(timer=timer, close=close, message=message_error, level=level) # fmt: skip + raise err + + return wrapped_action + + return dec + + +def echo_async_function( + *, + tag: str | None = None, + message: str | None = None, + level: LOG_LEVELS | int | None = None, + close: bool = True, + depth: int | None = None, +): + """ + Decorates an async method via logging before and after (including in the case of errors). + """ + + def dec( + action: Callable[PARAMS, Awaitable[RETURN]], + /, + ) -> Callable[PARAMS, Awaitable[RETURN]]: + # prepare the message + tag_ = tag or f"fct:{action.__name__}" + message_ = message or tag_ + + # modify function + @wraps(action) + async def wrapped_action(*_: PARAMS.args, **__: PARAMS.kwargs) -> RETURN: + timer = Timer(logger=None) + message_end, message_error = echo_beginning(_, __, timer=timer, depth=depth, close=close, message=message_, level=level) # fmt: skip + + try: + output = await action(*_, **__) + echo_end(timer=timer, close=close, message=message_end, level=level) # fmt: skip + return output + + except BaseException as err: + echo_end(timer=timer, close=close, message=message_error, level=level) # fmt: skip + raise err + + return wrapped_action + + return dec + + +# ---------------------------------------------------------------- +# AUXILIARY METHODS +# ---------------------------------------------------------------- + + +def echo_beginning( + posargs: tuple, + kwargs: dict, + /, + *, + timer: Timer, + depth: int | None, + close: bool, + message: str, + level: LOG_LEVELS | int | None, +) -> tuple[str, str]: + """ + Auxiliary method to be performed at the start of an echo-decorated method. + """ + depths = _DEPTH.get() or [0] + depth = depth or depths[-1] # either pick latest value or forced value + + message__ = safe_format_string(message, *posargs, **kwargs) + + message_start = "=" * (depth + 1) + "> [ ] " + message__ + message_end = message_error = "" + if close: + message_end = "=" * (depth + 1) + "> [/] " + message__ + " | elapsed: {t:.2f}s" # fmt: skip + message_error = "=" * (depth + 1) + "> [x] " + message__ + " | elapsed: {t:.2f}s" # fmt: skip + + log(message_start, level=level) + _DEPTH.set([*depths, depth + 1]) + timer.start() + + return message_end, message_error + + +def echo_end( + *, + timer: Timer, + close: bool, + message: str, + level: LOG_LEVELS | int | None, +) -> tuple[str, str]: + """ + Auxiliary method to be performed at the end of an echo-decorated method. + """ + depths = _DEPTH.get() + _DEPTH.set(depths[:-1] or [0]) # remove last value + if close: + msg = safe_format_string(message, t=timer.elapsed) + log(msg, level=level) diff --git a/src/_core/logging/errors.py b/src/_core/logging/errors.py new file mode 100644 index 0000000..7def0cd --- /dev/null +++ b/src/_core/logging/errors.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import traceback as tb + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "error_with_trace", + "error_with_trace_multiline", +] + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS/VARIABLES +# ---------------------------------------------------------------- + + +def error_with_trace_multiline(err: BaseException, /) -> list[str]: + """ + Adds tracestack to an error and returns as list of lines. + """ + return tb.format_exception( + type(err), + value=err, + tb=err.__traceback__, + ) + + +def error_with_trace(err: BaseException, /) -> str: + """ + Adds tracestack to an error and returns as single line. + """ + lines = error_with_trace_multiline(err) + return "\n".join(lines) diff --git a/src/_core/logging/special.py b/src/_core/logging/special.py new file mode 100644 index 0000000..24da937 --- /dev/null +++ b/src/_core/logging/special.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import logging +import os +import sys +from pathlib import Path +from typing import Any +from typing import Callable +from typing import TypeVar + +from .constants import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "log", + "log_console", + "log_debug_wrapped", + "log_debug_wrapped_args", + "log_dev", +] + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS/VARIABLES +# ---------------------------------------------------------------- + +T = TypeVar("T") + +# ---------------------------------------------------------------- +# ENUM +# ---------------------------------------------------------------- + + +def get_log_level(level: str | int, /) -> int: + if isinstance(level, str): + level = logging.getLevelNamesMapping().get(level, logging.INFO) + return level + + +# ---------------------------------------------------------------- +# METHODS - SPECIAL +# ---------------------------------------------------------------- + + +def log_debug_wrapped(cb: Callable[[], str], /): + """ + Performs logging.debug with the message is wrapped by a function call, + which is only called if DEBUG-mode is active. + + NOTE: used to save processing time + """ + if logging.DEBUG < logging.root.level: + return + message = cb() + for text in message.split("\n"): + logging.debug(text) + + +def log_debug_wrapped_args( + msg: Any, + *_, + **__, +): + """ + Performs logging.debug + with the computation of the message wrapped by a function call, + which is only called if DEBUG-mode is active. + + NOTE: used to save processing time + """ + if logging.DEBUG < logging.root.level: + return + + values = [str(value) for value in _] + values += [f"{key}: {value}" for key, value in __.items()] # fmt: skip + values_str = "; ".join(values) + + logging.debug(f"{msg} | {values_str}") + + +def log_console(*messages: Any): + for text in messages: + sys.stdout.write(f"{text}\n") + sys.stdout.flush() + + +def log_dev(*messages: Any, path: str): # pragma: no cover + p = Path(path) + if not p.exists(): + Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True) + p.touch(mode=0o644) + + with open(path, "a", encoding="utf-8") as fp: + print(*messages, file=fp) + + +# ---------------------------------------------------------------- +# METHODS - UNIVERSAL +# ---------------------------------------------------------------- + + +def log( + *messages: Any, + level: LOG_LEVELS | int | None = None, +): + level = get_log_level(level) + match level: + case None: + return log_console(*messages) + + case "DEBUG": + for text in messages: + logging.debug(text) + + case "WARN" | "WARNING": + for text in messages: + logging.warning(text) + + case "ERROR": + for text in messages: + logging.error(text) + + case "CRITICAL" | "FATAL": + message = "\n".join([str(text) for text in messages]) + logging.fatal(message) + exit(1) + + # case "INFO": + case _: + for text in messages: + logging.info(text) diff --git a/src/_core/utils/__init__.py b/src/_core/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/_core/utils/basic.py b/src/_core/utils/basic.py new file mode 100644 index 0000000..41f111c --- /dev/null +++ b/src/_core/utils/basic.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import json +import re +from datetime import datetime +from enum import Enum +from enum import StrEnum +from functools import reduce +from typing import Any +from typing import Callable +from typing import Iterable +from typing import Sequence +from typing import TypeVar +from typing import overload + +from flatdict import FlatDict + +from .code import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "as_flattened_dict", + "coerce_null", + "create_regex_from_prefix_pattern", + "extract_string", + "extract_strip", + "first_non_null", + "flatdict_to_dict", + "flatten", + "flatten_mixed", + "flatten_sets", + "indicator_function_factory", + "json_deserialise", + "merge_dicts", + "safe_format_string", + "split_string_list", + "validate_regex", +] + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS / VARIABLES +# ---------------------------------------------------------------- + +MAX_ITER = 1000 + +T = TypeVar("T") +_DATE_PATTERN = re.compile(pattern=r"^\d+-\d+-\d+$") +_TIME_PATTERN = re.compile(pattern=r"^\d+:\d+(:\d+(\.\d+)?)?$") + +# ---------------------------------------------------------------- +# METHODS - VALUES +# ---------------------------------------------------------------- + + +def first_non_null( + *values: T | None, + default: T, +) -> T: + for value in values: + if value is None: + continue + return value + return default + + +# ---------------------------------------------------------------- +# METHODS - STRINGS +# ---------------------------------------------------------------- + + +def safe_format_string( + text: str, + *pos_args: Any, + **kwargs: Any, +) -> str: + """ + Safely formats string leaving missing arguments alone. + """ + n_pos = len(pos_args) + for _ in range(MAX_ITER): + try: + return text.format(*pos_args, **kwargs) + + except IndexError as err: + if n_pos == 0: + text = re.sub(pattern="\\{\\}", repl="{{}}", string=text) + text = re.sub(pattern=f"\\{{{n_pos}\\}}", repl=f"{{{{{n_pos}}}}}", string=text) + n_pos += 1 + + except KeyError as err: + key = [*err.args, "?"][0] + text = re.sub(pattern=f"\\{{{key}\\}}", repl=f"{{{{{key}}}}}", string=text) + + raise Exception(f"could not safely format '{text}'") + + +@overload +def extract_string(x: None, /) -> None: ... + + +@overload +def extract_string(x: str | StrEnum, /) -> str: ... + + +@overload +def extract_string(x: Sequence[str | StrEnum] | set[str | StrEnum], /) -> list[str]: ... + + +@overload +def extract_string(x: dict[str | StrEnum, str | StrEnum], /) -> dict[str, str]: ... + + +def extract_string( + x: ( + None + | str + | StrEnum + | Sequence[str | StrEnum] + | set[str | StrEnum] + | dict[str | StrEnum, str | StrEnum] + ), + /, +): # -> None | str | list[str] | dict[str, str]: + """ + Returns the underlying string value of a string or string-enum. + + - Converts string/enum to string + - Converts list of strings/enums to list of strings + - Converts dictionary of strings/enums to dictionary of strings + """ + match x: + case None: + return None + + # DEV-NOTE: must prioritise Enum over str, since StrEnum extends str! + case Enum(): + return x.value + + case str(): + return x + + case dict(): + return {extract_string(key): extract_string(value) for key, value in x.items() } # fmt: skip + + case _: + return [extract_string(xx) for xx in x] + + +def extract_strip( + x: str | StrEnum, + /, + *, + left: str | None = None, + right: str | None = None, +) -> str: + """ + Performs left/right-strip to a string or string enum. + """ + x = extract_string(x) + if left is not None: + x = x.removeprefix(left) + if right is not None: + x = x.removesuffix(right) + return x + + +def json_deserialise(x: str) -> Any: + """ + Parses a JSON-ised string + """ + # if parses normally, return this + try: + return json.loads(x) + + except Exception as err: + pass + + # otherwise attempt to parse as time/date + try: + if re.match(pattern=_TIME_PATTERN, string=x): + return datetime.fromisoformat(f"2000-01-01 {x}").time() + + elif re.match(pattern=_DATE_PATTERN, string=x): + return datetime.fromisoformat(x).date() + + else: + return datetime.fromisoformat(x) + + except Exception as err: + pass + + raise Exception(f"failed to parse {x}") + + +def split_string_list( + value: str | None, + /, + *, + sep: str = ",", + remove_empty: bool = True, +) -> list[str]: + """ + Parses a (typically) comma-separated list of string as a list of strings, + optionally removes empty values (default `true`). + """ + if value is None: + return [] + + value = value.strip() + if value == "": + return [] + + values: list[str] = re.split(pattern=sep, string=value or "") + values = [x.strip() for x in values] + if remove_empty: + values = [x for x in values if x != ""] + + return values + + +@make_safe_none +def validate_regex(text: str, /) -> re.Pattern[str]: + result = re.compile(text) + return result + + +@make_safe_none +def substitute_regex(text: str, /) -> re.Pattern[str]: + result = re.compile(text) + return result + + +def create_regex_from_prefix_pattern(text: str | None, /) -> str | None: + r""" + Transforms strings that are not strict regex patterns + into proper regex patterns, e.g. + ```py + assert create_regex_from_pattern("H-AB") == r"^H-AB\b.*" + assert create_regex_from_pattern("H-AB*") == r"^H-AB\b.*" + assert create_regex_from_pattern("H-AB*,R*") == r"^H-AB\b.*|R\b.*" + assert create_regex_from_pattern("(H-AB,R)*") == r"^(H-AB|R)\b.*" + ``` + NOTE: If the input is null or text containing just white space, returns null. + """ + if not isinstance(text, str): + return None + + # remove spaces + text = re.sub(pattern=r"\s", repl="", string=text) + + # if the text just consisted of white space, return null + if text == "": + return None + + # if the text does not end in *, add it + if not text.endswith("*"): + text += "*" + + text = re.sub(pattern=r"\^", repl="", string=text) + text = re.sub(pattern=r",", repl=r"|", string=text) + # DEV-NOTE: unclear why, but the '\' needs to be escaped! + text = re.sub(pattern=r"([^\.])\.?\*", repl=r"\1\\b.*", string=text) + text = re.sub(pattern=r"^\.?\*", repl=r".*", string=text) + + if validate_regex(text) is None: + return None + + return f"^({text})$" + + +def coerce_null( + x: T | None, + /, + *, + default: T, +) -> T: + """ + Replaces value by a default if null + """ + if x is None: + x = default + return x + + +# ---------------------------------------------------------------- +# METHODS - FUNCTIONS +# ---------------------------------------------------------------- + + +def indicator_function_factory(value: T) -> Callable[[T], bool]: + """ + Returns a boolean-valued function + that returns `true` <==> the given value is assumed. + """ + + def indicator_function(x: T) -> bool: + return x == value + + return indicator_function + + +# ---------------------------------------------------------------- +# METHODS - ARRAYS +# ---------------------------------------------------------------- + + +def flatten(X: Iterable[list[T]], /) -> list[T]: + X_flat = [] + for XX in X: + X_flat.extend(XX) + return X_flat + + +def flatten_sets(X: Iterable[set[T]], /) -> set[T]: + X_flat = set([]) + for XX in X: + X_flat = X_flat.union(XX) + return X_flat + + +def flatten_mixed(X: Iterable[T | list[T]], /) -> list[T]: + X_flat = [] + for XX in X: + if isinstance(XX, list): + X_flat.extend(XX) + + else: + X_flat.append(XX) + + return X_flat + + +def merge_dicts(*objects: dict | FlatDict) -> dict: + """ + Merges dictionaries just like `dict1 | dict2`, + but prevents values being overwritten by None + + ```py + x = {'height': 200, 'name': 'Bob', 'colour': 'red'} + y = {'height': None, 'colour': 'blue', 'city': 'London'} + + # { 'height': None, 'name': 'Bob', 'colour': 'blue', 'city': 'London'} + print(x | y) + + # { 'height': 200, 'name': 'Bob', 'colour': 'blue', 'city': 'London'} + print(merge_dicts(x, y)) + ``` + """ + objects = map( + # wipes out keys with None-values + lambda x: {key: value for key, value in x.items() if value is not None}, + objects, + ) + # join cleaned objects together + result = reduce(lambda x, y: x | y, objects, {}) + return result + + +def flatdict_to_dict(object: FlatDict, /) -> dict: + """ + Convert FlatDict to a flattened dictionary. + """ + return {**object} + + +def as_flattened_dict(object: dict, /, *, delimiter: str = ":") -> dict: + """ + Convert a dictionary to a flattened dictionary. + """ + return flatdict_to_dict(FlatDict(object, delimiter=delimiter)) diff --git a/src/_core/utils/code.py b/src/_core/utils/code.py new file mode 100644 index 0000000..137fcbd --- /dev/null +++ b/src/_core/utils/code.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import logging +from functools import wraps +from typing import Any +from typing import Callable +from typing import Generic +from typing import Optional +from typing import ParamSpec +from typing import TypeVar +from typing import overload + +from lazy_load import lazy +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import Field +from safetywrap import Err +from safetywrap import Ok +from safetywrap import Result + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "TypeGuard", + "compute_once", + "flatten_safety_wrap", + "make_lazy", + "make_safe", + "make_safe_none", + "make_safe_none_verbose", + "safe_unwrap", + "value_of_model", + "wrap_result", +] + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS, VARIABLES +# ---------------------------------------------------------------- + +PARAMS = ParamSpec("PARAMS") +RETURN = TypeVar("RETURN") +T = TypeVar("T") +E = TypeVar("E") +ERR = TypeVar("ERR", bound=BaseException) +MODEL = TypeVar("MODEL", bound=BaseModel) + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def make_lazy(method: Callable[PARAMS, RETURN]) -> Callable[PARAMS, RETURN]: + """ + Decorates a method and makes it return a lazy-load output. + """ + + @wraps(method) + def wrapped_method(*_: PARAMS.args, **__: PARAMS.kwargs) -> RETURN: + return lazy(method, *_, **__) + + return wrapped_method + + +def compute_once(method: Callable[[], RETURN]) -> Callable[[], RETURN]: + """ + Decorates a possibly expensive method to ensure that it only computes once + and thereafter simply returns an internally stored value. + + If for some reason the value is destroyed, then recomputes this. + """ + _value = None + _first = True + + @wraps(method) + def wrapped_method() -> RETURN: + nonlocal _value + nonlocal _first + if _first or _value is None: + _value = method() + _first = False + return _value + + return wrapped_method + + +def value_of_model(m: MODEL): + return m.root + + +def flatten_safety_wrap( + method: Callable[PARAMS, Result[T, E] | RETURN], +) -> Callable[PARAMS, T | E]: + """ + Decorator removes Ok(...) | Err(...) wrapping. + + NOTE: Err-types will not result in errors. + """ + + @wraps(method) + def wrapped_method(*_: PARAMS.args, **__: PARAMS.kwargs): + output_wrapped = method(*_, **__) + output = output_wrapped.unwrap_or_else(lambda err: err) + return output + + return wrapped_method + + +@overload +def wrap_result( + method: Callable[PARAMS, Result[RETURN, ERR]], + /, +) -> Callable[PARAMS, Result[RETURN, ERR]]: ... + + +@overload +def wrap_result( + method: Callable[PARAMS, RETURN], + /, +) -> Callable[PARAMS, Result[RETURN, Exception]]: ... + + +def wrap_result( + method: Callable[PARAMS, RETURN] + | Callable[PARAMS, Result[RETURN, ERR]] + | Callable[PARAMS, RETURN | Result[RETURN, ERR]], + /, +) -> Callable[PARAMS, Result[RETURN, Exception]] | Callable[PARAMS, Result[RETURN, ERR]]: + """ + Uses the Ok/Err to wrap a method `f`. + Flattens any Ok/Err in the process + + | outcome of `f` | outcome of wrapped | + | :------------- | :----------------- | + | returns `Ok(x)` | returns `Ok(x)` | + | returns `Err(x)` | returns `Err(x)` | + | returns `x` | returns `Ok(x)` | + | raises Exception err | returns `Err(err)` | + | raises BaseException err | raises `err` | + """ + + @wraps(method) + def wrapped_fct(*_: PARAMS.args, **__: PARAMS.kwargs): + try: + value = method(*_, **__) + if isinstance(value, Result): + return value + return Ok(value) + + except Exception as err: + return Err(err) + + except BaseException as err: + raise err + + return wrapped_fct + + +def safe_unwrap( + method: Callable[[], RETURN], + default: E = None, + default_factory: Optional[Callable[[], E]] = None, + silent: bool = True, +) -> RETURN | E: + """ + Calls method and returns default if exception raised. + Only raises error in the case of interruptions/sys exit. + """ + try: + result = method() + return result + + except BaseException as err: + if isinstance(err, (KeyboardInterrupt, EOFError, SystemExit)): + raise err + + if not silent: + logging.error(err) + + if default_factory is not None: + result = default_factory() + return result + + return default + + +def make_safe( + default: E | None = None, + default_factory: Callable[[], E] | None = None, + silent: bool = True, +): + """ + Decorator which modifies funcitons + to make them return default values upon exceptions. + """ + + def dec(f: Callable[PARAMS, RETURN]) -> Callable[PARAMS, RETURN | E]: + @wraps(f) + def wrapped_fct(*_: PARAMS.args, **__: PARAMS.kwargs) -> RETURN | E: + return safe_unwrap( + lambda: f(*_, **__), + default=default, + default_factory=default_factory, + silent=silent, + ) + + return wrapped_fct + + return dec + + +def make_safe_none(f: Callable[PARAMS, RETURN]) -> Callable[PARAMS, RETURN | None]: + """ + Decorator which modifies functions + to make them return the default value None upon exceptions. + + NOTE: silently continues if an error occurs. + """ + + @wraps(f) + def wrapped_fct(*_: PARAMS.args, **__: PARAMS.kwargs) -> RETURN | None: + return safe_unwrap(lambda: f(*_, **__)) + + return wrapped_fct + + +def make_safe_none_verbose(f: Callable[PARAMS, RETURN]) -> Callable[PARAMS, RETURN | None]: + """ + Decorator which modifies functions + to make them return the default value None upon exceptions. + + NOTE: catches and logs errors if they occur, then continues. + """ + + @wraps(f) + def wrapped_fct(*_: PARAMS.args, **__: PARAMS.kwargs) -> RETURN | None: + return safe_unwrap(lambda: f(*_, **__), silent=False) + + return wrapped_fct + + +class TypeGuard(BaseModel, Generic[T, E]): + """ + Provides a method that asserts type or else returns a default value. + + The default of `default` is `null`. + + ## Usage ## + ```py + coerce = TypeGuard[int, None](type=int) + coerce(5) # 5 + coerce("cat") # None + + coerce = TypeGuard[int, int](type=int, default=-1) + coerce(5) # 5 + coerce("cat") # -1 + + coerce = TypeGuard[int, int](type=int, default_factory=lambda x: len(x)) + coerce(5) # 5 + coerce("cat") # 3 + + coerce = TypeGuard[int, str](type=int, default_factory=str) + coerce(5) # 5 + coerce("cat") # "cat" + coerce(7.1) # "7.1" + ``` + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + ) + type_: type = Field(..., alias="type") + default_: E | None = Field(default=None, alias="default") + default_factory_: Callable[[Any], E | None] | None = Field( + default=None, alias="default_factory" + ) + + def __call__(self, x: Any, /) -> T | E | None: + if isinstance(x, self.type_): + return x + f = self.default_factory_ + if self.default_factory_ is None: + return self.default_ + return f(x) diff --git a/src/_core/utils/config.py b/src/_core/utils/config.py new file mode 100644 index 0000000..210811f --- /dev/null +++ b/src/_core/utils/config.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import yaml + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "YamlIndentDumper", +] + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +class YamlIndentDumper(yaml.Dumper): + """ + PyYaml's `yaml.dump` for lists yields + ```yaml + key: + - value1 + - value2 + - ... + ``` + which currently does not match standard style, i.e. + ```yaml + key: + - value1 + - value2 + - ... + ``` + This class fixes this issue. + + Usage + ```py + yaml.dump(..., Dumper=YamlIndentDumper) + ``` + """ + + def increase_indent( + self, + flow=False, + indentless=False, + ): + return super(YamlIndentDumper, self).increase_indent(flow, False) diff --git a/src/_core/utils/io.py b/src/_core/utils/io.py new file mode 100644 index 0000000..2ca43cf --- /dev/null +++ b/src/_core/utils/io.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import json +from base64 import b64decode +from base64 import b64encode +from hashlib import sha256 +from io import BytesIO +from typing import Any + +import yaml + +from ..constants import * +from .io_yaml import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "BytesIOStream", + "decode_base_64", + "encode_base_64", + "hash_encode", + "parse_contents", + "read_yaml", + "read_yaml_from_contents", +] + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def hash_encode(text: str, encoding: ENCODING = "utf-8") -> bytes: + """ + Note: + A hash encoded value cannot (under current computational methods) + be effectively decoded. + They can 'only' be used to check if an entered value + matches another previously safely stored value (e.g. a password), + by comparing their hashes. + + """ + return sha256(text.encode(encoding)).hexdigest().encode("ascii") + + +def encode_base_64(text: str, encoding: ENCODING = "utf-8") -> str: + return b64encode(text.encode(encoding)).decode("ascii") + + +def decode_base_64(code: str, encoding: ENCODING = "utf-8") -> str: + try: + return b64decode(code.encode("ascii")).decode(encoding) + + except Exception as _: + return "" + + +class BytesIOStream: + """ + Provides context manager for a bytes stream. + """ + + _contents: bytes + + def __init__(self, contents: bytes): + self._contents = contents + + def __enter__(self): + """ + Context manager for BytesIO that deals with seeking. + """ + fp = BytesIO(self._contents).__enter__() + fp.seek(0) + return fp + + def __exit__(self, exc_type, exc_val, exc_tb): + return + + +def read_yaml(path: str): + """ + Reads yaml from a path and uses custom registered constructors for parsing. + """ + register_yaml_constructors() + with open(path, "rb") as fp: + assets = yaml.load(fp, Loader=yaml.FullLoader) + return assets + + +def read_yaml_from_contents(contents: bytes): + """ + Reads yaml from bytes and uses custom registered constructors for parsing. + """ + register_yaml_constructors() + with BytesIOStream(contents) as fp: + assets = yaml.load(fp, Loader=yaml.FullLoader) + return assets + + +def parse_contents( + contents: bytes, + /, + *, + format: BASIC_FILETYPES, +) -> Any: + match format: + case ".json": + # read from contents (assumed to be in yaml-format) + return json.loads(contents) + + case ".yaml": + # read from contents (assumed to be in yaml-format) + return read_yaml_from_contents(contents) + + case _: + raise ValueError(f"No read method developed for {format}") diff --git a/src/_core/utils/io_yaml.py b/src/_core/utils/io_yaml.py new file mode 100644 index 0000000..70dda9a --- /dev/null +++ b/src/_core/utils/io_yaml.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import re +from contextvars import ContextVar + +import yaml + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "register_yaml_constructors", +] + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS +# ---------------------------------------------------------------- + +_yaml_constructors_registered = ContextVar[bool]("constructors registered", default=False) + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def register_yaml_constructors(): + """ + Registers yaml-sugar to help parse .yaml files + """ + if _yaml_constructors_registered.get(): + return + + yaml.add_constructor(tag="!include", constructor=include_constructor) + yaml.add_constructor(tag="!not", constructor=not_constructor) + yaml.add_constructor(tag="!join", constructor=join_constructor) + yaml.add_constructor(tag="!tuple", constructor=tuple_constructor) + + _yaml_constructors_registered.set(True) + return + + +# ---------------------------------------------------------------- +# PARTS +# ---------------------------------------------------------------- + + +def include_constructor(loader: yaml.Loader, node: yaml.Node): + try: + value = loader.construct_yaml_str(node) + assert isinstance(value, str) + # parse argument + m = re.match(pattern=r"^(.*)\/#\/?(.*)$", string=value) + # read yaml from path + path = m.group(1) if m else value + register_yaml_constructors() + with open(path, "rb") as fp: + obj = yaml.load(fp, yaml.FullLoader) + # get part of yaml + keys_as_str = m.group(2) if m else "" + keys = keys_as_str.split("/") + for key in keys: + if key == "": + continue + obj = obj.get(key, dict()) + return obj + + except Exception as _: + return None + + +def not_constructor(loader: yaml.Loader, node: yaml.Node) -> bool: + try: + value = loader.construct_yaml_bool(node) + return not value + + except Exception as _: + return None + + +def join_constructor(loader: yaml.Loader, node: yaml.Node): + try: + values = loader.construct_sequence(node, deep=True) + sep, parts = str(values[0]), [str(_) for _ in values[1]] + return sep.join(parts) + + except Exception as _: + return "" + + +def tuple_constructor(loader: yaml.Loader, node: yaml.Node): + try: + value = loader.construct_sequence(node, deep=True) + return tuple(value) + + except Exception as _: + return None diff --git a/src/_core/utils/misc.py b/src/_core/utils/misc.py new file mode 100644 index 0000000..5ec6314 --- /dev/null +++ b/src/_core/utils/misc.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import re +from datetime import datetime +from functools import wraps +from textwrap import dedent as textwrap_dedent +from typing import Callable +from typing import TypeVar + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "dedent", + "dedent_full", + "get_date_stamp", + "get_datetime_stamp", + "get_timestamp", + "parse_datetime", + "strip_around", +] + + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def parse_datetime(stamp: str) -> datetime: + return datetime.fromisoformat(stamp.replace("Z", " +00:00")) + + +def get_timestamp(format: str = r"%Y-%m-%d %H:%M:%S%z") -> str: + return datetime.now().strftime(format) + + +def get_datetime_stamp(rounded: bool = False) -> str: + return get_timestamp(r"%Y-%m-%d %H:%M:%S%z" if rounded else r"%Y-%m-%d %H:%M:%S.%f%z") + + +def get_date_stamp() -> str: + return get_timestamp(r"%Y-%m-%d") + + +def strip_around( + text: str, + first: bool, + last: bool, + all: bool = True, +): + """ + Strips all initial/final 'empty' lines. + """ + lines = re.split(pattern=r"\n", string=text) + if all: + if first: + while len(lines) > 0 and lines[0].strip() == "": + lines = lines[1:] + if last: + while len(lines) > 0 and lines[-1].strip() == "": + lines = lines[:-1] + else: + if first: + lines = lines[1:] + if last: + lines = lines[:-1] + text = "\n".join(lines) + return text + + +def dec_prestrip(first: bool = True, last: bool = True, all: bool = False): + """ + Returns a decorator that modifies string -> string methods + """ + T = TypeVar("T") + + def dec(method: Callable[[str], T]) -> Callable[[str], T]: + """ + Performs method but first strips all initial/final 'empty' lines. + """ + + @wraps(method) + def wrapped_method(text: str) -> T: + text = strip_around(text, first=first, last=last, all=all) + return method(text) + + return wrapped_method + + return dec + + +@dec_prestrip(all=False) +def dedent(text: str) -> str: + r""" + Remove any common leading whitespace from every line in `text`. + + This can be used to make triple-quoted strings line up with the left + edge of the display, while still presenting them in the source code + in indented form. + + Note that tabs and spaces are both treated as whitespace, but they + are not equal: the lines " hello" and "\\thello" are + considered to have no common leading whitespace. + + Entirely blank lines are normalised to a newline character. + """ + return textwrap_dedent(text) + + +@dec_prestrip(all=True) +def dedent_full(text: str) -> str: + r""" + Remove any common leading whitespace from every line in `text`. + + This can be used to make triple-quoted strings line up with the left + edge of the display, while still presenting them in the source code + in indented form. + + Note that tabs and spaces are both treated as whitespace, but they + are not equal: the lines " hello" and "\\thello" are + considered to have no common leading whitespace. + + Entirely blank lines are normalised to a newline character. + + NOTE: this method completely strips all pre/post empty lines + (= lines containing at most only white spaces). + """ + return textwrap_dedent(text) diff --git a/src/_core/utils/time.py b/src/_core/utils/time.py new file mode 100644 index 0000000..9e3a24a --- /dev/null +++ b/src/_core/utils/time.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import re +import time +from datetime import datetime +from datetime import timedelta +from datetime import timezone +from typing import overload +from zoneinfo import ZoneInfo + +import pytz +from codetiming import Timer as TimerBasic +from pydantic import AwareDatetime +from pydantic import BaseModel +from pydantic import ConfigDict +from tzlocal import get_localzone + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "Timer", + "add_timezone", + "get_date_stamp", + "get_datetime_stamp", + "get_local_timezone", + "get_timestamp", + "get_timezone_from_name", + "parse_datetime", + "parse_duration", + "remove_timezone", + "timezone_as_gmt_offset", +] + +# ---------------------------------------------------------------- +# LOCAL VARIABLES +# ---------------------------------------------------------------- + +_TIME_PATTERN = re.compile(pattern=r"^(.*)([\+-])(\d+\:\d+)$") + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def parse_datetime(stamp: str) -> datetime: + return datetime.fromisoformat(stamp.replace("Z", " +00:00")) + + +def get_timestamp(format: str = r"%Y-%m-%d %H:%M:%S%z") -> str: + return datetime.now().strftime(format) + + +def get_datetime_stamp(rounded: bool = False) -> str: + return get_timestamp(r"%Y-%m-%d %H:%M:%S%z" if rounded else r"%Y-%m-%d %H:%M:%S.%f%z") + + +def get_date_stamp() -> str: + return get_timestamp(r"%Y-%m-%d") + + +def get_local_timezone() -> timezone: + """ + Returns the system timezone + """ + zone = get_localzone() + tz = get_timezone_from_name(zone) + return tz + + +def get_timezone_from_name(zone: str | ZoneInfo, /) -> timezone: + """ + Given a timezone as name e.g. UTC, CET, Asia/Tokyo, UTC+02:00 + determines a standard timezone. + """ + text = str(zone) + tz_file = pytz.timezone(text) + dt = datetime.now(tz_file).utcoffset() or timedelta() + tz = timezone(dt) + return tz + + +def timezone_as_gmt_offset(tz: timezone) -> str: + """ + Determines a universally acceptable format + for a timezone name. + """ + # compute offset as hours + t = datetime.now(tz) + dt = t.utcoffset() + offset = dt.total_seconds() / 3600 + hours = round(offset) + + if offset > 0: + return f"Etc/GMT+{hours:d}" + + elif offset < 0: + return f"Etc/GMT-{-hours:d}" + + return "GMT" + + +class Timer(TimerBasic): + _pause_time: float + + def start(self): + """ + Starts the timer. + """ + super().start() + self._pause_time = self._start_time + + @property + def laptime(self) -> float: + """ + Computes the time duration since last "lap" + (or since start of this is the first lap). + + NOTE: Does not pause or reset the timer. + """ + t0 = self._pause_time + t1 = time.perf_counter() + self._pause_time = t1 + return t1 - t0 + + @property + def elapsed(self) -> float: + """ + Returns the time duration since start. + + NOTE: Does not pause or reset the timer. + """ + self.last = time.perf_counter() - self._start_time + return self.last + + +@overload +def remove_timezone(t: None, /) -> None: ... + + +@overload +def remove_timezone(t: datetime, /) -> datetime: ... + + +def remove_timezone(t: datetime | None, /) -> datetime | None: + """ + Places in UTC and removes timezone information. + """ + match t: + case datetime(): + if t.tzinfo is not None: + t = t.astimezone(tz=timezone.utc) + t = t.replace(tzinfo=None) + + return t + + case _: + return None + + +@overload +def add_timezone( + t: None, + /, + *, + tz: timezone = timezone.utc, +) -> None: ... + + +@overload +def add_timezone( + t: datetime, + /, + *, + tz: timezone = timezone.utc, +) -> AwareDatetime: ... + + +def add_timezone( + t: datetime | None, + /, + *, + tz: timezone = timezone.utc, +) -> AwareDatetime | None: + """ + Adds timezone, ensuring UTC. + """ + match t: + case datetime(): + if t.tzinfo is None: + t = t.replace(tzinfo=tz) + + else: + t = t.astimezone(tz=tz) + + return t + + case _: + return None + + +@overload +def parse_duration(expr: str, /) -> timedelta: ... + + +@overload +def parse_duration(expr: None, /) -> None: ... + + +def parse_duration(expr: str | None) -> timedelta | None: + """ + Computes time duration based on a string expression + """ + + class TimeDuration(BaseModel): + model_config = ConfigDict( + extra="allow", + populate_by_name=True, + ) + value: timedelta | None + + try: + obj = TimeDuration.model_validate({"value": expr}) + return obj.value + + except Exception as _: + return None diff --git a/src/api.py b/src/api.py new file mode 100644 index 0000000..b5e4660 --- /dev/null +++ b/src/api.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Entry point to serve as FastAPI-application +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import os +import sys +from pathlib import Path + +os.chdir(Path(__file__).parent.parent) +sys.path.insert(0, os.getcwd()) + +import logging + +import uvicorn + +from .app.endpoints_fastapi import * +from .models.application import * +from .queries import * +from .queries._console.api import * +from .setup import * + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS +# ---------------------------------------------------------------- + +PID = os.getpid() +# NOTE: need this in case Azurite blob storage is connected +try: + logger = logging.getLogger("azure.core.pipeline.policies") + logger.setLevel(logging.WARNING) + +except Exception as _: + pass + +# ---------------------------------------------------------------- +# EXECUTION +# ---------------------------------------------------------------- + +if __name__ == "__main__": + args = CliArguments(config.INFO).parse(*sys.argv[1:]) + + config.pid.set(PID) + config.path_env.set(args.env) + config.path_logging.set(args.log) + config.path_config.set(args.config) + config.initialise_application( + name="app", + serialise=False, + log_to_files=True, + verbose=args.verbose, + ) + + # create app + route = "" # NOTE: only use "/xyz" do serve multiple application on the same port + app = create_ui(route=route, debug=args.verbose) + + # run app + uvicorn.run(app=app, host=config.http_ip.get(), port=config.http_port.get()) diff --git a/src/app/__init__.py b/src/app/__init__.py new file mode 100644 index 0000000..b67646c --- /dev/null +++ b/src/app/__init__.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +This submodule the logic of different modes of executing the application. +The submodules here draw on the resources in the remainder of the code base, +viz. + +- models +- queries +- setup + +in order to concentrate on the things that matter most. +""" diff --git a/src/app/endpoints/__init__.py b/src/app/endpoints/__init__.py new file mode 100644 index 0000000..1f1a69d --- /dev/null +++ b/src/app/endpoints/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +This submodule contains generalised endpoints +which can be used in any ui. +""" diff --git a/src/app/endpoints/endpoints_basic.py b/src/app/endpoints/endpoints_basic.py new file mode 100644 index 0000000..ebed5cb --- /dev/null +++ b/src/app/endpoints/endpoints_basic.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from ...setup import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "endpoint_ping", + "endpoint_version", +] + +# ---------------------------------------------------------------- +# ENDPOINTS +# ---------------------------------------------------------------- + + +def endpoint_ping() -> str: + return "success" + + +def endpoint_version() -> str: + return config.VERSION diff --git a/src/app/endpoints_fastapi/__init__.py b/src/app/endpoints_fastapi/__init__.py new file mode 100644 index 0000000..7a67696 --- /dev/null +++ b/src/app/endpoints_fastapi/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from .basic import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "create_ui", +] diff --git a/src/app/endpoints_fastapi/basic.py b/src/app/endpoints_fastapi/basic.py new file mode 100644 index 0000000..11420f2 --- /dev/null +++ b/src/app/endpoints_fastapi/basic.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Main script to creates the FastAPI instance, +including resources and endpoints. +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from fastapi import FastAPI +from fastapi.routing import APIRouter +from fastapi.security import HTTPBasic +from fastapi.templating import Jinja2Templates +from fastapi_offline import FastAPIOffline + +from ...setup import * +from .endpoints_basic import * +from .endpoints_features import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "create_ui", +] + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def create_ui( + *, + route: str = "", + debug: bool = False, +) -> FastAPI: + """ + Creates the API and adds endpoints + + **NOTE:** Uses `fastapi-offline` so that can be run offline. + """ + app = FastAPIOffline( + docs_url=f"{route}/docs", + title=config.INFO.name.title(), + description=config.INFO.description, + version=config.INFO.version, + debug=debug, + # see https://fastapi.tiangolo.com/how-to/configure-swagger-ui + # and https://swagger.io/docs/open-source-tools/swagger-ui/usage/configuration + swagger_ui_parameters={ + "docExpansion": "list", + "defaultModelsExpandDepth": 0, + "displayRequestDuration": True, + "syntaxHighlight": True, + "syntaxHighlight.theme": "obsidian", + }, + ) + router = APIRouter() + # add_resources(router, route=route) + add_endpoints(router, route=route) + app.include_router(router, prefix=route) + return app + + +# def add_resources( +# app: FastAPI | APIRouter, +# /, +# *, +# route: str, +# ): +# """ +# Connects static resources. +# """ +# app.mount(f"{route}/index.html", StaticFiles(directory="src/app/static", html=True), name="nodejs") +# return + + +def add_endpoints( + app: FastAPI | APIRouter, + /, + *, + route: str, +): + """ + Sets all the endpoints for the API. + """ + sec_http = HTTPBasic() + tmplt = Jinja2Templates(directory="src/app/static") + + add_endpoints_basic(app, tag="Basic", route=route, sec=sec_http, tmplt=tmplt) + add_endpoints_features(app, tag="Features", route=route, sec=sec_http, tmplt=tmplt) + return diff --git a/src/app/endpoints_fastapi/decorators.py b/src/app/endpoints_fastapi/decorators.py new file mode 100644 index 0000000..58edb5a --- /dev/null +++ b/src/app/endpoints_fastapi/decorators.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import logging +from datetime import datetime +from functools import wraps +from typing import Any +from typing import Awaitable +from typing import Callable +from typing import Concatenate +from typing import ParamSpec +from typing import TypeVar + +from fastapi import HTTPException +from fastapi import Request +from fastapi.responses import JSONResponse +from fastapi.security import HTTPBasicCredentials +from pydantic import BaseModel +from safetywrap import Err +from safetywrap import Ok + +from ..._core.constants import * +from ..._core.logging import * +from ...guards.http import * +from ...models.datasources import * +from ...models.filesmanager import * +from ...models.internal import * +from ...setup import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "add_http_auth", + "catch_internal_server_error", + "output_as_bytes", + "parse_payload", +] + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS/VARIABLES +# ---------------------------------------------------------------- + +PARAMS = ParamSpec("PARAMS") +T1 = TypeVar("T1") +T2 = TypeVar("T2") +MODEL = TypeVar("MODEL") +RETURN = TypeVar("RETURN") +CODE_DEFAULT = 500 + +# ---------------------------------------------------------------- +# DECORATORS +# ---------------------------------------------------------------- + + +def catch_internal_server_error( + action: Callable[PARAMS, Awaitable[RETURN]], + /, +): + """ + Decorates and endpoint by returning internal server error if error occurs. + """ + + # modify function + @wraps(action) + async def wrapped_action( + *_: PARAMS.args, + **__: PARAMS.kwargs, + ) -> RETURN: + try: + output = await action(*_, **__) + return output + + except TypeError as err: + logging.error(err) + code = 422 + err_str = str(err) + + except ExceptionWithData as err: + logging.error(err) + err_str = str(err) + code = err.code or CODE_DEFAULT + + except Exception as err: + logging.error(err) + err_str = error_with_trace(err) + err_str = str(err) + code = 500 + + except BaseException as err: + logging.error(err) + err_str = str(err) + code = 500 + + # NOTE: headers MUST be string-valued! + headers = dict( + code=str(code), + message=err_str, + ) + raise HTTPException(status_code=code, detail=err_str, headers=headers) + + return wrapped_action + + +def output_as_bytes( + action: Callable[ + PARAMS, + Awaitable[Any], + ], + /, +): + """ + Decorates endpoint with parsed query params and deserialised body + """ + + @wraps(action) + async def wrapped_action( + *_: PARAMS.args, + **__: PARAMS.kwargs, + ) -> JSONResponse: + # run method + result = await action(*_, **__) + + # unwrap safety + code = 200 + match result: + case Err() as err: + code = 500 + result = err.unwrap_err() + + case Ok(): + result = result.unwrap() + + # pre-handle jsonisable objects + match result: + case list(): + result = AnyArray(root=result) + + case dict(): + result = AnyDictionary(root=result) + + # serialise result + match result: + case None: + contents = None + + case BaseModel(): + contents = result.model_dump( + mode="json", + by_alias=True, + exclude_none=True, + exclude_unset=False, + exclude_defaults=False, + warnings="none", + ) + + case bool() | str() | int() | float() | datetime(): + contents = result + + case bytes(): + contents = result.decode() + + case None: + contents = None + + # prepare response + response = JSONResponse(contents, status_code=code) + + return response + + return wrapped_action + + +def add_http_auth( + action: Callable[ + Concatenate[HTTPBasicCredentials, PARAMS], + Awaitable[RETURN], + ], + /, +): + """ + Decorates and endpoint by adding basic http-authorisation to it. + + **DEV-NOTE:** + Signature cannot change when using `FastAPI`'s + `@app.get`, `@app.post`, etc. decorators. + Thus need to include all arguments needed by our decorators, + even if superfluous inside undecorated part. + """ + + # modify function - but with different signature! + @wraps(action) + async def wrapped_action( + http_cred: HTTPBasicCredentials, + *_: PARAMS.args, + **__: PARAMS.kwargs, + ) -> RETURN: + try: + guard_http_credentials(http_cred) + + except Exception as err: + logging.error(err) + raise HTTPException(status_code=401, detail=err) + + output = await action(http_cred, *_, **__) + return output + + return wrapped_action + + +def parse_payload( + type_: type[BaseModel], + /, +): + """ + Parsers arbitrary payloads + """ + + async def method( + request: Request, + /, + ) -> MODEL: + try: + content_type = request.headers.get("Content-Type") + fmt = MAP_MIME_TYPE_TO_FILETYPE.get(content_type) + contents = await request.body() + + parser = PayloadParser[MODEL](type_=type_, managers=config.get_managers()) + payload = parser.parse(contents, format=fmt) + return payload + + except Exception as err: + raise TypeError(f"could not parse payload in body - {err}") + + return method diff --git a/src/app/endpoints_fastapi/endpoints_basic.py b/src/app/endpoints_fastapi/endpoints_basic.py new file mode 100644 index 0000000..dfb6b50 --- /dev/null +++ b/src/app/endpoints_fastapi/endpoints_basic.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +API endpoints basic. +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from typing import Annotated + +from fastapi import Depends +from fastapi import FastAPI +from fastapi.responses import RedirectResponse +from fastapi.routing import APIRouter +from fastapi.security import HTTPBasic +from fastapi.security import HTTPBasicCredentials +from fastapi.templating import Jinja2Templates + +from ..endpoints import endpoints_basic as ep +from .decorators import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "add_endpoints_basic", +] + +# ---------------------------------------------------------------- +# ENDPOINTS +# ---------------------------------------------------------------- + + +def add_endpoints_basic( + app: FastAPI | APIRouter, + /, + *, + tag: str, + route: str, + sec: HTTPBasic, + tmplt: Jinja2Templates, +): + """ + Adds basic endpoints. + """ + + @app.get("/", summary="", tags=[], include_in_schema=False) + async def method(): + return RedirectResponse(f"{route}/docs") + + @app.get( + "/ping", + summary="Ping api", + tags=[tag], + include_in_schema=True, + ) + @catch_internal_server_error + @output_as_bytes + async def method(): + """ + An endpoint for debugging. + """ + return "success" + + @app.get( + "/version", + summary="Display the VERSION of the programme", + tags=[tag], + include_in_schema=True, + ) + @catch_internal_server_error + @add_http_auth + async def method( + # DEV-NOTE: add for @add_http_auth-decorator + http_cred: Annotated[HTTPBasicCredentials, Depends(sec)], + # end of decorator arguments + /, + ): + version = ep.endpoint_version() + return version diff --git a/src/app/endpoints_fastapi/endpoints_features.py b/src/app/endpoints_fastapi/endpoints_features.py new file mode 100644 index 0000000..78e3225 --- /dev/null +++ b/src/app/endpoints_fastapi/endpoints_features.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +API endpoints for main features. +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from typing import Annotated + +from fastapi import Depends +from fastapi import FastAPI +from fastapi import Request +from fastapi.routing import APIRouter +from fastapi.security import HTTPBasic +from fastapi.security import HTTPBasicCredentials +from fastapi.templating import Jinja2Templates + +from ...features import * +from ...models.application import * +from .decorators import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "add_endpoints_features", +] + +# ---------------------------------------------------------------- +# ENDPOINTS +# ---------------------------------------------------------------- + + +def add_endpoints_features( + app: FastAPI | APIRouter, + /, + *, + tag: str, + route: str, + sec: HTTPBasic, + tmplt: Jinja2Templates, +): + """ + Adds endpoints pertaining to the features of the repo. + """ + + @app.post( + "/feature/search-fs", + summary="Runs the feature SEARCH-FS", + tags=[tag], + include_in_schema=True, + ) + @catch_internal_server_error + @add_http_auth + @output_as_bytes + async def method( + # DEV-NOTE: add for @add_http_auth-decorator + http_cred: Annotated[HTTPBasicCredentials, Depends(sec)], + # end of decorator arguments + /, + *, + request: Request, + ): + # process body + parser = parse_payload(RequestsPayload) + contents: RequestsPayload = await parser(request) + tasks = parse_tasks(contents) + # perform feature + result = feat_searchfs.superfeature(tasks) + return result diff --git a/src/cli.py b/src/cli.py new file mode 100644 index 0000000..41a6cd3 --- /dev/null +++ b/src/cli.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Entry point to run application via CLI +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import os +import sys +from pathlib import Path + +os.chdir(Path(__file__).parent.parent) +sys.path.insert(0, os.getcwd()) + +from ._core.logging import * +from ._core.utils.basic import * +from .features import * +from .models.application import * +from .queries import * +from .queries._console.cli import * +from .setup import * + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS +# ---------------------------------------------------------------- + +PID = os.getpid() + +# ---------------------------------------------------------------- +# EXECUTION +# ---------------------------------------------------------------- + +if __name__ == "__main__": + args = CliArguments(config.INFO).parse(*sys.argv[1:]) + + # handle simple endpoints immediately + if args.mode == EnumFeatures.VERSION: + print(config.VERSION) + exit(0) + + config.pid.set(PID) + config.path_env.set(args.env) + config.path_logging.set(args.log) + config.path_config.set(args.config) + config.path_requests.set(args.requests) + config.initialise_application( + name="app", + serialise=False, + log_to_files=True, + verbose=args.verbose, + ) + + match args.mode: + case EnumFeatures.SEARCH_FS: + payload = config.parser_requests().parse() + tasks = parse_tasks(payload) + feat_searchfs.superfeature(tasks) + + case _ as mode: + raise NotImplementedError(f"no feature implemented for {extract_string(mode)}") diff --git a/src/features/__init__.py b/src/features/__init__.py new file mode 100644 index 0000000..deae3e7 --- /dev/null +++ b/src/features/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Submodule containing all features. +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +# NOTE: only import/export the submodules which are called as such +from . import feat_searchfs + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "feat_searchfs", +] diff --git a/src/features/feat_searchfs/__init__.py b/src/features/feat_searchfs/__init__.py new file mode 100644 index 0000000..00ff6fe --- /dev/null +++ b/src/features/feat_searchfs/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Submodule for the IFC-TO-{FILE} feature +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from .superfeature import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "superfeature_direct", + "superfeature_queue", +] diff --git a/src/features/feat_searchfs/feature.py b/src/features/feat_searchfs/feature.py new file mode 100644 index 0000000..7da09ff --- /dev/null +++ b/src/features/feat_searchfs/feature.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import logging + +from safetywrap import Err +from safetywrap import Ok +from safetywrap import Result + +from ..._core.logging import * +from ...models.application import * +from ...models.filesmanager import * +from ...models.internal.errors import * +from ...setup import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "feature", +] + +# ---------------------------------------------------------------- +# FEATURE +# ---------------------------------------------------------------- + + +@echo_function( + tag="FEATURE - SEARCH-FS | '{label}'", + level="INFO", + depth=0, +) +def feature( + *, + label: str, + ref_inputs: FileRef, + options: RequestTaskOptions, +) -> Result[str, str]: + """ + Feature `SEARCH-FS` + """ + try: + managers = config.get_managers() + cfg_general = config.parser_config().parse() + + raise NotImplementedError("feature SEARCH-FS not yet implemented") + + return Ok("success") + + except ExceptionWithData as err: + msg = f"task '{label}' failed with error code {err.code or 500} - {err}" # fmt: skip + logging.error(msg) + return Err(msg) + + except Exception as err: + msg = f"task '{label}' failed - {err}" # fmt: skip + logging.error(msg) + return Err(msg) + + except BaseException as err: + # DEV-NOTE: pass on all other kinds of exceptions + raise err diff --git a/src/features/feat_searchfs/superfeature.py b/src/features/feat_searchfs/superfeature.py new file mode 100644 index 0000000..9dbffc5 --- /dev/null +++ b/src/features/feat_searchfs/superfeature.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import logging + +from safetywrap import Err +from safetywrap import Ok +from safetywrap import Result + +from ..._core.utils.code import * +from ...models.application import * +from .feature import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "superfeature", +] + +# ---------------------------------------------------------------- +# WRAPPED FEATURES +# ---------------------------------------------------------------- + + +def superfeature( + tasks: list[RequestTask], + /, +) -> Result[str, list[str]]: + """ + Calls `SEARCH-FS` features for a list of tasks + """ + errors = list[str]() + n_tot = len(tasks) + for task in tasks: + result = feature( + label=task.label, + ref_inputs=task.data.inputs, + options=task.options, + ) + + if isinstance(result, Err): + errors.append(result.unwrap_err()) + + if (n := len(errors)) > 0: + match n, n_tot: + case 1, 1: + # NOTE: logging superfluous + pass + + case _, _ if n == n_tot: + logging.warning(f"all of the {n_tot} tasks failed") + + case _: + logging.warning(f"{n} of the {n_tot} tasks failed") + + return Err(errors) + + return Ok("success") diff --git a/src/guards/__init__.py b/src/guards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/guards/http.py b/src/guards/http.py new file mode 100644 index 0000000..4a3cf57 --- /dev/null +++ b/src/guards/http.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from fastapi.security import HTTPBasicCredentials + +from ..setup import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "guard_http_credentials", + "guard_http_password", + "guard_http_user", +] + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def guard_http_user(value: str, /): + """ + A guard which checks if http username is valid. + """ + value_expected = config.http_user() + if value != value_expected: + raise ValueError("Invalid http username") + + +def guard_http_password(value: str, /): + """ + A guard which checks if http password is valid. + """ + value_expected = config.http_password().get_secret_value() + if value != value_expected: + raise ValueError("Invalid http password!") + + +def guard_http_credentials( + cred: HTTPBasicCredentials, + /, +): + """ + A guard which checks if http credentials are valid. + """ + try: + guard_http_user(cred.username) + guard_http_password(cred.password) + + except Exception as err: + # msg = str(err) + msg = "Invalid http credentials!" + raise ValueError(msg) diff --git a/src/models/__init__.py b/src/models/__init__.py new file mode 100644 index 0000000..2e49883 --- /dev/null +++ b/src/models/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +This module contains various (nearly) self-contained models. +""" diff --git a/src/models/application/__init__.py b/src/models/application/__init__.py new file mode 100644 index 0000000..ec0afc1 --- /dev/null +++ b/src/models/application/__init__.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from ..generated.application import EnumFeatures +from ..generated.application import GeneralConfig +from ..generated.application import RepoInfo +from ..generated.application import RequestTask +from ..generated.application import RequestTaskData +from ..generated.application import RequestTaskOptions +from ..generated.application import RequestsPayload + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "EnumFeatures", + "GeneralConfig", + "RepoInfo", + "RequestTask", + "RequestTaskData", + "RequestTaskOptions", + "RequestsPayload", + "parse_tasks", +] + + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def parse_tasks(payload: RequestsPayload, /) -> list[RequestTask]: + """ + Given a payload retuns the list of tasks it encodes + """ + match payload.root: + case list() as tasks: + return tasks + + case _ as task: + return [task] diff --git a/src/models/datasources/__init__.py b/src/models/datasources/__init__.py new file mode 100644 index 0000000..978ab5c --- /dev/null +++ b/src/models/datasources/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from .any import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "AnyArray", + "AnyDataFrame", + "AnyDictionary", + "AnyEntity", +] diff --git a/src/models/datasources/any.py b/src/models/datasources/any.py new file mode 100644 index 0000000..a9591ba --- /dev/null +++ b/src/models/datasources/any.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from typing import Any + +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import RootModel + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "AnyArray", + "AnyDataFrame", + "AnyDictionary", + "AnyEntity", +] + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + + +class AnyEntity(BaseModel): + """ + Dummy model for parsing any entity + """ + + model_config = ConfigDict( + use_enum_values=True, + ) + + value: Any + + +class AnyDictionary(BaseModel): + """ + Dummy model for parsing dictionaries + """ + + model_config = ConfigDict( + extra="allow", + populate_by_name=True, + use_enum_values=True, + ) + + +class AnyArray(RootModel[list[BaseModel]]): + """ + Dummy model for parsing arrays + """ + + model_config = ConfigDict( + use_enum_values=True, + ) + + root: list[Any] + + +class AnyDataFrame(RootModel[list[AnyDictionary]]): + """ + Structure of tasks requests.yaml configuration file. + """ + + model_config = ConfigDict( + populate_by_name=True, + use_enum_values=True, + ) + root: list[AnyDictionary] diff --git a/src/models/filesmanager/__init__.py b/src/models/filesmanager/__init__.py new file mode 100644 index 0000000..b201ab4 --- /dev/null +++ b/src/models/filesmanager/__init__.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +This submodule provides the generic FilesManager interface +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from ..generated.application import EnumDataFileFormat +from ..generated.application import EnumFilesSystem +from ..generated.application import FileRef +from ..generated.application import MetaData +from ..generated.application import ProxyConfig +from .config import * +from .os import * +from .payloads import * +from .traits import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "ConfigLoader", + "EnumDataFileFormat", + "EnumFilesSystem", + "FileRef", + "FilesManager", + "FilesManagerFile", + "FilesManagerFolder", + "MetaData", + "OSFilesManager", + "OSFilesManagerFile", + "OSFilesManagerFolder", + "PayloadParser", + "ProxyConfig", +] diff --git a/src/models/filesmanager/config.py b/src/models/filesmanager/config.py new file mode 100644 index 0000000..41258df --- /dev/null +++ b/src/models/filesmanager/config.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import logging +from typing import Generic +from typing import TypeVar + +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import SkipValidation + +from ..._core.utils.io import * +from ..generated.application import EnumDataFileFormat +from ..generated.application import EnumFilesSystem +from ..generated.application import ProxyConfig +from .traits import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "ConfigLoader", +] + +# ---------------------------------------------------------------- +# CONSTANTS +# ---------------------------------------------------------------- + +T = TypeVar("T") + +# ---------------------------------------------------------------- +# CLASSES +# ---------------------------------------------------------------- + + +class ConfigLoader(BaseModel, Generic[T]): + """ + A generic class which provides a recursive method to load configs. + + ## Example load form file ## + ```py + loc = "..." # one off "OS", "SHAREPOINT", ... + path = "..." # path in location + managers = { + "OS": ..., + "SHAREPOINT": ..., + } + loader = ConfigLoader[GeneralConfig](managers=managers, type_=GeneralConfig) + cfg = loader.load_from_file(loc=loc, path=path) # will be type GeneralConfig + ``` + + ## Example load form file ## + ```py + contents = b"..." # contents of file as bytes + managers = { + "OS": ..., + "SHAREPOINT": ..., + } + loader = ConfigLoader[GeneralConfig](managers=managers, type_=GeneralConfig) + cfg = loader.load_from_contents(contents) # will be type GeneralConfig + ``` + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + arbitrary_types_allowed=True, + ) + + managers: dict[EnumFilesSystem, SkipValidation[FilesManager]] + type_: type[BaseModel] + + def get_file_contents( + self, + /, + *, + loc: EnumFilesSystem, + path: str, + ) -> tuple[bytes, EnumDataFileFormat]: + """ + Given a reference gets the file contents and file format. + + The absolute path is determined as follows: + """ + # determine path + manager = self.managers[loc] + + # get file extension -> format of file + _, _, ext = manager.__class__.path_split(path) + + # get file and contents + try: + file = manager.get_file(path) + contents = file.read_as_bytes() + + except Exception as err: + logging.error(f"could not load or read {loc}-file in '{path}' - {err}") + raise err + + try: + fmt = EnumDataFileFormat(ext) + + except Exception as err: + logging.error(f"unrecognised format {ext} - {err}") + raise err + + return contents, fmt + + def load_from_file( + self, + /, + *, + loc: EnumFilesSystem, + path: str, + fmt: EnumDataFileFormat | None = None, + chain: list[tuple[EnumFilesSystem, str]] | None = None, + ) -> T: + """ + Extracts config from file. + + NOTE: if `ref` attribute is set, will recursively extract referenced config-file. + """ + # for chain of references + if chain is None: + chain = [] + + # access file and read contents + contents, fmt_ = self.get_file_contents(loc=loc, path=path) + fmt = fmt or fmt_ + + # load (recursively) from contents + cfg = self.load_from_contents(contents, fmt=fmt, chain=chain) + + return cfg + + def load_from_contents( + self, + contents: bytes, + /, + *, + fmt: EnumDataFileFormat, + chain: list[tuple[EnumFilesSystem, str]] | None = None, + ) -> T: + """ + Extracts config from file-contents optionally parsed. + + NOTE: if `ref` attribute is set, will recursively extract referenced config-file. + """ + # for chain of references + if chain is None: + chain = [] + + # parse bytes -> dictionary + assets = parse_contents(contents, format=fmt.value) + + # parse dictionary -> model + try: + cfg = self.type_.model_validate(assets) + + except Exception as err: + try: + cfg = ProxyConfig.model_validate(assets) + + except Exception as _: + # raise first error! + raise err + + if isinstance(proxy := cfg, ProxyConfig): + cfg = self.load_from_proxy(proxy, chain=chain) + + return cfg + + def load_from_proxy( + self, + proxy: ProxyConfig, + /, + *, + chain: list[tuple[EnumFilesSystem, str]] | None = None, + ) -> T: + """ + Extracts config from a proxy to a file. + + NOTE: if `ref` attribute is set, will recursively extract referenced config-file. + """ + # for chain of references + if chain is None: + chain = [] + + loc = proxy.ref.location + path = proxy.ref.path + fmt = proxy.ref.format + if (loc, path) in chain: + chain_str = ' -> '.join([f"{loc_}/{path_}" for loc_, path_ in [*chain, (loc, path)]]) # fmt: skip + raise Exception(f"circular reference encountered whilst importing config {chain_str}!") # fmt: skip + + # recursive call + chain.append((loc, path)) + cfg = self.load_from_file(loc=loc, path=path, fmt=fmt, chain=chain) # fmt: skip + + return cfg diff --git a/src/models/filesmanager/os/__init__.py b/src/models/filesmanager/os/__init__.py new file mode 100644 index 0000000..d91e5a1 --- /dev/null +++ b/src/models/filesmanager/os/__init__.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +This submodule provides a realisation of the FilesManager interface for local operating system +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from .classes import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "OSFilesManager", + "OSFilesManagerFile", + "OSFilesManagerFolder", +] diff --git a/src/models/filesmanager/os/classes.py b/src/models/filesmanager/os/classes.py new file mode 100644 index 0000000..d8cde41 --- /dev/null +++ b/src/models/filesmanager/os/classes.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from __future__ import annotations + +import os +from datetime import datetime +from datetime import timezone +from pathlib import Path + +from pydantic import AwareDatetime + +from ...._core.constants import * +from ...._core.utils.code import * +from ...._core.utils.time import * +from ...generated.application import MetaData + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "OSFilesManager", + "OSFilesManagerFile", + "OSFilesManagerFolder", +] + +# ---------------------------------------------------------------- +# CLASSES +# ---------------------------------------------------------------- + + +class OSFilesManager: + """ + File system for a local operating system + """ + + _timezone: timezone | None + + def __init__(self, tz: timezone | None = None): + self._timezone = tz + return + + @staticmethod + def path_split(path: str, /) -> tuple[str, str, str]: + """ + Splits a full path into (absolute directory, basename, ext). + """ + path = path.strip().rstrip(r"\/") + filename = os.path.basename(path) + path = os.path.dirname(path) or "." + basename, ext = os.path.splitext(filename) + return path, basename, ext + + @staticmethod + def path_split_root(path: str, /) -> tuple[str, str]: + """ + Splits a full path into (root, relative path). + """ + return "", path + + @staticmethod + def path_join(*path: str) -> str: + """ + Static method to combine parts of path + """ + return Path(*path).as_posix() or "." + + @staticmethod + def path_rel(root: str, path: str, /) -> list[str]: + """ + Static method to compute series of subpaths from a root to a given path + """ + try: + rel = Path(path).relative_to(root) + parts = list(rel.parts) + return parts + + except Exception as _: + return [] + + def get_file(self, *path: str) -> OSFilesManagerFile: + tz = self._timezone + path_full = Path(*path).as_posix() or "." + path_full = path_full.strip().rstrip(r"\/") + return OSFilesManagerFile(path=path_full, tz=tz) + + def get_folder(self, *path: str) -> OSFilesManagerFolder: + tz = self._timezone + path_full = Path(*path).as_posix() or "." + path_full = path_full.strip().rstrip(r"\/") + return OSFilesManagerFolder(self, path=path_full, tz=tz) + + def create_folder(self, path: str, /) -> OSFilesManagerFolder: + """ + Use files manager to create folder by full path. + First checks if folder already exists. + """ + path = path.strip().rstrip(r"\/") + p = Path(path) + p.mkdir(parents=True, exist_ok=True) + return self.get_folder(path) + + def create_file( + self, + contents: bytes, + /, + *, + path: str, + chunk: int = 10 * SIZE_1_MB, + ) -> OSFilesManagerFile: + """ + Use files manager to create file by full path + """ + path, basename, ext = OSFilesManager.path_split(path) + filename = f"{basename}{ext}" + # first ensure folder exists + folder = self.create_folder(path) + # next create file within folder + file = folder.write_bytes(contents, name=filename, chunk=chunk) + return file + + +class OSFilesManagerFile: + """ + File manager for a local operating system + """ + + _path: str + _timezone: timezone | None + _object: Path + + def __init__( + self, + /, + *, + path: str, + tz: timezone | None, + ): + assert path != "", "Path cannot be empty!" + self._path = path + self._timezone = tz + return + + @staticmethod + def path_split(path: str, /) -> tuple[str, str, str]: + return OSFilesManager.path_split(path) + + @staticmethod + def path_split_root(path: str, /) -> tuple[str, str]: + """ + Splits a full path into (root, relative path). + """ + return OSFilesManager.path_split_root(path) + + @property + def exists(self) -> bool | None: + """ + Whether or not the file exists (unknown -> `None`) + """ + try: + return os.path.exists(self.path) + except Exception: + return None + + @property + def path(self) -> str: + return self._path + + @property + def directory(self) -> str: + """ + Gets basepath of file + """ + return os.path.dirname(self._path) + + @property + def filename(self) -> str: + """ + Gets basename of file (including extension) + """ + return os.path.basename(self._path) + + @property + def basename(self) -> str: + """ + Gets basename of file (including extension) + """ + basename, _ = os.path.splitext(self.filename) + return basename + + @property + def ext(self) -> str: + """ + Gets file extension + """ + _, ext = os.path.splitext(self._path) + return ext + + @property + def size(self) -> int: + meta = os.stat(self._path) + return meta.st_size + + @property + def author(self) -> str | None: + """ + Gets file author + """ + try: + # NOTE: only works on linux + p = Path(self._path) + return p.owner() + + except Exception as _: + return None + + @property + def author_id(self) -> int | None: + """ + Gets file author id + """ + return None + + @property + def date_created(self) -> AwareDatetime | None: + meta = os.stat(self._path) + + try: + # NOTE: only works for some OS's + t = datetime.fromtimestamp(meta.st_birthtime) + + except Exception as _: + t = datetime.fromtimestamp(meta.st_ctime) + return add_timezone(t, tz=self._timezone) + + @property + def date_modified(self) -> AwareDatetime | None: + meta = os.stat(self._path) + t = datetime.fromtimestamp(meta.st_mtime) + return add_timezone(t, tz=self._timezone) + + @make_lazy + def get_meta_data(self) -> MetaData: + """ + Gets bundled meta data associated to file. + """ + return MetaData( + filename=self.filename, + basename=self.basename, + ext=self.ext, + size=self.size, + author=self.author, + author_id=self.author_id, + time_created=self.date_created, + time_updated=self.date_modified, + ) + + def read_as_bytes(self) -> bytes: + """ + Downloads file contents as bytes + """ + with open(self._path, "rb") as fp: + contents = fp.read() + return contents + + def delete_self(self) -> bool: + """ + Deletes current file + """ + if not self.exists: + return True + try: + os.remove(self._path) + ex = self.exists + return False if ex is None else not ex + + except Exception: + return False + + +class OSFilesManagerFolder: + """ + Folder manager for a local operating system + """ + + _manager: OSFilesManager + _path: str + _timezone: timezone | None + _filenames: list[str] | None + + def __init__( + self, + manager: OSFilesManager, + /, + *, + path: str, + tz: timezone | None, + ): + assert path != "", "Path cannot be empty!" + self._manager = manager + self._path = path + self._timezone = tz + self._filenames = None + return + + @property + def exists(self) -> bool | None: + """ + Whether or not the folder exists (unknown -> `None`) + """ + try: + return os.path.exists(self.path) + except Exception: + return None + + @property + def path(self) -> str: + return self._path + + @property + def name(self) -> str: + return os.path.basename(self._path) + + @property + def subfolders(self) -> list[OSFilesManagerFolder]: + tz = self._timezone + names = os.listdir(self._path) + paths = [Path(self._path, name).as_posix() for name in names] + paths = [path for path in paths if Path(path).is_dir()] + return [OSFilesManagerFolder(self._manager, path=path, tz=tz) for path in paths] + + def get_subfolder(self, name: str) -> OSFilesManagerFolder: + return self._manager.get_folder(Path(self._path, name).as_posix()) + + @property + def files(self) -> list[OSFilesManagerFile]: + tz = self._timezone + names = os.listdir(self._path) + paths = [Path(self._path, name).as_posix() for name in names] + paths = [path for path in paths if Path(path).is_file()] + return [OSFilesManagerFile(path=path, tz=tz) for path in paths] + + @property + def filenames(self) -> list[str]: + names = os.listdir(self._path) + paths = [Path(self.path, name).as_posix() for name in names] + self._filenames = [name for name, path in zip(names, paths) if Path(path).is_file()] + return self._filenames + + def has_file(self, file: OSFilesManagerFile) -> bool: + return file.filename in self.filenames + + def get_file(self, name: str) -> OSFilesManagerFile: + return self._manager.get_file(Path(self._path, name).as_posix()) + + @make_lazy + def get_files_meta_data(self) -> list[MetaData]: + """ + Gets a list of metadata associated to files + """ + return [file.get_meta_data() for file in self.files] + + def write_bytes( + self, + contents: bytes, + /, + *, + name: str, + chunk: int = 10 * SIZE_1_MB, + ) -> OSFilesManagerFile: + path = Path(self._path, name).as_posix() + with open(path, "wb") as fp: + fp.write(contents) + + return OSFilesManagerFile(path=path, tz=self._timezone) + + def add_subfolder(self, name: str) -> OSFilesManagerFolder: + """ + Adds subfolder and returns a manager for it. + If subfolder already exists, it will not be created. + """ + path = Path(self._path, name).as_posix() + p = Path(path) + p.mkdir(parents=True, exist_ok=True) + return OSFilesManagerFolder( + self._manager, + path=path, + tz=self._timezone, + ) + + def clear_folder(self) -> bool: + """ + Removes all contents of current folder + """ + success = True + for file in self.files: + success = success and file.delete_self() + for subfolder in self.subfolders: + success = success and subfolder.delete_self() + return success + + def delete_self(self) -> bool: + """ + Deletes current folder + """ + if not self.exists: + return True + success = self.clear_folder() + if not success: + return False + if not self.exists: + return True + os.rmdir(self._path) + ex = self.exists + not_ex = False if ex is None else not ex + return success and not_ex diff --git a/src/models/filesmanager/payloads.py b/src/models/filesmanager/payloads.py new file mode 100644 index 0000000..eadc73c --- /dev/null +++ b/src/models/filesmanager/payloads.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + + +from functools import wraps +from typing import Callable +from typing import Concatenate +from typing import Generic +from typing import ParamSpec +from typing import TypeVar + +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import SkipValidation + +from ..._core.constants import * +from ..._core.utils.io import * +from ..generated.application import EnumDataFileFormat +from ..generated.application import EnumFilesSystem +from .config import * +from .traits import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "PayloadParser", +] + +# ---------------------------------------------------------------- +# CONSTANTS +# ---------------------------------------------------------------- + +T = TypeVar("T") +PARAMS = ParamSpec("PARAMS") +MODEL = TypeVar("MODEL") +RETURN = TypeVar("RETURN") + +# ---------------------------------------------------------------- +# CLASSES +# ---------------------------------------------------------------- + + +class PayloadParser(BaseModel, Generic[T]): + """ + Provides a method to parse payloads e.g. in endpoints. + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + arbitrary_types_allowed=True, + ) + + type_: type[BaseModel] + managers: dict[EnumFilesSystem, SkipValidation[FilesManager]] + location: EnumFilesSystem | None = None + root: str | None = None + + def parse( + self, + contents: bytes | T | None = None, + /, + *, + format: BASIC_FILETYPES | None = None, + ) -> T: + """ + Computes the payload for task insertion. There are 3 cases: + + - `contents ~ ` -> returns this + - `contents = null` -> reads from local file in setup/... + - `contents ~ bytes` -> parses payload. + """ + managers = self.managers + loc = self.location + root = self.root + + try: + fmt = EnumDataFileFormat(format) + + except Exception as _: + fmt = None + + match contents: + case BaseModel(): + assert isinstance(contents, self.type_) + return contents + + case None: + assert (loc is not None) and (root is not None), \ + f"need to set location and path in PayloadParser for {self.type_.__name__}" # fmt: skip + loader = ConfigLoader[T](managers=managers, type_=self.type_) + result = loader.load_from_file(loc=loc, path=root, fmt=fmt) + return result + + case _: + fmt = fmt or EnumDataFileFormat.FIELD_JSON + loader = ConfigLoader[T](managers=managers, type_=self.type_) + result = loader.load_from_contents(contents, fmt=fmt) + return result + + def add_config_from_path( + self, + action: Callable[Concatenate[MODEL, PARAMS], RETURN], + /, + ): + """ + Decorates method by adding config to it. + + NOTE: Only works for if obtaining configs from a path. + """ + + @wraps(action) + def wrapped_action(*_: PARAMS.args, **__: PARAMS.kwargs) -> RETURN: + cfg = self.parse() + result = action(cfg, *_, *__) + return result + + return wrapped_action diff --git a/src/models/filesmanager/traits.py b/src/models/filesmanager/traits.py new file mode 100644 index 0000000..8ccf0c3 --- /dev/null +++ b/src/models/filesmanager/traits.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +This module contains interfaces/types +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from __future__ import annotations + +from typing import Protocol + +from pydantic import AwareDatetime + +from ..._core.constants import * +from ..generated.application import MetaData + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "FilesManager", + "FilesManagerFile", + "FilesManagerFolder", +] + +# ---------------------------------------------------------------- +# CLASSES +# ---------------------------------------------------------------- + + +class FilesManager(Protocol): + """ + Interface for a generic file system manager + """ + + @staticmethod + def path_split(path: str, /) -> tuple[str, str, str]: + """ + Splits a full path into (absolute directory, basename, ext). + """ + ... + + @staticmethod + def path_split_root(path: str, /) -> tuple[str, str]: + """ + Splits a full path into (root, relative path). + """ + ... + + @staticmethod + def path_join(*path: str) -> str: + """ + Static method to combine parts of path + """ + ... + + @staticmethod + def path_rel(root: str, path: str, /) -> list[str]: + """ + Static method to compute series of subpaths from a root to a given path + """ + ... + + def get_folder(self, *path: str) -> FilesManagerFolder: + """ + Use files manager to get folder by full path + """ + ... + + def get_file(self, *path: str) -> FilesManagerFile: + """ + Use files manager to get file by full path + """ + ... + + def create_folder(self, path: str, /) -> FilesManagerFolder: + """ + Use files manager to create folder by full path. + First checks if folder already exists. + """ + ... + + def create_file( + self, + contents: bytes, + /, + *, + path: str, + chunk: int = 10 * SIZE_1_MB, + ) -> FilesManagerFile: + """ + Use files manager to create file by full path + """ + ... + + +class FilesManagerFile(Protocol): + """ + Interface for a generic file manager + """ + + @staticmethod + def path_split(path: str, /) -> tuple[str, str, str]: + """ + Splits a full path into (absolute directory, basename, ext). + """ + ... + + @staticmethod + def path_split_root(path: str, /) -> tuple[str, str]: + """ + Splits a full path into (root, relative path). + """ + ... + + @property + def exists(self) -> bool | None: + """ + Whether or not the file exists (unknown -> `None`) + """ + ... + + @property + def path(self) -> str: + """ + Gets path locator to file + """ + ... + + @property + def directory(self) -> str: + """ + Gets basepath of file + """ + ... + + @property + def filename(self) -> str: + """ + Gets filename of file (includes extension) + """ + ... + + @property + def basename(self) -> str: + """ + Gets basename of file (excludes extension) + """ + ... + + @property + def ext(self) -> str: + """ + Gets file extension + """ + ... + + @property + def size(self) -> int: + """ + Gets meta attribute - size of file + """ + ... + + @property + def author(self) -> str | None: + """ + Gets file author + """ + ... + + @property + def author_id(self) -> int | None: + """ + Gets file author id + """ + ... + + @property + def date_created(self) -> AwareDatetime | None: + """ + Gets meta attribute - date of creation + """ + ... + + @property + def date_modified(self) -> AwareDatetime | None: + """ + Gets meta attribute - date of (last) modification + """ + ... + + def get_meta_data(self) -> MetaData: + """ + Gets bundled meta data associated to file. + """ + ... + + def read_as_bytes(self) -> bytes: + """ + Reads file contents to bytes + """ + ... + + def delete_self(self) -> bool: + """ + Deletes current file + """ + ... + + +class FilesManagerFolder(Protocol): + """ + Interface for a generic folder manager + """ + + @property + def exists(self) -> bool | None: + """ + Whether or not the folder exists (unknown -> `None`) + """ + ... + + @property + def path(self) -> str: + """ + Gets path locator to folder + """ + ... + + @property + def name(self) -> str: + """ + Gets name identifier of folder + """ + ... + + @property + def subfolders(self) -> list[FilesManagerFolder]: + """ + Gets list of subfolder within folder + """ + ... + + def get_subfolder(self, name: str) -> FilesManagerFolder: + """ + Gets subfolder by name within folder + """ + ... + + @property + def files(self) -> list[FilesManagerFile]: + """ + Gets list of files within folder + """ + ... + + @property + def filenames(self) -> list[str]: + """ + Returns names of all files in folder. + """ + ... + + def has_file(self, file: FilesManagerFile) -> bool: + """ + Checks if file of given name exists in folder. + """ + ... + + def get_file(self, name: str) -> FilesManagerFile: + """ + Gets file by name within folder + """ + ... + + def get_files_meta_data(self) -> list[MetaData]: + """ + Gets a list of metadata associated to files + """ + ... + + def write_bytes( + self, + contents: bytes, + /, + *, + name: str, + chunk: int, + ) -> FilesManagerFile: + """ + Writes file contents to a folder given contents as bytes + """ + ... + + def add_subfolder(self, name: str) -> FilesManagerFolder: + """ + Adds subfolder and returns a manager for it. + If subfolder already exists, it will not be created. + """ + ... + + def clear_folder(self) -> bool: + """ + Removes all contents of current folder + """ + ... + + def delete_self(self) -> bool: + """ + Deletes current folder + """ + ... diff --git a/src/models/generated/application.py b/src/models/generated/application.py index 069fcfc..94026b1 100644 --- a/src/models/generated/application.py +++ b/src/models/generated/application.py @@ -4,8 +4,9 @@ from __future__ import annotations from enum import Enum +from typing import Any -from pydantic import AnyUrl, BaseModel, ConfigDict +from pydantic import AnyUrl, BaseModel, ConfigDict, Field, RootModel class Urls(BaseModel): @@ -30,11 +31,154 @@ class RepoInfo(BaseModel): urls: Urls +class GeneralConfig(BaseModel): + """ + Structure of configuration of application for use with features. + + NOTE: not yet implemented + """ + + model_config = ConfigDict( + extra="allow", + populate_by_name=True, + ) + version: str = Field( + default="X.Y.Z", + description="User defined version. Bump this value with every change to the config.", + ) + + +class RequestTaskOptions(BaseModel): + """ + Structure of requests payload > options + + NOTE: not yet implemented + """ + + model_config = ConfigDict( + extra="allow", + populate_by_name=True, + ) + + +class MetaData(BaseModel): + """ + Struct containing information about an object in a filesystem + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + ) + filename: str = Field( + ..., description="Filename (without path, but with extension)" + ) + basename: str = Field( + ..., description="Filename without path and without extension" + ) + ext: str = Field(..., description="Extension of file") + size: Any = Field(..., description="Size of file in bytes") + author: str | None = Field(default=None, description="Author of file") + author_id: str | None = Field(default=None, description="Id of author of file") + time_created: Any | None = Field(default=None, alias="time-created") + time_updated: Any | None = Field(default=None, alias="time-updated") + + +class EnumFeatures(str, Enum): + """ + Enumeration of features + """ + + VERSION = "version" + SEARCH_FS = "SEARCH-FS" + + +class EnumDataFileFormat(str, Enum): + """ + Enumeration of data file formats. + """ + + FIELD_JSON = ".json" + FIELD_YAML = ".yaml" + FIELD_TOML = ".toml" + FIELD_XML = ".xml" + FIELD_PARQUET = ".parquet" + FIELD_CSV = ".csv" + FIELD_XLSX = ".xlsx" + + class EnumFilesSystem(str, Enum): """ Location of file system """ OS = "OS" - BLOB = "BLOB" + BLOB_STORAGE = "BLOB-STORAGE" SHAREPOINT = "SHAREPOINT" + + +class FileRef(BaseModel): + """ + Structured reference to a file + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + ) + location: EnumFilesSystem = Field( + default=EnumFilesSystem.OS, + description="Which files management system is used to locate the file", + ) + path: str = Field(default=".", description="Absolute path to file.") + format: EnumDataFileFormat | None = Field( + default=None, + description='Optional format of file to be loaded (e.g. `".json"` or `".yaml"`).', + ) + + +class RequestTaskData(BaseModel): + """ + Structure of requests payload > data + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + ) + inputs: FileRef + + +class ProxyConfig(BaseModel): + """ + A proxy config which simply links to another config file. + """ + + model_config = ConfigDict( + extra="allow", + populate_by_name=True, + ) + ref: FileRef + + +class RequestTask(BaseModel): + """ + Structure of requests payload + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + ) + label: str = Field(..., description="Label of task") + options: RequestTaskOptions + data: RequestTaskData + + +class RequestsPayload(RootModel[RequestTask | list[RequestTask]]): + model_config = ConfigDict( + populate_by_name=True, + ) + root: RequestTask | list[RequestTask] = Field( + ..., description="Structure of requests payload" + ) diff --git a/src/models/internal/__init__.py b/src/models/internal/__init__.py new file mode 100644 index 0000000..69427a7 --- /dev/null +++ b/src/models/internal/__init__.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +This submodule provides the methods to be used in setup/config.py +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from .decorators import * +from .errors import * +from .temp import * +from .traits import * +from .trees import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "ExceptionWithData", + "GenericTree", + "Property", + "TempNameGenerator", + "TriggerProperty", + "convert_notes_to_exception", + "mark_errors", + "perform_action_on_error", + "temp_name", +] diff --git a/src/models/internal/decorators.py b/src/models/internal/decorators.py new file mode 100644 index 0000000..f1fdd61 --- /dev/null +++ b/src/models/internal/decorators.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from functools import wraps +from typing import Callable +from typing import Concatenate +from typing import ParamSpec +from typing import TypeVar + +from ..._core.logging import * +from .traits import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "mark_errors", + "perform_action_on_error", +] + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS +# ---------------------------------------------------------------- + +RETURN = TypeVar("RETURN") +T = TypeVar("T") +PARAMS = ParamSpec("PARAMS") + +# ---------------------------------------------------------------- +# CLASSES +# ---------------------------------------------------------------- + + +def mark_errors(has_error: TriggerProperty, /): + """ + Decorates method by intercepting and marking Exceptions. + """ + + def dec(method: Callable[PARAMS, RETURN]) -> Callable[PARAMS, RETURN]: + @wraps(method) + def wrapped_method(*_: PARAMS.args, **__: PARAMS.kwargs) -> RETURN: + try: + value = method(*_, **__) + return value + + except Exception as err: + has_error.set() + log_debug_wrapped_args(err, *_, **__) + raise err + + return wrapped_method + + return dec + + +def perform_action_on_error( + action: Callable[Concatenate[Exception, PARAMS], None], +): + """ + Decorates method by intercepting and marking Exceptions. + If an exception occurs, performs action, then raises error. + """ + + def dec(method: Callable[PARAMS, RETURN]) -> Callable[PARAMS, RETURN]: + @wraps(method) + def wrapped_method(*_: PARAMS.args, **__: PARAMS.kwargs) -> RETURN | None: + try: + value = method(*_, **__) + return value + + except Exception as err: + action(err, *_, **__) + raise err + + return wrapped_method + + return dec diff --git a/src/models/internal/errors.py b/src/models/internal/errors.py new file mode 100644 index 0000000..8fe71c2 --- /dev/null +++ b/src/models/internal/errors.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from functools import wraps +from typing import Callable +from typing import Generic +from typing import ParamSpec +from typing import TypeVar +from typing import Union + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "ExceptionWithData", + "convert_notes_to_exception", +] + +# ---------------------------------------------------------------- +# LOCAL TYPES +# ---------------------------------------------------------------- + +PARAMS = ParamSpec("PARAMS") +JSON_TYPE_BASIC = Union[None, bool, str, int, float] +JSON_TYPE = Union[JSON_TYPE_BASIC, list[JSON_TYPE_BASIC], dict[str, JSON_TYPE_BASIC]] +NOTES = TypeVar("NOTES", bound=JSON_TYPE) + +# ---------------------------------------------------------------- +# CLASSES +# ---------------------------------------------------------------- + + +class ExceptionWithData(Exception, Generic[NOTES]): + """ + Generic error class with data + """ + + _data: dict[str, NOTES] + + def __init__(self, *_, **__): + super(Exception, self).__init__(*_, **__) + self._data = dict[str, NOTES]() + + @property + def code(self) -> int | None: + """ + Error code attached to exception + """ + code = self.get_data("code") + if isinstance(code, int): + return int(code) + return None + + @code.setter + def code(self, x: int): + """ + Attach error code to exception + """ + self.add_data("code", None) + if isinstance(x, int): + self.add_data("code", x) + + @property + def data(self) -> dict[str, NOTES]: + """ + Data attached to exception + """ + return self._data + + @data.setter + def data(self, x: dict[str, NOTES], /): + """ + Attach data to exception + """ + self._data = x + + def get_data( + self, + key: str, + /, + ) -> NOTES: + """ + Get value of data by key + """ + return self._data.get(key) + + def add_data( + self, + key: str, + value: NOTES, + /, + ): + """ + Add data to exception + """ + self._data.update({key: value}) + + +# ---------------------------------------------------------------- +# DECORATORS +# ---------------------------------------------------------------- + + +def convert_notes_to_exception( + method: Callable[PARAMS, dict[str, NOTES]], + /, +): + """ + Decorates a validation method, converting notes into exception-notes + """ + + @wraps(method) + def wrapped_method(*_: PARAMS.args, **__: PARAMS.kwargs): + notes = method(*_, **__) + if len(notes) == 0: + return + + err = ExceptionWithData[NOTES]("data invalid") + err.data = notes + raise err + + return wrapped_method diff --git a/src/models/internal/temp.py b/src/models/internal/temp.py new file mode 100644 index 0000000..97a8ade --- /dev/null +++ b/src/models/internal/temp.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from typing import Iterable + +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import Field + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "TempNameGenerator", + "temp_name", +] + +# ---------------------------------------------------------------- +# METHODS - STRINGS +# ---------------------------------------------------------------- + + +class TempNameGenerator(BaseModel): + """ + Creates a generator for unused temporary names + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + arbitrary_types_allowed=True, + ) + + names: Iterable[str] + name: str = Field(default="tmp") + template: str = Field(default="tmp_{0}") + counter_: int = Field( + default=0, + init=False, + ) + used_: set[str] = Field(default_factory=set, init=False, repr=False) + new_: set[str] = Field(default_factory=set, init=False, repr=False) + + def model_post_init(self, __context): + self.used_ = {name for name in self.names} + self.new_ = set() + + def get_temporary(self) -> set[str]: + """ + Returns the newly created temp names + """ + return self.new_ + + def __call__(self) -> str: + """ + Generate a previously unused temporary name + """ + result = self.name + + while result in self.used_: + self.counter_ += 1 # iterate first to ensure 1st index is 1 + result = self.template.format(self.counter_) + + self.new_.add(result) + self.used_.add(result) + + return result + + +def temp_name( + names: Iterable[str], + /, + *, + name: str = "tmp", + template: str = "tmp_{0}", +) -> str: + gen = TempNameGenerator(names=names, name=name, template=template) + return gen() diff --git a/src/models/internal/traits.py b/src/models/internal/traits.py new file mode 100644 index 0000000..ab2c4bf --- /dev/null +++ b/src/models/internal/traits.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from contextvars import ContextVar +from typing import Any +from typing import Callable +from typing import Generic +from typing import TypeVar + +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import Field +from pydantic import SkipValidation + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "Property", + "TriggerProperty", +] + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS +# ---------------------------------------------------------------- + +T = TypeVar("T") + +# ---------------------------------------------------------------- +# CLASSES +# ---------------------------------------------------------------- + + +class Property(BaseModel, Generic[T]): + """ + A class allowing delayed setting of properties. + + Property clases are type-annotated + + ```py + temperature = Property[float](label="temp") # property of type + ... + value = temperature() # variable 'value' shows up with intellisense as type + ``` + + To set and get value, use as follows + + ```py + temperature = Property[float](label="temp") + temperature.set(273.15) + value = temperature() + print(value) # 273.15 + ``` + + Property instances are not final, i.e. can be set multiple times + + ```py + temperature = Property[float](label="temp") + temperature.set(273.15) + temperature.set(0.15) # allowed + ``` + + Can set a factory method + + ```py + name = Property[str](label="name", factory=lambda: 'Max Mustermann') + Property(value) # 'Max Mustermann' + ``` + + If a factory method is set, + then setting the value can still override it: + + ```py + # .set takes precedence + name = Property[str](label="name", factory=lambda: 'Max Mustermann') + name.set('Julia Musterfrau') + print(name()) # 'Julia Musterfrau' + + # .set overrides factory value + name = Property[str](label="name", factory=lambda: 'Max Mustermann') + print(name()) # 'Max Mustermann' + name.set('Julia Musterfrau') # allowed + print(name()) # 'Julia Musterfrau' + ``` + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + arbitrary_types_allowed=True, + ) + + label: str + default: SkipValidation[T] | None = Field(default=None) + factory: SkipValidation[Callable[[], T]] | None = Field(default=None) + value: SkipValidation[ContextVar[T]] = Field( + default_factory=lambda: ContextVar[T]("name"), + init=False, + ) + + def model_post_init(self, __context: Any) -> None: + self.value = ContextVar[T](self.label) + + def get_default(self) -> T: + if callable(self.factory): + return self.factory() + + if self.default is not None: + return self.default + + raise LookupError(f"Property {self.label} unset. Call {self.label}.set(...) first!") # fmt: skip + + def __call__(self) -> T: + return self.get() + + def get(self) -> T: + try: + value = self.value.get() + return value + + except LookupError as _: + value = self.get_default() + self.set(value) + return value + + def set(self, x: T): + self.value.set(x) + + +class TriggerProperty: + """ + Use to set a boolean value to `true` and maintain this value. + Initialises as false. + """ + + def __init__(self): + self._value = ContextVar[bool]("trigger") + self._value.set(False) + + @property + def value(self): + return self._value.get() + + def __call__(self) -> bool: + return self._value.get() + + def set(self): + """ + Permanently sets trigger value to `true`. + """ + self._value.set(True) diff --git a/src/models/internal/trees.py b/src/models/internal/trees.py new file mode 100644 index 0000000..45fa483 --- /dev/null +++ b/src/models/internal/trees.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from __future__ import annotations + +from typing import Generator +from typing import Generic +from typing import Literal +from typing import TypeVar + +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import Field + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "GenericTree", +] + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS, VARIABLES +# ---------------------------------------------------------------- + +T = TypeVar("T") +R = TypeVar("R") + +# ---------------------------------------------------------------- +# CLASSES +# ---------------------------------------------------------------- + + +class GenericTree(BaseModel, Generic[T]): + """ + A generic for handling trees + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + arbitrary_types_allowed=True, + ) + + root: T + children: list[T | GenericTree[T]] = Field(default_factory=list) + + def __str__(self) -> str: + lines = list(self._recursive_repr(self.root)) + return "\n".join(lines) + + def add(self, other: T | GenericTree[T]): + self.children.append(other) + + @staticmethod + def _repr_node( + node: T, + /, + *, + indent: str = " ", + lex: list[bool] = [], + sep: str = "", + ) -> str: + """ + Displays a single node + """ + prefix = "".join([ + (" " if is_last else "│") + indent + for is_last in lex[:-1] + ]) # fmt: skip + return f"{prefix}{sep}{node}" + + def _recursive_repr( + self, + node: T | None = None, + /, + *, + indent: str = " ", + lex: list[bool] = [], + sep: str = "", + ) -> Generator[str, None, None]: + """ + Method to recursive display elements of Tree + """ + if node is None: + node = self.root + + yield GenericTree._repr_node(node, indent=indent, lex=lex, sep=sep) + + n = len(self.children) + for k, child in enumerate(self.children): + is_final = k == n - 1 + sep = "╰──{conn}" if is_final else "├──{conn}" + if isinstance(child, GenericTree): + has_grandchildren = len(child.children) > 0 + conn = "╮ " if has_grandchildren else "─ " + yield from child._recursive_repr( + child.root, + indent=indent, + lex=[*lex, is_final], + sep=sep.format(conn=conn), + ) + + else: + conn = "─ " + yield GenericTree._repr_node( + child, + indent=indent, + lex=[*lex, is_final], + sep=sep.format(conn=conn), + ) + + return + + def walk( + self, + *, + mode: Literal["ROOT-FIRST", "CHILDREN-FIRST"] = "ROOT-FIRST", + include_root: bool = True, + ) -> Generator[T, None, None]: + """ + Traverses the tree + """ + if mode == "ROOT-FIRST" and include_root: + yield self.root + + for child in self.children: + if isinstance(child, GenericTree): + yield from child.walk(mode=mode, include_root=True) + + else: + yield child + if mode == "CHILDREN-FIRST" and include_root: + yield self.root + + return diff --git a/src/queries/__init__.py b/src/queries/__init__.py new file mode 100644 index 0000000..f5ef5e5 --- /dev/null +++ b/src/queries/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +This module contains various general methods for queries independent of the application at hand. +""" diff --git a/src/queries/_console/__init__.py b/src/queries/_console/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/queries/_console/api.py b/src/queries/_console/api.py new file mode 100644 index 0000000..e6bcd2a --- /dev/null +++ b/src/queries/_console/api.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from argparse import ArgumentParser + +from .basic import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "CliArguments", +] + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +class CliArguments(CliArgumentsBase): + _prog = "src/api.py" + _part = "APPLICATION" + + def create_parser(self) -> ArgumentParser: + parser = self.base_parser + parser.add_argument( + "--config", + type=str, + nargs="?", + help="set default path to general config", + default="setup/config.yaml", + ) + parser.add_argument( + "--env", + nargs="?", + type=str, + help="path to environment file", + default=".env", + ) + parser.add_argument( + "--log", + nargs="?", + type=str, + help="path to files for logging", + default="logs", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="more verbose console logging (force logging level to be DEBUG)", + ) + return parser diff --git a/src/queries/_console/basic.py b/src/queries/_console/basic.py new file mode 100644 index 0000000..3839ec2 --- /dev/null +++ b/src/queries/_console/basic.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import re +from argparse import ArgumentParser +from argparse import RawTextHelpFormatter + +from ..._core.utils.misc import * +from ...models.application import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "CliArgumentsBase", + "add_boolean_key_pair", +] + +# ---------------------------------------------------------------- +# CLASSES +# ---------------------------------------------------------------- + + +class CliArgumentsBase: + _parser = None + _info: RepoInfo + _prog: str = "main.py" + _part: str | None = None + + def __init__(self, info: RepoInfo): + self._info = info + + def parse(self, *cli_args: str): + return self.parser.parse_args(cli_args) + + @property + def parser(self) -> ArgumentParser: + if not isinstance(self._parser, ArgumentParser): + self._parser = self.create_parser() + return self._parser + + @property + def base_parser(self) -> ArgumentParser: + description = re.sub(pattern=r"(\r?\n)+", repl=" ", string=self._info.description) + part = "" if self._part is None else f" - {self._part}" + parser = ArgumentParser( + prog=self._prog, + description=dedent( + f""" + \x1b[1mProgramme: {self._info.name} @ v{self._info.version}{part}\x1b[0m + \x1b[2murl: \x1b[4m{self._info.urls.homepage}\x1b[0m + \x1b[2;3m{description}\x1b[0m + """ + ), + formatter_class=RawTextHelpFormatter, + ) + return parser + + def create_parser(self) -> ArgumentParser: + parser = self.base_parser + return parser + + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def add_boolean_key_pair( + parser: ArgumentParser, + key: str, + default: bool, + help_true: str | None = None, + help_false: str | None = None, +): + """ + Adds a pair of boolean switches to the argparser + """ + group = parser.add_mutually_exclusive_group(required=False) + key_safe = re.sub(pattern=r"-", repl=r"_", string=key) + group.add_argument(f"--{key}", dest=key_safe, action="store_true", help=help_true) + group.add_argument(f"--no-{key}", dest=key_safe, action="store_false", help=help_false) + parser.set_defaults(**{key_safe: default}) + return parser diff --git a/src/queries/_console/cli.py b/src/queries/_console/cli.py new file mode 100644 index 0000000..526441e --- /dev/null +++ b/src/queries/_console/cli.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from argparse import ArgumentParser + +from ..._core.utils.misc import * +from ...models.application import * +from .basic import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "CliArguments", +] + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +class CliArguments(CliArgumentsBase): + _prog = "src/cli.py" + _part = "APPLICATION" + + def create_parser(self) -> ArgumentParser: + parser = self.base_parser + parser.add_argument( + "mode", + type=EnumFeatures, + choices=[e.value for e in EnumFeatures], + help=dedent_full( + f""" + {EnumFeatures.VERSION.value} = show version of programme + {EnumFeatures.SEARCH_FS.value} = runs feature that searches a filesystem + """ + ), + ) + parser.add_argument( + "--config", + type=str, + nargs="?", + help="set default path to general config", + default="setup/config.yaml", + ) + parser.add_argument( + "--requests", + nargs="?", + type=str, + help="Set default path to requests payload", + default="setup/requests.yaml", + ) + parser.add_argument( + "--env", + nargs="?", + type=str, + help="path to environment file", + default=".env", + ) + parser.add_argument( + "--log", + nargs="?", + type=str, + help="path to files for logging", + default="logs", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="more verbose console logging (force logging level to be DEBUG)", + ) + return parser diff --git a/src/queries/environment/__init__.py b/src/queries/environment/__init__.py new file mode 100644 index 0000000..1e255a7 --- /dev/null +++ b/src/queries/environment/__init__.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from .basic import * +from .http import * +from .mode import * +from .rabbit import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "add_environment", + "get_environment", + "get_http_host_name_rabbit", + "get_http_ip", + "get_http_password", + "get_http_password_rabbit_admin", + "get_http_password_rabbit_guest", + "get_http_port", + "get_http_port_rabbit_queue", + "get_http_port_rabbit_web", + "get_http_user", + "get_http_user_rabbit_admin", + "get_http_user_rabbit_guest", + "get_path_logs", +] diff --git a/src/queries/environment/basic.py b/src/queries/environment/basic.py new file mode 100644 index 0000000..7a52255 --- /dev/null +++ b/src/queries/environment/basic.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import os +from functools import wraps +from typing import Callable +from typing import Concatenate +from typing import ParamSpec +from typing import TypeVar + +from dotenv import dotenv_values +from dotenv import load_dotenv + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "add_environment", + "get_environment", +] + +# ---------------------------------------------------------------- +# LOCAL CONSTANTS/VARIABLES +# ---------------------------------------------------------------- + +PARAMS = ParamSpec("PARAMS") +RETURN = TypeVar("RETURN") + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def get_environment(path: str, /) -> dict[str, str]: + """ + Loads environment variables from + + - bash session + - a given .env file + + Values in session are ignored if empty-like. + + If a key is in both the session and file, + then the session-value takes precedence, + allowing users to change environments on-the-fly. + """ + # values in file + environ_file = dotenv_values(path) + + # load from session + load_dotenv(dotenv_path=path) + env_from_session = { + key: value + for key, value in os.environ.items() + # NOTE: filter out empty/null values + if value not in [None, ""] + } + + # load from file + env_from_file = { + key: value + for key, value in environ_file.items() + # NOTE: allow file to include empty/null values + # if value not in [None, ""] + } + + # session env vars take precedence + # NOTE: left-right = low to higher precedence + env = env_from_file | env_from_session + + return dict(env) + + +def add_environment( + action: Callable[ + Concatenate[str, dict[str, str], PARAMS], + RETURN, + ], + /, +) -> Callable[Concatenate[str, PARAMS], RETURN]: + """ + Decorates method to make it get environment first. + Runs method with error wrapping, + catching errors with a ValueError + """ + + # modify function + @wraps(action) + def wrapped_action(path: str, *_: PARAMS.args, **__: PARAMS.kwargs) -> RETURN: + env = get_environment(path) + result = action(path, env, *_, **__) + return result + + return wrapped_action diff --git a/src/queries/environment/http.py b/src/queries/environment/http.py new file mode 100644 index 0000000..15fce38 --- /dev/null +++ b/src/queries/environment/http.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from typing import Any + +from pydantic import SecretStr + +from .basic import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "get_http_ip", + "get_http_password", + "get_http_port", + "get_http_user", +] + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +@add_environment +def get_http_route( + # DEV-NOTE: from decorator + path: str, + env: dict[str, str], + # end decorator args + default: str = "", +) -> str: + route = env.get("HTTP_ROUTE", default) + # ensure route starts with "/" + route = route.lstrip(" /") + route = f"/{route}" + # NOTE: in particular route = "/" is avoided + route = route.rstrip(" /") + return route + + +@add_environment +def get_http_ip( + # DEV-NOTE: from decorator + path: str, + env: dict[str, str], + # end decorator args + default: str = "0.0.0.0", +) -> str: + return env.get("HTTP_IP", default) + + +@add_environment +def get_http_port( + # DEV-NOTE: from decorator + path: str, + env: dict[str, str], + # end decorator args + default: int = 80, +) -> int: + value = env.get("HTTP_PORT", default) + return int(value) + + +@add_environment +def get_http_user( + # DEV-NOTE: from decorator + path: str, + env: dict[str, Any], + # end decorator args + default: str = "admin", +) -> str: + """ + Gets http user. + If value not set in .env, will raise a (Key)Exception. + """ + value = env["HTTP_USER"] + return value + + +@add_environment +def get_http_password( + # DEV-NOTE: from decorator + path: str, + env: dict[str, Any], + # end decorator args +) -> SecretStr: + """ + Gets http password. + If value not set in .env, will raise a (Key)Exception. + """ + value = env["HTTP_PASSWORD"] + return SecretStr(value) diff --git a/src/queries/environment/mode.py b/src/queries/environment/mode.py new file mode 100644 index 0000000..5036bf2 --- /dev/null +++ b/src/queries/environment/mode.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from .basic import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "get_path_logs", +] + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +@add_environment +def get_path_logs( + # DEV-NOTE: from decorator + path: str, + env: dict[str, str], + # end decorator args +) -> str: + value = env.get("PATH_LOGS", ".session") + return value diff --git a/src/queries/environment/rabbit.py b/src/queries/environment/rabbit.py new file mode 100644 index 0000000..3b3e85f --- /dev/null +++ b/src/queries/environment/rabbit.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from typing import Any + +from pydantic import SecretStr + +from .basic import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "get_http_host_name_rabbit", + "get_http_password_rabbit_admin", + "get_http_password_rabbit_guest", + "get_http_port_rabbit_queue", + "get_http_port_rabbit_web", + "get_http_user_rabbit_admin", + "get_http_user_rabbit_guest", +] + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +@add_environment +def get_http_host_name_rabbit( + # DEV-NOTE: from decorator + path: str, + env: dict[str, str], + # end decorator args +) -> str: + name = env["HTTP_HOST_NAME_RABBIT"] + return name + + +@add_environment +def get_http_port_rabbit_web( + # DEV-NOTE: from decorator + path: str, + env: dict[str, str], + # end decorator args +) -> int: + value = env["HTTP_PORT_RABBIT_WEB"] + return int(value) + + +@add_environment +def get_http_port_rabbit_queue( + # DEV-NOTE: from decorator + path: str, + env: dict[str, str], + # end decorator args +) -> int: + value = env["HTTP_PORT_RABBIT_QUEUE"] + return int(value) + + +@add_environment +def get_http_user_rabbit_admin( + # DEV-NOTE: from decorator + path: str, + env: dict[str, Any], + # end decorator args +) -> str: + """ + Gets http user. + If value not set in .env, will raise a (Key)Exception. + """ + value = env["HTTP_ADMIN_USER_RABBIT"] + return value + + +@add_environment +def get_http_password_rabbit_admin( + # DEV-NOTE: from decorator + path: str, + env: dict[str, Any], + # end decorator args +) -> SecretStr: + """ + Gets http password. + If value not set in .env, will raise a (Key)Exception. + """ + value = env["HTTP_ADMIN_PASSWORD_RABBIT"] + return SecretStr(value) + + +@add_environment +def get_http_user_rabbit_guest( + # DEV-NOTE: from decorator + path: str, + env: dict[str, Any], + # end decorator args +) -> str: + """ + Gets http user. + If value not set in .env, will raise a (Key)Exception. + """ + value = env["HTTP_GUEST_USER_RABBIT"] + return value + + +@add_environment +def get_http_password_rabbit_guest( + # DEV-NOTE: from decorator + path: str, + env: dict[str, Any], + # end decorator args +) -> SecretStr: + """ + Gets http password. + If value not set in .env, will raise a (Key)Exception. + """ + value = env["HTTP_GUEST_PASSWORD_RABBIT"] + return SecretStr(value) diff --git a/src/queries/filesmanager/__init__.py b/src/queries/filesmanager/__init__.py new file mode 100644 index 0000000..033ee19 --- /dev/null +++ b/src/queries/filesmanager/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from .basic import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "get_files_manager", +] diff --git a/src/queries/filesmanager/basic.py b/src/queries/filesmanager/basic.py new file mode 100644 index 0000000..f8e5f96 --- /dev/null +++ b/src/queries/filesmanager/basic.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +from datetime import timezone + +from ..._core.utils.basic import * +from ...models.filesmanager import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "get_files_manager", +] + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def get_files_manager( + location: EnumFilesSystem, + /, + *, + tz: timezone | None = None, +) -> FilesManager: + """ + Obtains files manager from user choice of system location. + """ + match location: + case EnumFilesSystem.OS: + return OSFilesManager(tz=tz) + + case EnumFilesSystem.SHAREPOINT: + raise NotImplementedError("FilesManager protocol not yet implemented for Sharepoint") # fmt: skip + + case EnumFilesSystem.BLOB_STORAGE: + raise NotImplementedError("FilesManager protocol not yet implemented for Blobstorage") # fmt: skip + + case _: + raise ValueError(f"No method determined for files system manager {extract_string(location)}.") # fmt: skip diff --git a/src/setup/__init__.py b/src/setup/__init__.py new file mode 100644 index 0000000..ee83184 --- /dev/null +++ b/src/setup/__init__.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +This module contains methods for setup purposes, +e.g. configuration of application. +""" + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +# NOTE: only import/export the submodules which are called as such +from . import config +from .config import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "INFO", + "TIMEZONE", + "VERSION", + "config", +] diff --git a/src/setup/config.py b/src/setup/config.py new file mode 100644 index 0000000..e0194a8 --- /dev/null +++ b/src/setup/config.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ---------------------------------------------------------------- +# IMPORTS +# ---------------------------------------------------------------- + +import logging +from contextvars import ContextVar +from pathlib import Path + +import toml +from pydantic import SecretStr + +from ..__paths__ import * +from .._core.constants import * +from .._core.logging import * +from .._core.utils.code import * +from .._core.utils.time import * +from ..models.application import * +from ..models.filesmanager import * +from ..models.internal import * +from ..queries.environment import * +from ..queries.filesmanager import * + +# ---------------------------------------------------------------- +# EXPORTS +# ---------------------------------------------------------------- + +__all__ = [ + "INFO", + "TIMEZONE", + "VERSION", +] + +# ---------------------------------------------------------------- +# GLOBAL PROPERTIES +# ---------------------------------------------------------------- + +pid = ContextVar[int]("pid") # fmt: skip +path_env = ContextVar[str]("path env", default=".env") # fmt: skip +path_logging = Property[str](label="path logging", factory=lambda: get_path_logs(path_env.get())) # fmt: skip +path_config = Property[str](label="path application config", factory=lambda: get_root_path("setup", "config.yaml")) # fmt: skip +path_requests = Property[str](label="path user requests", factory=lambda: get_root_path("setup", "requests.yaml")) # fmt: skip + +# for api server +http_ip = Property[str](label="http ip", factory=lambda: get_http_ip(path_env.get())) # fmt: skip +http_port = Property[int](label="http port", factory=lambda: get_http_port(path_env.get())) # fmt: skip +http_user = Property[str](label="http user", factory=lambda: get_http_user(path_env.get())) # fmt: skip +http_password = Property[SecretStr](label="http password", factory=lambda: get_http_password(path_env.get())) # fmt: skip + +# for rabbit/queue +http_host_name_rabbit = Property[str](label="host name of rabbit mq", factory=lambda: get_http_host_name_rabbit(path_env.get())) # fmt: skip +http_port_rabbit_queue = Property[int](label="port of rabbit queue", factory=lambda: get_http_port_rabbit_queue(path_env.get())) # fmt: skip +http_port_rabbit_web = Property[int](label="port of rabbit admin", factory=lambda: get_http_port_rabbit_web(path_env.get())) # fmt: skip +http_user_rabbit_admin = Property[str](label="admin username for rabbit mq", factory=lambda: get_http_user_rabbit_admin(path_env.get())) # fmt: skip +http_password_rabbit_admin = Property[SecretStr](label="admin password for rabbit mq", factory=lambda: get_http_password_rabbit_admin(path_env.get())) # fmt: skip +http_user_rabbit_guest = Property[str](label="guest username for rabbit mq", factory=lambda: get_http_user_rabbit_guest(path_env.get())) # fmt: skip +http_password_rabbit_guest = Property[SecretStr](label="guest password for rabbit mq", factory=lambda: get_http_password_rabbit_guest(path_env.get())) # fmt: skip + +# ---------------------------------------------------------------- +# METHODS +# ---------------------------------------------------------------- + + +def initialise_application( + *, + name: str, + title: str | None = None, + verbose: bool = False, + serialise: bool = True, + log_to_files: bool = False, +): + """ + Initialises logging and displays information about pid, cpus. + """ + level = "DEBUG" if verbose else "INFO" + path = path_logging.get() if log_to_files else None + configure_logging(name="root", level=level, path=path, serialise=serialise) # fmt: skip + + logging.info(f"running {title or name} v{INFO.version} on PID {pid.get()}") + return + + +# ---------------------------------------------------------------- +# QUERIES +# ---------------------------------------------------------------- + + +@compute_once +def load_repo_info() -> RepoInfo: + path = Path(get_root_path(), "pyproject.toml").as_posix() + with open(path, "r") as fp: + config_repo = toml.load(fp) + assets = config_repo.get("project", {}) + info = RepoInfo.model_validate(assets) + return info + + +@compute_once +def get_version() -> str: + info = load_repo_info() + return info.version + + +@compute_once +def get_managers() -> dict[EnumFilesSystem, FilesManager]: + """ + Returns managers to access files in different locations. + """ + return { + EnumFilesSystem.OS: get_files_manager(EnumFilesSystem.OS, tz=TIMEZONE), + # TODO: implement use of credentials and add protocols for other file systems + # EnumFilesSystem.SHAREPOINT: get_files_manager(EnumFilesSystem.SHAREPOINT, tz=TIMEZONE), + # EnumFilesSystem.BLOB_STORAGE: get_files_manager(EnumFilesSystem.BLOB_STORAGE, tz=TIMEZONE), + } + + +# ---------------------------------------------------------------- +# LAZY LOADED RESOURCES / PROPERTIES +# ---------------------------------------------------------------- + +INFO = load_repo_info() +VERSION = get_version() +TIMEZONE = get_local_timezone() + +parser_requests = Property[PayloadParser[RequestsPayload]]( + label="parser:requests payload", + factory=lambda: PayloadParser(type_=RequestsPayload, managers=get_managers(), location="OS", root=path_requests.get()), +) # fmt: skip + +parser_config = Property[PayloadParser[GeneralConfig]]( + label="parser:general application config", + factory=lambda: PayloadParser(type_=GeneralConfig, managers=get_managers(), location="OS", root=path_config.get()), +) # fmt: skip diff --git a/templates/template-config.yaml b/templates/template-config.yaml index e69de29..a0a06d4 100644 --- a/templates/template-config.yaml +++ b/templates/template-config.yaml @@ -0,0 +1,2 @@ +# NOTE: user versioning to keep track of changes of their config files +version: 0.0.0 diff --git a/templates/template-requests-multiple.yaml b/templates/template-requests-multiple.yaml new file mode 100644 index 0000000..a7e4ac8 --- /dev/null +++ b/templates/template-requests-multiple.yaml @@ -0,0 +1,20 @@ +- label: 'First task' + options: {} + data: + inputs: + location: OS + path: 'path/to/directory1' + +- label: 'Second task' + options: {} + data: + inputs: + location: OS + path: 'path/to/directory2' + +- label: 'Third task' + options: {} + data: + inputs: + location: OS + path: 'path/to/directory3' diff --git a/templates/template-requests.yaml b/templates/template-requests.yaml index 6ce9cc1..31243e3 100644 --- a/templates/template-requests.yaml +++ b/templates/template-requests.yaml @@ -1,3 +1,11 @@ -ref: - location: OS # enum for file system - path: 'relative/or absolute path to directory' +# For logging purposes - a recognisable label for the task +label: 'Some label' + +# NOTE: not yet implemented +options: {} + +# The main request +data: + inputs: + location: OS # enum for file system + path: 'relative/or absolute path to directory' diff --git a/uv.lock b/uv.lock index 1555286..23b7c80 100644 --- a/uv.lock +++ b/uv.lock @@ -541,7 +541,7 @@ wheels = [ [[package]] name = "example-rabbit-mq" -version = "0.0.0" +version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "argparse" }, @@ -572,6 +572,7 @@ dependencies = [ { name = "tabulate" }, { name = "toml" }, { name = "tzdata" }, + { name = "tzlocal" }, { name = "uvicorn" }, ] @@ -621,6 +622,7 @@ requires-dist = [ { name = "tabulate", specifier = ">=0.9.0" }, { name = "toml", specifier = ">=0.10.2" }, { name = "tzdata", specifier = ">=2025.2" }, + { name = "tzlocal", specifier = ">=5.3.1" }, { name = "uvicorn", specifier = ">=0.38.0" }, ] @@ -2201,6 +2203,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, ] +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + [[package]] name = "ujson" version = "5.11.0"