diff --git a/.gitignore b/.gitignore index 3efd499..16cb9af 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,6 @@ dmypy.json # IDE .idea/ + +# Agents +.codex diff --git a/README.md b/README.md index 136d4c8..9ccc21d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Please generously STAR★ our project or donate to us! ## Getting started -Refer to[📚 Document & 中文文档](https://aloha-python.readthedocs.io/) for detailed introduction. +Refer to 📚 [Document](https://aloha-python.readthedocs.io/) & [中文文档](https://aloha-python.readthedocs.io/zh-cn/main/) for detailed introduction. ```shell pip install aloha[all] diff --git a/doc/.readthedocs.yaml b/doc/.readthedocs.yaml index 04cc1b1..426052d 100644 --- a/doc/.readthedocs.yaml +++ b/doc/.readthedocs.yaml @@ -11,6 +11,8 @@ build: jobs: pre_create_environment: - echo "Command run at 'pre_create_environment' step" + pre_build: + - 'case "${READTHEDOCS_LANGUAGE}" in zh*) cp ./doc/mkdocs.zh.yml ./doc/mkdocs.yml ;; esac' post_build: - echo "Command run at 'post_build' step $(date)" @@ -22,3 +24,5 @@ python: - requirements: ./doc/requirements.txt - method: pip path: ./src + extra_requirements: + - all diff --git a/doc/en/README-cli.md b/doc/en/README-cli.md index 75f9d6d..875fb53 100644 --- a/doc/en/README-cli.md +++ b/doc/en/README-cli.md @@ -19,17 +19,34 @@ Note: this `module_name` **must** include a function named `main()`. ## Compile Python code into binary -Sometimes you need to compile Python source code into binary libraries to protect source code. +Sometimes you need to distribute Python code without shipping the raw `.py` files. This command turns most Python modules into compiled extension libraries (`.so` files on Linux/macOS), which can help with source-code protection and reduce direct readability of implementation details. -Aloha helps you build Python source code into binaries using `Cython`: +Aloha uses `Cython` for this workflow: ```bash aloha compile --base=./app --dist=./build --keep='main.py' ``` +How it works: + +1. Scan the source tree under `--base`. +2. Skip hidden folders, the output folder, excluded paths, `.pyc` files, and `.pyx` files. +3. Copy non-Python files directly into the target tree. +4. Convert regular `.py` modules into compiled extension modules. +5. Keep files listed in `--keep` as plain Python files. +6. Move the final build result into `--dist` and clean up intermediate Cython artifacts. + +Notes: + +- `__init__.py` files are copied as plain Python files. +- All python submodule folders MUST contain a `__init__.py` file, otherwise python files in this submodule/folder will be compiled and built to root folder. +- Files passed through `--keep` are not compiled. +- `Cython` must be installed first. +- The command writes a temporary build tree under `/tmp/build/` before moving the result to `--dist`. + Available options: - `--base`: root folder that includes source code to build - `--dist`: (default=`build`) target folder for binary code -- `--exclude`: files/folders to exclude (use multiple times for multiple paths) -- `--keep`: source files to keep as-is instead of converting to dynamic libraries (use multiple times) +- `--exclude`: files/folders to exclude; pass one or more paths after the flag +- `--keep`: source files to keep as-is instead of converting to dynamic libraries; pass one or more paths after the flag diff --git a/doc/en/README-develop.md b/doc/en/README-develop.md index 8602233..3ee78e5 100644 --- a/doc/en/README-develop.md +++ b/doc/en/README-develop.md @@ -24,5 +24,5 @@ build_image app_common latest tool/app.Dockerfile ## Develop docs ```bash -mkdocs serve -f mkdocs.yml -a 0.0.0.0:80 +mkdocs serve -f mkdocs.yml -a 0.0.0.0:3000 ``` diff --git a/doc/en/README-get-start.md b/doc/en/README-get-start.md index a635528..c82617b 100644 --- a/doc/en/README-get-start.md +++ b/doc/en/README-get-start.md @@ -6,8 +6,44 @@ pip install aloha[all] ``` -## Step 2. Start your project based on the template +## Step 2. Use this repository as a boilerplate -You can refer to the `app` folder in the GitHub repository to start using `aloha` in your project: +The `app/` folder in this repository is a boilerplate/template project built on top of `aloha`. +It gives you a ready-to-use application layout, development scripts, and containerized tooling so you can start building instead of assembling the project skeleton yourself. + +### What this template gives you + +- A containerized development environment based on Docker and Docker Compose +- Pre-installed Python and project dependencies +- An application entry point you can extend directly +- A conventional layout for source code, documentation, notebooks, and tooling + +### Recommended workflow + +1. Clone this repository. +2. Open the `app/` directory to inspect the starter application structure. +3. Use the scripts under `tool/cicd/` to start the development container when you want a reproducible environment. +4. Put your own application code in the template structure and grow from there. + +### Launch the development environment + +If you want the full boilerplate experience, start the containerized DEV environment: + +```bash +./tool/cicd/run-dev.sh up +./tool/cicd/run-dev.sh enter +``` + +The `up` command creates or starts the development container. The `enter` command opens an interactive shell inside that container. + +### Project structure + +The template is organized around a few common folders: + +- `app/`: application code and entry points +- `src/`: the `aloha` library source code +- `doc/`: documentation source files +- `notebook/`: Jupyter notebooks for experimentation +- `tool/`: scripts and Docker assets for development and CI/CD [:octicons-mark-github-16: Go to Template Project](https://github.com/LabNow-ai/aloha-python/tree/main/app){ .md-button } diff --git a/doc/en/api.md b/doc/en/api.md deleted file mode 100644 index 1dd9377..0000000 --- a/doc/en/api.md +++ /dev/null @@ -1,5 +0,0 @@ -# API Reference - -To be added. - -::: aloha diff --git a/doc/en/api/core.md b/doc/en/api/core.md new file mode 100644 index 0000000..3913fb3 --- /dev/null +++ b/doc/en/api/core.md @@ -0,0 +1,11 @@ +# Core and Configuration + +::: aloha + +::: aloha.base + +::: aloha.settings + +::: aloha.config.paths + +::: aloha.config.hocon diff --git a/doc/en/api/db.md b/doc/en/api/db.md new file mode 100644 index 0000000..d0ddfa4 --- /dev/null +++ b/doc/en/api/db.md @@ -0,0 +1,23 @@ +# Database + +::: aloha.db + +::: aloha.db.base + +::: aloha.db.duckdb + +::: aloha.db.elasticsearch + +::: aloha.db.kafka + +::: aloha.db.mongo + +::: aloha.db.mysql + +::: aloha.db.oracle + +::: aloha.db.postgres + +::: aloha.db.redis + +::: aloha.db.sqlite diff --git a/doc/en/api/encrypt.md b/doc/en/api/encrypt.md new file mode 100644 index 0000000..f386563 --- /dev/null +++ b/doc/en/api/encrypt.md @@ -0,0 +1,19 @@ +# Encryption + +::: aloha.encrypt + +::: aloha.encrypt.hash + +::: aloha.encrypt.jwt + +::: aloha.encrypt.aes + +::: aloha.encrypt.rsa + +::: aloha.encrypt.vault + +::: aloha.encrypt.vault.base + +::: aloha.encrypt.vault.plain + +::: aloha.encrypt.vault.cyberark diff --git a/doc/en/api/index.md b/doc/en/api/index.md new file mode 100644 index 0000000..3b1e8f2 --- /dev/null +++ b/doc/en/api/index.md @@ -0,0 +1,15 @@ +# API Reference + +The API documentation is split by package so each page stays focused and easier to maintain. + +Choose a section below: + +- Core and configuration +- Logging +- Command-line scripts +- Service layer +- Encryption +- Database +- Time helpers +- Utilities +- Testing utilities diff --git a/doc/en/api/logging.md b/doc/en/api/logging.md new file mode 100644 index 0000000..410e34c --- /dev/null +++ b/doc/en/api/logging.md @@ -0,0 +1,7 @@ +# Logging + +::: aloha.logger + +::: aloha.logger.logger + +::: aloha.logger.handler diff --git a/doc/en/api/scripts.md b/doc/en/api/scripts.md new file mode 100644 index 0000000..1ddf726 --- /dev/null +++ b/doc/en/api/scripts.md @@ -0,0 +1,9 @@ +# Command-Line Scripts + +::: aloha.script.base + +::: aloha.script.info + +::: aloha.script.start + +::: aloha.script.compile diff --git a/doc/en/api/service.md b/doc/en/api/service.md new file mode 100644 index 0000000..8c21528 --- /dev/null +++ b/doc/en/api/service.md @@ -0,0 +1,23 @@ +# Service Layer + +::: aloha.service + +::: aloha.service.app + +::: aloha.service.web + +::: aloha.service.http.base_api_client + +::: aloha.service.http.base_api_handler + +::: aloha.service.http.plain_http_handler + +::: aloha.service.http.files + +::: aloha.service.openapi.client + +::: aloha.service.api.v0 + +::: aloha.service.api.v1 + +::: aloha.service.api.v2 diff --git a/doc/en/api/testing.md b/doc/en/api/testing.md new file mode 100644 index 0000000..2e81faf --- /dev/null +++ b/doc/en/api/testing.md @@ -0,0 +1,9 @@ +# Testing Utilities + +::: aloha.testing + +::: aloha.testing.unit + +::: aloha.testing.service_v1 + +::: aloha.testing.service_v2 diff --git a/doc/en/api/time.md b/doc/en/api/time.md new file mode 100644 index 0000000..10be8c0 --- /dev/null +++ b/doc/en/api/time.md @@ -0,0 +1,9 @@ +# Time Helpers + +::: aloha.time + +::: aloha.time.timeout_async + +::: aloha.time.timeout_asyncio + +::: aloha.time.timeout_signal diff --git a/doc/en/api/util.md b/doc/en/api/util.md new file mode 100644 index 0000000..ae72830 --- /dev/null +++ b/doc/en/api/util.md @@ -0,0 +1,15 @@ +# Utilities + +::: aloha.util + +::: aloha.util.html + +::: aloha.util.json + +::: aloha.util.random + +::: aloha.util.sys_cuda + +::: aloha.util.sys_gpu + +::: aloha.util.sys_info diff --git a/doc/mkdocs.yml b/doc/mkdocs.yml index d2cf6c5..a13d63d 100644 --- a/doc/mkdocs.yml +++ b/doc/mkdocs.yml @@ -1,7 +1,8 @@ site_name: Aloha - a versatile Python library to build microservice. -repo_url: https://github.com/LabNow-ai/aloha -edit_uri: blob/main/doc/ -docs_dir: ./ +repo_url: https://github.com/LabNow-ai/aloha-python +edit_uri: blob/main/doc/en/ +site_url: !ENV READTHEDOCS_CANONICAL_URL +docs_dir: ./en theme: name: material highlightjs: true @@ -30,25 +31,6 @@ theme: name: Switch to light mode plugins: - search - - i18n: - docs_structure: folder - reconfigure_material: true - languages: - - locale: en - name: English - default: true - build: true - - locale: zh - name: 简体中文 - build: true - nav_translations: - zh: - Introduction: 介绍 - Get Started: 快速开始 - CLI Commands: CLI 命令 - Configs: 配置说明 - Development Guide: 开发文档 - API Reference: API 参考 - mkdocstrings: handlers: # See: https://mkdocstrings.github.io/python/usage/ python: @@ -80,4 +62,14 @@ nav: - CLI Commands: "README-cli.md" - Configs: "README-config.md" - Development Guide: "README-develop.md" - - API Reference: "api.md" + - API Reference: + - Overview: "api/index.md" + - Core and Config: "api/core.md" + - Logging: "api/logging.md" + - CLI Scripts: "api/scripts.md" + - Service: "api/service.md" + - Encryption: "api/encrypt.md" + - Database: "api/db.md" + - Time: "api/time.md" + - Utilities: "api/util.md" + - Testing: "api/testing.md" diff --git a/doc/mkdocs.zh.yml b/doc/mkdocs.zh.yml new file mode 100644 index 0000000..202fbe2 --- /dev/null +++ b/doc/mkdocs.zh.yml @@ -0,0 +1,75 @@ +site_name: Aloha - a versatile Python library to build microservice. +repo_url: https://github.com/LabNow-ai/aloha-python +edit_uri: blob/main/doc/zh/ +site_url: !ENV READTHEDOCS_CANONICAL_URL +docs_dir: ./zh +theme: + name: material + highlightjs: true + features: + - navigation.tabs + - navigation.tabs.sticky + - navigation.expand + - navigation.top + - toc.integrate + - content.tabs.link + - content.code.annotate + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + primary: black + accent: blues + toggle: + icon: material/brightness-7 + name: 切换到暗色模式 + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: black + accent: blue + toggle: + icon: material/brightness-4 + name: 切换到亮色模式 +plugins: + - search + - mkdocstrings: + handlers: + python: + options: + docstring_style: sphinx +markdown_extensions: + - admonition + - attr_list + - def_list + - toc: + permalink: true + - markdown_include.include: + base_path: . + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.emoji: + emoji_index: !!python/name:materialx.emoji.twemoji + emoji_generator: !!python/name:materialx.emoji.to_svg +nav: + - 介绍: "index.md" + - 快速开始: "README-get-start.md" + - CLI 命令: "README-cli.md" + - 配置说明: "README-config.md" + - 开发文档: "README-develop.md" + - API 参考: + - 概览: "api/index.md" + - 核心与配置: "api/core.md" + - 日志: "api/logging.md" + - 命令行脚本: "api/scripts.md" + - 服务层: "api/service.md" + - 加密: "api/encrypt.md" + - 数据库: "api/db.md" + - 时间工具: "api/time.md" + - 工具函数: "api/util.md" + - 测试工具: "api/testing.md" diff --git a/doc/requirements.txt b/doc/requirements.txt index 162fc8b..b0be5a1 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -36,7 +36,6 @@ pytest-cov # docs mkdocs -mkdocs-static-i18n mkdocstrings[python] markdown-include mkdocs-material diff --git a/doc/zh/README-cli.md b/doc/zh/README-cli.md index 8663559..8e4b887 100644 --- a/doc/zh/README-cli.md +++ b/doc/zh/README-cli.md @@ -19,17 +19,34 @@ aloha start package_name.module_name ## 将 Python 代码编译为二进制 -在某些场景下，你可能需要把 Python 源码编译为二进制库，以保护源码。 +在某些场景下，你可能希望发布可运行的 Python 项目，但不直接分发原始 `.py` 源码。这个命令会把大部分 Python 模块编译为扩展库文件（Linux/macOS 上通常是 `.so`），从而提升源码保护能力，也减少实现细节的直接可读性。 -Aloha 支持通过 `Cython` 进行构建： +Aloha 通过 `Cython` 完成这个流程： ```bash aloha compile --base=./app --dist=./build --keep='main.py' ``` +它的执行逻辑如下： + +1. 扫描 `--base` 指定的源码目录。 +2. 跳过隐藏目录、输出目录、被排除路径、`.pyc` 文件和 `.pyx` 文件。 +3. 将非 Python 文件原样复制到目标目录。 +4. 将普通的 `.py` 模块编译为扩展模块。 +5. 将 `--keep` 指定的文件保留为普通 Python 文件，不参与编译。 +6. 将最终产物移动到 `--dist`，并清理中间生成的 Cython 文件。 + +需要注意： + +- `__init__.py` 会被当作普通 Python 文件复制，不会编译。 +- 所有的作为python submodule的文件夹中，必须包含`__init__.py`文件，否则该submodule模块下的.py文件会编译后会被放置到根目录下。 +- `--keep` 指定的文件不会被编译。 +- 使用前需要先安装 `Cython`。 +- 该命令会先在 `/tmp/build/<项目名>` 下生成临时构建目录，再把结果移动到 `--dist`。 + 可用参数： - `--base`：待构建源码的根目录 - `--dist`：二进制产物目录（默认值为 `build`） -- `--exclude`：需要排除的文件/目录（可多次传入） -- `--keep`：保留为源码、不转为动态库的文件（可多次传入） +- `--exclude`：需要排除的文件/目录，参数后可直接跟一个或多个路径 +- `--keep`：保留为源码、不转为动态库的文件，参数后可直接跟一个或多个路径 diff --git a/doc/zh/README-develop.md b/doc/zh/README-develop.md index 1d114f0..4d3fba1 100644 --- a/doc/zh/README-develop.md +++ b/doc/zh/README-develop.md @@ -24,5 +24,5 @@ build_image app_common latest tool/app.Dockerfile ## 开发文档 ```bash -mkdocs serve -f mkdocs.yml -a 0.0.0.0:80 +mkdocs serve -f mkdocs.yml -a 0.0.0.0:3000 ``` diff --git a/doc/zh/README-get-start.md b/doc/zh/README-get-start.md index c6b9c07..c35c18e 100644 --- a/doc/zh/README-get-start.md +++ b/doc/zh/README-get-start.md @@ -6,7 +6,45 @@ pip install aloha[all] ``` -## 第二步：基于模板启动项目 +## 第二步：把本仓库当作 boilerplate 使用 + +仓库中的 `app/` 目录就是基于 `aloha` 的 boilerplate / 模板项目。 +它已经准备好了可直接使用的项目结构、开发脚本和容器化工具，你可以直接在这个骨架上开始开发，而不需要从零搭建工程。 + +### 这个模板提供了什么 + +- 基于 Docker 和 Docker Compose 的容器化开发环境 +- 预装好的 Python 运行环境和项目依赖 +- 可直接扩展的应用入口 +- 适合持续开发的常见目录结构：源码、文档、Notebook、工具脚本 + +### 推荐使用方式 + +1. 克隆本仓库。 +2. 打开 `app/` 目录，查看模板项目的结构。 +3. 需要可复现开发环境时，使用 `tool/cicd/` 里的脚本启动开发容器。 +4. 在模板结构上放入你自己的业务代码，并逐步扩展。 + +### 启动开发环境 + +如果你想直接使用完整的 boilerplate 开发环境，可以启动容器化 DEV 环境： + +```bash +./tool/cicd/run-dev.sh up +./tool/cicd/run-dev.sh enter +``` + +其中 `up` 用于创建或启动开发容器，`enter` 用于进入容器内部的交互式 Shell。 + +### 项目结构 + +这个模板围绕几个常见目录组织： + +- `app/`：应用代码和入口 +- `src/`：`aloha` 库源码 +- `doc/`：文档源码 +- `notebook/`：用于实验和探索的 Jupyter Notebook +- `tool/`：开发与 CI/CD 相关脚本和 Docker 资源 你可以参考 GitHub 仓库中的 `app` 目录，在自己的项目中开始使用 `aloha`： diff --git a/doc/zh/api.md b/doc/zh/api.md deleted file mode 100644 index db64ea8..0000000 --- a/doc/zh/api.md +++ /dev/null @@ -1,5 +0,0 @@ -# API 参考 - -待补充。 - -::: aloha diff --git a/doc/zh/api/core.md b/doc/zh/api/core.md new file mode 100644 index 0000000..8cdda12 --- /dev/null +++ b/doc/zh/api/core.md @@ -0,0 +1,11 @@ +# 核心与配置 + +::: aloha + +::: aloha.base + +::: aloha.settings + +::: aloha.config.paths + +::: aloha.config.hocon diff --git a/doc/zh/api/db.md b/doc/zh/api/db.md new file mode 100644 index 0000000..90f2fc1 --- /dev/null +++ b/doc/zh/api/db.md @@ -0,0 +1,23 @@ +# 数据库 + +::: aloha.db + +::: aloha.db.base + +::: aloha.db.duckdb + +::: aloha.db.elasticsearch + +::: aloha.db.kafka + +::: aloha.db.mongo + +::: aloha.db.mysql + +::: aloha.db.oracle + +::: aloha.db.postgres + +::: aloha.db.redis + +::: aloha.db.sqlite diff --git a/doc/zh/api/encrypt.md b/doc/zh/api/encrypt.md new file mode 100644 index 0000000..aad95c8 --- /dev/null +++ b/doc/zh/api/encrypt.md @@ -0,0 +1,19 @@ +# 加密 + +::: aloha.encrypt + +::: aloha.encrypt.hash + +::: aloha.encrypt.jwt + +::: aloha.encrypt.aes + +::: aloha.encrypt.rsa + +::: aloha.encrypt.vault + +::: aloha.encrypt.vault.base + +::: aloha.encrypt.vault.plain + +::: aloha.encrypt.vault.cyberark diff --git a/doc/zh/api/index.md b/doc/zh/api/index.md new file mode 100644 index 0000000..50cbeca --- /dev/null +++ b/doc/zh/api/index.md @@ -0,0 +1,15 @@ +# API 参考 + +API 文档按包拆分为多个页面，这样每一页都更聚焦，也更方便维护。 + +下面是可查看的章节： + +- 核心与配置 +- 日志 +- 命令行脚本 +- 服务层 +- 加密 +- 数据库 +- 时间工具 +- 工具函数 +- 测试工具 diff --git a/doc/zh/api/logging.md b/doc/zh/api/logging.md new file mode 100644 index 0000000..63a1824 --- /dev/null +++ b/doc/zh/api/logging.md @@ -0,0 +1,7 @@ +# 日志 + +::: aloha.logger + +::: aloha.logger.logger + +::: aloha.logger.handler diff --git a/doc/zh/api/scripts.md b/doc/zh/api/scripts.md new file mode 100644 index 0000000..69a3c99 --- /dev/null +++ b/doc/zh/api/scripts.md @@ -0,0 +1,9 @@ +# 命令行脚本 + +::: aloha.script.base + +::: aloha.script.info + +::: aloha.script.start + +::: aloha.script.compile diff --git a/doc/zh/api/service.md b/doc/zh/api/service.md new file mode 100644 index 0000000..92130f3 --- /dev/null +++ b/doc/zh/api/service.md @@ -0,0 +1,23 @@ +# 服务层 + +::: aloha.service + +::: aloha.service.app + +::: aloha.service.web + +::: aloha.service.http.base_api_client + +::: aloha.service.http.base_api_handler + +::: aloha.service.http.plain_http_handler + +::: aloha.service.http.files + +::: aloha.service.openapi.client + +::: aloha.service.api.v0 + +::: aloha.service.api.v1 + +::: aloha.service.api.v2 diff --git a/doc/zh/api/testing.md b/doc/zh/api/testing.md new file mode 100644 index 0000000..6bce8f6 --- /dev/null +++ b/doc/zh/api/testing.md @@ -0,0 +1,9 @@ +# 测试工具 + +::: aloha.testing + +::: aloha.testing.unit + +::: aloha.testing.service_v1 + +::: aloha.testing.service_v2 diff --git a/doc/zh/api/time.md b/doc/zh/api/time.md new file mode 100644 index 0000000..05d84bc --- /dev/null +++ b/doc/zh/api/time.md @@ -0,0 +1,9 @@ +# 时间工具 + +::: aloha.time + +::: aloha.time.timeout_async + +::: aloha.time.timeout_asyncio + +::: aloha.time.timeout_signal diff --git a/doc/zh/api/util.md b/doc/zh/api/util.md new file mode 100644 index 0000000..c106668 --- /dev/null +++ b/doc/zh/api/util.md @@ -0,0 +1,15 @@ +# 工具函数 + +::: aloha.util + +::: aloha.util.html + +::: aloha.util.json + +::: aloha.util.random + +::: aloha.util.sys_cuda + +::: aloha.util.sys_gpu + +::: aloha.util.sys_info diff --git a/src/README.md b/src/README.md index 4f4839b..10342a5 100644 --- a/src/README.md +++ b/src/README.md @@ -21,7 +21,7 @@ Please generously STAR★ our project or donate to us! ## Getting started -Refer to[📚 Document & 中文文档](https://aloha-python.readthedocs.io/) for detailed introduction. +Refer to 📚 [Document](https://aloha-python.readthedocs.io/) & [中文文档](https://aloha-python.readthedocs.io/zh-cn/main/) for detailed introduction. ```shell pip install aloha[all] diff --git a/src/aloha/__init__.py b/src/aloha/__init__.py index 8d09f69..d374cc5 100644 --- a/src/aloha/__init__.py +++ b/src/aloha/__init__.py @@ -1,3 +1,3 @@ -__all__ = ("__version__",) - from ._version import __version__ + +__all__ = ("__version__",) diff --git a/src/aloha/config/paths.py b/src/aloha/config/paths.py index 0a6a913..8875331 100644 --- a/src/aloha/config/paths.py +++ b/src/aloha/config/paths.py @@ -1,9 +1,9 @@ -__all__ = ("get_resource_dir", "get_config_dir", "get_current_module_dir", "get_project_base_dir", "path_join") - import os import sys import warnings +__all__ = ("get_resource_dir", "get_config_dir", "get_current_module_dir", "get_project_base_dir", "path_join") + def path_join(*args) -> str: """ diff --git a/src/aloha/db/duckdb.py b/src/aloha/db/duckdb.py index 191a14b..fdc8373 100644 --- a/src/aloha/db/duckdb.py +++ b/src/aloha/db/duckdb.py @@ -1,4 +1,4 @@ -__all__ = ("DuckOperator",) +"""DuckDB connection helpers.""" from pathlib import Path @@ -8,11 +8,16 @@ from aloha.logger import LOG +__all__ = ("DuckOperator",) + LOG.debug("duckdb version = %s, duckdb_engine = %s " % (duckdb.__version__, duckdb_engine.__version__)) class DuckOperator: + """Create and use a DuckDB connection through SQLAlchemy.""" + def __init__(self, db_config, **kwargs): + """Build a DuckDB engine, creating the database file if necessary.""" """db_config example: { "path": "/path/to/db.duckdb", # file path of duckdb, use ":memory:" for in-memory mode @@ -79,6 +84,7 @@ def _prepare_database(self): raise RuntimeError(f"Failed to create database file '{path}': {e}") def _initialize_schema(self): + """Create or select the requested schema.""" if self._config["schema"] == "main": return @@ -104,6 +110,7 @@ def connection(self): conn = connection def execute_query(self, sql, *args, **kwargs): + """Execute a SQL statement and return the cursor result.""" with self.engine.connect() as conn: cur = conn.execute(text(sql), *args, *kwargs) if self._config.get("auto_commit", True): @@ -112,4 +119,5 @@ def execute_query(self, sql, *args, **kwargs): @property def connection_str(self) -> str: + """Return a human-readable connection string.""" return f"duckdb:///{self._config['path']} [schema={self._config['schema']}, read_only={self._config['read_only']}]" diff --git a/src/aloha/db/elasticsearch.py b/src/aloha/db/elasticsearch.py index b6412c4..f360d6f 100644 --- a/src/aloha/db/elasticsearch.py +++ b/src/aloha/db/elasticsearch.py @@ -1,4 +1,4 @@ -__all__ = ("ElasticSearchOperator",) +"""Elasticsearch connection helpers.""" import json @@ -7,9 +7,14 @@ from ..logger import LOG from .base import PasswordVault +__all__ = ("ElasticSearchOperator",) + class ElasticSearchOperator: + """Create and use an Elasticsearch client with optional index helpers.""" + def __init__(self, config, index_config=None): + """Build the client and optionally load the index configuration.""" self.es_config = config password_vault = PasswordVault.get_vault(config.get("vault_type"), config.get("vault_config")) @@ -36,6 +41,7 @@ def __init__(self, config, index_config=None): @staticmethod def _load_config(config): + """Load an index configuration from a dict or JSON file.""" if isinstance(config, dict): return config @@ -48,6 +54,7 @@ def _load_config(config): raise ValueError("Invalid ES config data type") def put_mapping(self, index_name=None, index_type=None, index_config: dict | None = None): + """Apply a mapping definition to the current index.""" return self.es.indices.put_mapping( index=index_name or self.index_name, doc_type=index_type or self.index_type, @@ -55,6 +62,7 @@ def put_mapping(self, index_name=None, index_type=None, index_config: dict | Non ) def build_index(self, index_name=None, index_config=None, raise_if_exist=False): + """Create the index if it does not already exist.""" if self.es.indices.exists(index=index_name or self.index_name) is not True: res = self.es.indices.create(index=index_name or self.index_name, body=index_config or self.index_config) return res @@ -67,10 +75,13 @@ def build_index(self, index_name=None, index_config=None, raise_if_exist=False): return False def search(self, query, index_name=None, index_type=None): + """Execute a search query.""" return self.es.search(index=index_name or self.index_name, doc_type=index_type or self.index_type, body=query) def msearch(self, body): + """Execute a multi-search request.""" return self.es.msearch(body=body) def insert(self, doc, index_name=None, index_type=None, id=None): + """Insert or replace a document.""" return self.es.index(index=index_name or self.index_name, doc_type=index_type or self.index_type, id=id, body=doc) diff --git a/src/aloha/db/kafka.py b/src/aloha/db/kafka.py index 54d7403..0817d4f 100644 --- a/src/aloha/db/kafka.py +++ b/src/aloha/db/kafka.py @@ -1,4 +1,4 @@ -__all__ = ("KafkaOperator",) +"""Kafka connection helpers.""" import json import typing @@ -8,10 +8,14 @@ from ..logger import LOG +__all__ = ("KafkaOperator",) + LOG.debug("Version of confluent_kafka client = %s" % kafka.__version__) class KafkaOperator: + """Create Kafka admin, producer, and consumer clients.""" + def __init__(self, kafka_config): """ Parameter reference: https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md @@ -30,11 +34,13 @@ def __init__(self, kafka_config): LOG.debug("Kafka connection info: " + str(self._config)) def admin_client(self, *args, **kwargs): + """Return a configured Kafka AdminClient.""" config_admin = {**self._config} a = kafka_admin.AdminClient(config_admin) return a def create_topic(self, topic: str, num_partitions=3, replication_factor=1, *args, **kwargs): + """Create a Kafka topic and wait for the broker response.""" """Note: In a multi-cluster production scenario, it is more typical to use a replication_factor of 3 for durability.""" a = self.admin_client() new_topic = kafka_admin.NewTopic(topic, num_partitions=num_partitions, replication_factor=replication_factor) @@ -56,6 +62,7 @@ def create_topic(self, topic: str, num_partitions=3, replication_factor=1, *args return True def producer_deliver(self, topic: str, generator: typing.Iterator[str], func_callback: callable = None, *args, **kwargs): + """Stream messages from an iterator into a Kafka topic.""" # func_callback should be a function that takes two arguments: err and msg config_producer = {**self._config} p = kafka.Producer(config_producer) @@ -85,6 +92,7 @@ def delivery_report(err, msg): def consumer_generator( self, topics_subscribe: list, group_id: str | None = None, poll_timeout: float = 1.0, *args, **kwargs ) -> typing.Iterator[str]: + """Yield decoded messages from the subscribed Kafka topics.""" config_consumer = {"auto.offset.reset": "earliest", **self._config} if group_id is not None: config_consumer["group.id"] = group_id diff --git a/src/aloha/db/mongo.py b/src/aloha/db/mongo.py index 5bb03a9..91ba9e3 100644 --- a/src/aloha/db/mongo.py +++ b/src/aloha/db/mongo.py @@ -1,4 +1,4 @@ -__all__ = ("MongoOperator",) +"""MongoDB connection helpers.""" import ipaddress import json @@ -8,6 +8,8 @@ from ..logger import LOG from .base import PasswordVault +__all__ = ("MongoOperator",) + def _is_ip_addr(s): try: @@ -21,6 +23,7 @@ def _is_ip_addr(s): def MongoOperator(config): + """Return a cached MongoDB operation wrapper for the given config.""" db_name = config.get("db_name") collection_name = config.get("collection_name") @@ -37,7 +40,10 @@ def MongoOperator(config): class _MongoDBOperation: + """MongoDB collection helper built on top of pymongo.""" + def __init__(self, config, db_name=None, collection_name=None): + """Create a MongoClient and optionally bind a default collection.""" self.db_name, self.collection_name = db_name, collection_name host = config["host"] @@ -74,6 +80,7 @@ def __init__(self, config, db_name=None, collection_name=None): LOG.exception(e) def set_collection(self, collection_name): + """Switch the active collection after verifying it exists.""" if collection_name not in self.db.list_collection_names(): raise Exception("Collection[%s] does not exist in [%s]" % (self.collection_name, self.db_name)) self.collection_name = collection_name @@ -81,15 +88,7 @@ def set_collection(self, collection_name): return True def check_and_get_collection(self, collection_name=None, raise_if_not_exists=True): - """ - 检查collection是否存在，如果存在则返回collection对象，否则抛出异常 - Args: - @param collection_name: str或unicode collection名称 - Returns: - @return: collection对象 - Raises: - @raise Exception: 如果collection在对应db中不存在，则 - """ + """Return the active collection, switching it when requested.""" self.db = self.conn[self.db_name] if self.collection_name is not None: @@ -108,6 +107,7 @@ def check_and_get_collection(self, collection_name=None, raise_if_not_exists=Tru return self.collection def insert(self, doc_or_docs, check_keys=False, collection_name=None): + """Insert a single document or a list of documents.""" try: collection = self.check_and_get_collection(collection_name) return collection.insert(doc_or_docs, check_keys=check_keys) @@ -115,6 +115,7 @@ def insert(self, doc_or_docs, check_keys=False, collection_name=None): LOG.exception(e) def insert_many(self, docs, collection_name=None): + """Insert many documents at once.""" try: collection = self.check_and_get_collection(collection_name) return collection.insert_many(docs) @@ -122,6 +123,7 @@ def insert_many(self, docs, collection_name=None): LOG.exception(e) def insert_one(self, doc, collection_name=None): + """Insert one document.""" try: collection = self.check_and_get_collection(collection_name) return collection.insert_one(doc) @@ -129,6 +131,7 @@ def insert_one(self, doc, collection_name=None): LOG.exception(e) def delete_many(self, field_filter, collection_name=None): + """Delete all documents matching the filter.""" try: collection = self.check_and_get_collection(collection_name) return collection.delete_many(filter=field_filter) @@ -136,6 +139,7 @@ def delete_many(self, field_filter, collection_name=None): LOG.exception(e) def delete_one(self, field_filter, collection_name=None): + """Delete one document matching the filter.""" try: collection = self.check_and_get_collection(collection_name) return collection.delete_one(filter=field_filter) @@ -153,6 +157,7 @@ def update_one( session=None, collection_name=None, ): + """Update one document and return whether the update succeeded.""" try: collection = self.check_and_get_collection(collection_name) collection.update_one( @@ -180,6 +185,7 @@ def update_many( session=None, collection_name=None, ): + """Update many documents matching the filter.""" try: collection = self.check_and_get_collection(collection_name) return collection.update_many( @@ -195,17 +201,7 @@ def update_many( LOG.exception(e) def query(self, field_filter=None, sort=None, limit=40, skip=0, collection_name=None): - """ - 从mongo查询数据 - Args: - @param field_filter: dict 根据mongo查询语法构造查询条件 - @param sort: array 排序条件，例如：[("company_name",pymongo.ASCENDING), ("_id",pymongo.ASCENDING)] - @param limit: int 查询条数 - @param skip: int controls the starting point of the results set - @param collection_name: str collection名称，在建立完连接后可动态更换要查询的collection - Returns: - @return: 返回查询结果的游标对象 - """ + """Query documents with optional sorting, limit, and skip.""" try: collection = self.check_and_get_collection(collection_name) if sort: @@ -217,13 +213,7 @@ def query(self, field_filter=None, sort=None, limit=40, skip=0, collection_name= LOG.exception(e) def find_many(self, field_filter=None, projection=None, collection_name=None, *args, **kwargs): - """ - 从mongo查询数据，返回所有返回数据的游标 - Args: - @param field_filter: dict 根据mongo查询语法构造查询条件 - @param projection: dict 限制返回的字段 exmaple: { name: 1, contribs: 1, _id: 0 } - @param collection_name: str collection名称，在建立完连接后可动态更换要查询的collection - """ + """Return a cursor for a MongoDB query.""" try: collection = self.check_and_get_collection(collection_name) result = collection.find(field_filter or {}, projection, *args, **kwargs) @@ -232,13 +222,7 @@ def find_many(self, field_filter=None, projection=None, collection_name=None, *a LOG.exception(e) def find_one(self, field_filter=None, projection=None, collection_name=None, *args, **kwargs): - """ - 从mongo查询数据,只返回单个结果 - Args: - @param field_filter: dict 根据mongo查询语法构造查询条件 - @param projection: dict 限制返回的字段 exmaple: { name: 1, contribs: 1, _id: 0 } - @param collection_name: str collection名称，在建立完连接后可动态更换要查询的collection - """ + """Return a single matching MongoDB document.""" try: collection = self.check_and_get_collection(collection_name) result = collection.find_one(field_filter or {}, projection, *args, **kwargs) diff --git a/src/aloha/db/mysql.py b/src/aloha/db/mysql.py index e3aeef9..aff7632 100644 --- a/src/aloha/db/mysql.py +++ b/src/aloha/db/mysql.py @@ -1,4 +1,4 @@ -__all__ = ("MySqlOperator",) +"""MySQL connection helpers.""" import pymysql from sqlalchemy import create_engine @@ -7,11 +7,16 @@ from ..logger import LOG from .base import PasswordVault +__all__ = ("MySqlOperator",) + LOG.debug("Version of pymysql = %s" % pymysql.__version__) class MySqlOperator: + """Create and use a SQLAlchemy-backed MySQL connection.""" + def __init__(self, db_config, **kwargs): + """Build a connection pool from the provided database config.""" password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config")) self._config = { "host": db_config["host"], @@ -39,10 +44,12 @@ def connection(self): return self.db def execute_query(self, sql, *args, **kwargs): + """Execute a SQL statement and return the cursor result.""" with self.db.connect() as conn: cur = conn.execute(text(sql), *args, **kwargs) return cur @property def connection_str(self) -> str: + """Return a human-readable connection string.""" return "mysql://{user}:{password}@{host}:{port}/{dbname}".format(**self._config) diff --git a/src/aloha/db/oracle.py b/src/aloha/db/oracle.py index 62f9905..197dd9b 100644 --- a/src/aloha/db/oracle.py +++ b/src/aloha/db/oracle.py @@ -1,4 +1,4 @@ -__all__ = ("OracledbOperator",) +"""Oracle DB connection helpers.""" import oracledb from sqlalchemy import create_engine @@ -7,11 +7,16 @@ from ..logger import LOG from .base import PasswordVault +__all__ = ("OracledbOperator",) + LOG.debug("oracledb version = %s" % oracledb.__version__) class OracledbOperator: + """Create and use a SQLAlchemy-backed Oracle connection.""" + def __init__(self, db_config, **kwargs): + """Build an Oracle connection pool from the provided config.""" """example of db_config: { "host": "192.168.1.100", @@ -73,10 +78,12 @@ def connection(self): return self.engine def execute_query(self, sql, *args, **kwargs): + """Execute a SQL statement and return the cursor result.""" with self.engine.connect() as conn: cur = conn.execute(text(sql), *args, **kwargs) return cur @property def connection_str(self) -> str: + """Return a human-readable connection string.""" return "oracle://{user}@{host}:{port}".format(**self._config) diff --git a/src/aloha/db/postgres.py b/src/aloha/db/postgres.py index f036b60..ede289e 100644 --- a/src/aloha/db/postgres.py +++ b/src/aloha/db/postgres.py @@ -1,4 +1,4 @@ -__all__ = ("PostgresOperator",) +"""PostgreSQL connection helpers.""" import psycopg from sqlalchemy import create_engine @@ -7,11 +7,16 @@ from ..logger import LOG from .base import PasswordVault +__all__ = ("PostgresOperator",) + LOG.debug("postgres: psycopg version = %s" % psycopg.__version__) class PostgresOperator: + """Create and use a SQLAlchemy-backed PostgreSQL connection.""" + def __init__(self, db_config, **kwargs): + """Build a PostgreSQL connection pool from the database config.""" password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config")) self._config = { "host": db_config["host"], @@ -44,10 +49,12 @@ def connection(self): return self.engine def execute_query(self, sql, *args, **kwargs): + """Execute a SQL statement and return the cursor result.""" with self.engine.connect() as conn: cur = conn.execute(text(sql), *args, **kwargs) return cur @property def connection_str(self) -> str: + """Return a human-readable connection string.""" return "postgresql://{user}:{password}@{host}:{port}/{dbname}".format(**self._config) diff --git a/src/aloha/db/redis.py b/src/aloha/db/redis.py index 3f086fd..11fdb1d 100644 --- a/src/aloha/db/redis.py +++ b/src/aloha/db/redis.py @@ -1,4 +1,4 @@ -__all__ = ("RedisOperator",) +"""Redis connection helpers.""" import redis from packaging import version @@ -6,9 +6,14 @@ from ..logger import LOG from .base import PasswordVault +__all__ = ("RedisOperator",) + class RedisOperator: + """Create Redis connections with version-checked redis-py.""" + def __init__(self, config): + """Normalize Redis connection settings and build connection metadata.""" self._check_redis_version() password_vault = PasswordVault.get_vault(config.get("vault_type"), config.get("vault_config")) @@ -30,6 +35,7 @@ def __init__(self, config): @staticmethod def _check_redis_version() -> bool: + """Ensure a redis-py version new enough for the helpers is installed.""" ver_min = version.parse("4.1.0") valid = False try: @@ -50,7 +56,7 @@ def _check_redis_version() -> bool: @property def connection_generic(self): - """https://github.com/redis/redis-py/blob/master/redis/client.py""" + """Return a standard Redis client.""" LOG.debug("StrictRedis connection info: {host}:{port}".format(**self._config)) if self._pool is None: @@ -59,5 +65,6 @@ def connection_generic(self): @property def connection_cluster(self): + """Return a Redis Cluster client.""" LOG.debug("RedisCluster connection info: {host}:{port}".format(**self._config)) return redis.RedisCluster(**self._config) diff --git a/src/aloha/db/sqlite.py b/src/aloha/db/sqlite.py index 4925422..8e7c460 100644 --- a/src/aloha/db/sqlite.py +++ b/src/aloha/db/sqlite.py @@ -1,4 +1,4 @@ -__all__ = ("SqliteOperator",) +"""SQLite connection helpers.""" import sqlite3 @@ -8,9 +8,14 @@ from ..logger import LOG from .base import PasswordVault +__all__ = ("SqliteOperator",) + class SqliteOperator: + """Create and use a SQLAlchemy-backed SQLite connection.""" + def __init__(self, db_config, **kwargs): + """Build a SQLite or SQLCipher engine from the provided config.""" self._connection_pattern = "sqlite://{dbname}" dbname = db_config.get("dbname", "") if len(dbname) > 0: @@ -42,10 +47,12 @@ def connection(self): return self.db def execute_query(self, sql, *args, **kwargs): + """Execute a SQL statement and return the cursor result.""" with self.db.connect() as conn: cur = conn.execute(text(sql), *args, **kwargs) return cur @property def connection_str(self) -> str: + """Return the SQLAlchemy connection URL used by the engine.""" return self._connection_pattern.format(**self._config) diff --git a/src/aloha/encrypt/aes.py b/src/aloha/encrypt/aes.py index b3461d5..36c0924 100644 --- a/src/aloha/encrypt/aes.py +++ b/src/aloha/encrypt/aes.py @@ -1,3 +1,5 @@ +"""AES encrypt/decrypt helpers.""" + import base64 import binascii from typing import Callable, Optional, Union @@ -15,6 +17,7 @@ def _generate_key(key_size: int, method="const") -> bytes: + """Generate an AES key using a deterministic or random strategy.""" if method == "const": return b"0" * key_size # b'b6046801716aec00' elif method == "random": @@ -23,9 +26,12 @@ def _generate_key(key_size: int, method="const") -> bytes: class AesEncryptor: + """Encrypt and decrypt strings with a selectable AES cipher profile.""" + supported_cipher_methods = _AES_CIPHER_METHODS def __init__(self, key: Union[str, bytes] = None, key_size: int = 16, cipher_name: str = "AES/ECB/PKCS5Padding"): + """Initialize the AES key and cipher settings.""" _key = key if key is None: _key = _generate_key(key_size) @@ -44,6 +50,7 @@ def __init__(self, key: Union[str, bytes] = None, key_size: int = 16, cipher_nam self.cipher_name = cipher_name def encrypt(self, text: str, output_format="hex", func_pad: Optional[Callable] = None) -> Union[str, bytes]: + """Encrypt a UTF-8 string and return hex, base64, or raw bytes.""" dict_params, pad_style = _AES_CIPHER_METHODS.get(self.cipher_name) if not callable(func_pad): @@ -71,6 +78,7 @@ def _func_pad(x): def decrypt( self, text: Union[str, bytes], input_format: str = "hex", func_unpad: Optional[Callable] = None ) -> Union[str, bytes]: + """Decrypt ciphertext produced by :meth:`encrypt`.""" text += (len(text) % 4) * "=" if input_format == "hex": crypt = binascii.a2b_hex(text) @@ -94,6 +102,7 @@ def _func_unpad(x): def main(): + """Small self-test for the AES helper.""" a = AesEncryptor() src = "hello~" enc = a.encrypt(src, output_format="base64") diff --git a/src/aloha/encrypt/hash.py b/src/aloha/encrypt/hash.py index 0ac608b..3eecaa8 100644 --- a/src/aloha/encrypt/hash.py +++ b/src/aloha/encrypt/hash.py @@ -1,20 +1,26 @@ +"""Hash helpers used by the signing utilities.""" + import hashlib import json def get_md5_of_str(string): + """Return the MD5 hex digest of a string.""" return hashlib.md5(string.encode()).hexdigest() def get_sha256_of_str(string): + """Return the SHA-256 hex digest of a string.""" return hashlib.sha256(string.encode()).hexdigest() def hash_dict(dic): + """Hash a dictionary after JSON normalization.""" s = json.dumps(dict(dic), sort_keys=True, ensure_ascii=False, default=str) return hashlib.md5(s.encode()).hexdigest() def hash_obj(obj): + """Hash an arbitrary JSON-serializable object.""" s = json.dumps(obj, sort_keys=True, ensure_ascii=False, default=str) return hashlib.md5(s.encode()).hexdigest() diff --git a/src/aloha/encrypt/jwt.py b/src/aloha/encrypt/jwt.py index d7d5103..5de4ebb 100644 --- a/src/aloha/encrypt/jwt.py +++ b/src/aloha/encrypt/jwt.py @@ -1,3 +1,5 @@ +"""JWT encode/decode helpers.""" + import jwt from ..logger import LOG @@ -6,11 +8,13 @@ def encode(secret_key: str, payload: dict, headers: dict = None, **kwargs): + """Encode a payload into a JWT token.""" token = jwt.encode(payload=payload, key=secret_key, headers=headers, **kwargs) return token def decode(secret_key: str, token: str, **kwargs): + """Decode a JWT token and return either the payload or an error string.""" try: resp = jwt.decode(jwt=token, key=secret_key, algorithms=["HS256"], **kwargs) except jwt.ExpiredSignatureError as e: diff --git a/src/aloha/encrypt/rsa.py b/src/aloha/encrypt/rsa.py index d0f08f5..5d37847 100644 --- a/src/aloha/encrypt/rsa.py +++ b/src/aloha/encrypt/rsa.py @@ -1,4 +1,4 @@ -__all__ = ("RsaEncryptor",) +"""RSA encrypt/decrypt and signing helpers.""" import base64 from functools import lru_cache @@ -9,6 +9,8 @@ from Crypto.PublicKey import RSA from Crypto.Signature import pss +__all__ = ("RsaEncryptor",) + t_cipher_module = Union[PKCS1_v1_5.PKCS115_Cipher, PKCS1_OAEP.PKCS1OAEP_Cipher] _RSA_CIPHER_METHODS = { # FULL_CIPHER_NAME: (module, dict_params) @@ -19,6 +21,8 @@ class RsaEncryptor: + """Encrypt, decrypt, and convert RSA keys and payloads.""" + _dict_cache_cipher = {} _dict_cache_decipher = {} supported_cipher_methods = _RSA_CIPHER_METHODS @@ -27,6 +31,7 @@ class RsaEncryptor: def __init__( self, key_private: str | None = None, key_public: str | None = None, cipher_name: str = "RSA/ECB/PKCS1Padding" ): + """Load optional keys and select a cipher profile.""" self.key_private, self.key_public = self.load_keys_from_string(key_private=key_private, key_public=key_public) assert self._get_cipher_module(cipher_name) is not None, "Invalid cipher_name!" self.cipher_name = cipher_name @@ -40,12 +45,14 @@ def _get_cipher_module(full_cipher_name: str | None = None) -> Optional[Tuple]: @staticmethod def generate_key_pair(size: int = 1024) -> Tuple[str, str]: + """Generate a PEM-encoded RSA key pair.""" key_pair = RSA.generate(size) key_private, key_public = key_pair.exportKey(), key_pair.publickey().exportKey() return key_private.decode("ascii"), key_public.decode("ascii") @lru_cache def get_cipher(self, key_public: str | None = None, cipher_name="RSA/ECB/PKCS1Padding") -> t_cipher_module: + """Return a cached public-key cipher instance.""" if key_public is None: key_pub = self.key_public else: @@ -61,6 +68,7 @@ def get_cipher(self, key_public: str | None = None, cipher_name="RSA/ECB/PKCS1Pa @lru_cache def get_decipher(self, key_private: str | None = None, cipher_name="RSA/ECB/PKCS1Padding") -> t_cipher_module: + """Return a cached private-key decipher instance.""" if key_private is None: key_pri = self.key_private else: @@ -78,6 +86,7 @@ def get_decipher(self, key_private: str | None = None, cipher_name="RSA/ECB/PKCS def load_keys_from_binary( key_private: bytes | None = None, key_public: bytes | None = None ) -> Tuple[Optional[RSA.RsaKey], Optional[RSA.RsaKey]]: + """Load RSA keys from PEM/DER bytes.""" _key_private, _key_public = None, None if key_private is not None: @@ -98,6 +107,7 @@ def load_keys_from_binary( def load_keys_from_string( key_private: str | None = None, key_public: str | None = None ) -> Tuple[Optional[RSA.RsaKey], Optional[RSA.RsaKey]]: + """Load RSA keys from PEM-like strings.""" _key_private, _key_public = None, None if key_private is not None: @@ -125,6 +135,7 @@ def load_keys_from_string( def encrypt_with_public_key( self, message: Union[str, bytes], key_public: str | None = None, cipher_name: str = None ) -> bytes: + """Encrypt a message with a public key.""" data = message if isinstance(message, bytes) else message.encode("UTF-8") cipher = self.get_cipher(key_public=key_public, cipher_name=cipher_name or self.cipher_name) return cipher.encrypt(data) @@ -132,20 +143,24 @@ def encrypt_with_public_key( def decrypt_with_private_key( self, ciphertext: Union[str, bytes], key_private: str | None = None, cipher_name: str | None = None, **kwargs ) -> bytes: + """Decrypt ciphertext with a private key.""" data = ciphertext if isinstance(ciphertext, bytes) else ciphertext.encode("ascii") decipher = self.get_decipher(key_private=key_private, cipher_name=cipher_name or self.cipher_name) return decipher.decrypt(data, **kwargs) @staticmethod def convert_bytes_to_base64(data: bytes) -> str: + """Encode raw bytes as base64 text.""" return base64.b64encode(data).decode() @staticmethod def convert_base64_to_bytes(data: str) -> bytes: + """Decode base64 text into raw bytes.""" return base64.decodebytes(data.encode("ascii")) def main(): + """Small self-test for the RSA helper.""" key_pairs = [ # private_key, public_key RsaEncryptor.generate_key_pair(), ] diff --git a/src/aloha/encrypt/vault/__init__.py b/src/aloha/encrypt/vault/__init__.py index fd85018..2ccd294 100644 --- a/src/aloha/encrypt/vault/__init__.py +++ b/src/aloha/encrypt/vault/__init__.py @@ -1,5 +1,5 @@ -__all__ = ("BaseVault", "DummyVault", "AesVault", "CyberArkVault") - from .base import BaseVault, DummyVault from .cyberark import CyberArkVault from .plain import AesVault + +__all__ = ("BaseVault", "DummyVault", "AesVault", "CyberArkVault") diff --git a/src/aloha/encrypt/vault/base.py b/src/aloha/encrypt/vault/base.py index 59b394d..52aa2db 100644 --- a/src/aloha/encrypt/vault/base.py +++ b/src/aloha/encrypt/vault/base.py @@ -1,34 +1,19 @@ +"""Password vault abstraction used by database and service helpers.""" + import abc from urllib.parse import quote_plus as urlquote class BaseVault(abc.ABC): - """ - Abstract base class for password vault implementations. - - Defines the interface for password vaults that can decrypt passwords. - """ + """Abstract base class for password vault implementations.""" @abc.abstractmethod def decrypt_password(self, *args, **kwargs): - """ - Decrypt a password. - - :param args: Additional arguments - :param kwargs: Additional keyword arguments, should contain 'password' - :return: Decrypted password - """ + """Decrypt a password and return the plain-text value.""" return kwargs.get("password") def get_password(self, password, *args, **kwargs): - """ - Get a password, optionally URL-encoded. - - :param password: Password or dict containing password - :param args: Additional arguments - :param kwargs: Additional keyword arguments, can include 'url_encode' - :return: Password, optionally URL-encoded - """ + """Return a password, optionally URL-encoded.""" kwargs.update(password if isinstance(password, dict) else {"password": password}) url_quote = kwargs.get("url_encode", True) @@ -43,27 +28,15 @@ def get_password(self, password, *args, **kwargs): class DummyVault(BaseVault): - """ - Dummy vault implementation that returns passwords as-is. - - No actual encryption/decryption is performed. - """ + """Vault implementation that returns passwords unchanged.""" def decrypt_password(self, *args, **kwargs): - """ - Return password without decryption. - - :param args: Additional arguments - :param kwargs: Additional keyword arguments, should contain 'password' - :return: Original password - """ + """Return the original password without decryption.""" return kwargs.get("password") def main(): - """ - Test function for DummyVault. - """ + """Small self-test for the dummy vault.""" vault = DummyVault() ret = vault.get_password(None, url_quote=True) # print(ret) diff --git a/src/aloha/encrypt/vault/cyberark.py b/src/aloha/encrypt/vault/cyberark.py index 4500d24..e6c7954 100644 --- a/src/aloha/encrypt/vault/cyberark.py +++ b/src/aloha/encrypt/vault/cyberark.py @@ -1,3 +1,5 @@ +"""CyberArk-based password vault helpers.""" + import hashlib from binascii import a2b_hex from urllib.parse import quote_plus as urlquote @@ -16,20 +18,25 @@ class CyberArkVault(BaseVault, AesEncryptor): + """Fetch and decrypt passwords from a CyberArk-compatible endpoint.""" + _cached: dict = {} def __init__(self, url: str, app_id: str, key: str | None = None, safe: str = "AIM_ELIS_LAS", folder: str = "root"): + """Initialize the vault with the CyberArk endpoint and credentials.""" super().__init__(key) self.key, self.url, self.app_id, self.safe, self.folder = key, url, app_id, safe, folder @staticmethod def get_sign(appid, keyvalue): + """Generate the CyberArk request signature.""" hash_string = appid + "&" + keyvalue sha256 = hashlib.sha256() sha256.update(hash_string.encode("utf8")) return sha256.hexdigest() def decrypt_password(self, text): + """Decrypt the AES-encrypted password returned by CyberArk.""" if text is None: return None @@ -39,6 +46,7 @@ def decrypt_password(self, text): return s.decode() def get_cyberark_password(self, object: str | None = None, **kwargs): + """Request and decrypt a password from the CyberArk endpoint.""" assert isinstance(object, str) kwargs.update({"object": object}) @@ -76,6 +84,7 @@ def get_cyberark_password(self, object: str | None = None, **kwargs): return None def get_password(self, object=None, **kwargs): + """Return a cached CyberArk password, optionally URL-encoded.""" key_for_cache = "{app_id};{safe};{folder};{key};{object}".format( app_id=self.app_id, safe=self.safe, folder=self.folder, key=self.key, object=object ) @@ -94,6 +103,7 @@ def get_password(self, object=None, **kwargs): def main(): + """Small self-test scaffold for the CyberArk vault.""" cfg_cyberark = dict( url="https://localhost/pidms/rest/pwd/getPassword", # to fill properly app_id="", diff --git a/src/aloha/encrypt/vault/plain.py b/src/aloha/encrypt/vault/plain.py index b37dcd4..7a6dea9 100644 --- a/src/aloha/encrypt/vault/plain.py +++ b/src/aloha/encrypt/vault/plain.py @@ -1,3 +1,5 @@ +"""AES-backed password vault helpers.""" + import pyhocon from ...encrypt.aes import AesEncryptor @@ -5,20 +7,26 @@ def _is_empty_str(s): + """Return True when the value should be treated as empty.""" return s is None or isinstance(s, pyhocon.config_tree.NoneValue) or s == "None" or s == "" class AesVault(AesEncryptor, BaseVault): + """Password vault that stores encrypted secrets with AES.""" + def __init__(self, key: str | None = None): + """Initialize the vault with an optional AES key.""" super().__init__(key) def decrypt_password(self, pwd): + """Decrypt the stored password and return plain text.""" if _is_empty_str(pwd): return None return self.decrypt(pwd) def main(): + """Small self-test for the AES vault.""" vault = AesVault() pwd = vault.get_password(None, url_quote=True) # print(pwd) diff --git a/src/aloha/logger/__init__.py b/src/aloha/logger/__init__.py index 32bfbf8..1459143 100644 --- a/src/aloha/logger/__init__.py +++ b/src/aloha/logger/__init__.py @@ -1,8 +1,7 @@ -__all__ = ("LOG", "get_logger", "getLogger") - from ..settings import SETTINGS from .logger import get_logger, getLogger LOG = get_logger( level=SETTINGS.config.get("deploy", {}).get("log_level", 10), # 10 = logging.DEBUG ) +__all__ = ("LOG", "get_logger", "getLogger") diff --git a/src/aloha/script/compile.py b/src/aloha/script/compile.py index 2a45d66..7d4f0cf 100644 --- a/src/aloha/script/compile.py +++ b/src/aloha/script/compile.py @@ -4,8 +4,6 @@ Example: python -m aloha.script.compile --base=./ --dist=../build --keep='main.py' """ -__all__ = ("build", "package") - import argparse import glob import os @@ -19,6 +17,8 @@ except ImportError: raise RuntimeError("Please pip install Cython first!") +__all__ = ("build", "package") + def _expand(patterns: list | None = None): files = [] diff --git a/src/aloha/service/__init__.py b/src/aloha/service/__init__.py index 5747748..35bf1d2 100644 --- a/src/aloha/service/__init__.py +++ b/src/aloha/service/__init__.py @@ -1,5 +1,4 @@ -__all__ = ("DefaultHandler404", "v0", "v1", "v2") - - from .api import v0, v1, v2 from .http import DefaultHandler404 + +__all__ = ("DefaultHandler404", "v0", "v1", "v2") diff --git a/src/aloha/service/api/v0.py b/src/aloha/service/api/v0.py index b239cfc..f5bf570 100644 --- a/src/aloha/service/api/v0.py +++ b/src/aloha/service/api/v0.py @@ -1,4 +1,9 @@ -__all__ = ('APIHandler', 'APICaller',) +"""Version 0 JSON API helpers. + +This module defines the simplest request/response protocol used by aloha: +request bodies are passed directly to the handler method and the response is +serialized as a JSON object with a `code` and `message` field. +""" import json import logging @@ -6,32 +11,45 @@ from ..http import AbstractApiClient, AbstractApiHandler +__all__ = ("APIHandler", "APICaller") + class APIHandler(AbstractApiHandler, ABC): - MAP_ERROR_INFO = { - 'BAD_REQUEST': {'code': '5101', 'message': ['Bad request: fail to parse body as JSON object!']} - } + """Base Tornado handler for v0 JSON endpoints. + + Subclasses implement :meth:`response`, which receives parsed request data + and returns a Python object that can be JSON-serialized. + """ + + MAP_ERROR_INFO = {"BAD_REQUEST": {"code": "5101", "message": ["Bad request: fail to parse body as JSON object!"]}} async def post(self, *args, **kwargs): + """Parse the request body, call :meth:`response`, and return JSON.""" req_body = self.request_body if req_body is not None: # body_arguments kwargs.update(req_body) - resp = dict(code=5200, message=['success']) + resp = dict(code=5200, message=["success"]) try: result = self.response(*args, **kwargs) # this call may throw TypeError when argument missing - resp['data'] = result + resp["data"] = result except Exception as e: if self.LOG.level == logging.DEBUG: self.LOG.error(e, exc_info=True) - return self.finish({'code': 5201, 'message': [repr(e)]}) + return self.finish({"code": 5201, "message": [repr(e)]}) - resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(',', ':')) + resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(",", ":")) return self.finish(resp) class APICaller(AbstractApiClient): + """Client helper for v0 endpoints. + + The payload is sent as-is, without signature wrapping or token exchange. + """ + def wrap_request_data(self, data: dict) -> dict: + """Return the request body unchanged.""" assert isinstance(data, dict), "Data object must be a dict!" return data diff --git a/src/aloha/service/api/v1.py b/src/aloha/service/api/v1.py index bc14f26..8b8ce76 100644 --- a/src/aloha/service/api/v1.py +++ b/src/aloha/service/api/v1.py @@ -1,4 +1,8 @@ -__all__ = ("APIHandler", "APICaller", "sign_data", "sign_check") +"""Version 1 signed JSON API helpers. + +Version 1 adds request signing with `app_id`, `salt_uuid`, and `sign` fields. +Handlers validate the signature before dispatching to the service logic. +""" import json import logging @@ -9,6 +13,8 @@ from ...settings import SETTINGS from ..http import AbstractApiClient, AbstractApiHandler +__all__ = ("APIHandler", "APICaller", "sign_data", "sign_check") + APP_ID_KEYS = SETTINGS.config.get("APP_ID_KEYS", {}) APP_OPTIONS = SETTINGS.config.get("APP_OPTIONS", {}) FUNC_SIGN_CHECK = {"md5": get_md5_of_str, "sha256": get_sha256_of_str} @@ -16,6 +22,8 @@ class APIHandler(AbstractApiHandler, ABC): + """Signed API handler for v1 endpoints.""" + MAP_ERROR_INFO = { "BAD_REQUEST": {"code": "5101", "message": ["Bad request: fail to parse body as JSON object!"]}, "MISSING_ARGS": {"code": "5102", "message": ["Required argument field(s) missing..."]}, @@ -23,6 +31,7 @@ class APIHandler(AbstractApiHandler, ABC): } async def post(self): + """Validate the signature and dispatch the wrapped payload.""" body_arguments = self.request_body try: @@ -52,11 +61,20 @@ async def post(self): class APICaller(AbstractApiClient): + """Client helper that wraps payloads with v1 signing metadata.""" + APP_ID_KEYS = AbstractApiClient.config.get("APP_ID_KEYS", {}) def wrap_request_data( - self, data, app_id: str = None, app_key: str = None, salt_uuid: str = None, sign: str = None, sign_method: str = None + self, + data, + app_id: str | None = None, + app_key: str | None = None, + salt_uuid: str | None = None, + sign: str | None = None, + sign_method: str | None = None, ): + """Wrap the payload with signature fields expected by v1 handlers.""" if app_id is None: # if len(APP_ID_KEYS) != 1: # raise RuntimeError('Please specify 1 and only 1 in APP_ID_KEYS in configurations!') @@ -73,6 +91,10 @@ def wrap_request_data( def sign_data(salt_uuid: str, app_id: str, app_key: str, data, sign_method: str = None): + """Generate the v1 signature for a payload. + + The signature is based on `app_id + salt_uuid + data + app_key`. + """ data_str = str(json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(",", ":"))) public_key = app_id + salt_uuid + data_str + app_key @@ -84,15 +106,7 @@ def sign_data(salt_uuid: str, app_id: str, app_key: str, data, sign_method: str def sign_check(salt_uuid: str, app_id: str, sign: str, data, sign_method: str = None, date_time=None): - """Sign Validation - :param salt_uuid: Universal Unified ID for 1) Signature, 2) Log tracing - :param app_id: APP ID - :param sign: sing = hash(app_id + salt_uuid + data + app_key) - :param data: data object, will be serialized to JSON string - :param date_time: not used for now - :param sign_method: Sign method, one of the following: md5, sha256 - :return: If the signature passed validation - """ + """Validate a v1 request signature.""" func_sign_check = func_sign_check_default if sign_method is None else FUNC_SIGN_CHECK.get(sign_method) if func_sign_check is None: diff --git a/src/aloha/service/api/v2.py b/src/aloha/service/api/v2.py index d338d69..bb17eac 100644 --- a/src/aloha/service/api/v2.py +++ b/src/aloha/service/api/v2.py @@ -1,82 +1,96 @@ -__all__ = ('APIHandler', 'APICaller',) +"""Version 2 token-based JSON API helpers. + +Version 2 uses an access token in the request header and a request-id header +for tracing. It keeps the same request/response shape as the earlier API +generations while adding header-based authentication. +""" import json import logging from abc import ABC from datetime import datetime, timedelta -from typing import Optional, Awaitable +from typing import Awaitable, Optional -from ..http import AbstractApiClient, AbstractApiHandler from ...encrypt import jwt from ...settings import SETTINGS +from ..http import AbstractApiClient, AbstractApiHandler + +__all__ = ("APIHandler", "APICaller") class APIHandler(AbstractApiHandler, ABC): - async def prepare(self, ) -> Optional[Awaitable[None]]: - access_token = self.request.headers.get('Access-Token') + """Token-authenticated API handler for v2 endpoints.""" + + async def prepare( + self, + ) -> Optional[Awaitable[None]]: + """Validate the access token before handling the request.""" + access_token = self.request.headers.get("Access-Token") if access_token is None: - return self.finish({ - 'msg': 'Invalid Access-Token in request header!' - }) + return self.finish({"msg": "Invalid Access-Token in request header!"}) else: - secret_key = SETTINGS.config['APP_SECRET_KEY'] + secret_key = SETTINGS.config["APP_SECRET_KEY"] # options = None # TODO: if not validate expiration options = {"verify_exp": False} access_token = jwt.decode(secret_key, access_token, options=options) if not isinstance(access_token, dict): - self.LOG.error('Invalid Access-Token found in request for [%s]: %s' % ( - str(self.request.full_url()), access_token - )) - return self.finish({ - 'msg': access_token - }) - self.set_header('Request-ID', self.request_id) + self.LOG.error( + "Invalid Access-Token found in request for [%s]: %s" % (str(self.request.full_url()), access_token) + ) + return self.finish({"msg": access_token}) + self.set_header("Request-ID", self.request_id) async def post(self, *args, **kwargs): + """Handle POST requests with JSON request bodies.""" body_arguments = self.request_body kwargs.update(body_arguments) try: if self.LOG.level == logging.DEBUG: s_kwargs = json.dumps(kwargs, ensure_ascii=False) - self.LOG.debug('POST Request [%s]: %s' % (self.request_id, s_kwargs[:1000])) + self.LOG.debug("POST Request [%s]: %s" % (self.request_id, s_kwargs[:1000])) self.api_args, self.api_kwargs = args or (), kwargs or {} resp = self.response(*self.api_args, **self.api_kwargs) # this call may throw TypeError when argument missing except Exception as e: self.LOG.error(e, exc_info=True) - self.LOG.info('POST Request [%s]: %s' % (self.request_id, self.request.body)) - return self.finish({'status': 'error', 'message': [str(e)]}) + self.LOG.info("POST Request [%s]: %s" % (self.request_id, self.request.body)) + return self.finish({"status": "error", "message": [str(e)]}) if isinstance(resp, (dict, list)): - resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(',', ':')) + resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(",", ":")) return self.finish(resp) async def get(self, *args, **kwargs): + """Handle GET requests with query-string arguments.""" query_arguments = self.request_param kwargs.update(query_arguments) try: - self.LOG.debug('GET Request [%s]: %s' % (self.request_id, kwargs)) + self.LOG.debug("GET Request [%s]: %s" % (self.request_id, kwargs)) self.api_args, self.api_kwargs = args or (), kwargs or {} resp = self.response(*self.api_args, **self.api_kwargs) # this call may throw TypeError when argument missing except Exception as e: self.LOG.error(e, exc_info=True) - self.LOG.info('GET Request [%s]: %s' % (self.request_id, kwargs)) - return self.finish({'status': 'error', 'message': [repr(e)]}) + self.LOG.info("GET Request [%s]: %s" % (self.request_id, kwargs)) + return self.finish({"status": "error", "message": [repr(e)]}) if isinstance(resp, (dict, list)): - resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(',', ':')) + resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(",", ":")) return self.finish(resp) class APICaller(AbstractApiClient): - APP_ID_KEYS = AbstractApiClient.config.get('APP_ID_KEYS', {}) - APP_SECRET_KEY = AbstractApiClient.config.get('APP_SECRET_KEY') + """Client helper that adds v2 access-token headers automatically.""" + + APP_ID_KEYS = AbstractApiClient.config.get("APP_ID_KEYS", {}) + APP_SECRET_KEY = AbstractApiClient.config.get("APP_SECRET_KEY") def wrap_request_data(self, data: dict) -> dict: + """Return the request body unchanged.""" assert isinstance(data, dict), "Data object must be a dict!" return data def get_headers(self, app_id: str = None, app_key: str = None) -> dict: + """Build the HTTP headers expected by v2 handlers.""" if app_id is None: # if len(APP_ID_KEYS) != 1: # raise RuntimeError('Please specify 1 and only 1 in APP_ID_KEYS in configurations!') @@ -84,14 +98,8 @@ def get_headers(self, app_id: str = None, app_key: str = None) -> dict: expire_time = datetime.now() + timedelta(days=1) - access_token = jwt.encode( - secret_key=self.APP_SECRET_KEY, - payload={ - 'exp': int(expire_time.timestamp()), - 'aid': app_id - } - ) + access_token = jwt.encode(secret_key=self.APP_SECRET_KEY, payload={"exp": int(expire_time.timestamp()), "aid": app_id}) headers = super().get_headers() - headers.update({'Access-Token': access_token}) + headers.update({"Access-Token": access_token}) return headers diff --git a/src/aloha/service/app.py b/src/aloha/service/app.py index 1a2318c..00c5043 100644 --- a/src/aloha/service/app.py +++ b/src/aloha/service/app.py @@ -1,4 +1,4 @@ -__all__ = ('Application',) +"""Service application bootstrap utilities.""" import asyncio @@ -8,52 +8,38 @@ import uvloop from tornado.platform.asyncio import AsyncIOMainLoop - LOG.info('Using uvloop == %s for service event loop...' % uvloop.__version__) + LOG.info("Using uvloop == %s for service event loop..." % uvloop.__version__) asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) AsyncIOMainLoop().install() except ImportError: - LOG.info('[uvloop] NOT installed, fallback to asyncio loop! Consider `pip install uvloop`!') + LOG.info("[uvloop] NOT installed, fallback to asyncio loop! Consider `pip install uvloop`!") from tornado.options import options from ..settings import SETTINGS from .web import WebApplication +__all__ = ("Application",) + class Application: - """ - Main application class for aloha service. - - Wraps a WebApplication and manages the event loop lifecycle. - Tries to use uvloop if available for better performance. - """ + """Bootstrap and run an aloha web service.""" + def __init__(self, *args, **kwargs): - """ - Initialize the application. - - :param args: Additional arguments - :param kwargs: Additional keyword arguments - """ - options['log_file_prefix'] = 'access.log' + """Create the service application wrapper.""" + options["log_file_prefix"] = "access.log" settings = dict(SETTINGS.config) self.web_app = WebApplication(settings) def start(self): - """ - Start the application and run the event loop. - - Starts the web application and enters the event loop. - The event loop must not be running before calling this method. - - :raises RuntimeError: If event loop is already running - """ + """Start the web app and run the asyncio event loop.""" try: self.web_app.start() event_loop = asyncio.get_event_loop() if event_loop.is_running(): # notice: the event loop MUST NOT be initialized before web_app starts (as it may fork process) # ref: https://github.com/tornadoweb/tornado/issues/2426#issuecomment-400895086 - raise RuntimeError('Event loop already running before WebApp starts!') + raise RuntimeError("Event loop already running before WebApp starts!") else: event_loop.run_forever() except KeyboardInterrupt: @@ -64,9 +50,7 @@ def start(self): pass def stop(self): - """ - Stop the application and event loop. - """ + """Stop the event loop if it is currently running.""" event_loop = asyncio.get_event_loop() if event_loop.is_running(): event_loop.stop() diff --git a/src/aloha/service/http/__init__.py b/src/aloha/service/http/__init__.py index 7f08a4f..1099a97 100644 --- a/src/aloha/service/http/__init__.py +++ b/src/aloha/service/http/__init__.py @@ -1,5 +1,10 @@ -__all__ = ("AbstractApiClient", "AbstractApiHandler", "DefaultHandler404", "PlainHttpHandler") - from .base_api_client import AbstractApiClient from .base_api_handler import AbstractApiHandler, DefaultHandler404 from .plain_http_handler import PlainHttpHandler + +__all__ = ( + "AbstractApiClient", + "AbstractApiHandler", + "DefaultHandler404", + "PlainHttpHandler", +) diff --git a/src/aloha/service/http/base_api_client.py b/src/aloha/service/http/base_api_client.py index ddc8f45..d09f76c 100644 --- a/src/aloha/service/http/base_api_client.py +++ b/src/aloha/service/http/base_api_client.py @@ -1,3 +1,5 @@ +"""Base HTTP client helpers for aloha API clients.""" + import uuid from abc import ABC, abstractmethod from urllib.parse import urljoin @@ -10,17 +12,21 @@ class AbstractApiClient(ABC): + """Common client behavior for aloha HTTP APIs.""" + LOG = LOG RETRY_METHOD_WHITELIST: frozenset = frozenset(['GET', 'POST']) RETRY_STATUS_FORCELIST: frozenset = frozenset({413, 429, 503, 502, 504}) config = SETTINGS.config def __init__(self, url_endpoint: str = None, *args, **kwargs): + """Store the endpoint used by the client.""" self.url_endpoint = url_endpoint or '' LOG.debug('API Caller URL endpoint set to: %s' % self.url_endpoint) @classmethod def get_request_session(cls, total_retries: int = 3, *args, **kwargs) -> requests.Session: + """Create a requests session with retry support.""" session = requests.Session() # https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry.DEFAULT_ALLOWED_METHODS retries = Retry( @@ -31,6 +37,7 @@ def get_request_session(cls, total_retries: int = 3, *args, **kwargs) -> request return session def get_headers(self, *args, **kwargs) -> dict: + """Build the default request headers used by aloha clients.""" headers = { 'Content-Type': 'application/json', 'Request-ID': str(uuid.uuid1()), @@ -39,18 +46,13 @@ def get_headers(self, *args, **kwargs) -> dict: @abstractmethod def wrap_request_data(self, data: dict) -> dict: + """Transform the request payload before sending it.""" assert isinstance(data, dict), "Data object must be a dict!" raise NotImplementedError() # return data def call(self, api_url: str, data: dict = None, timeout=5, **kwargs): - """Trigger API call - :param api_url: do NOT start with slash (/) - :param data: a dictionary which includes the request data - :param timeout: requests timeout in seconds - :param kwargs: keywords arguments which will be updated to data - :return: - """ + """Call a remote API and return the parsed JSON response.""" body = data or dict() body.update(kwargs) payload = self.wrap_request_data(data=body) diff --git a/src/aloha/service/http/base_api_handler.py b/src/aloha/service/http/base_api_handler.py index 0cbd406..c4cdd41 100644 --- a/src/aloha/service/http/base_api_handler.py +++ b/src/aloha/service/http/base_api_handler.py @@ -1,3 +1,5 @@ +"""Base Tornado handlers used by aloha services.""" + import json from abc import ABC from datetime import datetime @@ -9,17 +11,21 @@ class AbstractApiHandler(web.RequestHandler, ABC): + """Shared request parsing and response helpers for JSON APIs.""" + LOG = LOG MAP_ERROR_INFO: dict = { 'BAD_REQUEST': {'code': '5101', 'message': ['Bad request: fail to parse body as JSON object!']} } def __init__(self, *args, **kwargs): + """Initialize request state used by subclasses.""" self.api_args: Optional[tuple] = None self.api_kwargs: Optional[dict] = None super().__init__(*args, **kwargs) def on_finish(self) -> None: + """Invoke any stored callback after the request finishes.""" func_callback = getattr(self, 'callback', None) if callable(func_callback) \ and isinstance(self.api_args, tuple) \ @@ -27,26 +33,32 @@ def on_finish(self) -> None: func_callback(*self.api_args, **self.api_kwargs) def response(self, *args, **kwargs) -> dict: + """Subclasses must implement the business response.""" raise NotImplementedError() def set_default_headers(self) -> None: + """Set the JSON content type for API responses.""" self.set_header('Content-Type', 'application/json; charset=utf-8') def data_received(self, chunk: bytes) -> Optional[Awaitable[None]]: + """Accept streaming request bodies when Tornado calls back.""" pass @property def request_header_content_type(self) -> str: + """Return the request content type with a JSON default.""" return self.request.headers.get('Content-Type', 'application/json; charset=utf-8') @property def request_id(self): + """Return or create a request identifier for tracing.""" if 'Request-ID' not in self.request.headers: self.request.headers['Request-ID'] = datetime.now().strftime('%Y%m%d-%H%M%S-%f') return self.request.headers.get('Request-ID') @property def request_body(self) -> dict: + """Parse the request body as JSON or multipart form data.""" content_type: str = self.request_header_content_type body_arguments: dict = Optional[None] @@ -62,6 +74,7 @@ def request_body(self) -> dict: @property def request_param(self) -> dict: + """Parse query/body arguments into a JSON-friendly dict.""" ret: dict = {} for k, v in self.request.arguments.items(): val = v[0].decode('utf-8') @@ -75,10 +88,14 @@ def request_param(self) -> dict: class DefaultHandler404(AbstractApiHandler): + """Default JSON 404 handler used by aloha services.""" + def response(self, *args, **kwargs) -> Optional[dict]: + """Return the default 404 response payload.""" return self.prepare() def prepare(self): # for all methods + """Finalize the 404 response.""" msg = { "code": 404, "status": "error", diff --git a/src/aloha/service/http/files.py b/src/aloha/service/http/files.py index 79fe60c..46a4056 100644 --- a/src/aloha/service/http/files.py +++ b/src/aloha/service/http/files.py @@ -1,3 +1,5 @@ +"""Helpers for handling multipart upload files and remote file inputs.""" + import time import requests @@ -6,6 +8,12 @@ def iter_over_request_files(request, url_files): + """Yield uploaded files and optional remote files as normalized tuples. + + Each yielded item is `(field_name, file_name, content_type, body_bytes)`. + Files can come from multipart form uploads or from URLs listed in + `url_files`. + """ for file_key, files in request.files.items(): # iter over files uploaded by multipart for f in files: file_name, content_type = f["filename"], f["content_type"] diff --git a/src/aloha/service/http/plain_http_handler.py b/src/aloha/service/http/plain_http_handler.py index bbdcef9..4813c12 100644 --- a/src/aloha/service/http/plain_http_handler.py +++ b/src/aloha/service/http/plain_http_handler.py @@ -1,13 +1,19 @@ +"""Plain Tornado handler with permissive CORS defaults.""" + from typing import Optional, Awaitable from tornado import web class PlainHttpHandler(web.RequestHandler): + """Minimal handler that exposes JSON-friendly CORS headers.""" + def data_received(self, chunk: bytes) -> Optional[Awaitable[None]]: + """Accept streamed body chunks without additional processing.""" pass def set_default_headers(self): + """Enable permissive cross-origin access for simple APIs.""" self.set_header('Access-Control-Allow-Origin', '*') self.set_header('Access-Control-Allow-Headers', '*') self.set_header('Access-Control-Max-Age', 1000) diff --git a/src/aloha/service/openapi/__init__.py b/src/aloha/service/openapi/__init__.py index 7150467..2a1291c 100644 --- a/src/aloha/service/openapi/__init__.py +++ b/src/aloha/service/openapi/__init__.py @@ -1,3 +1,3 @@ -__all__ = ("OpenApiClient",) - from .client import OpenApiClient + +__all__ = ("OpenApiClient",) diff --git a/src/aloha/service/openapi/client.py b/src/aloha/service/openapi/client.py index 5d95996..cbabb36 100644 --- a/src/aloha/service/openapi/client.py +++ b/src/aloha/service/openapi/client.py @@ -1,3 +1,5 @@ +"""Client helper for OpenAPI-style services protected by tokens.""" + import json from datetime import datetime, timedelta from typing import Optional @@ -14,10 +16,13 @@ class OpenApiClient: + """Simple HTTP client that acquires and caches an access token.""" + retry_method_whitelist = frozenset(['GET', 'POST']) retry_status_forcelist = frozenset({413, 429, 503, 502, 504}) def __init__(self, url_oauth_get_token: str, client_id: str, client_secret: str, grant_type: str = 'client_credentials'): + """Store OAuth-style client credentials and token endpoint.""" self.url_oauth_get_token = url_oauth_get_token self.client_id = client_id self.client_secret = client_secret @@ -28,6 +33,7 @@ def __init__(self, url_oauth_get_token: str, client_id: str, client_secret: str, @classmethod def get_request_session(cls, total_retries: int = 10, *args, **kwargs) -> Session: + """Create a retry-enabled requests session.""" session = Session() # https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry.DEFAULT_ALLOWED_METHODS retries = Retry( @@ -38,6 +44,7 @@ def get_request_session(cls, total_retries: int = 10, *args, **kwargs) -> Sessio return session def get_access_token(self) -> str: + """Fetch or refresh the cached access token.""" now = datetime.now() if self.expires_at is None or self.expires_at > now: @@ -63,6 +70,7 @@ def get_access_token(self) -> str: return self.access_token def _get_request_url(self, url: str): + """Attach access token and request id to the target URL.""" request_url = '{url}?access_token={access_token}&request_id={request_id}'.format( url=url, access_token=self.get_access_token(), request_id=datetime.now().strftime('%Y%m%d-%H%M%S-%f') ) @@ -70,6 +78,7 @@ def _get_request_url(self, url: str): @staticmethod def _get_data_from_esg_response(resp) -> Optional[dict]: + """Parse a JSON response and unwrap legacy ESG payloads.""" try: return resp.json() except (json.JSONDecodeError, JSONDecodeError): # requests may use `simplejson` @@ -83,6 +92,7 @@ def _get_data_from_esg_response(resp) -> Optional[dict]: raise ValueError(msg) def post(self, url_api: str, body: dict, headers: dict = None, timeout: int = 5): + """Send a POST request to the remote API.""" url = self._get_request_url(url_api) LOG.debug('Calling ESG POST: %s' % url) try: @@ -92,6 +102,7 @@ def post(self, url_api: str, body: dict, headers: dict = None, timeout: int = 5) LOG.error('Error calling ESG API POST [%s]: %s' % (url, str(e))) def get(self, url_api: str, body: dict, headers: dict = None, timeout: int = 5): + """Send a GET request to the remote API.""" url = self._get_request_url(url_api) LOG.debug('Calling ESG GET: %s' % url) try: diff --git a/src/aloha/service/web.py b/src/aloha/service/web.py index ab977da..e62676d 100644 --- a/src/aloha/service/web.py +++ b/src/aloha/service/web.py @@ -1,3 +1,5 @@ +"""Tornado web application assembly for aloha services.""" + import logging import os @@ -16,7 +18,7 @@ def _load_handlers(name): - """Load the (URL pattern, handler) tuples for each component.""" + """Load `(URL pattern, handler)` tuples from a service module.""" mod = __import__(name, fromlist=["default_handlers"]) handlers = [] for url, handler in mod.default_handlers: @@ -27,13 +29,17 @@ def _load_handlers(name): class WebApplication(web.Application): + """Tornado application that loads handlers from configured service modules.""" + def __init__(self, config: dict, *args, **kwargs): + """Create the application and its HTTP server.""" handlers = self.init_handlers(config) super().__init__(handlers=handlers, **config) self.http_server = httpserver.HTTPServer(self) @staticmethod def init_handlers(config: dict): + """Collect and normalize all handlers from configured service modules.""" settings = config.get("service", {}) modules = settings.get("modules", []) handlers = [] @@ -50,6 +56,7 @@ def init_handlers(config: dict): return [(HostMatches("(.*)"), handlers)] def start(self): + """Bind the configured port and start the HTTP server.""" service_settings = self.settings.get("service", {}) port = service_settings.get("port") or int(os.environ.get("PORT_SVC", 80)) diff --git a/src/aloha/util/html.py b/src/aloha/util/html.py index dce696b..e9867a8 100644 --- a/src/aloha/util/html.py +++ b/src/aloha/util/html.py @@ -1,9 +1,12 @@ +"""HTML extraction helpers.""" + import re from lxml import etree def extract_img_url(string): + """Extract the first image source URL from an HTML fragment.""" try: if string is None: return None @@ -16,6 +19,7 @@ def extract_img_url(string): def extract_text(raw_data): + """Extract visible text from an HTML fragment.""" if raw_data is not None: raw_data = re.sub(r"", "", raw_data) html = etree.HTML(raw_data) diff --git a/src/aloha/util/random.py b/src/aloha/util/random.py index b9d47b9..2ba4579 100644 --- a/src/aloha/util/random.py +++ b/src/aloha/util/random.py @@ -1,3 +1,7 @@ +"""Random helper aliases built on top of `secrets.SystemRandom`.""" + +from secrets import SystemRandom + __all__ = ( "random", "random_bool", @@ -9,28 +13,16 @@ "random_seed", ) -from secrets import SystemRandom - random = SystemRandom() - -def random_bool(): - return random.choice([True, False]) - - random_choice = random.choice - - random_int = random.randint - - random_ratio = random.random - - random_uniform = random.uniform - - random_sample = random.sample +random_seed = random.seed -random_seed = random.seed +def random_bool(): + """Return a random boolean value.""" + return random.choice([True, False]) diff --git a/src/aloha/util/sys_cuda.py b/src/aloha/util/sys_cuda.py index dbd1962..181b422 100644 --- a/src/aloha/util/sys_cuda.py +++ b/src/aloha/util/sys_cuda.py @@ -1,9 +1,10 @@ -__all__ = ("get_cuda_info",) - from collections import namedtuple from ..logger import LOG +__all__ = ("get_cuda_info",) + + Status = namedtuple("Status", "version,gpu_availability") diff --git a/src/aloha/util/sys_gpu.py b/src/aloha/util/sys_gpu.py index 0416ef8..a718645 100644 --- a/src/aloha/util/sys_gpu.py +++ b/src/aloha/util/sys_gpu.py @@ -1,5 +1,3 @@ -__all__ = ("get_gpu_info",) - # ! `pip install pynvml`, reference: https://github.com/gpuopenanalytics/pynvml from collections import namedtuple @@ -15,6 +13,8 @@ LOG.warn("Package `pynvml` NOT installed! Cannot get GPU info.") nvml = nvidia_smi = None +__all__ = ("get_gpu_info",) + Device = namedtuple("Device", field_names="index,name,arch") DeviceStatus = namedtuple("DeviceStatus", field_names="mem_total,mem_free,mem_used,gpu_rate,mem_rate") diff --git a/src/aloha/util/sys_info.py b/src/aloha/util/sys_info.py index bdcf342..05eafab 100644 --- a/src/aloha/util/sys_info.py +++ b/src/aloha/util/sys_info.py @@ -1,5 +1,3 @@ -__all__ = ("get_sys_info",) - import platform from datetime import datetime @@ -7,6 +5,8 @@ from ..logger import LOG +__all__ = ("get_sys_info",) + LOG.debug("Using psutil == %s" % psutil.__version__) diff --git a/src/setup.py b/src/setup.py index 0201493..db25938 100644 --- a/src/setup.py +++ b/src/setup.py @@ -14,12 +14,22 @@ dict_extra_requires = { "build": ["Cython"], "service": ["requests", "tornado", "psutil", "pyjwt", "fastapi", "httpx"], - "db": ["sqlalchemy", "psycopg[binary]", "pymysql", "elasticsearch", "pymongo", "redis"], + "db": [ + "sqlalchemy", + "psycopg[binary]", + "pymysql", + "elasticsearch", + "pymongo", + "redis", + "duckdb", + "duckdb-engine", + "oracledb", + ], "stream": ["confluent_kafka"], - "data": ["pandas"], + "data": ["pandas", "lxml"], "report": ["openpyxl", "XlsxWriter"], "test": ["pytest-cov"], - "docs": ["mkdocs", "mkdocs-static-i18n", "mkdocstrings[python]", "markdown-include", "mkdocs-material"], + "docs": ["mkdocs", "mkdocstrings[python]", "markdown-include", "mkdocs-material"], } setup( @@ -30,7 +40,7 @@ license="Apache-2.0", url="https://github.com/LabNow.ai/aloha", project_urls={ - "Source": "https://github.com/LabNow-ai/aloha", + "Source": "https://github.com/LabNow-ai/aloha-python", "CI Pipeline": "https://github.com/LabNow-ai/aloha-python/actions", "Documentation": "https://aloha-python.readthedocs.io/", },