diff --git a/.github/dependabot.yml b/.github/dependabot.yml index bf5d4eb..2bdd9e0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,11 +1,25 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 + updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "sunday" + groups: + all-github-actions: + patterns: + - "*" + - package-ecosystem: "pip" # See documentation for possible values - directory: "docs/" # Location of package manifests + directory: "./" # Location of package manifests schedule: interval: "weekly" + day: "sunday" + groups: + all-pip: + patterns: + - "*" diff --git a/.github/workflows/pip.yml b/.github/workflows/build.yml similarity index 57% rename from .github/workflows/pip.yml rename to .github/workflows/build.yml index b240870..deb742d 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/build.yml @@ -1,28 +1,54 @@ # Secret Variables required in GitHub secrets: TWINE_USERNAME, TWINE_PASSWORD / TWINE_USERNAME_TEST, TWINE_PASSWORD_TEST -name: build-pip-publish +name: build-python-package on: push: - branches: [ main ] - paths-ignore: [ "*.md" ] + branches: ["main"] + tags: ["v*"] + paths-ignore: ["*.md"] pull_request: - branches: [ main ] - paths-ignore: [ "*.md" ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: + branches: ["main"] + paths-ignore: ["*.md"] + workflow_dispatch: # Allows you to run this workflow manually from the Actions tab +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: - build-pypi-package: - # The type of runner that the job will run on + job-ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - run: pip install ruff && ruff check ./src + + job-semgrep: runs-on: ubuntu-latest + container: + image: semgrep/semgrep:latest + continue-on-error: true + if: (github.actor != 'dependabot[bot]') + steps: + - uses: actions/checkout@v6 + - run: | + semgrep ci --verbose \ + --config p/ci \ + --config p/security-audit \ + --config p/python \ + --config p/javascript \ + --config p/react \ + --config p/owasp-top-ten + build-pypi-package: + runs-on: ubuntu-latest steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # sudo python setup.py install clean --all - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: pip-install-test run: | diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 6175c1d..48dabbd 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -6,5 +6,10 @@ "ms-python.python", "ms-python.vscode-pylance" ], - "unwantedRecommendations": [] + "unwantedRecommendations": [ + "ms-python.flake8", + "ms-python.pylint", + "ms-python.black-formatter", + "ms-python.autopep8" + ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index fb1ea69..ecbf80e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,8 @@ { "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll": "always" + }, "files.exclude": { "**/.git": true, "**/node_modules": true, @@ -7,20 +10,25 @@ "**/*.pyc": true }, "[python]": { - "editor.defaultFormatter": "ms-python.autopep8", + "editor.defaultFormatter": "charliermarsh.ruff", "editor.codeActionsOnSave": { - "source.organizeImports": "explicit", - "source.fixAll.ruff": "explicit" - } + "source.organizeImports.ruff": "always", + "source.fixAll.ruff": "always" + }, + "editor.formatOnSave": true }, "[sql]": { "editor.defaultFormatter": "dbcode.dbcode", // mtxr.sqltools, dbcode.dbcode, ReneSaarsoo.sql-formatter-vsc "editor.formatOnSave": true }, + "ruff.enable": true, + "ruff.lint.enable": true, + "ruff.nativeServer": "on", "ruff.lineLength": 128, "ruff.configuration": { "format": { "quote-style": "double" } - } + }, + "markdown.validate.enabled": true } diff --git a/README.md b/README.md index 050c39a..cb83cb4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,19 @@ -# Aloha! +# Aloha ## What is it? `aloha` is a versatile Python utility package for building microservices. -[![License](https://img.shields.io/github/license/QPod/aloha)](https://github.com/QPod/aloha/blob/main/LICENSE) -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/QPod/aloha-python/pip.yml?branch=main)](https://github.com/QPod/aloha-python/actions) -[![Code Activity](https://img.shields.io/github/commit-activity/m/QPod/aloha)](https://github.com/QPod/aloha/pulse) -[![Recent Code Update](https://img.shields.io/github/last-commit/QPod/docker-images.svg)](https://github.com/QPod/aloha/stargazers) - +[![License](https://img.shields.io/github/license/QPod/aloha-python)](https://github.com/QPod/aloha-python/blob/main/LICENSE) +[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/QPod/aloha-python/build.yml?branch=main)](https://github.com/QPod/aloha-python/actions) +[![Join the Gitter Chat](https://img.shields.io/gitter/room/nwjs/nw.js.svg)](https://gitter.im/QPod/) [![PyPI version](https://img.shields.io/pypi/v/aloha)](https://pypi.python.org/pypi/aloha/) [![PyPI Downloads](https://img.shields.io/pypi/dm/aloha)](https://pepy.tech/badge/aloha/) - ---- +[![Code Activity](https://img.shields.io/github/commit-activity/m/QPod/aloha-python)](https://github.com/QPod/aloha-python/pulse) +[![Recent Code Update](https://img.shields.io/github/last-commit/QPod/docker-images.svg)](https://github.com/QPod/aloha-python/stargazers) Please generously STAR★ our project or donate to us! -[![GitHub Starts](https://img.shields.io/github/stars/QPod/aloha.svg?label=Stars&style=social)](https://github.com/QPod/aloha/stargazers) +[![GitHub Starts](https://img.shields.io/github/stars/QPod/aloha-python.svg?label=Stars&style=social)](https://github.com/QPod/aloha-python/stargazers) [![Donate-PayPal](https://img.shields.io/badge/Donate-PayPal-blue.svg)](https://paypal.me/haobibo) [![Donate-AliPay](https://img.shields.io/badge/Donate-Alipay-blue.svg)](https://raw.githubusercontent.com/wiki/haobibo/resources/img/Donate-AliPay.png) [![Donate-WeChat](https://img.shields.io/badge/Donate-WeChat-green.svg)](https://raw.githubusercontent.com/wiki/haobibo/resources/img/Donate-WeChat.png) @@ -32,7 +30,8 @@ Refer to[📚 Document & 中文文档](https://aloha-python.readthedocs.io/) for pip install aloha[all] ``` -And then: +And then: + ```python from aloha.logger import LOG from aloha.settings import SETTINGS as S diff --git a/app/README.md b/app/README.md new file mode 100644 index 0000000..3032011 --- /dev/null +++ b/app/README.md @@ -0,0 +1,179 @@ +# App Demo + +Using `aloha` to develop your project - a boilerplate/template project. + +## Overview + +This project provides a containerized development environment using Docker and Docker Compose. It sets up a complete development workspace with all necessary dependencies pre-installed, allowing you to focus on writing code rather than configuring your environment. + +### Key Components + +- **Docker**: Containerization platform that packages applications with all their dependencies +- **Docker Compose**: Tool for defining and running multi-container Docker applications +- **Development Container**: A pre-configured environment with Python, Node.js, and database clients + +## How to quickly setup and start DEV environment + +### Prerequisites + +Before getting started, ensure you have the following installed: + +- Docker Engine +- Docker Compose +- Git (for cloning the repository) + +You can verify Docker installation by running: + +```bash +docker --version +docker-compose --version +``` + +### Step 1: Launch the Development Environment + +Run this command in your terminal: + +```bash +./tool/cicd/run-dev.sh up +``` + +**What happens when you run this command:** + +1. **Port Availability Check**: The script first verifies that the required ports are not already in use on your system. The ports are dynamically assigned based on your user ID (UID) to avoid conflicts with other developers. + +2. **Docker Image Build**: If the Docker image doesn't exist yet, Docker Compose will build it using: + - `tool/cicd/docker-compose.app-demo.DEV.yml`: Defines the container configuration + - `tool/dev-demo.Dockerfile`: Specifies how to build the Docker image + + The build process includes: + - Installing Node.js package manager (pnpm) + - Setting up Python with JupyterLab + - Installing project dependencies from `app/requirements.txt` + - Adding PostgreSQL database client tools + +3. **Container Start**: Docker Compose starts the container with the following features: + - **Volume Mounts**: Your local code directories are mounted into the container, enabling live development (changes on your host are immediately visible in the container): + - `doc/` → `/root/doc` + - `notebook/` → `/root/notebook` + - `src/` → `/root/src` + - `app/` → `/root/app` + - **Port Forwarding**: Exposes ports for your application and web interface + - **Persistent Process**: The container runs `tail -f /dev/null` to stay active + +### Step 2: Enter the Development Container + +Once the environment is running, execute: + +```bash +./tool/cicd/run-dev.sh enter +``` + +**What this command does:** + +- Uses `docker exec -it` to create an interactive terminal session +- Attaches you to the running container with a bash shell +- You'll be logged in as the root user inside the container +- Your working directory will be `/root` + +**What you can do inside the container:** + +- Run Python scripts and applications +- Use JupyterLab for interactive development +- Install additional packages with pip or npm +- Access the PostgreSQL database using the client tools +- Edit files (changes will be reflected on your host machine) + +### Step 3: Manage the Environment + +The `run-dev.sh` script provides several commands to manage your development environment: + +| Command | Description | +| -------------------------------- | ------------------------------------------- | +| `./tool/cicd/run-dev.sh up` | Start or create the development environment | +| `./tool/cicd/run-dev.sh restart` | Restart the running container | +| `./tool/cicd/run-dev.sh logs` | View and follow container logs | +| `./tool/cicd/run-dev.sh enter` | Access the container's bash shell | +| `./tool/cicd/run-dev.sh down` | Stop and remove the container | + +### Understanding the Port Assignment + +The script dynamically assigns ports to avoid conflicts: + +- **Base App Port**: 30000 (as specified in the `run-dev.sh`)+ your UID +- **Base Web Port**: 33000 (as specified in the `run-dev.sh`)+ your UID + +Your specific ports will be displayed when you run any `run-dev.sh` command: + +``` +---------------------------------------- +User: yourusername (UID: 1000) +Project Name: dev-app-demo-yourusername +Container: dev-app-demo-yourusername +App Port Expose: 31000 +Web Port Expose: 34000 +Action: up +Compose: /path/to/docker-compose.app-demo.DEV.yml +---------------------------------------- +``` + +### Tearing Down the Environment + +When you're done working, you can stop and remove the container: + +```bash +./tool/cicd/run-dev.sh down +``` + +**Note:** This command only removes the container, not the Docker image. If you want to reclaim disk space by removing the image as well, run: + +```bash +docker rmi $(docker images | grep dev-app-demo | awk '{print $3}') +``` + +## Project Structure + +``` +aloha-python/ +├── app/ # Application code +│ ├── main.py # Main application entry point +│ ├── requirements.txt # Python dependencies +│ └── app_common/ # Common application utilities +├── src/ # Source code for the aloha library +├── doc/ # Documentation files +├── notebook/ # Jupyter notebooks +└── tool/ # Development tools + ├── cicd/ # CI/CD scripts and configs + │ ├── run-dev.sh # Main development environment script + │ └── docker-compose.app-demo.DEV.yml + ├── dev-demo.Dockerfile + └── app-demo.Dockerfile +``` + +## Troubleshooting + +### "Port is already in use" error + +If you see this error, another process is using the assigned ports. You can: + +1. Identify and stop the conflicting process +2. Work with a system administrator to free up the ports + +### Container won't start + +- Check Docker logs: `./tool/cicd/run-dev.sh logs` +- Ensure Docker service is running: `systemctl status docker` (Linux) or check Docker Desktop (Windows/macOS) + +### Changes not reflecting + +- Verify your files are in the mounted directories +- Check that you're editing files on your host machine (not just inside the container) +- Restart any running services inside the container if needed + +## Next Steps + +Once inside the container, you can: + +1. Explore the `app/` directory to understand the application structure +2. Check out the Jupyter notebooks in the `notebook/` directory +3. Review the documentation in the `doc/` directory +4. Start developing your application! diff --git a/demo/app_common/__init__.py b/app/app_common/__init__.py similarity index 100% rename from demo/app_common/__init__.py rename to app/app_common/__init__.py diff --git a/demo/app_common/api/__init__.py b/app/app_common/api/__init__.py similarity index 100% rename from demo/app_common/api/__init__.py rename to app/app_common/api/__init__.py diff --git a/demo/app_common/api/api_common_query_postgres.py b/app/app_common/api/api_common_query_postgres.py similarity index 100% rename from demo/app_common/api/api_common_query_postgres.py rename to app/app_common/api/api_common_query_postgres.py diff --git a/demo/app_common/api/api_common_sys_info.py b/app/app_common/api/api_common_sys_info.py similarity index 100% rename from demo/app_common/api/api_common_sys_info.py rename to app/app_common/api/api_common_sys_info.py diff --git a/demo/app_common/api/api_multipart.py b/app/app_common/api/api_multipart.py similarity index 100% rename from demo/app_common/api/api_multipart.py rename to app/app_common/api/api_multipart.py diff --git a/demo/app_common/debug.py b/app/app_common/debug.py similarity index 63% rename from demo/app_common/debug.py rename to app/app_common/debug.py index 59f4b68..6425923 100644 --- a/demo/app_common/debug.py +++ b/app/app_common/debug.py @@ -9,17 +9,19 @@ def main(): "app_common.api.api_multipart", ] - if 'service' not in SETTINGS.config: - SETTINGS.config['service'] = {} + if "service" not in SETTINGS.config: + SETTINGS.config["service"] = {} # load the service modules from SETTINGS.config['service']['modules'] - SETTINGS.config['service'].update({ - 'modules': modules_to_load, - 'debug': True, - }) + SETTINGS.config["service"].update( + { + "modules": modules_to_load, + "debug": True, + } + ) # Use self defined 404 handler - SETTINGS.config['default_handler_class'] = DefaultHandler404 + SETTINGS.config["default_handler_class"] = DefaultHandler404 app = Application() @@ -27,5 +29,5 @@ def main(): app.start() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/demo/app_common/main.py b/app/app_common/main.py similarity index 100% rename from demo/app_common/main.py rename to app/app_common/main.py diff --git a/demo/main.py b/app/main.py similarity index 86% rename from demo/main.py rename to app/main.py index 7a44fb9..771972a 100644 --- a/demo/main.py +++ b/app/main.py @@ -10,7 +10,8 @@ print(usage) exit(-1) -m = importlib.import_module(sys.argv[1]) +sys.argv.pop(0) +m = importlib.import_module(sys.argv[0]) f_main = getattr(m, "main") if f_main is None: diff --git a/app/requirements.txt b/app/requirements.txt new file mode 100644 index 0000000..b9b47b6 --- /dev/null +++ b/app/requirements.txt @@ -0,0 +1,27 @@ +# aloha[all] +attrdict3 +pyhocon +pycryptodome +packaging +Cython +requests +tornado +psutil +pyjwt +fastapi +httpx +sqlalchemy +psycopg[binary] +pymysql +elasticsearch +pymongo +redis +confluent_kafka +pandas +openpyxl>=3 +XlsxWriter +pytest-cov +mkdocs +mkdocstrings[python] +markdown-include +mkdocs-material diff --git a/demo/resource/config/deploy-DEV.conf b/app/resource/config/deploy-DEV.conf similarity index 100% rename from demo/resource/config/deploy-DEV.conf rename to app/resource/config/deploy-DEV.conf diff --git a/demo/resource/config/main.conf b/app/resource/config/main.conf similarity index 83% rename from demo/resource/config/main.conf rename to app/resource/config/main.conf index 813becd..c501d81 100644 --- a/demo/resource/config/main.conf +++ b/app/resource/config/main.conf @@ -1,6 +1,7 @@ include required("deploy-DEV.conf") -APP_MODUEL = "Aloha" +APP_MODULE = "Aloha" +APP_MODULE = ${?APP_MODULE} APP_DOMAIN = { LOCAL = "http://localhost:9999" diff --git a/demo/requirements.txt b/demo/requirements.txt deleted file mode 100644 index 43c6e50..0000000 --- a/demo/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -aloha[all] diff --git a/.readthedocs.yaml b/doc/.readthedocs.yaml similarity index 86% rename from .readthedocs.yaml rename to doc/.readthedocs.yaml index 89ed9f6..4f09fd3 100644 --- a/.readthedocs.yaml +++ b/doc/.readthedocs.yaml @@ -7,9 +7,10 @@ version: 2 # Set the version of Python and other tools you might need build: - os: ubuntu-22.04 + os: ubuntu-lts-latest tools: - python: "3.11" + python: "3" + # nodejs: "latest" # apt_packages: [] jobs: pre_create_environment: @@ -18,7 +19,7 @@ build: - echo "Command run at 'post_build' step $(date)" mkdocs: - configuration: mkdocs.yml + configuration: ./doc/mkdocs.yml # Optionally declare the Python requirements required to build your docs python: diff --git a/doc/README-tasks.md b/doc/README-cli.md similarity index 83% rename from doc/README-tasks.md rename to doc/README-cli.md index baa860f..14aaf47 100644 --- a/doc/README-tasks.md +++ b/doc/README-cli.md @@ -24,12 +24,12 @@ Sometime, you need to compile your python source code into binary libraries to p Aloha helps you build your python source code into binary using `Cython`. You can run the following command to build your code. ```bash -aloha compile --base=./demo --dist=./build --keep='main.py' +aloha compile --base=./app --dist=./build --keep='main.py' ``` The following options can be used: - `--base`: the root folder which includes source code to build - `--dist`: (default='build') target folder for the binary code -- `--exclude`: a collection of files/folders to exclude (you can specify multiple excludes by using this option multiple times) +- `--exclude`: a collection of files/folders to exclude (you can specify multiple excludes by using this option multiple times) - `--keep`: source files keep as is and not converting to dynamic library (you can specify multiple excludes by using this option multiple times) diff --git a/doc/README-develop.md b/doc/README-develop.md index 66b690a..6dbdc98 100644 --- a/doc/README-develop.md +++ b/doc/README-develop.md @@ -3,14 +3,14 @@ ## Live debug on source code using docker ```bash -# Firstly, cd to the project root folder (which includes `src`) of the project, and then: +# Firstly, cd to the project root folder (which includes `src`) of the project, and then: docker run -it \ -v $(pwd):/root/app/ \ -w /root/app/src \ --name="app-$(whoami)" \ -p 8080:80 \ docker.io/qpod/base:latest bash - + python -m aloha.script.start app_common.debug ``` diff --git a/doc/README-get-start.md b/doc/README-get-start.md index 6501e0e..69386df 100644 --- a/doc/README-get-start.md +++ b/doc/README-get-start.md @@ -2,12 +2,12 @@ ## Step 1. install the python package -``` title="Install aloha and all it's extra requirments" +```title="Install aloha and all it's extra requirments" pip install aloha[all] ``` ## Step 2. Start your project based on the template -You can refer to the `demo` folder of the GitHub repo to start using `aloha` in your project: +You can refer to the `app` folder of the GitHub repo to start using `aloha` in your project: -[:octicons-mark-github-16: Go to TemplateProject](https://github.com/QPod/aloha/tree/main/demo){ .md-button } +[:octicons-mark-github-16: Go to TemplateProject](https://github.com/QPod/aloha/tree/main/app){ .md-button } diff --git a/doc/index.md b/doc/index.md index 2b9f125..150ba4c 100644 --- a/doc/index.md +++ b/doc/index.md @@ -2,25 +2,26 @@ Aloha! Thanks for your interesting in this python package. -[![License](https://img.shields.io/github/license/QPod/aloha)](https://github.com/QPod/aloha/blob/main/LICENSE) -[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/QPod/aloha/build)](https://github.com/QPod/aloha/actions) +[![License](https://img.shields.io/github/license/QPod/aloha-python)](https://github.com/QPod/aloha-python/blob/main/LICENSE) +[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/QPod/aloha-python/build.yml?branch=main)](https://github.com/QPod/aloha-python/actions) [![Join the Gitter Chat](https://img.shields.io/gitter/room/nwjs/nw.js.svg)](https://gitter.im/QPod/) [![PyPI version](https://img.shields.io/pypi/v/aloha)](https://pypi.python.org/pypi/aloha/) [![PyPI Downloads](https://img.shields.io/pypi/dm/aloha)](https://pepy.tech/badge/aloha/) -[![Code Activity](https://img.shields.io/github/commit-activity/m/QPod/aloha)](https://github.com/QPod/aloha/pulse) -[![Recent Code Update](https://img.shields.io/github/last-commit/QPod/docker-images.svg)](https://github.com/QPod/aloha/stargazers) +[![Code Activity](https://img.shields.io/github/commit-activity/m/QPod/aloha-python)](https://github.com/QPod/aloha-python/pulse) +[![Recent Code Update](https://img.shields.io/github/last-commit/QPod/docker-images.svg)](https://github.com/QPod/aloha-python/stargazers) -Please generously STAR★ our project or donate to us! [![GitHub Starts](https://img.shields.io/github/stars/QPod/aloha.svg?label=Stars&style=social)](https://github.com/QPod/aloha/stargazers) +Please generously STAR★ our project or donate to us! [![GitHub Starts](https://img.shields.io/github/stars/QPod/aloha-python.svg?label=Stars&style=social)](https://github.com/QPod/aloha-python/stargazers) [![Donate-PayPal](https://img.shields.io/badge/Donate-PayPal-blue.svg)](https://paypal.me/haobibo) [![Donate-AliPay](https://img.shields.io/badge/Donate-Alipay-blue.svg)](https://raw.githubusercontent.com/wiki/haobibo/resources/img/Donate-AliPay.png) [![Donate-WeChat](https://img.shields.io/badge/Donate-WeChat-green.svg)](https://raw.githubusercontent.com/wiki/haobibo/resources/img/Donate-WeChat.png) ## What is it? + The python package `aloha` is a cute and versatile to build python microservices. The package encapsulates commonly used components and features, such as: - === "English" + - Rapidly create RESTful APIs and start services - Logging utils - Manage different environments,configuration files, and resource files @@ -28,6 +29,7 @@ The package encapsulates commonly used components and features, such as: - Detect and monitor application runtime environment === "中文" + `aloha`是一个用来快速构建微服务的Python包，它包含了创建微服务常用的组件和功能： - 快速创建微服务的RESTful API并启动服务 @@ -38,35 +40,35 @@ The package encapsulates commonly used components and features, such as: ## Installation -``` title="It's easy to install aloha using the following command" +```title="It's easy to install aloha using the following command" pip install aloha[all] ``` Notice the `[all]` after the package is a set of (one or more) extra requirements, which enables additional features. === "English" + These extra requirements can include: - `all`: everything below are included. - `service`: components/packages used to build RESTful APIs -- aloha use tornado to support services. - `build`: used to build python code into binary files, which is particularly useful to protect source code. - `db`: connect to popular databases, like MySQL / PostgreSQL / Redis. - - `stream`: processing steram data using confluent_kafka. + - `stream`: processing steram data using confluent_kafka. - `data`: processing data or doing data science tasks using packages like pandas. - `report`: utilites to export data and report to Excel files. - `test`: unit test utilites. - `docs`: utilites used to build documentations. +=== "中文" + 请留意，上述安装命令中包名后的`[all]`是额外的安装依赖，这些额外的安装内容在使用某些模块的时候会用到。 -=== "中文" - 请留意，上述安装命令中包名后的`[all]`是额外的安装依赖，这些额外的安装内容在使用某些模块的时候会用到。 - - `all`: 包含了下面的所有的依赖包 - `service`: 用于创建RESTful APIs的依赖，aloha基于torndao来构建服务； - `build`: 用于将Python代码编译为二进制包或类库，这对于需要进行源代码保护的场景十分有用； - `db`: 连接到常用的数据库，如MySQL、PostgreSQL、Redis等； - - `stream`: 基于confluent_kafka进行流数据处理； + - `stream`: 基于confluent_kafka进行流数据处理； - `data`: 使用pandas等package进行数据处理； - `report`: 将数据导出为Excel格式的报告； - `test`: 进行单元测试的功能； diff --git a/mkdocs.yml b/doc/mkdocs.yml similarity index 87% rename from mkdocs.yml rename to doc/mkdocs.yml index 0cf7584..ae4eeab 100644 --- a/mkdocs.yml +++ b/doc/mkdocs.yml @@ -1,7 +1,7 @@ site_name: Aloha - a versatile Python library to build microservice. repo_url: https://github.com/QPod/aloha edit_uri: blob/main/doc/ -docs_dir: doc +docs_dir: ./ theme: name: material highlightjs: true @@ -56,9 +56,9 @@ markdown_extensions: emoji_index: !!python/name:materialx.emoji.twemoji emoji_generator: !!python/name:materialx.emoji.to_svg nav: - - Introduction: 'index.md' - - Get Started: 'README-get-start.md' - - Common Tasks: 'README-tasks.md' - - Configs: 'README-config.md' - - Development Guide: 'README-develop.md' - - API Reference: 'api.md' + - Introduction: "index.md" + - Get Started: "README-get-start.md" + - CLI Commands: "README-cli.md" + - Configs: "README-config.md" + - Development Guide: "README-develop.md" + - API Reference: "api.md" diff --git a/notebook/test-api-query-postgresql.ipynb b/notebook/test-api-query-postgresql.ipynb index b74335a..7d5910f 100644 --- a/notebook/test-api-query-postgresql.ipynb +++ b/notebook/test-api-query-postgresql.ipynb @@ -24,8 +24,8 @@ "import sys\n", "\n", "sys.path.insert(0, '../src/')\n", - "sys.path.insert(0, '../demo/')\n", - "os.environ['DIR_RESOURCE'] = '../demo/resource/'" + "sys.path.insert(0, '../app/')\n", + "os.environ['DIR_RESOURCE'] = '../app/resource/'" ] }, { diff --git a/notebook/test-db-postgres.ipynb b/notebook/test-db-postgres.ipynb index 834a1a0..1e75169 100644 --- a/notebook/test-db-postgres.ipynb +++ b/notebook/test-db-postgres.ipynb @@ -19,7 +19,7 @@ "source": [ "import os, sys\n", "sys.path.insert(0, '../src/')\n", - "os.environ['DIR_RESOURCE'] = '../demo/resource'" + "os.environ['DIR_RESOURCE'] = '../app/resource'" ] }, { diff --git a/src/README.md b/src/README.md index 050c39a..cb83cb4 100644 --- a/src/README.md +++ b/src/README.md @@ -1,21 +1,19 @@ -# Aloha! +# Aloha ## What is it? `aloha` is a versatile Python utility package for building microservices. -[![License](https://img.shields.io/github/license/QPod/aloha)](https://github.com/QPod/aloha/blob/main/LICENSE) -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/QPod/aloha-python/pip.yml?branch=main)](https://github.com/QPod/aloha-python/actions) -[![Code Activity](https://img.shields.io/github/commit-activity/m/QPod/aloha)](https://github.com/QPod/aloha/pulse) -[![Recent Code Update](https://img.shields.io/github/last-commit/QPod/docker-images.svg)](https://github.com/QPod/aloha/stargazers) - +[![License](https://img.shields.io/github/license/QPod/aloha-python)](https://github.com/QPod/aloha-python/blob/main/LICENSE) +[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/QPod/aloha-python/build.yml?branch=main)](https://github.com/QPod/aloha-python/actions) +[![Join the Gitter Chat](https://img.shields.io/gitter/room/nwjs/nw.js.svg)](https://gitter.im/QPod/) [![PyPI version](https://img.shields.io/pypi/v/aloha)](https://pypi.python.org/pypi/aloha/) [![PyPI Downloads](https://img.shields.io/pypi/dm/aloha)](https://pepy.tech/badge/aloha/) - ---- +[![Code Activity](https://img.shields.io/github/commit-activity/m/QPod/aloha-python)](https://github.com/QPod/aloha-python/pulse) +[![Recent Code Update](https://img.shields.io/github/last-commit/QPod/docker-images.svg)](https://github.com/QPod/aloha-python/stargazers) Please generously STAR★ our project or donate to us! -[![GitHub Starts](https://img.shields.io/github/stars/QPod/aloha.svg?label=Stars&style=social)](https://github.com/QPod/aloha/stargazers) +[![GitHub Starts](https://img.shields.io/github/stars/QPod/aloha-python.svg?label=Stars&style=social)](https://github.com/QPod/aloha-python/stargazers) [![Donate-PayPal](https://img.shields.io/badge/Donate-PayPal-blue.svg)](https://paypal.me/haobibo) [![Donate-AliPay](https://img.shields.io/badge/Donate-Alipay-blue.svg)](https://raw.githubusercontent.com/wiki/haobibo/resources/img/Donate-AliPay.png) [![Donate-WeChat](https://img.shields.io/badge/Donate-WeChat-green.svg)](https://raw.githubusercontent.com/wiki/haobibo/resources/img/Donate-WeChat.png) @@ -32,7 +30,8 @@ Refer to[📚 Document & 中文文档](https://aloha-python.readthedocs.io/) for pip install aloha[all] ``` -And then: +And then: + ```python from aloha.logger import LOG from aloha.settings import SETTINGS as S diff --git a/src/aloha/__init__.py b/src/aloha/__init__.py index 8dee4bf..8d09f69 100644 --- a/src/aloha/__init__.py +++ b/src/aloha/__init__.py @@ -1 +1,3 @@ +__all__ = ("__version__",) + from ._version import __version__ diff --git a/src/aloha/_version.py b/src/aloha/_version.py index 3845ff8..01dc7a8 100644 --- a/src/aloha/_version.py +++ b/src/aloha/_version.py @@ -1 +1 @@ -__version__ = "2022.1120.0945" +__version__ = "2026.0421.2100" diff --git a/src/aloha/base.py b/src/aloha/base.py index bd678d1..3109c2a 100644 --- a/src/aloha/base.py +++ b/src/aloha/base.py @@ -5,5 +5,13 @@ class BaseModule(ABC): + """ + Abstract base class for all modules in aloha. + + Provides common attributes to all modules: + - config: Global configuration object + - LOG: Logger instance + """ + config = SETTINGS.config LOG = LOG diff --git a/src/aloha/config/hocon.py b/src/aloha/config/hocon.py index b0da4e1..0636e4e 100644 --- a/src/aloha/config/hocon.py +++ b/src/aloha/config/hocon.py @@ -3,11 +3,27 @@ def load_config_from_hocon(config_file): + """ + Load configuration from a single HOCON file. + + :param config_file: Path to the HOCON configuration file + :return: Configuration as an ordered dictionary + """ config = ConfigFactory.parse_file(config_file).as_plain_ordered_dict() return config def load_config_from_hocon_files(config_files: list, base_dir: str): + """ + Load configuration from multiple HOCON files. + + Combines multiple HOCON files using include directives and returns + the result as an AttrDict for attribute-style access. + + :param config_files: List of HOCON configuration file names + :param base_dir: Base directory for resolving relative paths + :return: Configuration as an AttrDict object + """ s = [] for config_file in config_files: f = 'include required("%s")' % config_file diff --git a/src/aloha/config/paths.py b/src/aloha/config/paths.py index 5e89ea0..0a6a913 100644 --- a/src/aloha/config/paths.py +++ b/src/aloha/config/paths.py @@ -1,4 +1,4 @@ -__all__ = ('get_resource_dir', 'get_config_dir', 'get_current_module_dir', 'get_project_base_dir', 'path_join') +__all__ = ("get_resource_dir", "get_config_dir", "get_current_module_dir", "get_project_base_dir", "path_join") import os import sys @@ -6,6 +6,14 @@ def path_join(*args) -> str: + """ + Join path components and normalize the result. + Performs path joining, user expansion, environment variable expansion, + and conversion to absolute path. + + :param args: Path components to join + :return: Normalized absolute path string + """ p = os.path.join(*args) p = os.path.expanduser(p) p = os.path.expandvars(p) @@ -14,16 +22,34 @@ def path_join(*args) -> str: def get_resource_dir(*args) -> str: + """ + Get the resource directory path. + The resource directory is determined by: + 1. If DIR_RESOURCE environment variable is set, use it; + 2. Otherwise, default to 'resource' directory under current working directory. + + :param args: Additional path components to append to the resource directory + :return: Resource directory path + """ dir_cwd = os.getcwd() - dir_resource = os.environ.get('DIR_RESOURCE', 'resource') + dir_resource = os.environ.get("DIR_RESOURCE", "resource") return path_join(dir_cwd, dir_resource, *args) def get_config_dir(*args) -> str: - dir_config = os.environ.get('DIR_CONFIG') + """ + Get the configuration directory path. + The config directory is determined by: + 1. If DIR_CONFIG environment variable is set, use it; + 2. Otherwise, default to 'config' directory under resource directory. + + :param args: Additional path components to append to the config directory + :return: Config directory path + """ + dir_config = os.environ.get("DIR_CONFIG") dir_resource = get_resource_dir() if dir_config is None or len(dir_config.strip()) == 0: - dir_config = 'config' + dir_config = "config" dir_config = path_join(dir_resource, dir_config, *args) return dir_config @@ -38,44 +64,58 @@ def get_config_files() -> list: (b) If environment variable `ENV_PROFILE` is not defined, the entry config file will be "main.conf". :return: list of string, which are file names of config files """ - files_config = os.environ.get('FILES_CONFIG', None) + files_config = os.environ.get("FILES_CONFIG", None) if files_config is None: - env_profile = os.environ.get('ENV_PROFILE', None) + env_profile = os.environ.get("ENV_PROFILE", None) if env_profile is None: - files_config = 'main.conf' + files_config = "main.conf" else: - files_config = 'main-%s.conf' % env_profile + files_config = "main-%s.conf" % env_profile - files = files_config.split(',') + files = files_config.split(",") ret = [] msgs = [] for f in files: file = get_config_dir(f) if not os.path.exists(file): - msgs.append('Expecting config file [%s] but it does not exists!' % file) + msgs.append("Expecting config file [%s] but it does not exists!" % file) else: - print(' ---> Loading config file [%s]' % file, file=sys.stderr) + print(" ---> Loading config file [%s]" % file, file=sys.stderr) ret.append(os.path.expandvars(f)) if len(ret) == 0: - msgs.append('No config files set properly, EMPTY config will be used!') + msgs.append("No config files set properly, EMPTY config will be used!") if len(msgs) > 0: - warnings.warn('\n'.join(msgs)) + warnings.warn("\n".join(msgs)) return ret def get_current_module_dir(file_caller: str) -> str: + """ + Get the directory containing the given module file. + + :param file_caller: Path to the module file (usually __file__) + :return: Directory path of the module + """ dirs = file_caller.split(os.sep) return os.sep.join(dirs[:-1]) def get_project_base_dir(file_caller: str) -> str: + """ + Get the project base directory by searching upwards for __init__.py. + Starts from the given file path and moves up until a directory without + __init__.py is found, indicating the project root. + + :param file_caller: Path to the starting file (usually __file__) + :return: Project base directory path + """ dirs = file_caller.split(os.sep) - dir_base = '' + dir_base = "" for i in range(len(dirs)): - dir_base = os.sep.join(dirs[:-1 - i]) - file_init = path_join(dir_base, '__init__.py') + dir_base = os.sep.join(dirs[: -1 - i]) + file_init = path_join(dir_base, "__init__.py") if not os.path.exists(file_init): break return dir_base diff --git a/src/aloha/db/base.py b/src/aloha/db/base.py index b6d69e0..f508d53 100644 --- a/src/aloha/db/base.py +++ b/src/aloha/db/base.py @@ -4,24 +4,46 @@ class PasswordVault: + """ + Password vault manager that provides access to password vault implementations. + + Caches vault instances for performance. + """ + _dict_cache_vault = {} @staticmethod - def get_vault(vault_type: str = None, vault_config: dict = None, *args, **kwargs) -> vault.BaseVault: - encryption_method = vault_type or SETTINGS.config.get('PASSWORD_ENCRYPTION') - LOG.debug('Using password vault: %s', encryption_method) + def get_vault(vault_type: str | None = None, vault_config: dict | None = None, **kwargs) -> vault.BaseVault: + """ + Get a password vault instance. + + Supports multiple vault types: + - 'plain' or 'aes': AES-based vault (default fallback) + - 'cyberark': CyberArk vault + - Other/None: Dummy vault (plain text) - cache_key = '%s:%s' % (encryption_method, str(vault_config)) + :param vault_type: Type of vault to use (overrides config) + :param vault_config: Vault configuration dictionary + :param args: Additional arguments + :param kwargs: Additional keyword arguments + :return: Vault instance implementing BaseVault interface + :raises RuntimeError: If CyberArk vault is requested but config is missing + """ + encryption_method = vault_type or SETTINGS.config.get("PASSWORD_ENCRYPTION") + LOG.debug("Using password vault: %s", encryption_method) # nosemgrep + + cache_key = "%s:%s" % (encryption_method, str(vault_config)) if cache_key not in PasswordVault._dict_cache_vault: - if encryption_method in ('plain', 'aes') or encryption_method is True: + if encryption_method in ("plain", "aes") or encryption_method is True: v = vault.AesVault(**(vault_config or {})) - elif encryption_method == 'cyberark': - config_cyberark = vault_config or SETTINGS.config.get('CYBERARK_CONFIG') + elif encryption_method == "cyberark": + config_cyberark = vault_config or SETTINGS.config.get("CYBERARK_CONFIG") if config_cyberark is None: - raise RuntimeError('Missing [CYBERARK_CONFIG] in config!') + raise RuntimeError("Missing [CYBERARK_CONFIG] in config!") v = vault.CyberArkVault(**config_cyberark) else: - LOG.info('Using plain password vault as unknown value of PASSWORD_ENCRYPTION=%s in config.', encryption_method) + msg = "Using plain password vault as unknown value of PASSWORD_ENCRYPTION=%s in config." % encryption_method + LOG.info(msg) # nosemgrep v = vault.DummyVault(**(vault_config or {})) PasswordVault._dict_cache_vault[cache_key] = v @@ -29,13 +51,19 @@ def get_vault(vault_type: str = None, vault_config: dict = None, *args, **kwargs def main(): + """ + Command-line tool to decrypt passwords from config. + + Usage: python -m aloha.db.base + """ import sys + config_key = sys.argv[-1] - LOG.debug('Getting pwd for deploy key [deploy.%s]' % config_key) + LOG.debug("Getting pwd for deploy key [deploy.%s]" % config_key) try: - db_config = SETTINGS.config['deploy'][config_key] + db_config = SETTINGS.config["deploy"][config_key] password_vault = PasswordVault.get_vault() - p = password_vault.get_password(db_config.get('password')) + p = password_vault.get_password(db_config.get("password")) LOG.debug("Decrypted PWD: %s" % p) except KeyError: - LOG.error('Please make sure config key [deploy.%s] exists!' % config_key) + LOG.error("Please make sure config key [deploy.%s] exists!" % config_key) diff --git a/src/aloha/db/duckdb.py b/src/aloha/db/duckdb.py index 384506c..191a14b 100644 --- a/src/aloha/db/duckdb.py +++ b/src/aloha/db/duckdb.py @@ -1,13 +1,14 @@ -__all__ = ('DuckOperator',) +__all__ = ("DuckOperator",) from pathlib import Path import duckdb import duckdb_engine -from aloha.logger import LOG from sqlalchemy import create_engine, text -LOG.debug('duckdb version = %s, duckdb_engine = %s ' % (duckdb.__version__, duckdb_engine.__version__)) +from aloha.logger import LOG + +LOG.debug("duckdb version = %s, duckdb_engine = %s " % (duckdb.__version__, duckdb_engine.__version__)) class DuckOperator: @@ -21,19 +22,19 @@ def __init__(self, db_config, **kwargs): } """ self._config = { - 'path': db_config.get('path', ':memory:'), - 'schema': db_config.get('schema', 'main'), - 'read_only': bool(db_config.get('read_only', False)), - 'config': db_config.get('config', {}), - 'auto_commit': db_config.get('auto_commit', True), + "path": db_config.get("path", ":memory:"), + "schema": db_config.get("schema", "main"), + "read_only": bool(db_config.get("read_only", False)), + "config": db_config.get("config", {}), + "auto_commit": db_config.get("auto_commit", True), } - if not self._config['path'] or self._config['path'] == ':memory:': # in-memory mode - self._config['path'] = ':memory:' + if not self._config["path"] or self._config["path"] == ":memory:": # in-memory mode + self._config["path"] = ":memory:" - if self._config['read_only']: # in-memory mode cannot be read-only + if self._config["read_only"]: # in-memory mode cannot be read-only LOG.warning("In-memory database cannot be read-only. Setting read_only=False.") - self._config['read_only'] = False + self._config["read_only"] = False else: self._prepare_database() @@ -42,11 +43,8 @@ def __init__(self, db_config, **kwargs): str_connection = f"duckdb:///{self._config['path']}" self.engine = create_engine( str_connection, - connect_args={ - 'read_only': self._config['read_only'], - 'config': self._config['config'] - }, - **kwargs + connect_args={"read_only": self._config["read_only"], "config": self._config["config"]}, + **kwargs, ) self._initialize_schema() @@ -54,19 +52,17 @@ def __init__(self, db_config, **kwargs): LOG.debug(msg) except Exception as e: LOG.exception(e) - raise RuntimeError('Failed to connect to DuckDB') + raise RuntimeError("Failed to connect to DuckDB") def _prepare_database(self): """Prepare the database file and its parent directory.""" - path = self._config['path'] + path = self._config["path"] path_obj = Path(path) parent_dir = path_obj.parent if not parent_dir.exists(): - if self._config['read_only']: - raise RuntimeError( - f"Directory '{parent_dir}' does not exist and read_only=True" - ) + if self._config["read_only"]: + raise RuntimeError(f"Directory '{parent_dir}' does not exist and read_only=True") try: parent_dir.mkdir(parents=True, exist_ok=True) LOG.debug(f"Created directory: {parent_dir}") @@ -74,10 +70,8 @@ def _prepare_database(self): raise RuntimeError(f"Failed to create directory '{parent_dir}': {e}") if not path_obj.exists(): - if self._config['read_only']: - raise RuntimeError( - f"DuckDB file '{path}' does not exist and read_only=True" - ) + if self._config["read_only"]: + raise RuntimeError(f"DuckDB file '{path}' does not exist and read_only=True") try: LOG.debug(f"Database file not found, creating: {path}") duckdb.connect(path).close() @@ -85,25 +79,23 @@ def _prepare_database(self): raise RuntimeError(f"Failed to create database file '{path}': {e}") def _initialize_schema(self): - if self._config['schema'] == 'main': + if self._config["schema"] == "main": return try: - if self._config['read_only']: + if self._config["read_only"]: result = self.engine.connext().execute( text("SELECT schema_name FROM information_schema.schemata WHERE schema_name = :schema"), - {'schema': self._config['schema']} + {"schema": self._config["schema"]}, ) if not result.fetchone(): - raise RuntimeError( - f"Schema '{self._config['schema']}' does not exist and read_only=True" - ) + raise RuntimeError(f"Schema '{self._config['schema']}' does not exist and read_only=True") else: self.engine.connect().execute(text(f"CREATE SCHEMA IF NOT EXISTS {self._config['schema']}")) self.engine.connect().execute(text(f"SET schema '{self._config['schema']}'")) except Exception as e: - raise RuntimeError(f'Failed to initialize schema: {e}') + raise RuntimeError(f"Failed to initialize schema: {e}") @property def connection(self): @@ -114,7 +106,7 @@ def connection(self): def execute_query(self, sql, *args, **kwargs): with self.engine.connect() as conn: cur = conn.execute(text(sql), *args, *kwargs) - if self._config.get('auto_commit', True): + if self._config.get("auto_commit", True): conn.commit() return cur diff --git a/src/aloha/db/elasticsearch.py b/src/aloha/db/elasticsearch.py index bf534c7..b6412c4 100644 --- a/src/aloha/db/elasticsearch.py +++ b/src/aloha/db/elasticsearch.py @@ -1,33 +1,33 @@ -__all__ = ('ElasticSearchOperator',) +__all__ = ("ElasticSearchOperator",) import json from elasticsearch import Elasticsearch -from .base import PasswordVault + from ..logger import LOG +from .base import PasswordVault class ElasticSearchOperator: def __init__(self, config, index_config=None): self.es_config = config - password_vault = PasswordVault.get_vault(config.get('vault_type'), config.get('vault_config')) - username = config.get('username') - password = password_vault.get_password(config.get('password')) + password_vault = PasswordVault.get_vault(config.get("vault_type"), config.get("vault_config")) + username = config.get("username") + password = password_vault.get_password(config.get("password")) self._config = { - 'http_auth': (username, password) if username is not None and password is not None else None, - 'hosts': config.get('host', 'localhost'), - - 'timeout': config.get("timeout", 0.1), - 'max_retries': config.get("max_retries", 3), - 'retry_on_timeout': config.get("retry_on_timeout", True) + "http_auth": (username, password) if username is not None and password is not None else None, + "hosts": config.get("host", "localhost"), + "timeout": config.get("timeout", 0.1), + "max_retries": config.get("max_retries", 3), + "retry_on_timeout": config.get("retry_on_timeout", True), } - LOG.debug("ElasticSearch connection info: " + str(self._config['hosts'])) + LOG.debug("ElasticSearch connection info: " + str(self._config["hosts"])) self.index_config = index_config - self.index_name = self.es_config.get('index_name') - self.index_type = self.es_config.get('index_type') + self.index_name = self.es_config.get("index_name") + self.index_type = self.es_config.get("index_type") self.es = Elasticsearch(**self._config) @@ -36,30 +36,27 @@ def __init__(self, config, index_config=None): @staticmethod def _load_config(config): - if type(config) == dict: + if isinstance(config, dict): return config - elif type(config) == str and ".json" in config: + elif isinstance(config, str) and ".json" in config: with open(config, "r", encoding="utf-8") as f: config = json.load(f) return config else: - raise Exception("Invalid ES config data type") + raise ValueError("Invalid ES config data type") - def put_mapping(self, index_name=None, index_type=None, index_config=None): + def put_mapping(self, index_name=None, index_type=None, index_config: dict | None = None): return self.es.indices.put_mapping( index=index_name or self.index_name, doc_type=index_type or self.index_type, - body=index_config["mappings"][index_type or self.index_type] + body=index_config["mappings"][index_type or self.index_type], ) def build_index(self, index_name=None, index_config=None, raise_if_exist=False): if self.es.indices.exists(index=index_name or self.index_name) is not True: - res = self.es.indices.create( - index=index_name or self.index_name, - body=index_config or self.index_config - ) + res = self.es.indices.create(index=index_name or self.index_name, body=index_config or self.index_config) return res else: msg = "Index [%s] already exits" % self.index_name diff --git a/src/aloha/db/kafka.py b/src/aloha/db/kafka.py index 11880d1..54d7403 100644 --- a/src/aloha/db/kafka.py +++ b/src/aloha/db/kafka.py @@ -1,4 +1,4 @@ -__all__ = ('KafkaOperator',) +__all__ = ("KafkaOperator",) import json import typing @@ -8,7 +8,7 @@ from ..logger import LOG -LOG.debug('Version of confluent_kafka client = %s' % kafka.__version__) +LOG.debug("Version of confluent_kafka client = %s" % kafka.__version__) class KafkaOperator: @@ -23,9 +23,9 @@ def __init__(self, kafka_config): """ self._config = json.loads(json.dumps(kafka_config, ensure_ascii=False)) # deep copy - if 'host' in kafka_config: + if "host" in kafka_config: self._config = { - 'bootstrap.servers': ','.join(['{host}:{port}'.format(**i) for i in kafka_config.pop('host')]), + "bootstrap.servers": ",".join(["{host}:{port}".format(**i) for i in kafka_config.pop("host")]), } LOG.debug("Kafka connection info: " + str(self._config)) @@ -61,11 +61,11 @@ def producer_deliver(self, topic: str, generator: typing.Iterator[str], func_cal p = kafka.Producer(config_producer) def delivery_report(err, msg): - """ Called once for each message produced to indicate delivery result. Triggered by poll() or flush(). """ + """Called once for each message produced to indicate delivery result. Triggered by poll() or flush().""" if err is not None: - LOG.error('Kafka msg delivery failed: {}'.format(err)) + LOG.error("Kafka msg delivery failed: {}".format(err)) else: - LOG.debug('Kafka msg delivered to {} [{}]'.format(msg.topic(), msg.partition())) + LOG.debug("Kafka msg delivered to {} [{}]".format(msg.topic(), msg.partition())) if func_callback is None: func_callback = delivery_report @@ -77,15 +77,17 @@ def delivery_report(err, msg): # Asynchronously produce a message, the delivery report callback # will be triggered from poll() above, or flush() below, when the message has # been successfully delivered or failed permanently. - p.produce(topic, data.encode('utf-8'), callback=func_callback) + p.produce(topic, data.encode("utf-8"), callback=func_callback) # Wait for any outstanding messages to be delivered and delivery report callbacks to be triggered. p.flush() - def consumer_generator(self, topics_subscribe: list, group_id: str = None, poll_timeout: float = 1.0, *args, **kwargs) -> typing.Iterator[str]: - config_consumer = {'auto.offset.reset': 'earliest', **self._config} + def consumer_generator( + self, topics_subscribe: list, group_id: str | None = None, poll_timeout: float = 1.0, *args, **kwargs + ) -> typing.Iterator[str]: + config_consumer = {"auto.offset.reset": "earliest", **self._config} if group_id is not None: - config_consumer['group.id'] = group_id + config_consumer["group.id"] = group_id c = kafka.Consumer(config_consumer) c.subscribe(topics_subscribe) @@ -101,8 +103,8 @@ def consumer_generator(self, topics_subscribe: list, group_id: str = None, poll_ LOG.error("Kafka consumer: {}".format(msg.error())) continue - data = msg.value().decode('utf-8') - LOG.debug('Received message: {}'.format(data)) + data = msg.value().decode("utf-8") + LOG.debug("Received message: {}".format(data)) yield data c.close() diff --git a/src/aloha/db/mongo.py b/src/aloha/db/mongo.py index 38b1a92..5bb03a9 100644 --- a/src/aloha/db/mongo.py +++ b/src/aloha/db/mongo.py @@ -1,12 +1,12 @@ -__all__ = ('MongoOperator',) +__all__ = ("MongoOperator",) import ipaddress import json import pymongo -from .base import PasswordVault from ..logger import LOG +from .base import PasswordVault def _is_ip_addr(s): @@ -21,15 +21,11 @@ def _is_ip_addr(s): def MongoOperator(config): - db_name = config.get('db_name') - collection_name = config.get('collection_name') + db_name = config.get("db_name") + collection_name = config.get("collection_name") _config = {k: v for k, v in config.items() if v is not None} - key = '%s:%s:%s' % ( - json.dumps(_config, sort_keys=True, ensure_ascii=False), - db_name or '', - collection_name or '' - ) + key = "%s:%s:%s" % (json.dumps(_config, sort_keys=True, ensure_ascii=False), db_name or "", collection_name or "") if key not in _conn: try: @@ -44,27 +40,27 @@ class _MongoDBOperation: def __init__(self, config, db_name=None, collection_name=None): self.db_name, self.collection_name = db_name, collection_name - host = config['host'] + host = config["host"] - if config.get('port') is None and isinstance(host, list): - hosts = ['{host}:{port}'.format(**h) for h in host] + if config.get("port") is None and isinstance(host, list): + hosts = ["{host}:{port}".format(**h) for h in host] else: - hosts = ['{host}:{port}'.format(host=host, port=config['port'])] + hosts = ["{host}:{port}".format(host=host, port=config["port"])] - replicaSet = config.get('replicaSet') - if replicaSet is None and not _is_ip_addr(hosts[0].split(':')[0]): + replicaSet = config.get("replicaSet") + if replicaSet is None and not _is_ip_addr(hosts[0].split(":")[0]): # if `replicaSet` not defined, and host in config is domain name (not IP) - replicaSet = hosts[0].split('.')[0] # use the first segment of domain name as replicaSet + replicaSet = hosts[0].split(".")[0] # use the first segment of domain name as replicaSet - password_vault = PasswordVault.get_vault(config.get('vault_type'), config.get('vault_config')) + password_vault = PasswordVault.get_vault(config.get("vault_type"), config.get("vault_config")) _config = { - 'host': 'mongodb://%s' % ','.join(hosts), - 'port': config.get('port'), - 'replicaSet': replicaSet, - 'username': config['username'], - 'password': password_vault.get_password(config.get('password')), - 'maxPoolSize': config.get('maxPoolSize'), - 'authSource': config.get('authSource', db_name) + "host": "mongodb://%s" % ",".join(hosts), + "port": config.get("port"), + "replicaSet": replicaSet, + "username": config["username"], + "password": password_vault.get_password(config.get("password")), + "maxPoolSize": config.get("maxPoolSize"), + "authSource": config.get("authSource", db_name), } LOG.debug(_config) @@ -146,23 +142,55 @@ def delete_one(self, field_filter, collection_name=None): except Exception as e: LOG.exception(e) - def update_one(self, field_filter, update, upsert=False, bypass_document_validation=False, - collation=None, array_filters=None, session=None, collection_name=None): + def update_one( + self, + field_filter, + update, + upsert=False, + bypass_document_validation=False, + collation=None, + array_filters=None, + session=None, + collection_name=None, + ): try: collection = self.check_and_get_collection(collection_name) - collection.update_one(filter=field_filter, update=update, upsert=upsert, bypass_document_validation=bypass_document_validation, - collation=collation, array_filters=array_filters, session=session) + collection.update_one( + filter=field_filter, + update=update, + upsert=upsert, + bypass_document_validation=bypass_document_validation, + collation=collation, + array_filters=array_filters, + session=session, + ) return True except Exception as e: LOG.exception(e) return False - def update_many(self, field_filter, update, upsert=False, bypass_document_validation=False, - collation=None, array_filters=None, session=None, collection_name=None): + def update_many( + self, + field_filter, + update, + upsert=False, + bypass_document_validation=False, + collation=None, + array_filters=None, + session=None, + collection_name=None, + ): try: collection = self.check_and_get_collection(collection_name) - return collection.update_many(filter=field_filter, update=update, upsert=upsert, bypass_document_validation=bypass_document_validation, - collation=collation, array_filters=array_filters, session=session) + return collection.update_many( + filter=field_filter, + update=update, + upsert=upsert, + bypass_document_validation=bypass_document_validation, + collation=collation, + array_filters=array_filters, + session=session, + ) except Exception as e: LOG.exception(e) @@ -234,4 +262,4 @@ def count(self, field_filter=None, collection_name=None): def check_connected(self): if not self.conn.connected: - raise NameError('stat:connected Error') + raise NameError("stat:connected Error") diff --git a/src/aloha/db/mysql.py b/src/aloha/db/mysql.py index ca03d07..e3aeef9 100644 --- a/src/aloha/db/mysql.py +++ b/src/aloha/db/mysql.py @@ -1,35 +1,38 @@ -__all__ = ('MySqlOperator',) +__all__ = ("MySqlOperator",) import pymysql from sqlalchemy import create_engine from sqlalchemy.sql import text -from .base import PasswordVault from ..logger import LOG +from .base import PasswordVault -LOG.debug('Version of pymysql = %s' % pymysql.__version__) +LOG.debug("Version of pymysql = %s" % pymysql.__version__) class MySqlOperator: def __init__(self, db_config, **kwargs): - password_vault = PasswordVault.get_vault(db_config.get('vault_type'), db_config.get('vault_config')) + password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config")) self._config = { - 'host': db_config['host'], - 'port': db_config['port'], - 'user': db_config['user'], - 'password': password_vault.get_password(db_config['password']), - 'dbname': db_config['dbname'], + "host": db_config["host"], + "port": db_config["port"], + "user": db_config["user"], + "password": password_vault.get_password(db_config["password"]), + "dbname": db_config["dbname"], } try: self.db = create_engine( - 'mysql+pymysql://{user}:{password}@{host}:{port}/{dbname}'.format(**self._config), - pool_size=50, pool_recycle=500, pool_pre_ping=True, **kwargs + "mysql+pymysql://{user}:{password}@{host}:{port}/{dbname}".format(**self._config), + pool_size=50, + pool_recycle=500, + pool_pre_ping=True, + **kwargs, ) LOG.debug("MySQL connected: {host}:{port}/{dbname}".format(**self._config)) except Exception as e: LOG.exception(e) - raise RuntimeError('Failed to connect to MySQL') + raise RuntimeError("Failed to connect to MySQL") @property def connection(self): @@ -42,4 +45,4 @@ def execute_query(self, sql, *args, **kwargs): @property def connection_str(self) -> str: - return 'mysql://{user}:{password}@{host}:{port}/{dbname}'.format(**self._config) + return "mysql://{user}:{password}@{host}:{port}/{dbname}".format(**self._config) diff --git a/src/aloha/db/oracle.py b/src/aloha/db/oracle.py index b929000..62f9905 100644 --- a/src/aloha/db/oracle.py +++ b/src/aloha/db/oracle.py @@ -1,11 +1,11 @@ -__all__ = ('OracledbOperator',) +__all__ = ("OracledbOperator",) import oracledb from sqlalchemy import create_engine from sqlalchemy.sql import text -from .base import PasswordVault from ..logger import LOG +from .base import PasswordVault LOG.debug("oracledb version = %s" % oracledb.__version__) @@ -26,17 +26,17 @@ def __init__(self, db_config, **kwargs): } """ - password_vault = PasswordVault.get_vault(db_config.get('vault_type'), db_config.get('vault_config')) + password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config")) self._config = { - 'host': db_config['host'], - 'port': db_config['port'], - 'user': db_config['user'], - 'password': password_vault.get_password(db_config.get('password')), + "host": db_config["host"], + "port": db_config["port"], + "user": db_config["user"], + "password": password_vault.get_password(db_config.get("password")), } - if 'lib_dir' in db_config: # use Thick mode + if "lib_dir" in db_config: # use Thick mode try: - oracledb.init_oracle_client(lib_dir=db_config['lib_dir']) + oracledb.init_oracle_client(lib_dir=db_config["lib_dir"]) LOG.info("Oracle client initialized in THICK mode from: %s" % db_config["lib_dir"]) except Exception as e: LOG.warning(f"Warning: {e}") @@ -46,7 +46,7 @@ def __init__(self, db_config, **kwargs): sid = db_config.get("sid") if service_name: # using service_name (recommended) - dsn = oracledb.makedsn(db_config["host"], db_config["port"], service_name=service_name) + dsn = oracledb.makedsn(db_config["host"], db_config["port"], service_name=service_name) elif sid: # using SID dsn = oracledb.makedsn(db_config["host"], db_config["port"], sid=sid) else: @@ -57,13 +57,16 @@ def __init__(self, db_config, **kwargs): self.engine = create_engine( "oracle+oracledb://{user}:{password}@".format(**self._config), connect_args={"dsn": dsn}, - pool_size=20, max_overflow=10, pool_pre_ping=True, **kwargs + pool_size=20, + max_overflow=10, + pool_pre_ping=True, + **kwargs, ) msg = "OracleDB connected: {host}:{port}".format(**self._config) print(msg) except Exception as e: LOG.error(e) - raise RuntimeError(f"Failed to connect to OracleDB") + raise RuntimeError("Failed to connect to OracleDB") @property def connection(self): diff --git a/src/aloha/db/postgres.py b/src/aloha/db/postgres.py index ba4f460..f036b60 100644 --- a/src/aloha/db/postgres.py +++ b/src/aloha/db/postgres.py @@ -1,34 +1,38 @@ -__all__ = ('PostgresOperator',) +__all__ = ("PostgresOperator",) import psycopg from sqlalchemy import create_engine from sqlalchemy.sql import text -from .base import PasswordVault from ..logger import LOG +from .base import PasswordVault -LOG.debug('postgres: psycopg version = %s' % psycopg.__version__) +LOG.debug("postgres: psycopg version = %s" % psycopg.__version__) class PostgresOperator: def __init__(self, db_config, **kwargs): - password_vault = PasswordVault.get_vault(db_config.get('vault_type'), db_config.get('vault_config')) + password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config")) self._config = { - 'host': db_config['host'], - 'port': db_config['port'], - 'user': db_config['user'], - 'password': password_vault.get_password(db_config.get('password')), - 'dbname': db_config['dbname'], + "host": db_config["host"], + "port": db_config["port"], + "user": db_config["user"], + "password": password_vault.get_password(db_config.get("password")), + "dbname": db_config["dbname"], } connect_args = {} - if 'schema' in db_config: - connect_args['options'] = '-csearch_path={}'.format(db_config['schema']) + if "schema" in db_config: + connect_args["options"] = "-csearch_path={}".format(db_config["schema"]) try: self.engine = create_engine( - 'postgresql+psycopg://{user}:{password}@{host}:{port}/{dbname}'.format(**self._config), - connect_args=connect_args, client_encoding='utf8', - pool_size=20, max_overflow=10, pool_pre_ping=True, **kwargs + "postgresql+psycopg://{user}:{password}@{host}:{port}/{dbname}".format(**self._config), + connect_args=connect_args, + client_encoding="utf8", + pool_size=20, + max_overflow=10, + pool_pre_ping=True, + **kwargs, ) LOG.debug("PostgresSQL connected: {host}:{port}/{dbname}".format(**self._config)) except Exception as e: diff --git a/src/aloha/db/redis.py b/src/aloha/db/redis.py index b28d711..3f086fd 100644 --- a/src/aloha/db/redis.py +++ b/src/aloha/db/redis.py @@ -1,48 +1,48 @@ -__all__ = ('RedisOperator',) +__all__ = ("RedisOperator",) import redis from packaging import version -from .base import PasswordVault from ..logger import LOG +from .base import PasswordVault class RedisOperator: def __init__(self, config): self._check_redis_version() - password_vault = PasswordVault.get_vault(config.get('vault_type'), config.get('vault_config')) + password_vault = PasswordVault.get_vault(config.get("vault_type"), config.get("vault_config")) _config = { - 'host': config['host'], - 'port': config.get('port', '6379'), - 'password': password_vault.get_password(config.get('password', None)), - 'decode_responses': config.get('decode_responses', True), - 'retry_on_timeout': True, - 'max_connections': config.get('max_connections', 1000), - 'socket_timeout': 3, - 'socket_connect_timeout': 1, + "host": config["host"], + "port": config.get("port", "6379"), + "password": password_vault.get_password(config.get("password", None)), + "decode_responses": config.get("decode_responses", True), + "retry_on_timeout": True, + "max_connections": config.get("max_connections", 1000), + "socket_timeout": 3, + "socket_connect_timeout": 1, } - if 'db' in config: - _config['db'] = config['db'] + if "db" in config: + _config["db"] = config["db"] self._config = _config self._pool = None @staticmethod def _check_redis_version() -> bool: - ver_min = version.parse('4.1.0') + ver_min = version.parse("4.1.0") valid = False try: ver_cur = version.parse(redis.__version__) if ver_cur >= ver_min: valid = True - LOG.debug('Using redis version = %s' % redis.__version__) + LOG.debug("Using redis version = %s" % redis.__version__) except Exception as e: - LOG.error('Failed to obtain redis version!') + LOG.error("Failed to obtain redis version!") LOG.error(str(e)) if not valid: - msg = 'Invalid version of `redis-py`, version >4.1.0 required!' + msg = "Invalid version of `redis-py`, version >4.1.0 required!" LOG.fatal(msg) raise ImportError(msg) diff --git a/src/aloha/db/sqlite.py b/src/aloha/db/sqlite.py index a8ee26d..4925422 100644 --- a/src/aloha/db/sqlite.py +++ b/src/aloha/db/sqlite.py @@ -1,43 +1,41 @@ -__all__ = ('SqliteOperator',) +__all__ = ("SqliteOperator",) import sqlite3 from sqlalchemy import create_engine from sqlalchemy.sql import text -from .base import PasswordVault from ..logger import LOG +from .base import PasswordVault class SqliteOperator: def __init__(self, db_config, **kwargs): self._connection_pattern = "sqlite://{dbname}" - dbname = db_config.get('dbname', '') + dbname = db_config.get("dbname", "") if len(dbname) > 0: - dbname = '/%s' % dbname - self._config = {'dbname': dbname} + dbname = "/%s" % dbname + self._config = {"dbname": dbname} - if 'password' in db_config: + if "password" in db_config: try: import sqlcipher3 except ImportError: - raise RuntimeError('Python package required for encrypted sqlite3: sqlcipher3-binary') - LOG.debug('Version of sqlcipher3 = %s' % sqlcipher3.sqlite_version) - password_vault = PasswordVault.get_vault(db_config.get('vault_type'), db_config.get('vault_config')) - password = password_vault.get_password(db_config.get('password', None)) - self._config['password'] = password + raise RuntimeError("Python package required for encrypted sqlite3: sqlcipher3-binary") + LOG.debug("Version of sqlcipher3 = %s" % sqlcipher3.sqlite_version) + password_vault = PasswordVault.get_vault(db_config.get("vault_type"), db_config.get("vault_config")) + password = password_vault.get_password(db_config.get("password", None)) + self._config["password"] = password self._connection_pattern = "sqlite+pysqlcipher://:{password}@/{dbname}" else: - LOG.debug('Version of sqlite = %s' % sqlite3.sqlite_version) + LOG.debug("Version of sqlite = %s" % sqlite3.sqlite_version) try: - self.db = create_engine( - self._connection_pattern.format(**self._config), **kwargs - ) + self.db = create_engine(self._connection_pattern.format(**self._config), **kwargs) LOG.debug("Sqlite connected: %s" % self.connection_str) except Exception as e: LOG.exception(e) - raise RuntimeError('Failed to connect to sqlite') + raise RuntimeError("Failed to connect to sqlite") @property def connection(self): diff --git a/src/aloha/encrypt/aes.py b/src/aloha/encrypt/aes.py index 10b8121..b3461d5 100644 --- a/src/aloha/encrypt/aes.py +++ b/src/aloha/encrypt/aes.py @@ -1,86 +1,102 @@ import base64 import binascii -from typing import Union, Callable, Optional +from typing import Callable, Optional, Union from Crypto.Cipher import AES from Crypto.Random import get_random_bytes from Crypto.Util.Padding import pad, unpad _AES_CIPHER_METHODS = { # FULL_CIPHER_NAME: (dict_params, pad_style) - "AES/ECB/PKCS5Padding": ({'mode': AES.MODE_ECB}, 'pkcs7'), - "AES/ECB/NoPadding": ({'mode': AES.MODE_ECB}, 'pkcs7'), - "AES/CBC/PKCS7Padding": ({'mode': AES.MODE_CBC, 'iv': b'0000000000000000'}, 'pkcs7'), - "AES/CBC/NoPadding": ({'mode': AES.MODE_CBC, 'iv': b'0000000000000000'}, 'x923'), + "AES/ECB/PKCS5Padding": ({"mode": AES.MODE_ECB}, "pkcs7"), + "AES/ECB/NoPadding": ({"mode": AES.MODE_ECB}, "pkcs7"), + "AES/CBC/PKCS7Padding": ({"mode": AES.MODE_CBC, "iv": b"0000000000000000"}, "pkcs7"), + "AES/CBC/NoPadding": ({"mode": AES.MODE_CBC, "iv": b"0000000000000000"}, "x923"), } -def _generate_key(key_size: int, method='const') -> bytes: - if method == 'const': - return b'0' * key_size # b'b6046801716aec00' - elif method == 'random': +def _generate_key(key_size: int, method="const") -> bytes: + if method == "const": + return b"0" * key_size # b'b6046801716aec00' + elif method == "random": return get_random_bytes(key_size) - raise ValueError('Invalid AES key generate method: [%s]' % method) + raise ValueError("Invalid AES key generate method: [%s]" % method) class AesEncryptor: supported_cipher_methods = _AES_CIPHER_METHODS - def __init__(self, key: Union[str, bytes] = None, key_size: int = 16, cipher_name: str = 'AES/ECB/PKCS5Padding'): + def __init__(self, key: Union[str, bytes] = None, key_size: int = 16, cipher_name: str = "AES/ECB/PKCS5Padding"): _key = key if key is None: _key = _generate_key(key_size) elif isinstance(key, str): _key = key.encode() - if len(_key) not in (16, 24, 32,): + if len(_key) not in ( + 16, + 24, + 32, + ): raise ValueError("Invalid key size/length [%s] for AesEncryptor!" % len(_key)) self.key_aes, self.block_size = _key, AES.block_size # https://pycryptodome.readthedocs.io/en/latest/src/util/util.html self.cipher_name = cipher_name - def encrypt(self, text: str, output_format='hex', func_pad: Optional[Callable] = None) -> Union[str, bytes]: + def encrypt(self, text: str, output_format="hex", func_pad: Optional[Callable] = None) -> Union[str, bytes]: dict_params, pad_style = _AES_CIPHER_METHODS.get(self.cipher_name) if not callable(func_pad): - func_pad = lambda x: pad(data, block_size=self.block_size, style=pad_style) + + def _func_pad(x): + return pad(x, block_size=self.block_size, style=pad_style) + + if not callable(func_pad): + func_pad = _func_pad data = text.encode() padded = func_pad(data) cipher = AES.new(key=self.key_aes, **dict_params) bytes_crypt = cipher.encrypt(padded) - if output_format == 'hex': + if output_format == "hex": crypt = binascii.b2a_hex(bytes_crypt).decode() - elif output_format == 'base64': + elif output_format == "base64": crypt = base64.b64encode(bytes_crypt).decode() - elif output_format in ('bytes', 'bin'): + elif output_format in ("bytes", "bin"): crypt = bytes_crypt else: - raise ValueError('Unknown output_type [%s]' % output_format) + raise ValueError("Unknown output_type [%s]" % output_format) return crypt - def decrypt(self, text: Union[str, bytes], input_format: str = 'hex', func_unpad: Optional[Callable] = None) -> Union[str, bytes]: - text += (len(text) % 4) * '=' - if input_format == 'hex': + def decrypt( + self, text: Union[str, bytes], input_format: str = "hex", func_unpad: Optional[Callable] = None + ) -> Union[str, bytes]: + text += (len(text) % 4) * "=" + if input_format == "hex": crypt = binascii.a2b_hex(text) - elif input_format == 'base64': + elif input_format == "base64": crypt = base64.b64decode(text) - elif input_format in ('bytes', 'bin'): + elif input_format in ("bytes", "bin"): crypt = text else: - raise ValueError('Unknown output_type [%s]' % input_format) + raise ValueError("Unknown output_type [%s]" % input_format) dict_params, pad_style = _AES_CIPHER_METHODS.get(self.cipher_name) cipher = AES.new(key=self.key_aes, **dict_params) data = cipher.decrypt(crypt) + + def _func_unpad(x): + return unpad(x, block_size=self.block_size, style=pad_style) + if not callable(func_unpad): - func_unpad = lambda x: unpad(x, block_size=self.block_size, style=pad_style) + func_unpad = _func_unpad data = func_unpad(data) - return data.decode('UTF-8') + return data.decode("UTF-8") def main(): a = AesEncryptor() - src = 'hello~' - enc = a.encrypt(src, output_format='base64') - dec = a.decrypt(enc, input_format='base64') + src = "hello~" + enc = a.encrypt(src, output_format="base64") + dec = a.decrypt(enc, input_format="base64") + assert src == dec # print(enc, dec, src == dec) diff --git a/src/aloha/encrypt/jwt.py b/src/aloha/encrypt/jwt.py index 0d5356c..d7d5103 100644 --- a/src/aloha/encrypt/jwt.py +++ b/src/aloha/encrypt/jwt.py @@ -2,26 +2,17 @@ from ..logger import LOG -LOG.debug('Using pyjwt == %s' % jwt.__version__.__str__()) +LOG.debug("Using pyjwt == %s" % jwt.__version__.__str__()) -def encode( - secret_key: str, - payload: dict, - headers: dict = None, - **kwargs -): +def encode(secret_key: str, payload: dict, headers: dict = None, **kwargs): token = jwt.encode(payload=payload, key=secret_key, headers=headers, **kwargs) return token -def decode( - secret_key: str, - token: str, - **kwargs -): +def decode(secret_key: str, token: str, **kwargs): try: - resp = jwt.decode(jwt=token, key=secret_key, algorithms=['HS256'], **kwargs) + resp = jwt.decode(jwt=token, key=secret_key, algorithms=["HS256"], **kwargs) except jwt.ExpiredSignatureError as e: resp = str(e) except jwt.PyJWTError as e: diff --git a/src/aloha/encrypt/rsa.py b/src/aloha/encrypt/rsa.py index 16c22f4..d0f08f5 100644 --- a/src/aloha/encrypt/rsa.py +++ b/src/aloha/encrypt/rsa.py @@ -1,8 +1,8 @@ -__all__ = ('RsaEncryptor',) +__all__ = ("RsaEncryptor",) import base64 from functools import lru_cache -from typing import Tuple, Optional, Union +from typing import Optional, Tuple, Union from Crypto.Cipher import PKCS1_OAEP, PKCS1_v1_5 from Crypto.Hash import SHA1, SHA256 @@ -12,9 +12,9 @@ t_cipher_module = Union[PKCS1_v1_5.PKCS115_Cipher, PKCS1_OAEP.PKCS1OAEP_Cipher] _RSA_CIPHER_METHODS = { # FULL_CIPHER_NAME: (module, dict_params) - "RSA/ECB/PKCS1Padding": (PKCS1_v1_5, {'randfunc': None}), - "RSA/ECB/OAEPWithSHA-1AndMGF1Padding": (PKCS1_OAEP, {'hashAlgo': SHA1, 'mgfunc': lambda x, y: pss.MGF1(x, y, SHA1)}), - "RSA/ECB/OAEPWithSHA-256AndMGF1Padding": (PKCS1_OAEP, {'hashAlgo': SHA256, 'mgfunc': lambda x, y: pss.MGF1(x, y, SHA1)}), + "RSA/ECB/PKCS1Padding": (PKCS1_v1_5, {"randfunc": None}), + "RSA/ECB/OAEPWithSHA-1AndMGF1Padding": (PKCS1_OAEP, {"hashAlgo": SHA1, "mgfunc": lambda x, y: pss.MGF1(x, y, SHA1)}), + "RSA/ECB/OAEPWithSHA-256AndMGF1Padding": (PKCS1_OAEP, {"hashAlgo": SHA256, "mgfunc": lambda x, y: pss.MGF1(x, y, SHA1)}), } @@ -24,33 +24,35 @@ class RsaEncryptor: supported_cipher_methods = _RSA_CIPHER_METHODS # ref: https://cryptobook.nakov.com/asymmetric-key-ciphers/rsa-encrypt-decrypt-examples - def __init__(self, key_private: str = None, key_public: str = None, cipher_name: str = 'RSA/ECB/PKCS1Padding'): + def __init__( + self, key_private: str | None = None, key_public: str | None = None, cipher_name: str = "RSA/ECB/PKCS1Padding" + ): self.key_private, self.key_public = self.load_keys_from_string(key_private=key_private, key_public=key_public) - assert self._get_cipher_module(cipher_name) is not None, 'Invalid cipher_name!' + assert self._get_cipher_module(cipher_name) is not None, "Invalid cipher_name!" self.cipher_name = cipher_name @staticmethod - def _get_cipher_module(full_cipher_name: str = None) -> Optional[Tuple]: + def _get_cipher_module(full_cipher_name: str | None = None) -> Optional[Tuple]: try: return _RSA_CIPHER_METHODS[full_cipher_name] except KeyError: - raise ValueError('Unsupported full cipher name, supported ones: %s.' % ','.join(sorted(_RSA_CIPHER_METHODS))) + raise ValueError("Unsupported full cipher name, supported ones: %s." % ",".join(sorted(_RSA_CIPHER_METHODS))) @staticmethod def generate_key_pair(size: int = 1024) -> Tuple[str, str]: key_pair = RSA.generate(size) key_private, key_public = key_pair.exportKey(), key_pair.publickey().exportKey() - return key_private.decode('ascii'), key_public.decode('ascii') + return key_private.decode("ascii"), key_public.decode("ascii") @lru_cache - def get_cipher(self, key_public: str = None, cipher_name='RSA/ECB/PKCS1Padding') -> t_cipher_module: + def get_cipher(self, key_public: str | None = None, cipher_name="RSA/ECB/PKCS1Padding") -> t_cipher_module: if key_public is None: key_pub = self.key_public else: _, key_pub = self.load_keys_from_string(key_public=key_public) if key_pub is None: - raise RuntimeError('Public Key not set!') - cache_key = key_pub.export_key(format='OpenSSH') + raise RuntimeError("Public Key not set!") + cache_key = key_pub.export_key(format="OpenSSH") if cache_key not in RsaEncryptor._dict_cache_cipher: pkcs_module, dict_param = self._get_cipher_module(cipher_name) RsaEncryptor._dict_cache_cipher[cache_key] = pkcs_module.new(key_pub, **dict_param) @@ -58,14 +60,14 @@ def get_cipher(self, key_public: str = None, cipher_name='RSA/ECB/PKCS1Padding') return RsaEncryptor._dict_cache_cipher[cache_key] @lru_cache - def get_decipher(self, key_private: str = None, cipher_name='RSA/ECB/PKCS1Padding') -> t_cipher_module: + def get_decipher(self, key_private: str | None = None, cipher_name="RSA/ECB/PKCS1Padding") -> t_cipher_module: if key_private is None: key_pri = self.key_private else: key_pri, _ = self.load_keys_from_string(key_private=key_private) if key_pri is None: - raise RuntimeError('Private Key not set!') - cache_key = key_pri.export_key(format='OpenSSH') + raise RuntimeError("Private Key not set!") + cache_key = key_pri.export_key(format="OpenSSH") if cache_key not in RsaEncryptor._dict_cache_decipher: pkcs_module, dict_param = self._get_cipher_module(cipher_name) RsaEncryptor._dict_cache_decipher[cache_key] = pkcs_module.new(key_pri, **dict_param) @@ -73,56 +75,64 @@ def get_decipher(self, key_private: str = None, cipher_name='RSA/ECB/PKCS1Paddin return RsaEncryptor._dict_cache_decipher[cache_key] @staticmethod - def load_keys_from_binary(key_private: bytes = None, key_public: bytes = None) -> Tuple[Optional[RSA.RsaKey], Optional[RSA.RsaKey]]: + def load_keys_from_binary( + key_private: bytes | None = None, key_public: bytes | None = None + ) -> Tuple[Optional[RSA.RsaKey], Optional[RSA.RsaKey]]: _key_private, _key_public = None, None if key_private is not None: try: _key_private = RSA.import_key(key_private) - except ValueError as e: - raise ValueError('RSA pri key format error: [%s]' % key_private) + except ValueError: + raise ValueError("RSA pri key format error: [%s]" % key_private) if key_public is not None: try: _key_public = RSA.import_key(key_public) - except ValueError as e: - raise ValueError('RSA pub key format error: [%s]' % key_public) + except ValueError: + raise ValueError("RSA pub key format error: [%s]" % key_public) return _key_private, _key_public @staticmethod - def load_keys_from_string(key_private: str = None, key_public: str = None) -> Tuple[Optional[RSA.RsaKey], Optional[RSA.RsaKey]]: + def load_keys_from_string( + key_private: str | None = None, key_public: str | None = None + ) -> Tuple[Optional[RSA.RsaKey], Optional[RSA.RsaKey]]: _key_private, _key_public = None, None if key_private is not None: - if not key_private.startswith('-----'): - key_pri = f'-----BEGIN RSA PRIVATE KEY-----\n{key_private}\n-----END RSA PRIVATE KEY-----' + if not key_private.startswith("-----"): + key_pri = f"-----BEGIN RSA PRIVATE KEY-----\n{key_private}\n-----END RSA PRIVATE KEY-----" else: key_pri = key_private try: _key_private = RSA.import_key(key_pri) - except ValueError as e: - raise ValueError('RSA private key format error: [%s]' % key_pri) + except ValueError: + raise ValueError("RSA private key format error: [%s]" % key_pri) if key_public is not None: - if not key_public.startswith('-----'): - key_pub = f'-----BEGIN PUBLIC KEY-----\n{key_public}\n-----END PUBLIC KEY-----' + if not key_public.startswith("-----"): + key_pub = f"-----BEGIN PUBLIC KEY-----\n{key_public}\n-----END PUBLIC KEY-----" else: key_pub = key_public try: _key_public = RSA.import_key(key_pub) - except ValueError as e: - raise ValueError('RSA public key format error: [%s]' % key_pub) + except ValueError: + raise ValueError("RSA public key format error: [%s]" % key_pub) return _key_private, _key_public - def encrypt_with_public_key(self, message: Union[str, bytes], key_public: str = None, cipher_name: str = None) -> bytes: - data = message if isinstance(message, bytes) else message.encode('UTF-8') + def encrypt_with_public_key( + self, message: Union[str, bytes], key_public: str | None = None, cipher_name: str = None + ) -> bytes: + data = message if isinstance(message, bytes) else message.encode("UTF-8") cipher = self.get_cipher(key_public=key_public, cipher_name=cipher_name or self.cipher_name) return cipher.encrypt(data) - def decrypt_with_private_key(self, ciphertext: Union[str, bytes], key_private: str = None, cipher_name: str = None, **kwargs) -> bytes: - data = ciphertext if isinstance(ciphertext, bytes) else ciphertext.encode('ascii') + def decrypt_with_private_key( + self, ciphertext: Union[str, bytes], key_private: str | None = None, cipher_name: str | None = None, **kwargs + ) -> bytes: + data = ciphertext if isinstance(ciphertext, bytes) else ciphertext.encode("ascii") decipher = self.get_decipher(key_private=key_private, cipher_name=cipher_name or self.cipher_name) return decipher.decrypt(data, **kwargs) @@ -132,7 +142,7 @@ def convert_bytes_to_base64(data: bytes) -> str: @staticmethod def convert_base64_to_bytes(data: str) -> bytes: - return base64.decodebytes(data.encode('ascii')) + return base64.decodebytes(data.encode("ascii")) def main(): @@ -140,7 +150,7 @@ def main(): RsaEncryptor.generate_key_pair(), ] - for str_src in ('aloha!', '😄', '{"x": 1}'): + for str_src in ("aloha!", "😄", '{"x": 1}'): for i, (key_pri, key_pub) in enumerate(key_pairs): rsa_enc = RsaEncryptor() x_enc = rsa_enc.encrypt_with_public_key(str_src, key_public=key_pub) @@ -149,8 +159,10 @@ def main(): rsa_dec = RsaEncryptor() y_bin = rsa_dec.convert_base64_to_bytes(x_txt) y_dec = rsa_dec.decrypt_with_private_key(y_bin, key_private=key_pri) - y_txt = y_dec.decode('UTF-8') + y_txt = y_dec.decode("UTF-8") - print('[test {i_case} success = {status}] {src} -> {enc}'.format( - i_case=i, status=(y_txt == str_src), src=y_txt, enc=x_txt - )) + print( + "[test {i_case} success = {status}] {src} -> {enc}".format( + i_case=i, status=(y_txt == str_src), src=y_txt, enc=x_txt + ) + ) diff --git a/src/aloha/encrypt/vault/__init__.py b/src/aloha/encrypt/vault/__init__.py index 32b091e..fd85018 100644 --- a/src/aloha/encrypt/vault/__init__.py +++ b/src/aloha/encrypt/vault/__init__.py @@ -1,4 +1,4 @@ -__all__ = ('BaseVault', 'DummyVault', 'AesVault', 'CyberArkVault',) +__all__ = ("BaseVault", "DummyVault", "AesVault", "CyberArkVault") from .base import BaseVault, DummyVault from .cyberark import CyberArkVault diff --git a/src/aloha/encrypt/vault/base.py b/src/aloha/encrypt/vault/base.py index 9311d78..59b394d 100644 --- a/src/aloha/encrypt/vault/base.py +++ b/src/aloha/encrypt/vault/base.py @@ -1,16 +1,36 @@ import abc - from urllib.parse import quote_plus as urlquote class BaseVault(abc.ABC): + """ + Abstract base class for password vault implementations. + + Defines the interface for password vaults that can decrypt passwords. + """ + @abc.abstractmethod def decrypt_password(self, *args, **kwargs): - return kwargs.get('password') + """ + Decrypt a password. + + :param args: Additional arguments + :param kwargs: Additional keyword arguments, should contain 'password' + :return: Decrypted password + """ + return kwargs.get("password") def get_password(self, password, *args, **kwargs): - kwargs.update(password if isinstance(password, dict) else {'password': password}) - url_quote = kwargs.get('url_encode', True) + """ + Get a password, optionally URL-encoded. + + :param password: Password or dict containing password + :param args: Additional arguments + :param kwargs: Additional keyword arguments, can include 'url_encode' + :return: Password, optionally URL-encoded + """ + kwargs.update(password if isinstance(password, dict) else {"password": password}) + url_quote = kwargs.get("url_encode", True) pwd = self.decrypt_password(*args, **kwargs) if pwd is None: @@ -23,11 +43,27 @@ def get_password(self, password, *args, **kwargs): class DummyVault(BaseVault): + """ + Dummy vault implementation that returns passwords as-is. + + No actual encryption/decryption is performed. + """ + def decrypt_password(self, *args, **kwargs): - return kwargs.get('password') + """ + Return password without decryption. + + :param args: Additional arguments + :param kwargs: Additional keyword arguments, should contain 'password' + :return: Original password + """ + return kwargs.get("password") def main(): + """ + Test function for DummyVault. + """ vault = DummyVault() ret = vault.get_password(None, url_quote=True) # print(ret) diff --git a/src/aloha/encrypt/vault/cyberark.py b/src/aloha/encrypt/vault/cyberark.py index 5d3345d..4500d24 100644 --- a/src/aloha/encrypt/vault/cyberark.py +++ b/src/aloha/encrypt/vault/cyberark.py @@ -6,19 +6,19 @@ from Crypto.Cipher import AES from requests.packages.urllib3.exceptions import InsecureRequestWarning -from .base import BaseVault from ...encrypt.aes import AesEncryptor from ...logger import LOG +from .base import BaseVault requests.packages.urllib3.disable_warnings(InsecureRequestWarning) -if hasattr(requests.packages.urllib3.util.ssl_, 'DEFAULT_CIPHERS'): - requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += ':HIGHT:!DH:!aNULL' +if hasattr(requests.packages.urllib3.util.ssl_, "DEFAULT_CIPHERS"): + requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += ":HIGHT:!DH:!aNULL" class CyberArkVault(BaseVault, AesEncryptor): _cached: dict = {} - def __init__(self, url: str, app_id: str, key: str = None, safe: str = 'AIM_ELIS_LAS', folder: str = 'root'): + def __init__(self, url: str, app_id: str, key: str | None = None, safe: str = "AIM_ELIS_LAS", folder: str = "root"): super().__init__(key) self.key, self.url, self.app_id, self.safe, self.folder = key, url, app_id, safe, folder @@ -35,31 +35,36 @@ def decrypt_password(self, text): cryptor = AES.new(self.key_aes, AES.MODE_ECB) s = cryptor.decrypt(a2b_hex(text.encode())) - s = s[0: -s[-1]] + s = s[0 : -s[-1]] return s.decode() - def get_cyberark_password(self, object: str = None, **kwargs): + def get_cyberark_password(self, object: str | None = None, **kwargs): assert isinstance(object, str) - kwargs.update({'object': object}) + kwargs.update({"object": object}) - app_id = kwargs.get('app_id', self.app_id) + app_id = kwargs.get("app_id", self.app_id) data = { "appId": app_id, - "safe": kwargs.get('safe', self.safe), - "folder": kwargs.get('folder', self.folder), - "object": kwargs.get('object', object), - "sign": self.get_sign(app_id, self.key) + "safe": kwargs.get("safe", self.safe), + "folder": kwargs.get("folder", self.folder), + "object": kwargs.get("object", object), + "sign": self.get_sign(app_id, self.key), } retry = 5 while retry: try: - LOG.debug('POST CyberArk: %s with data: %s', self.url, data) - resp = requests.post(self.url, json=data, verify=False, headers={'Content-Type': 'application/json'}) + LOG.debug("POST CyberArk: %s with data: %s", self.url, data) + resp = requests.post( + self.url, + json=data, + headers={"Content-Type": "application/json"}, + # verify=False, + ) tmp = resp.json() - if resp.status_code == 200 and int(tmp['code']) == 200: - LOG.debug('Got data from CyberArk: %s', tmp) - return self.decrypt_password(tmp['password']) + if resp.status_code == 200 and int(tmp["code"]) == 200: + LOG.debug("Got data from CyberArk: %s", tmp) + return self.decrypt_password(tmp["password"]) else: raise RuntimeError(resp.text) except Exception as e: @@ -67,34 +72,38 @@ def get_cyberark_password(self, object: str = None, **kwargs): if retry == 0: raise e else: - LOG.error('CyberArk request error: {}'.format(e)) + LOG.error("CyberArk request error: {}".format(e)) return None def get_password(self, object=None, **kwargs): - key_for_cache = '{app_id};{safe};{folder};{key};{object}'.format( + key_for_cache = "{app_id};{safe};{folder};{key};{object}".format( app_id=self.app_id, safe=self.safe, folder=self.folder, key=self.key, object=object ) if key_for_cache not in self._cached: - kwargs.update(object if isinstance(object, dict) else {'object': object}) - url_quote = kwargs.get('url_encode', True) + kwargs.update(object if isinstance(object, dict) else {"object": object}) + url_quote = kwargs.get("url_encode", True) pwd = self.get_cyberark_password(**kwargs) if url_quote: # quote/escape password by default pwd = urlquote(pwd) self._cached[key_for_cache] = pwd else: - LOG.debug('Using cached CyberArk key: %s' % key_for_cache) + LOG.debug("Using cached CyberArk key: %s" % key_for_cache) return self._cached[key_for_cache] def main(): cfg_cyberark = dict( - url='https://localhost/pidms/rest/pwd/getPassword', # to fill properly - app_id='', safe='', folder='root', key='', + url="https://localhost/pidms/rest/pwd/getPassword", # to fill properly + app_id="", + safe="", + folder="root", + key="", ) # from ...settings import SETTINGS # cfg_cyberark = SETTINGS.config['CYBERARK_CONFIG'] vault = CyberArkVault(**cfg_cyberark) - pwd = vault.get_password({'object': 'PG_'}) + pwd = vault.get_password({"object": "PG_"}) + assert pwd is not None # print(pwd) diff --git a/src/aloha/encrypt/vault/plain.py b/src/aloha/encrypt/vault/plain.py index 85344f6..b37dcd4 100644 --- a/src/aloha/encrypt/vault/plain.py +++ b/src/aloha/encrypt/vault/plain.py @@ -1,7 +1,7 @@ import pyhocon -from .base import BaseVault from ...encrypt.aes import AesEncryptor +from .base import BaseVault def _is_empty_str(s): @@ -9,7 +9,7 @@ def _is_empty_str(s): class AesVault(AesEncryptor, BaseVault): - def __init__(self, key: str = None): + def __init__(self, key: str | None = None): super().__init__(key) def decrypt_password(self, pwd): diff --git a/src/aloha/logger/__init__.py b/src/aloha/logger/__init__.py index e21cda3..32bfbf8 100644 --- a/src/aloha/logger/__init__.py +++ b/src/aloha/logger/__init__.py @@ -1,9 +1,8 @@ -__all__ = ("LOG", "get_logger") +__all__ = ("LOG", "get_logger", "getLogger") -from .logger import get_logger from ..settings import SETTINGS +from .logger import get_logger, getLogger LOG = get_logger( - level=SETTINGS.config.get('deploy', {}).get('log_level', 10), # 10 = logging.DEBUG - # logger_name='default', + level=SETTINGS.config.get("deploy", {}).get("log_level", 10), # 10 = logging.DEBUG ) diff --git a/src/aloha/logger/handler.py b/src/aloha/logger/handler.py index 43b206e..447cfcb 100644 --- a/src/aloha/logger/handler.py +++ b/src/aloha/logger/handler.py @@ -10,12 +10,12 @@ class MultiProcessSafeDailyRotatingFileHandler(BaseRotatingHandler): - Utc not supported """ - def __init__(self, filename: str, encoding='utf8', delay=False, utc=False, **kwargs): + def __init__(self, filename: str, encoding="utf8", delay=False, utc=False, **kwargs): self.utc = utc self.suffix = "%Y-%m%d" self.baseFilename = filename self.currentFileName = self._compute_fn() - BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay) + BaseRotatingHandler.__init__(self, filename, "a", encoding, delay) def shouldRollover(self, record): if self.currentFileName != self._compute_fn(): @@ -31,9 +31,7 @@ def doRollover(self): self.stream = self._open() def _compute_fn(self): - return self.baseFilename.replace(".log", "") + "_" \ - + time.strftime(self.suffix, time.localtime()) \ - + '.log' + return self.baseFilename.replace(".log", "") + "_" + time.strftime(self.suffix, time.localtime()) + ".log" def _open(self): return open(self.currentFileName, mode=self.mode, encoding=self.encoding) diff --git a/src/aloha/logger/logger.py b/src/aloha/logger/logger.py index 5e78430..5bb0160 100644 --- a/src/aloha/logger/logger.py +++ b/src/aloha/logger/logger.py @@ -6,23 +6,45 @@ from .handler import MultiProcessSafeDailyRotatingFileHandler -def setup_logger(logger: logging.Logger, level: int = logging.DEBUG, logger_name: str = None, module: str = None, formatter_str: str = None): +def setup_logger( + logger: logging.Logger, + level: int = logging.DEBUG, + logger_name: str | None = None, + module: str | None = None, + formatter_str: str | None = None, +): + """ + Set up a logger with file and stream handlers. + + Configures the logger with: + - A multi-process safe daily rotating file handler + - A console stream handler + - A standard log format + + :param logger: Logger instance to set up + :param level: Logging level (default: DEBUG) + :param logger_name: Name of the logger (optional) + :param module: Module name for log file naming (optional) + :param formatter_str: Custom log format string (optional) + """ if not logger.handlers: - formatter = logging.Formatter(formatter_str or '%(levelname)s> %(asctime)s> %(module)s:%(lineno)s> %(message)s') + formatter = logging.Formatter(formatter_str or "%(levelname)s> %(asctime)s> %(module)s:%(lineno)s> %(message)s") - folder = os.environ.get('DIR_LOG', 'logs') + folder = os.environ.get("DIR_LOG", "logs") os.makedirs(folder, exist_ok=True) if module is None: from ..settings import SETTINGS - module = SETTINGS.config.get('APP_MODULE') or os.environ.get('APP_MODULE', 'default') - # if logger_name is not None and len(logger_name) > 0: - # module = '%s_%s' % (logger_name, module) + module = SETTINGS.config.get("APP_MODULE") or os.environ.get("APP_MODULE", None) + + if logger_name is not None and len(logger_name) > 0: + logger_name = logger_name.strip().replace(" ", "_") + + path_file = [module, logger_name, socket.gethostname(), "p%s" % os.getpid()] # module, logger_name, hostname, pid + path_file = "_".join(str(i) for i in path_file if i is not None and len(str(i)) > 0) + path_file = pjoin(folder, "%s.log" % path_file) - path_file = [module, socket.gethostname(), 'p%s' % os.getpid()] # module, hostname, pid - path_file = '_'.join(str(i) for i in path_file if i is not None) - path_file = pjoin(folder, '%s.log' % path_file) file_handler = MultiProcessSafeDailyRotatingFileHandler(path_file) file_handler.setFormatter(formatter) logger.addHandler(file_handler) @@ -34,14 +56,27 @@ def setup_logger(logger: logging.Logger, level: int = logging.DEBUG, logger_name logger.setLevel(level) -def get_logger(level=logging.DEBUG, logger_name: str | None = None, *args, **kwargs) -> logging.Logger: - if logger_name is None: - logger_name = 'default' - +def get_logger(logger_name: str | None = None, level=logging.DEBUG, **kwargs) -> logging.Logger: + """ + Get a configured logger instance. + + Creates or retrieves a logger by name and sets it up with file and stream handlers. + Accepts both string and integer log levels. + + :param level: Logging level (int or str, default: DEBUG) + :param logger_name: Name of the logger (default: 'default') + :param args: Additional arguments passed to setup_logger + :param kwargs: Additional keyword arguments passed to setup_logger + :return: Configured logger instance + """ + logger = logging.getLogger(logger_name) if isinstance(level, str): level = getattr(logging, str(level).upper(), 10) - setup_logger(logger, level=level, logger_name=logger_name, *args, **kwargs) + setup_logger(logger, level=level, logger_name=logger_name, **kwargs) return logger + + +getLogger = get_logger diff --git a/src/aloha/script/base.py b/src/aloha/script/base.py index 40855f6..1a05a92 100644 --- a/src/aloha/script/base.py +++ b/src/aloha/script/base.py @@ -1,31 +1,31 @@ -import sys import argparse import importlib +import sys def main(): - if '' not in sys.path: # if start from script, cwd is not include in sys.path - sys.path.insert(0, '') + if "" not in sys.path: # if start from script, cwd is not include in sys.path + sys.path.insert(0, "") parser = argparse.ArgumentParser() - parser.add_argument('cmd') + parser.add_argument("cmd") args, _ = parser.parse_known_args() cmd = args.cmd - module = '%s.%s' % (__package__, cmd) + module = "%s.%s" % (__package__, cmd) try: module = importlib.import_module(module) except ImportError as e: - print('Invalid sub-command: %s\n\tFailed to import: %s' % (cmd, module)) + print("Invalid sub-command: %s\n\tFailed to import: %s" % (cmd, module)) print(str(e)) exit(-1) sys.argv.pop(0) - print('aloha command options: %s' % ''.join(sys.argv)) - func_main = getattr(module, 'main') + print("aloha command options: %s" % "".join(sys.argv)) + func_main = getattr(module, "main") exit(func_main()) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/aloha/script/compile.py b/src/aloha/script/compile.py index 552a064..2a45d66 100644 --- a/src/aloha/script/compile.py +++ b/src/aloha/script/compile.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 -# This script build a given python package into a package of dynamic library (.so) files. +""" +This script build a given python package into a package of dynamic library (.so) files. +Example: python -m aloha.script.compile --base=./ --dist=../build --keep='main.py' +""" -__all__ = ('build', 'package') +__all__ = ("build", "package") import argparse import glob @@ -14,12 +17,12 @@ try: from Cython.Build import cythonize except ImportError: - raise RuntimeError('Please pip install Cython first!') + raise RuntimeError("Please pip install Cython first!") -def _expand(patterns: list = None): +def _expand(patterns: list | None = None): files = [] - for pattern in patterns: + for pattern in patterns or []: for file in glob.glob(pattern): files.append(os.path.abspath(file)) files = sorted(set(files)) @@ -27,7 +30,7 @@ def _expand(patterns: list = None): def _delete(file_path: str, ignore_errors=True): - print('Removing file/folder: %s' % file_path) + print("Removing file/folder: %s" % file_path) try: if os.path.isfile(file_path) or os.path.islink(file_path): os.unlink(file_path) @@ -38,8 +41,10 @@ def _delete(file_path: str, ignore_errors=True): raise e -def build(base: str = None, dist: str = 'build', exclude: list = None, keep: list = None, copy_others=True): - path_base = os.path.abspath(base) +def build( + base: str | None = None, dist: str = "build", exclude: list | None = None, keep: list | None = None, copy_others=True +): + path_base = os.path.abspath(base or "./") path_build = os.path.abspath(dist) files_exclude = _expand(exclude or []) # sorted(set(os.path.abspath(i) for i in files_exclude)) files_keep = _expand(keep or []) # sorted(set((os.path.abspath(i) for i in files_keep))) @@ -50,7 +55,7 @@ def build(base: str = None, dist: str = 'build', exclude: list = None, keep: lis dir_name = dir_path.split(os.sep)[-1] # name of the current directory flag_skip: bool = False - if dir_name.startswith('.') or (os.sep + '.' in dir_path): + if dir_name.startswith(".") or (os.sep + "." in dir_path): flag_skip = True # hidden folders and sub-folders elif dir_path.startswith(path_build): flag_skip = True # skip: folder for build output, and excluded folders @@ -66,15 +71,15 @@ def build(base: str = None, dist: str = 'build', exclude: list = None, keep: lis for file in file_names: (name, extension), path = os.path.splitext(file), os.path.join(dir_path, file) - if path in files_exclude or extension in ('.pyc', 'pyx') or name.startswith('.'): + if path in files_exclude or extension in (".pyc", "pyx") or name.startswith("."): continue # skip: excluded files, pyc/pyx files, and hidden files path_full = os.path.abspath(path) - action = 'copy' - if extension in ('.py',): - if not name.startswith('__') and path_full not in files_keep: - action = 'cythonize' + action = "copy" + if extension in (".py",): + if not name.startswith("__") and path_full not in files_keep: + action = "cythonize" elif not copy_others: continue # if not copying other files, skip the file @@ -82,11 +87,11 @@ def build(base: str = None, dist: str = 'build', exclude: list = None, keep: lis act_list[action].append((path, dst)) - for (src, dst) in act_list.get('copy', ()) + act_list.get('cythonize', ()): + for src, dst in act_list.get("copy", ()) + act_list.get("cythonize", ()): os.makedirs(os.path.dirname(dst), exist_ok=True) shutil.copyfile(src, dst) - target_cythonize = [dst for (src, dst) in act_list.get('cythonize', ())] + target_cythonize = [dst for (src, dst) in act_list.get("cythonize", ())] n_parallel = os.cpu_count() or 8 @@ -94,9 +99,9 @@ def build(base: str = None, dist: str = 'build', exclude: list = None, keep: lis cythonized = cythonize(target_cythonize, nthreads=n_parallel, language_level=3) # c code -> dynamic library file - path_build_tmp = os.path.join(path_build, '.tmp') + path_build_tmp = os.path.join(path_build, ".tmp") script_args = ["build_ext", "-b", path_build, "-t", path_build_tmp, "-j", n_parallel] - print('Build args: %s' % ' '.join(str(s) for s in script_args)) + print("Build args: %s" % " ".join(str(s) for s in script_args)) setup(ext_modules=cythonized, script_args=script_args) # clean up @@ -108,18 +113,25 @@ def build(base: str = None, dist: str = 'build', exclude: list = None, keep: lis _delete(path_build_tmp) -def package(base: str = None, dist: str = 'build', exclude: list = None, keep: list = None, copy_others=True, *args, **kwargs): +def package( + base: str | None = None, + dist: str = "build", + exclude: list | None = None, + keep: list | None = None, + copy_others=True, + **kwargs, +): t = time.time() - path_base = os.path.abspath(base or './') + path_base = os.path.abspath(base or "./") path_dist = os.path.abspath(dist) os.makedirs(path_dist, exist_ok=True) - if len(glob.glob(path_dist + '/*')) > 0: - raise ValueError('Dist folder [%s] MUST be an empty directory or an non-existing folder!' % path_dist) + if len(glob.glob(path_dist + "/*")) > 0: + raise ValueError("Dist folder [%s] MUST be an empty directory or an non-existing folder!" % path_dist) folder_name = os.getcwd().split(os.sep)[-1] - folder_temp = os.path.join('/tmp/build/', folder_name) - print('Building project to path:', folder_temp) + folder_temp = os.path.join("/tmp/build/", folder_name) + print("Building project to path:", folder_temp) _delete(folder_temp, ignore_errors=True) # clear the folder first build( @@ -127,28 +139,29 @@ def package(base: str = None, dist: str = 'build', exclude: list = None, keep: l dist=folder_temp, # target directory for build files exclude=exclude, # exclude this file by default, this is a collection of files/folders to exclude keep=keep, # source files keep as is and not converting to dynamic library - copy_others=copy_others + copy_others=copy_others, ) - [shutil.move(f, os.path.join(path_dist, f.split(os.sep)[-1])) for f in glob.glob(folder_temp + '/*')] + [shutil.move(f, os.path.join(path_dist, f.split(os.sep)[-1])) for f in glob.glob(folder_temp + "/*")] t = time.time() - t - print('\n\nTime consumed to build code: %.2f seconds.' % t) + print("\n\nTime consumed to build code: %.2f seconds." % t) print("Successfully finished building package to: ", path_dist) def main(): p = argparse.ArgumentParser() - p.add_argument('--base', type=str, help='root folder which includes source code to build') - p.add_argument('--dist', type=str, default='build', help='target folder for the binary code') - p.add_argument('--exclude', type=str, nargs='*', default=(), help='a collection of files/folders to exclude') - p.add_argument('--keep', type=str, nargs='*', default=(), help='source files keep as is and not converting to dynamic library') + p.add_argument("--base", type=str, help="root folder which includes source code to build") + p.add_argument("--dist", type=str, default="build", help="target folder for the binary code") + p.add_argument("--exclude", type=str, nargs="*", default=(), help="a collection of files/folders to exclude") + p.add_argument( + "--keep", type=str, nargs="*", default=(), help="source files keep as is and not converting to dynamic library" + ) args = p.parse_args() args = vars(args) for k, v in args.items(): - print('%s = %s' % (k, v)) + print("%s = %s" % (k, v)) package(**args) -if __name__ == '__main__': - """python -m aloha.script.compile --base=./ --dist=../build --keep='main.py'""" +if __name__ == "__main__": main() diff --git a/src/aloha/script/start.py b/src/aloha/script/start.py index 0d1aba9..aefd0b6 100644 --- a/src/aloha/script/start.py +++ b/src/aloha/script/start.py @@ -4,17 +4,17 @@ def main(): - print('\n'.join(sorted('%s=%s' % (k, v) for k, v in os.environ.items()))) + print("\n".join(sorted("%s=%s" % (k, v) for k, v in os.environ.items()))) - usage = ''' + usage = """ Usage: `python main.py app_common.main` ; or set environment variable `ENTRYPOINT` the `module.name` should be a python package file or package which include a `main()` function - ''' + """ try: module_name = sys.argv[1] except IndexError: - module_name = os.environ.get('ENTRYPOINT') + module_name = os.environ.get("ENTRYPOINT") if module_name is None: print(usage) @@ -23,20 +23,20 @@ def main(): try: m = importlib.import_module(module_name) except ImportError: - raise ValueError('Invalid entrypoint: %s' % module_name) + raise ValueError("Invalid entrypoint: %s" % module_name) - f_main = getattr(m, 'main') + f_main = getattr(m, "main") if f_main is None: - print('Given module does not provides a `main()` function!') + print("Given module does not provides a `main()` function!") else: - print('Starting module: %s' % module_name) + print("Starting module: %s" % module_name) ret = f_main() if ret: print(ret) -if __name__ == '__main__': +if __name__ == "__main__": """aloha start module.name""" """python -m aloha.script.start module.name""" main() diff --git a/src/aloha/service/__init__.py b/src/aloha/service/__init__.py index a425d22..5747748 100644 --- a/src/aloha/service/__init__.py +++ b/src/aloha/service/__init__.py @@ -1,8 +1,5 @@ -import sys +__all__ = ("DefaultHandler404", "v0", "v1", "v2") + from .api import v0, v1, v2 from .http import DefaultHandler404 - -for module in (v0, v1, v2): - full_name = '{}.{}'.format(__package__, module.__name__.rsplit('.')[-1]) - sys.modules[full_name] = sys.modules[module.__name__] diff --git a/src/aloha/service/api/v1.py b/src/aloha/service/api/v1.py index 3edb5b1..bc14f26 100644 --- a/src/aloha/service/api/v1.py +++ b/src/aloha/service/api/v1.py @@ -1,61 +1,61 @@ -__all__ = ('APIHandler', 'APICaller', 'sign_data', 'sign_check') +__all__ = ("APIHandler", "APICaller", "sign_data", "sign_check") import json import logging import uuid from abc import ABC -from ..http import AbstractApiClient, AbstractApiHandler from ...encrypt.hash import get_md5_of_str, get_sha256_of_str from ...settings import SETTINGS +from ..http import AbstractApiClient, AbstractApiHandler -APP_ID_KEYS = SETTINGS.config.get('APP_ID_KEYS', {}) -APP_OPTIONS = SETTINGS.config.get('APP_OPTIONS', {}) -FUNC_SIGN_CHECK = {'md5': get_md5_of_str, 'sha256': get_sha256_of_str} -func_sign_check_default = FUNC_SIGN_CHECK.get(APP_OPTIONS.get('sign_method', 'md5')) +APP_ID_KEYS = SETTINGS.config.get("APP_ID_KEYS", {}) +APP_OPTIONS = SETTINGS.config.get("APP_OPTIONS", {}) +FUNC_SIGN_CHECK = {"md5": get_md5_of_str, "sha256": get_sha256_of_str} +func_sign_check_default = FUNC_SIGN_CHECK.get(APP_OPTIONS.get("sign_method", "md5")) class APIHandler(AbstractApiHandler, ABC): MAP_ERROR_INFO = { - 'BAD_REQUEST': {'code': '5101', 'message': ['Bad request: fail to parse body as JSON object!']}, - 'MISSING_ARGS': {'code': '5102', 'message': ['Required argument field(s) missing...']}, - 'SIGN_CHECK_FAIL': {'code': '5104', 'message': ['Invalid sign, sign check failed!']}, + "BAD_REQUEST": {"code": "5101", "message": ["Bad request: fail to parse body as JSON object!"]}, + "MISSING_ARGS": {"code": "5102", "message": ["Required argument field(s) missing..."]}, + "SIGN_CHECK_FAIL": {"code": "5104", "message": ["Invalid sign, sign check failed!"]}, } async def post(self): body_arguments = self.request_body try: - salt_uuid = body_arguments.pop('salt_uuid') - app_id = body_arguments.pop('app_id') - sign = body_arguments.pop('sign') - data = body_arguments.pop('data') + salt_uuid = body_arguments.pop("salt_uuid") + app_id = body_arguments.pop("app_id") + sign = body_arguments.pop("sign") + data = body_arguments.pop("data") except KeyError: # cannot find default key from parsed body - return self.finish(self.MAP_ERROR_INFO['MISSING_ARGS']) + return self.finish(self.MAP_ERROR_INFO["MISSING_ARGS"]) is_valid_req = sign_check(salt_uuid=salt_uuid, app_id=app_id, sign=sign, data=data) # , sign_method='sha256' if not is_valid_req: - return self.finish(self.MAP_ERROR_INFO['SIGN_CHECK_FAIL']) + return self.finish(self.MAP_ERROR_INFO["SIGN_CHECK_FAIL"]) - resp = dict(code=5200, message=['success']) + resp = dict(code=5200, message=["success"]) try: result = self.response(**data) # this call may throw TypeError when argument missing - resp['data'] = result - resp['salt_uuid'] = salt_uuid + resp["data"] = result + resp["salt_uuid"] = salt_uuid except Exception as e: if self.LOG.level == logging.DEBUG: self.LOG.error(e, exc_info=True) - return self.finish({'code': 5201, 'message': [repr(e)]}) + return self.finish({"code": 5201, "message": [repr(e)]}) - resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(',', ':')) + resp = json.dumps(resp, ensure_ascii=False, default=str, separators=(",", ":")) return self.finish(resp) class APICaller(AbstractApiClient): - APP_ID_KEYS = AbstractApiClient.config.get('APP_ID_KEYS', {}) + APP_ID_KEYS = AbstractApiClient.config.get("APP_ID_KEYS", {}) def wrap_request_data( - self, data, app_id: str = None, app_key: str = None, salt_uuid: str = None, sign: str = None, sign_method: str = None + self, data, app_id: str = None, app_key: str = None, salt_uuid: str = None, sign: str = None, sign_method: str = None ): if app_id is None: # if len(APP_ID_KEYS) != 1: @@ -67,23 +67,18 @@ def wrap_request_data( app_id=app_id, app_key=app_key or self.APP_ID_KEYS.get(app_id), data=data, - sign_method=sign_method + sign_method=sign_method, ) - return { - 'salt_uuid': salt_uuid, - 'app_id': app_id, - 'sign': sign, - 'data': data - } + return {"salt_uuid": salt_uuid, "app_id": app_id, "sign": sign, "data": data} def sign_data(salt_uuid: str, app_id: str, app_key: str, data, sign_method: str = None): - data_str = str(json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(',', ':'))) + data_str = str(json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(",", ":"))) public_key = app_id + salt_uuid + data_str + app_key func_sign_check = func_sign_check_default if sign_method is None else FUNC_SIGN_CHECK.get(sign_method) if func_sign_check is None: - raise ValueError('Invalid `sign_method`: %s' % sign_method) + raise ValueError("Invalid `sign_method`: %s" % sign_method) sign = func_sign_check(public_key) return sign @@ -101,13 +96,13 @@ def sign_check(salt_uuid: str, app_id: str, sign: str, data, sign_method: str = func_sign_check = func_sign_check_default if sign_method is None else FUNC_SIGN_CHECK.get(sign_method) if func_sign_check is None: - raise ValueError('Invalid `sign_method`: %s' % sign_method) + raise ValueError("Invalid `sign_method`: %s" % sign_method) app_key = APP_ID_KEYS.get(app_id) if app_key is None: # APP_ID not in the dict, unknown APP_ID return False - data_str = str(json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(',', ':'))) + # data_str = str(json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(',', ':'))) # --> Compatible with older version API right_sign = func_sign_check(app_id + salt_uuid + app_key) @@ -115,7 +110,7 @@ def sign_check(salt_uuid: str, app_id: str, sign: str, data, sign_method: str = return True # <-- - public_key = str(json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(',', ':'))) + public_key = str(json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(",", ":"))) public_key = app_id + salt_uuid + public_key + app_key right_sign = func_sign_check(public_key) return sign == right_sign diff --git a/src/aloha/service/app.py b/src/aloha/service/app.py index 2f027ce..1a2318c 100644 --- a/src/aloha/service/app.py +++ b/src/aloha/service/app.py @@ -14,19 +14,39 @@ except ImportError: LOG.info('[uvloop] NOT installed, fallback to asyncio loop! Consider `pip install uvloop`!') -from .web import WebApplication -from ..settings import SETTINGS - from tornado.options import options +from ..settings import SETTINGS +from .web import WebApplication + class Application: + """ + Main application class for aloha service. + + Wraps a WebApplication and manages the event loop lifecycle. + Tries to use uvloop if available for better performance. + """ def __init__(self, *args, **kwargs): + """ + Initialize the application. + + :param args: Additional arguments + :param kwargs: Additional keyword arguments + """ options['log_file_prefix'] = 'access.log' settings = dict(SETTINGS.config) self.web_app = WebApplication(settings) def start(self): + """ + Start the application and run the event loop. + + Starts the web application and enters the event loop. + The event loop must not be running before calling this method. + + :raises RuntimeError: If event loop is already running + """ try: self.web_app.start() event_loop = asyncio.get_event_loop() @@ -44,6 +64,9 @@ def start(self): pass def stop(self): + """ + Stop the application and event loop. + """ event_loop = asyncio.get_event_loop() if event_loop.is_running(): event_loop.stop() diff --git a/src/aloha/service/http/__init__.py b/src/aloha/service/http/__init__.py index 9eee633..7f08a4f 100644 --- a/src/aloha/service/http/__init__.py +++ b/src/aloha/service/http/__init__.py @@ -1,3 +1,5 @@ +__all__ = ("AbstractApiClient", "AbstractApiHandler", "DefaultHandler404", "PlainHttpHandler") + from .base_api_client import AbstractApiClient from .base_api_handler import AbstractApiHandler, DefaultHandler404 from .plain_http_handler import PlainHttpHandler diff --git a/src/aloha/service/openapi/__init__.py b/src/aloha/service/openapi/__init__.py index 184ccdc..7150467 100644 --- a/src/aloha/service/openapi/__init__.py +++ b/src/aloha/service/openapi/__init__.py @@ -1 +1,3 @@ +__all__ = ("OpenApiClient",) + from .client import OpenApiClient diff --git a/src/aloha/service/web.py b/src/aloha/service/web.py index b91072e..ab977da 100644 --- a/src/aloha/service/web.py +++ b/src/aloha/service/web.py @@ -1,7 +1,7 @@ import logging import os -from tornado import web, httpserver +from tornado import httpserver, web from tornado.routing import HostMatches from ..logger import LOG @@ -9,20 +9,19 @@ from ..settings import SETTINGS setup_logger( - logging.getLogger("tornado.access") - , formatter_str='A> %(asctime)s> %(message)s' - , module='access_%s' % (SETTINGS.config.get('APP_MODULE') or os.environ.get('APP_MODULE', 'default')) + logging.getLogger("tornado.access"), + formatter_str="A> %(asctime)s> %(message)s", + module="access_%s" % (SETTINGS.config.get("APP_MODULE") or os.environ.get("APP_MODULE", "default")), ) def _load_handlers(name): """Load the (URL pattern, handler) tuples for each component.""" - mod = __import__(name, fromlist=['default_handlers']) + mod = __import__(name, fromlist=["default_handlers"]) handlers = [] for url, handler in mod.default_handlers: - - if not url.startswith('/'): - url = '/' + url + if not url.startswith("/"): + url = "/" + url handlers.append((url, handler)) return handlers @@ -35,31 +34,28 @@ def __init__(self, config: dict, *args, **kwargs): @staticmethod def init_handlers(config: dict): - settings = config.get('service', {}) - modules = settings.get('modules', []) + settings = config.get("service", {}) + modules = settings.get("modules", []) handlers = [] for m in modules: _handlers = _load_handlers(m) for h in _handlers: (url, class_handler) = h handlers.append(h) - s_log_msg = 'Loaded API module %-50s' % url + s_log_msg = "Loaded API module %-50s" % url if LOG.level < logging.INFO: # more verbose information - s_log_msg += '\t from class %s' % str(class_handler) + s_log_msg += "\t from class %s" % str(class_handler) LOG.info(s_log_msg) - return [ - (HostMatches('(.*)'), handlers) - ] + return [(HostMatches("(.*)"), handlers)] def start(self): - service_settings = self.settings.get('service', {}) + service_settings = self.settings.get("service", {}) - port = service_settings.get('port', int(os.environ.get('PORT_SVC', 80))) - # if overwrite port in param - port = os.environ.get('port', port) + port = service_settings.get("port") or int(os.environ.get("PORT_SVC", 80)) + port = os.environ.get("port", port) # if overwrite port in param - num_process = int(service_settings.get('num_process', 0)) - LOG.info('Starting service with [%s] process at port [%s]...', num_process or 'undefined', port) + num_process = int(service_settings.get("num_process") or 0) + LOG.info("Starting service with [%s] process at port [%s]...", num_process or "undefined", port) self.http_server.bind(port) self.http_server.start(num_processes=num_process) diff --git a/src/aloha/settings.py b/src/aloha/settings.py index 72ea403..4e70ec6 100644 --- a/src/aloha/settings.py +++ b/src/aloha/settings.py @@ -2,19 +2,42 @@ class Settings: + """ + Global settings management class for aloha. + + Manages configuration loading and provides access to common directories. + """ + def __init__(self): self._config = None @property def resource_dir(self): + """ + Get the resource directory path. + + :return: Resource directory path + """ return paths.get_resource_dir() @property def config_dir(self): + """ + Get the configuration directory path. + + :return: Config directory path + """ return paths.get_config_dir() @property def config(self): + """ + Get the global configuration object. + + Lazily loads configuration from HOCON files on first access. + + :return: Configuration object + """ if self._config is None: config_files = paths.get_config_files() # by default, use the `main.conf` file in the config_dir self._config = hocon.load_config_from_hocon_files(config_files, base_dir=paths.get_config_dir()) @@ -22,6 +45,12 @@ def config(self): return self._config def __getitem__(self, item): + """ + Get a configuration value by key. + + :param item: Configuration key + :return: Configuration value + """ return self.config[item] diff --git a/src/aloha/testing/service_v1.py b/src/aloha/testing/service_v1.py index d040f01..2310b2a 100644 --- a/src/aloha/testing/service_v1.py +++ b/src/aloha/testing/service_v1.py @@ -4,6 +4,7 @@ import requests from aloha.service.api.v1 import APICaller + from .unit import UnitTestCase @@ -12,7 +13,7 @@ class ServiceTestCase(UnitTestCase, ABC, APICaller): @classmethod def setUpClass(cls) -> None: - cls.api_url_base = ServiceTestCase.config.get('service', {}).get('port', 80) + cls.api_url_base = ServiceTestCase.config.get("service", {}).get("port", 80) @classmethod def request_api(cls, api_url, timeout=5, **kwargs): @@ -24,10 +25,8 @@ def request_api(cls, api_url, timeout=5, **kwargs): :return: """ payload = cls.wrap_request_data(data=kwargs) - url = 'http://localhost:%s/%s' % (cls.api_url_base, api_url) + url = "http://localhost:%s/%s" % (cls.api_url_base, api_url) cls.LOG.debug("POST %s %s" % (url, json.dumps(payload, ensure_ascii=False, sort_keys=True))) - resp = requests.post( - url, json=payload, timeout=timeout, headers={'Content-Type': 'application/json'} - ).json() + resp = requests.post(url, json=payload, timeout=timeout, headers={"Content-Type": "application/json"}).json() cls.LOG.debug(resp) return resp diff --git a/src/aloha/testing/service_v2.py b/src/aloha/testing/service_v2.py index a4d3bee..2451e61 100644 --- a/src/aloha/testing/service_v2.py +++ b/src/aloha/testing/service_v2.py @@ -1,6 +1,7 @@ from abc import ABC from aloha.service.api.v2 import APICaller + from .unit import UnitTestCase @@ -9,7 +10,7 @@ class ServiceTestCase(UnitTestCase, ABC, APICaller): @classmethod def setUpClass(cls) -> None: - cls.api_url_port = ServiceTestCase.config.get('service', {}).get('port', 80) + cls.api_url_port = ServiceTestCase.config.get("service", {}).get("port", 80) @classmethod def request_api(cls, api_url, timeout=5, **kwargs): @@ -20,7 +21,7 @@ def request_api(cls, api_url, timeout=5, **kwargs): :param kwargs: request data :return: """ - url = 'http://localhost:%s/%s' % (cls.api_url_port, api_url) + url = "http://localhost:%s/%s" % (cls.api_url_port, api_url) # cls.LOG.debug("POST %s %s" % (url, json.dumps(kwargs, ensure_ascii=False, sort_keys=True))) # resp = requests.post( # url, json=kwargs, timeout=timeout, headers={'Content-Type': 'application/json'} diff --git a/src/aloha/times/__init__.py b/src/aloha/time/__init__.py similarity index 100% rename from src/aloha/times/__init__.py rename to src/aloha/time/__init__.py diff --git a/src/aloha/times/timeout_async.py b/src/aloha/time/timeout_async.py similarity index 92% rename from src/aloha/times/timeout_async.py rename to src/aloha/time/timeout_async.py index 8ca7f7b..6b53803 100644 --- a/src/aloha/times/timeout_async.py +++ b/src/aloha/time/timeout_async.py @@ -1,5 +1,5 @@ -""" Refer to: https://github.com/aio-libs/async-timeout/blob/master/async_timeout/__init__.py -""" +"""Refer to: https://github.com/aio-libs/async-timeout/blob/master/async_timeout/__init__.py""" + import asyncio import enum import warnings @@ -68,9 +68,7 @@ class Timeout: __slots__ = ("_deadline", "_loop", "_state", "_timeout_handler") - def __init__( - self, deadline: Optional[float], loop: asyncio.AbstractEventLoop - ) -> None: + def __init__(self, deadline: Optional[float], loop: asyncio.AbstractEventLoop) -> None: self._loop = loop self._state = _State.INIT @@ -90,10 +88,10 @@ def __enter__(self) -> "Timeout": return self def __exit__( - self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType], + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], ) -> Optional[bool]: self._do_exit(exc_type) return None @@ -103,10 +101,10 @@ async def __aenter__(self) -> "Timeout": return self async def __aexit__( - self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType], + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], ) -> Optional[bool]: self._do_exit(exc_type) return None diff --git a/src/aloha/times/timeout_asyncio.py b/src/aloha/time/timeout_asyncio.py similarity index 92% rename from src/aloha/times/timeout_asyncio.py rename to src/aloha/time/timeout_asyncio.py index f6f2048..eae421b 100644 --- a/src/aloha/times/timeout_asyncio.py +++ b/src/aloha/time/timeout_asyncio.py @@ -2,7 +2,7 @@ import enum import sys import warnings -from functools import wraps, partial +from functools import partial, wraps from types import TracebackType from typing import Any, Optional, Type @@ -71,9 +71,7 @@ class Timeout: __slots__ = ("_deadline", "_loop", "_state", "_task", "_timeout_handler") - def __init__( - self, deadline: Optional[float], loop: asyncio.AbstractEventLoop - ) -> None: + def __init__(self, deadline: Optional[float], loop: asyncio.AbstractEventLoop) -> None: self._loop = loop self._state = _State.INIT @@ -96,10 +94,10 @@ def __enter__(self) -> "Timeout": return self def __exit__( - self, - exc_type: Type[BaseException], - exc_val: BaseException, - exc_tb: TracebackType, + self, + exc_type: Type[BaseException], + exc_val: BaseException, + exc_tb: TracebackType, ) -> Optional[bool]: self._do_exit(exc_type) return None @@ -109,10 +107,10 @@ async def __aenter__(self) -> "Timeout": return self async def __aexit__( - self, - exc_type: Type[BaseException], - exc_val: BaseException, - exc_tb: TracebackType, + self, + exc_type: Type[BaseException], + exc_val: BaseException, + exc_tb: TracebackType, ) -> Optional[bool]: self._do_exit(exc_type) return None @@ -166,9 +164,7 @@ def shift_to(self, deadline: float) -> None: else: # state is ENTER raise asyncio.CancelledError - self._timeout_handler = self._loop.call_at( - deadline, self._on_timeout, self._task - ) + self._timeout_handler = self._loop.call_at(deadline, self._on_timeout, self._task) def _do_enter(self) -> None: if self._state != _State.INIT: @@ -199,7 +195,7 @@ def _current_task(loop: asyncio.AbstractEventLoop) -> "Optional[asyncio.Task[Any def _get_running_loop() -> asyncio.AbstractEventLoop: loop = asyncio.get_running_loop() if loop is None: - print('--' * 20) + print("--" * 20) loop = asyncio.get_event_loop() if sys.version_info >= (3, 7): diff --git a/src/aloha/times/timeout_signal.py b/src/aloha/time/timeout_signal.py similarity index 90% rename from src/aloha/times/timeout_signal.py rename to src/aloha/time/timeout_signal.py index 7272c12..2a5496d 100644 --- a/src/aloha/times/timeout_signal.py +++ b/src/aloha/time/timeout_signal.py @@ -26,6 +26,7 @@ def something_that_should_not_exceed_ten_seconds(): print('Maybe exceeded 10 seconds, but finished either way') ``` """ + import contextlib import errno import os @@ -35,7 +36,9 @@ def something_that_should_not_exceed_ten_seconds(): class TimeOutRestriction(contextlib.ContextDecorator): - def __init__(self, milliseconds: float, *, timeout_message: str = DEFAULT_TIMEOUT_MESSAGE, suppress_timeout_errors: bool = False): + def __init__( + self, milliseconds: float, *, timeout_message: str = DEFAULT_TIMEOUT_MESSAGE, suppress_timeout_errors: bool = False + ): self.millisecond = milliseconds self.timeout_message = timeout_message self.suppress = bool(suppress_timeout_errors) diff --git a/src/aloha/util/html.py b/src/aloha/util/html.py index 2ea12ba..dce696b 100644 --- a/src/aloha/util/html.py +++ b/src/aloha/util/html.py @@ -9,7 +9,7 @@ def extract_img_url(string): return None html = etree.HTML(string) for ii in html: - images = ii.xpath('p/img/@src') + images = ii.xpath("p/img/@src") return images[0] except Exception as e: print(e, string) @@ -22,14 +22,22 @@ def extract_text(raw_data): content = [] if html is not None: - html_data = html.xpath('/html/body/*//text()') + html_data = html.xpath("/html/body/*//text()") for data in html_data: - tmp = data.strip(' \n\r').replace('\n', '').replace('\t', '').replace(u'\u3000', u'') \ - .replace(u'\xa0', u'').replace('\r', '').replace(u'\u2028', u'').replace(u'\u2029', u'') + tmp = ( + data.strip(" \n\r") + .replace("\n", "") + .replace("\t", "") + .replace("\u3000", "") + .replace("\xa0", "") + .replace("\r", "") + .replace("\u2028", "") + .replace("\u2029", "") + ) if tmp: content.append(tmp) - item_article = ''.join(content) + item_article = "".join(content) return item_article else: return None diff --git a/src/aloha/util/json.py b/src/aloha/util/json.py index adc9b0a..efd61c8 100644 --- a/src/aloha/util/json.py +++ b/src/aloha/util/json.py @@ -6,7 +6,22 @@ class ObjectWithDateTimeEncoder(json.JSONEncoder): + """ + JSON encoder that handles datetime and pandas timestamp objects. + + Converts: + - pandas NaT to None + - pandas Timestamp to Unix timestamp (float) + - datetime objects to Unix timestamp (float) + """ + def default(self, obj): + """ + Convert objects to JSON serializable types. + + :param obj: Object to encode + :return: JSON serializable representation + """ if isinstance(obj, NaTType): # notice: NaTType is a subclass of datetime return None if isinstance(obj, Timestamp): diff --git a/src/aloha/util/random.py b/src/aloha/util/random.py index 095856c..b9d47b9 100644 --- a/src/aloha/util/random.py +++ b/src/aloha/util/random.py @@ -1,3 +1,14 @@ +__all__ = ( + "random", + "random_bool", + "random_choice", + "random_int", + "random_ratio", + "random_uniform", + "random_sample", + "random_seed", +) + from secrets import SystemRandom random = SystemRandom() @@ -7,25 +18,19 @@ def random_bool(): return random.choice([True, False]) -def random_choice(*args, **kwargs): - return random.choice(*args, **kwargs) +random_choice = random.choice -def random_int(a, b): - return random.randint(a, b) +random_int = random.randint -def random_ratio(): - return random.random() +random_ratio = random.random -def random_uniform(*args): - return random.uniform(*args) +random_uniform = random.uniform -def random_sample(*args): - return random.sample(*args) +random_sample = random.sample -def random_seed(*args, **kwargs): - return random.seed(*args, **kwargs) +random_seed = random.seed diff --git a/src/aloha/util/sys_cuda.py b/src/aloha/util/sys_cuda.py index 0c22adb..dbd1962 100644 --- a/src/aloha/util/sys_cuda.py +++ b/src/aloha/util/sys_cuda.py @@ -1,23 +1,21 @@ -__all__ = ('get_cuda_info',) +__all__ = ("get_cuda_info",) from collections import namedtuple from ..logger import LOG -Status = namedtuple('Status', 'version,gpu_availability') +Status = namedtuple("Status", "version,gpu_availability") def get_gpu_status_for_tf(*args, **kwargs) -> dict: status = Status(version=None, gpu_availability=False) try: import tensorflow as tf - LOG.info('tensorflow version = %s' % tf.__version__) - status = Status( - version=tf.__version__, - gpu_availability=tf.test.is_gpu_available() - ) + + LOG.info("tensorflow version = %s" % tf.__version__) + status = Status(version=tf.__version__, gpu_availability=tf.test.is_gpu_available()) except Exception as e: - LOG.error('Error detecting CUDA availability for tensorflow') + LOG.error("Error detecting CUDA availability for tensorflow") LOG.error(str(e)) return status._asdict() @@ -26,13 +24,11 @@ def get_gpu_status_for_torch(*args, **kwargs) -> dict: status = Status(version=None, gpu_availability=False) try: import torch - LOG.info('torch version = %s' % torch.__version__) - status = Status( - version=torch.__version__, - gpu_availability=torch.cuda.is_available() - ) + + LOG.info("torch version = %s" % torch.__version__) + status = Status(version=torch.__version__, gpu_availability=torch.cuda.is_available()) except Exception as e: - LOG.error('Error detecting CUDA availability for torch') + LOG.error("Error detecting CUDA availability for torch") LOG.error(str(e)) return status._asdict() @@ -41,14 +37,12 @@ def get_gpu_status_for_paddle(*args, **kwargs) -> dict: status = Status(version=None, gpu_availability=False) try: import paddle - LOG.info('Paddlepaddle version = %s' % paddle.__version__) + + LOG.info("Paddlepaddle version = %s" % paddle.__version__) paddle.utils.run_check() - status = Status( - version=paddle.__version__, - gpu_availability=True - ) + status = Status(version=paddle.__version__, gpu_availability=True) except Exception as e: - LOG.error('Error detecting CUDA availability for paddle') + LOG.error("Error detecting CUDA availability for paddle") LOG.error(str(e)) return status._asdict() @@ -64,8 +58,9 @@ def get_cuda_info(*args, **kwargs) -> dict: def main(*args, **kwargs): data = get_cuda_info() import json + print(json.dumps(data, ensure_ascii=False, indent=2)) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/aloha/util/sys_gpu.py b/src/aloha/util/sys_gpu.py index dab5186..0416ef8 100644 --- a/src/aloha/util/sys_gpu.py +++ b/src/aloha/util/sys_gpu.py @@ -1,4 +1,4 @@ -__all__ = ('get_gpu_info',) +__all__ = ("get_gpu_info",) # ! `pip install pynvml`, reference: https://github.com/gpuopenanalytics/pynvml @@ -10,13 +10,13 @@ import pynvml as nvml from pynvml.smi import nvidia_smi - LOG.debug('Using pynvml == %s' % nvml.__version__) + LOG.debug("Using pynvml == %s" % nvml.__version__) except ImportError: - LOG.warn('Package `pynvml` NOT installed! Cannot get GPU info.') + LOG.warn("Package `pynvml` NOT installed! Cannot get GPU info.") nvml = nvidia_smi = None -Device = namedtuple('Device', field_names='index,name,arch') -DeviceStatus = namedtuple('DeviceStatus', field_names='mem_total,mem_free,mem_used,gpu_rate,mem_rate') +Device = namedtuple("Device", field_names="index,name,arch") +DeviceStatus = namedtuple("DeviceStatus", field_names="mem_total,mem_free,mem_used,gpu_rate,mem_rate") class NvInfo: @@ -26,9 +26,9 @@ def __init__(self): return try: nvml.nvmlInit() - LOG.debug('NVML loaded and initialized successfully.') - except Exception as e: - LOG.error('Fail to initialize NVML!') + LOG.debug("NVML loaded and initialized successfully.") + except Exception: + LOG.error("Fail to initialize NVML!") nvml = None def __del__(self): @@ -36,25 +36,25 @@ def __del__(self): if nvml is not None: nvml.nvmlShutdown() except Exception as e: - LOG.error('Exception removing NvInfo: %s' % e) + LOG.error("Exception removing NvInfo: %s" % e) @staticmethod def get_driver_version() -> str: if nvml is not None: try: ver: bytes = nvml.nvmlSystemGetDriverVersion() - LOG.debug('GPU driver version %s' % str(ver)) - return ver.decode(encoding='UTF-8') + LOG.debug("GPU driver version %s" % str(ver)) + return ver.decode(encoding="UTF-8") except Exception as e: - LOG.info('NVML library error: %s' % str(e)) - return 'Unknown' + LOG.info("NVML library error: %s" % str(e)) + return "Unknown" @staticmethod def get_device_count() -> int: try: return nvml.nvmlDeviceGetCount() except Exception as e: - LOG.info('NVML library error: %s' % str(e)) + LOG.info("NVML library error: %s" % str(e)) return 0 def get_device_list(self) -> list: @@ -64,19 +64,19 @@ def get_device_list(self) -> list: name, arch = None, None try: - name = nvml.nvmlDeviceGetName(handler).decode(encoding='UTF-8') + name = nvml.nvmlDeviceGetName(handler).decode(encoding="UTF-8") except Exception as e: - LOG.info('Failed to get device name!') + LOG.info("Failed to get device name!") LOG.info(str(e)) try: arch = nvml.nvmlDeviceGetArchitecture(handler) except Exception as e: - LOG.info('Failed to get device architecture!') + LOG.info("Failed to get device architecture!") LOG.info(str(e)) device = Device(index=i, name=name, arch=arch) - LOG.debug('Found device {index} info: name={name}; arch={arch}'.format(**device._asdict())) + LOG.debug("Found device {index} info: name={name}; arch={arch}".format(**device._asdict())) list_device.append(device) return list_device @@ -93,7 +93,7 @@ def get_device_status(self, index_device=0) -> DeviceStatus: mem_free=mem_info.free, mem_used=mem_info.used, gpu_rate=util_info.gpu, - mem_rate=util_info.memory + mem_rate=util_info.memory, ) @staticmethod @@ -103,7 +103,7 @@ def get_smi(): try: return nvidia_smi.getInstance() except Exception as e: - LOG.warn('Failed to get smi: %s' % str(e)) + LOG.warn("Failed to get smi: %s" % str(e)) return @@ -113,10 +113,7 @@ def get_gpu_info(*args, **kwargs) -> dict: list_devices = nv_info.get_device_list() for i, d in enumerate(list_devices): status = nv_info.get_device_status(index_device=i) - item = { - 'device': d._asdict(), - 'status': status._asdict() - } + item = {"device": d._asdict(), "status": status._asdict()} list_device.append(item) ret = { @@ -127,7 +124,7 @@ def get_gpu_info(*args, **kwargs) -> dict: smi = nv_info.get_smi() if smi is not None: if len(args) == 0: - args = 'name;vbios_version;inforom.oem;compute-apps'.split(';') + args = "name;vbios_version;inforom.oem;compute-apps".split(";") for k in args: ret[k] = smi.DeviceQuery(k) @@ -137,8 +134,9 @@ def get_gpu_info(*args, **kwargs) -> dict: def main(*args, **kwargs): info_gpus = get_gpu_info() import json + print(json.dumps(info_gpus, ensure_ascii=False, indent=2)) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/aloha/util/sys_info.py b/src/aloha/util/sys_info.py index fde9a14..bdcf342 100644 --- a/src/aloha/util/sys_info.py +++ b/src/aloha/util/sys_info.py @@ -1,4 +1,4 @@ -__all__ = ('get_sys_info',) +__all__ = ("get_sys_info",) import platform from datetime import datetime @@ -7,14 +7,20 @@ from ..logger import LOG -LOG.debug('Using psutil == %s' % psutil.__version__) +LOG.debug("Using psutil == %s" % psutil.__version__) def get_size(bytes, suffix="B"): - """Scale bytes to its proper format + """ + Scale bytes to its proper format. + e.g: 1253656 => '1.20MB' 1253656678 => '1.17GB' + + :param bytes: Number of bytes to format + :param suffix: Unit suffix (default: "B") + :return: Formatted size string """ factor = 1024 for unit in ["", "K", "M", "G", "T", "P"]: @@ -24,10 +30,15 @@ def get_size(bytes, suffix="B"): def get_os_info(*args, **kwargs) -> dict: + """ + Get operating system information. + + :return: Dictionary containing OS information including boot time and platform details + """ ret = {} boot_time = datetime.fromtimestamp(psutil.boot_time()) - ret['boot_time'] = boot_time.strftime('%Y-%m-%d %H:%M:%S.%f') + ret["boot_time"] = boot_time.strftime("%Y-%m-%d %H:%M:%S.%f") uname = platform.uname() ret.update(uname._asdict()) @@ -35,6 +46,11 @@ def get_os_info(*args, **kwargs) -> dict: def get_cpu_info(*args, **kwargs) -> dict: + """ + Get CPU information. + + :return: Dictionary containing CPU information including core count, frequency, and usage percentage + """ cpu_freq = psutil.cpu_freq() # CPU frequencies ret = { "num_cores_physical": psutil.cpu_count(logical=False), @@ -45,12 +61,17 @@ def get_cpu_info(*args, **kwargs) -> dict: "cpu_percent_total": f"{psutil.cpu_percent()}%", } for i, percentage in enumerate(psutil.cpu_percent(percpu=True, interval=1)): - ret['cpu_percent_core_%02d' % i] = f"{percentage}%" + ret["cpu_percent_core_%02d" % i] = f"{percentage}%" return ret def get_mem_info(*args, **kwargs) -> dict: + """ + Get memory information. + + :return: Dictionary containing virtual memory and swap space information + """ svmem = psutil.virtual_memory() swap = psutil.swap_memory() @@ -59,7 +80,6 @@ def get_mem_info(*args, **kwargs) -> dict: "vm_available": f"{get_size(svmem.available)}", "vm_used": f"{get_size(svmem.used)}", "vm_percent": f"{svmem.percent}%", - "swap_total": f"{get_size(swap.total)}", "swap_free": f"{get_size(swap.free)}", "swap_used": f"{get_size(swap.used)}", @@ -68,6 +88,11 @@ def get_mem_info(*args, **kwargs) -> dict: def get_disk_info(*args, **kwargs) -> dict: + """ + Get disk information. + + :return: Dictionary containing disk I/O statistics and partition information + """ # get IO statistics since boot disk_io = psutil.disk_io_counters() partitions = psutil.disk_partitions() @@ -75,7 +100,7 @@ def get_disk_info(*args, **kwargs) -> dict: ret = { "io_total_read": f"{get_size(disk_io.read_bytes)}", "io_total_write": f"{get_size(disk_io.write_bytes)}", - "partitions": [] + "partitions": [], } for partition in partitions: @@ -87,12 +112,14 @@ def get_disk_info(*args, **kwargs) -> dict: try: partition_usage = psutil.disk_usage(partition.mountpoint) - part.update({ - "size_total": f"{get_size(partition_usage.total)}", - "size_used": f"{get_size(partition_usage.used)}", - "size_free": f"{get_size(partition_usage.free)}", - "percent_used": f"{partition_usage.percent}%", - }) + part.update( + { + "size_total": f"{get_size(partition_usage.total)}", + "size_used": f"{get_size(partition_usage.used)}", + "size_free": f"{get_size(partition_usage.free)}", + "percent_used": f"{partition_usage.percent}%", + } + ) except PermissionError: pass # this can be caught due to the disk that isn't ready @@ -102,6 +129,11 @@ def get_disk_info(*args, **kwargs) -> dict: def get_net_info(*args, **kwargs) -> dict: + """ + Get network information. + + :return: Dictionary containing network I/O statistics and interface information + """ # get IO statistics since boot net_io = psutil.net_io_counters() @@ -111,26 +143,33 @@ def get_net_info(*args, **kwargs) -> dict: ret = { "net_total_sent": f"{get_size(net_io.bytes_sent)}", "net_total_received": f"{get_size(net_io.bytes_recv)}", - "interfaces": [] + "interfaces": [], } for interface_name, interface_addresses in if_addresses.items(): interface = {"name": interface_name} for address in interface_addresses: - family = str(address.family).split('.')[-1] - family = {'AF_LINK': 'mac', 'AF_INET': 'ipv4', 'AF_INET6': 'ipv6'}.get(family, family) + family = str(address.family).split(".")[-1] + family = {"AF_LINK": "mac", "AF_INET": "ipv4", "AF_INET6": "ipv6"}.get(family, family) - interface['%s_address' % family] = address.address - interface['%s_netmask' % family] = address.netmask - interface['%s_broadcast' % family] = address.broadcast + interface["%s_address" % family] = address.address + interface["%s_netmask" % family] = address.netmask + interface["%s_broadcast" % family] = address.broadcast - ret['interfaces'].append(interface) + ret["interfaces"].append(interface) return ret def get_sys_info(*args, **kwargs) -> dict: + """ + Get comprehensive system information. + + Combines information from OS, CPU, memory, disk, and network subsystems. + + :return: Dictionary containing complete system information + """ return { "os_info": get_os_info(), "cpu_info": get_cpu_info(), @@ -141,11 +180,15 @@ def get_sys_info(*args, **kwargs) -> dict: def main(): + """ + Print system information as JSON to stdout. + """ data = get_sys_info() import json + data = json.dumps(data, ensure_ascii=False, indent=2) print(data) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/setup.py b/src/setup.py index b0df72c..bd4bb1b 100644 --- a/src/setup.py +++ b/src/setup.py @@ -6,63 +6,56 @@ long_description = fh.read() _now = datetime.now() -_version = '%s.%02d%02d.%02d%02d' % (_now.year, _now.month, _now.day, _now.hour, _now.minute) +_version = "%s.%02d%02d.%02d%02d" % (_now.year, _now.month, _now.day, _now.hour, _now.minute) -with open('./aloha/_version.py', 'wt') as fp: +with open("./aloha/_version.py", "wt") as fp: fp.write('__version__ = "%s"\n' % _version) dict_extra_requires = { - 'build': ['Cython'], - 'service': ['requests', 'tornado', 'psutil', 'pyjwt', 'fastapi', 'httpx'], - 'db': ['sqlalchemy', 'psycopg[binary]', 'pymysql', 'elasticsearch', 'pymongo', 'redis'], - 'stream': ['confluent_kafka'], - 'data': ['pandas'], - 'report': ['openpyxl>=3', 'XlsxWriter'], - 'test': ['pytest-cov'], - 'docs': ['mkdocs', 'mkdocstrings[python]', 'markdown-include', 'mkdocs-material'], + "build": ["Cython"], + "service": ["requests", "tornado", "psutil", "pyjwt", "fastapi", "httpx"], + "db": ["sqlalchemy", "psycopg[binary]", "pymysql", "elasticsearch", "pymongo", "redis"], + "stream": ["confluent_kafka"], + "data": ["pandas"], + "report": ["openpyxl>=3", "XlsxWriter"], + "test": ["pytest-cov"], + "docs": ["mkdocs", "mkdocstrings[python]", "markdown-include", "mkdocs-material"], } setup( - name='aloha', + name="aloha", version=_version, - author='QPod', - author_email='45032326+QPod0@users.noreply.github.com', - license='Apache Software License', - url='https://github.com/QPod/aloha', + author="LabNow.ai", + author_email="postmaster@labnow.ai", + license="Apache Software License", + url="https://github.com/LabNow.ai/aloha", project_urls={ - 'Source': 'https://github.com/QPod/aloha', - 'CI Pipeline': 'https://github.com/QPod/aloha/actions', - 'Documentation': 'https://aloha-python.readthedocs.io/', + "Source": "https://github.com/LabNow-ai/aloha", + "CI Pipeline": "https://github.com/LabNow-ai/aloha-python/actions", + "Documentation": "https://aloha-python.readthedocs.io/", }, - packages=find_packages(where=".", exclude=("app_common*",)), include_package_data=True, package_data={}, - platforms='Linux, Mac OS X, Windows', + platforms="Linux, Mac OS X, Windows", zip_safe=False, - install_requires=['attrdict3', 'pyhocon', 'pycryptodome', 'packaging'], + install_requires=["attrdict3", "pyhocon", "pycryptodome", "packaging"], extras_require={ **dict_extra_requires, - 'all': sorted(y for x in dict_extra_requires.values() for y in x), - }, - python_requires='>=3.6', - entry_points={ - 'console_scripts': [ - 'aloha = aloha.script.base:main' - ] + "all": sorted(y for x in dict_extra_requires.values() for y in x), }, - + python_requires=">=3.6", + entry_points={"console_scripts": ["aloha = aloha.script.base:main"]}, data_files=[], - description='Aloha - a versatile Python utility package for building services', + description="Aloha - a versatile Python utility package for building services", long_description=long_description, long_description_content_type="text/markdown", - classifiers=[ - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Intended Audience :: Science/Research', + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Intended Audience :: Science/Research", "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", - ] + ], ) diff --git a/tool/cicd/docker-compose.app-demo.DEV.yml b/tool/cicd/docker-compose.app-demo.DEV.yml index ca06e80..53338fe 100644 --- a/tool/cicd/docker-compose.app-demo.DEV.yml +++ b/tool/cicd/docker-compose.app-demo.DEV.yml @@ -1,6 +1,5 @@ name: ${PROJECT_NAME:-DEV-app-demo} - services: dev-app-demo: container_name: ${CONTAINER_NAME:-dev-app-demo} @@ -10,21 +9,20 @@ services: dockerfile: tool/dev-demo.Dockerfile args: - ENABLE_CODE_BUILD=false - - PROFILE_LOCALIZE=${PROFILE_LOCALIZE:-default} + # - PROFILE_LOCALIZE=${PROFILE_LOCALIZE:-default} pull: true restart: unless-stopped environment: - - PYTHONUNBUFFERED=1 - - PYTHONDONTWRITEBYTECODE=1 + # - ENV_PROFILE=${ENV_PROFILE:-DEV} - PROFILE_LOCALIZE=${PROFILE_LOCALIZE:-default} - - ENV_PROFILE=${ENV_PROFILE:-DEV} - # env_file: ["../credentials/DEV-app-demo.env"] + - PYTHONPATH=/root/app:/root/src:/root/notebook + # env_file: ["../credentials/DEV-app-demo.env"] user: "0:0" - ports: ["${PORT_APP:-9000}:9000", "${PORT_WEB:-3000}:3000"] + ports: [ "${PORT_APP:-9000}:9000", "${PORT_WEB:-3000}:3000" ] volumes: - ../../doc:/root/doc:rw - ../../notebook:/root/notebook:rw - ../../src:/root/src:rw - - ../../demo:/root/demo:rw + - ../../app:/root/app:rw working_dir: /root command: "tail -f /dev/null" diff --git a/tool/cicd/docker-compose.db.yml b/tool/cicd/docker-compose.db.yml index 709b5a6..87af693 100644 --- a/tool/cicd/docker-compose.db.yml +++ b/tool/cicd/docker-compose.db.yml @@ -5,12 +5,11 @@ networks: name: net-db-common external: false - services: db-postgres-common: container_name: db-postgres-common hostname: db-postgres-common - image: quay.io/labnow/postgres-16-ext + image: quay.io/labnow/postgres-17-ext pull_policy: if_not_present restart: unless-stopped environment: @@ -24,6 +23,6 @@ services: # sudo chmod -R 777 /data/data-pg-common - ../../src/resource/db:/data/mapp-ihealth/sql:rw # - ./sql:/docker-entrypoint-initdb.d:rw - networks: ["net-db-common"] - ports: ["5432:5432"] + networks: [ "net-db-common" ] + ports: [ "5432:5432" ] # su postgres && psql -d postgres -U postgres diff --git a/tool/cicd/run-dev.sh b/tool/cicd/run-dev.sh index d0909f0..3905528 100755 --- a/tool/cicd/run-dev.sh +++ b/tool/cicd/run-dev.sh @@ -7,8 +7,8 @@ USERNAME="$(whoami)" USERID="$(id -u)" -BASE_APP_PORT=40000 -BASE_WEB_PORT=43000 +BASE_APP_PORT=30000 +BASE_WEB_PORT=33000 export PORT_APP=$((BASE_APP_PORT + USERID)) export PORT_WEB=$((BASE_WEB_PORT + USERID)) export PROJECT_NAME="dev-app-demo-${USERNAME}" diff --git a/tool/dev-demo.Dockerfile b/tool/dev-demo.Dockerfile index 7730e46..b3e6187 100644 --- a/tool/dev-demo.Dockerfile +++ b/tool/dev-demo.Dockerfile @@ -8,7 +8,7 @@ FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} AS dev ARG PROFILE_LOCALIZE -# COPY src/requirements.txt /tmp/ +COPY app/requirements.txt /tmp/ USER root RUN set -eux && pwd && ls -alh \