diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index e7d1ce6..7ce5d56 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -12,10 +12,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.10" - name: Install dependencies run: | diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5d98182..16e0df0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,10 +12,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.10" - name: Install dependencies run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3876945..28b6f11 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,10 +19,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.10" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -38,7 +38,6 @@ jobs: - name: Test with pytest run: | python run_pytests.py tests --N_RANDOM_TESTS_PER_CASE=10 --run_slow=False - coverage-lcov Run-tests-on-Windows: name: Run tests on Windows-latest @@ -46,10 +45,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.10" - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8b86107 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +FROM python:3.10-slim AS base_stage +ENV ROOT_DIR=/PythonProject-Template +ARG MODE=release +# MODE: 'release' or 'it' (will be interpreted as debug). This is used in the docker_entrypoint.sh script +ENV MODE=${MODE} + +# Build stage +FROM base_stage AS build_stage + +ARG DEBIAN_FRONTEND=noninteractive + +# System dependencies +RUN apt update -y && \ + apt install -y --no-install-recommends build-essential git dos2unix && \ + rm -rf /var/lib/apt/lists/* + +# Files +WORKDIR "$ROOT_DIR" +COPY *_requirements.txt requirements.txt *.toml *.lock *.py ./ +COPY src src +COPY dist dist +COPY docker_files docker_files +RUN dos2unix docker_files/*.sh + +# Virtual environment +RUN python3 -m venv ./venv && \ + . ./venv/bin/activate && \ + pip install --upgrade pip && \ + pip install poetry && \ + poetry install --no-interaction --no-ansi + +# rm unnecessary files +RUN rm -rf *_requirements.txt requirements.txt *.toml *.lock setup.py run_pytests.py src/*.egg-info dist + +# Main stage +FROM base_stage AS main_stage + +# Files +WORKDIR $ROOT_DIR +COPY --from=build_stage $ROOT_DIR ./ +VOLUME data + +# Entrypoint +ENV PATH="venv/bin:$PATH" +ENTRYPOINT ["bash", "docker_files/docker_entrypoint.sh"] diff --git a/change_project_name.py b/change_project_name.py index 01f2443..44517a2 100644 --- a/change_project_name.py +++ b/change_project_name.py @@ -174,6 +174,19 @@ def update_docs_yml(args): return 0 +def update_dockerfile(args): + if not os.path.exists("Dockerfile"): + return 0 + with open("Dockerfile", "r") as f: + content = f.read() + root_dir = os.path.basename(os.path.dirname(__file__)) + print(f"ROOT_DIR=/{root_dir}") + content = re.sub(r'ENV ROOT_DIR=/(.*?)\n', f'ENV ROOT_DIR=/{root_dir}\n', content) + with open("Dockerfile", "w") as f: + f.write(content) + return 0 + + def main(): parser = get_parser() args = parser.parse_args() @@ -198,7 +211,7 @@ def main(): update_readme_md(args) update_license(args) update_docs_yml(args) - + update_dockerfile(args) return 0 diff --git a/data/.keep b/data/.keep new file mode 100644 index 0000000..e69de29 diff --git a/docker_files/build_image_and_run.py b/docker_files/build_image_and_run.py new file mode 100644 index 0000000..68548e6 --- /dev/null +++ b/docker_files/build_image_and_run.py @@ -0,0 +1,143 @@ +import os +import argparse +import subprocess +import json +import time +from typing import Optional + + +def get_args_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--dockerfiles_folder", + type=str, + default=os.path.join(os.path.dirname(__file__), ".."), + ) + parser.add_argument( + "--tag", + type=str, + default="latest", + ) + parser.add_argument( + "--image_name", + type=str, + default=None, + ) + parser.add_argument( + "--mode", + type=str, + default="release", + choices=["release", "data", "trainings", "analysis", "it"], + help="Mode of the project to run the container for. \n" + "- release: This mode is responsible for running the __main__.py of the current package. \n" + "- it: This is a development mode that is responsible for running the container in the interactive mode.", + ) + parser.add_argument( + "--args", + type=str, + default=None, + help="Arguments to pass to the container as a JSON string " + ) + parser.add_argument( + "--only_run", + action=argparse.BooleanOptionalAction, + default=False, + help="If set, only runs the container without building the image.", + ) + return parser + + +def build_images( + *, + dockerfiles_folder: str = ".", + tag: str = "latest", + image_name: Optional[str] = None, +): + if not image_name: + image_name = os.path.basename(os.path.dirname(os.path.dirname(__file__))) + images = [] + for root, dirs, files in os.walk(dockerfiles_folder): + for file in files: + if not file.startswith("Dockerfile"): + continue + post_name = file.replace("Dockerfile", "") + full_image_name = f"{image_name}{post_name}:{tag}" + print(f"Building image {full_image_name} from {os.path.join(root, file)}") + cmd = f"docker build -t {full_image_name} -f {os.path.join(root, file)} ." + print(f"{cmd = }") + subprocess.run(cmd, shell=True, check=True, cwd=root) + print(f"Built image {full_image_name}") + images.append(full_image_name) + print(f"Pruning old images") + cmd = "docker image prune --force" + print(f"{cmd = }") + subprocess.run(cmd, shell=True, check=True) + print(f"Pruned old images") + return images + +def run_container( + *, + container_name: str = None, + image_name: str = None, + tag: str = "latest", + args: list = None, + run_args: list = None, +): + root_folder_name = os.path.basename(os.path.dirname(os.path.dirname(__file__))) + if not container_name: + container_name = f"{image_name}-container" + if not image_name: + image_name = root_folder_name + if ":" in image_name: + image_name, tag = image_name.split(":") + print(f"Removing container {container_name} if it exists") + cmd = f"docker rm --force {container_name}" + print(f"{cmd = }") + subprocess.run(cmd, shell=True, check=False) + print(f"Removed container {container_name}") + time.sleep(1) + print("-" * 80) + print(f"Running container {container_name} from image {image_name}:{tag}") + container_args = [ + "--name", container_name, + "--detach", + f"--volume=" + f"{os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'data'))}" + f":/{root_folder_name}/data", + ] + run_args_str = " ".join([str(arg) for arg in run_args]) if run_args else "" + cmd = f"docker run {' '.join(container_args)} {run_args_str} {image_name}:{tag}" + if args: + cmd += " " + " ".join([str(arg) for arg in args]) + print(f"{cmd = }") + subprocess.run(cmd, shell=True, check=True) + print(f"Container {container_name} is now running in the background.") + return + +def main(): + args = get_args_parser().parse_args() + if not args.image_name: + args.image_name = os.path.basename(os.path.dirname(os.path.dirname(__file__))) + print(f"Running with args: {json.dumps(vars(args), indent=4)}") + if args.only_run: + images = [f"{args.image_name}:{args.tag}"] + else: + images = build_images( + dockerfiles_folder=args.dockerfiles_folder, + tag=args.tag, + image_name=args.image_name, + ) + for image in images: + run_container( + image_name=image, + tag=args.tag, + args=json.loads(args.args) if args.args else None, + container_name=f"{args.image_name}-{args.mode}", + run_args=["-e", f"MODE={args.mode}"], + ) + return 0 + + +if __name__ == "__main__": + exit(main()) + diff --git a/docker_files/docker_entrypoint.sh b/docker_files/docker_entrypoint.sh new file mode 100644 index 0000000..965fac1 --- /dev/null +++ b/docker_files/docker_entrypoint.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +if [ -z "$ROOT_DIR" ]; then + echo "ROOT_DIR is not set. Exiting." + exit 1 +fi + +if [ -z "$MODE" ]; then + MODE="release" +fi + +source venv/bin/activate +echo "Working directory: $(pwd)" +echo "Running in mode $MODE." + +if [ "$MODE" = "release" ]; then + python3 -m "$ROOT_DIR" "$@" +fi +if [ "$MODE" = "it" ]; then + bash +fi +echo "Completed in mode $MODE." diff --git a/pyproject.toml b/pyproject.toml index 12dde80..4b5c3cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,12 +8,12 @@ classifiers = [ "Intended Audience :: Science/Research", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Scientific/Engineering", "Topic :: Software Development :: Libraries :: Python Modules", "Operating System :: OS Independent", ] -requires-python = ">=3.8" +requires-python = ">=3.10" dependencies = [ "numpy>=1.29.5", "setuptools>=65.5.1", @@ -21,7 +21,8 @@ dependencies = [ "pytest-cov>=4.1.0", "pytest_json_report>=1.5.0", "pythonbasictools>=0.0.1a11", - "psutil>=5.9.6" + "psutil>=5.9.6", + "docutils>=0.21.2", ] license={file="LICENSE"}