diff --git a/README.md b/README.md index 919bb5ae..cacbe8c1 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Command line tool that allows the user to execute data operations on the platfor Linux example for each environment: - pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.8/site-packages ./app/pilotcli.py -n + pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.10/site-packages ./app/pilotcli.py -n Note: Building for ARM Mac may require a newer version of `pyinstaller`. diff --git a/app/configs/app_config.py b/app/configs/app_config.py index 7c902ef6..54791112 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -28,10 +28,6 @@ class Env: greenroom_bucket_prefix = 'gr' # the number of items to active interative mode interative_threshold = 10 - # set hard limit for pending jobs, otherwise cli will consume all memory - # to cache jobs. If later on the speed of chunk deliver become faster, we - # can increase the concurrency number. - num_of_jobs = ConfigClass.concurrent_job_limit github_url = 'PilotDataPlatform/cli' diff --git a/app/configs/config.py b/app/configs/config.py index 977bc270..634d8337 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -37,7 +37,6 @@ class Settings(BaseSettings): apikey_endpoint: str = 'api-key' upload_batch_size: int = 100 - concurrent_job_limit: int = 10 upload_chunk_size: int = 1024 * 1024 * 20 # 20MB def __init__(self, **data): diff --git a/app/configs/user_config.py b/app/configs/user_config.py index 2a868e68..86dac7e9 100644 --- a/app/configs/user_config.py +++ b/app/configs/user_config.py @@ -42,6 +42,11 @@ class UserConfig(metaclass=Singleton): This user config is global. """ + _api_key: str + _access_token: str + _refresh_token: str + _username: str + def __init__( self, config_path: Union[str, Path, None] = None, @@ -53,7 +58,6 @@ def __init__( This adjustment is made to prevent complications with mounted NFS volumes where all files have root ownership. """ - if config_path is None: config_path = ConfigClass.config_path if config_filename is None: @@ -106,6 +110,12 @@ def __init__( } self.save() + # load all config into memory + self._api_key = decryption(self.config['USER']['api_key'], self.secret) + self._access_token = decryption(self.config['USER']['access_token'], self.secret) + self._refresh_token = decryption(self.config['USER']['refresh_token'], self.secret) + self._username = decryption(self.config['USER']['username'], self.secret) + def _check_user_permissions(self, path: Path, expected_bits: Iterable[int]) -> Union[str, None]: """Check if file or folder is owned by the user and has proper access mode.""" @@ -147,42 +157,37 @@ def is_access_token_exists(self) -> bool: @property def username(self): - return decryption(self.config['USER']['username'], self.secret) + return self._username @username.setter def username(self, val): self.config['USER']['username'] = encryption(val, self.secret) - @property - def password(self): - return decryption(self.config['USER']['password'], self.secret) - - @password.setter - def password(self, val): - self.config['USER']['password'] = encryption(val, self.secret) - @property def api_key(self): - return decryption(self.config['USER']['api_key'], self.secret) + return self._api_key @api_key.setter def api_key(self, val): + self._api_key = val self.config['USER']['api_key'] = encryption(val, self.secret) @property def access_token(self): - return decryption(self.config['USER'].get('access_token', ''), self.secret) + return self._access_token @access_token.setter def access_token(self, val): + self._access_token = val self.config['USER']['access_token'] = encryption(val, self.secret) @property def refresh_token(self): - return decryption(self.config['USER']['refresh_token'], self.secret) + return self._refresh_token @refresh_token.setter def refresh_token(self, val): + self._refresh_token = val self.config['USER']['refresh_token'] = encryption(val, self.secret) @property diff --git a/app/services/clients/base_auth_client.py b/app/services/clients/base_auth_client.py index 8bcf551f..01c47b68 100644 --- a/app/services/clients/base_auth_client.py +++ b/app/services/clients/base_auth_client.py @@ -17,11 +17,12 @@ class BaseAuthClient(BaseClient): token_manager: SrvTokenManager - user = UserConfig() + user: UserConfig def __init__(self, endpoint: str, timeout: int = 10) -> None: super().__init__(endpoint, timeout) + self.user = UserConfig() self.token_manager = SrvTokenManager() self.headers = { 'Authorization': 'Bearer ' + self.user.access_token, diff --git a/app/services/clients/base_client.py b/app/services/clients/base_client.py index b8423d52..9ddc289d 100644 --- a/app/services/clients/base_client.py +++ b/app/services/clients/base_client.py @@ -44,6 +44,7 @@ def _request( if response.status_code not in self.retry_status: response.raise_for_status() return response + time.sleep(self.retry_interval) logger.debug(f'failed with over {self.retry_count} retries.') diff --git a/app/services/file_manager/file_manifests.py b/app/services/file_manager/file_manifests.py index 2e304a7d..24bdca53 100644 --- a/app/services/file_manager/file_manifests.py +++ b/app/services/file_manager/file_manifests.py @@ -28,11 +28,13 @@ def dupe_checking_hook(pairs): class SrvFileManifests(BaseAuthClient, metaclass=MetaService): app_config = AppConfig() - user = UserConfig() + user: UserConfig def __init__(self, interactive=True): super().__init__(self.app_config.Connections.url_bff) + self.user = UserConfig() + self.interactive = interactive self.endpoint = self.app_config.Connections.url_bff + '/v1' diff --git a/app/services/file_manager/file_tag.py b/app/services/file_manager/file_tag.py index 5007c266..711dcffd 100644 --- a/app/services/file_manager/file_tag.py +++ b/app/services/file_manager/file_tag.py @@ -17,10 +17,11 @@ class SrvFileTag(metaclass=MetaService): appconfig = AppConfig() - user = UserConfig() + user: UserConfig def __init__(self, interactive=True): self.interactive = interactive + self.user = UserConfig() @staticmethod def validate_tag(tag): diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 33ab4551..93abdc3d 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -233,12 +233,7 @@ def simple_upload( # noqa: C901 # now loop over each file under the folder and start # the chunk upload - - # thread number +1 reserve one thread to refresh token - # and remove the token decorator in functions - - pool = ThreadPool(num_of_thread + 1) - pool.apply_async(upload_client.upload_token_refresh) + pool = ThreadPool(num_of_thread) on_success_res = [] file_object: FileObject @@ -338,11 +333,7 @@ def resume_upload( mhandler.SrvOutPutHandler.resume_check_success() # lastly, start resumable upload for the rest of the chunks - # thread number +1 reserve one thread to refresh token - # and remove the token decorator in functions - - pool = ThreadPool(num_of_thread + 1) - pool.apply_async(upload_client.upload_token_refresh) + pool = ThreadPool(num_of_thread) on_success_res = [] for file_object in unfinished_items: upload_client.stream_upload(file_object, pool) diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index 6e4b28de..6c5ad80f 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -8,7 +8,6 @@ import math import os import threading -import time from logging import getLogger from multiprocessing.pool import ThreadPool from typing import Any @@ -22,7 +21,6 @@ import app.services.output_manager.message_handler as mhandler from app.configs.app_config import AppConfig -from app.configs.config import ConfigClass from app.configs.user_config import UserConfig from app.models.upload_form import generate_on_success_form from app.services.clients.base_auth_client import BaseAuthClient @@ -31,7 +29,6 @@ from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import SrvErrorHandler from app.services.user_authentication.decorator import require_valid_token -from app.services.user_authentication.token_manager import SrvTokenManager from app.utils.aggregated import get_file_info_by_geid from .exception import INVALID_CHUNK_ETAG @@ -92,7 +89,6 @@ def __init__( # for tracking the multi-threading chunk upload self.active_jobs = 0 self.lock = threading.Lock() - self.chunk_upload_done = threading.Event() def generate_meta(self, local_path: str) -> Tuple[int, int]: """ @@ -309,14 +305,15 @@ def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> None: been uploaded. """ count = 0 - semaphore = threading.Semaphore(AppConfig.Env.num_of_jobs) + semaphore = threading.Semaphore(pool._processes + 1) + chunk_upload_done = threading.Event() def on_complete(result): semaphore.release() with self.lock: self.active_jobs -= 1 if self.active_jobs == 0: - self.chunk_upload_done.set() + chunk_upload_done.set() # process on the file content f = open(file_object.local_path, 'rb') @@ -355,8 +352,14 @@ def on_complete(result): count += 1 + # for resumable check ONLY if user resume the upload at 100% + # just check if there is any active job, if not, set the event + while not chunk_upload_done.wait(timeout=60): + if self.active_jobs == 0: + chunk_upload_done.set() + logger.warning('Waiting for all the chunks to be uploaded, remaining jobs: %s', file_object.progress) + f.close() - self.chunk_upload_done.wait() def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str, etag: str, chunk_size: int) -> None: """ @@ -466,16 +469,3 @@ def check_status(self, file_object: FileObject) -> bool: def set_finish_upload(self): self.finish_upload = True - - def upload_token_refresh(self, azp: str = ConfigClass.keycloak_device_client_id): - token_manager = SrvTokenManager() - DEFAULT_INTERVAL = 2 # seconds to check if the upload is finished - total_count = 0 # when total_count equals token_refresh_interval, refresh token - while self.finish_upload is not True: - if total_count >= AppConfig.Env.token_refresh_interval: - token_manager.refresh(azp) - total_count = 0 - - # if not then sleep for DEFAULT_INTERVAL seconds - time.sleep(DEFAULT_INTERVAL) - total_count = total_count + DEFAULT_INTERVAL diff --git a/app/services/user_authentication/token_manager.py b/app/services/user_authentication/token_manager.py index 6200e81d..8340456c 100644 --- a/app/services/user_authentication/token_manager.py +++ b/app/services/user_authentication/token_manager.py @@ -20,11 +20,7 @@ class SrvTokenManager(BaseClient, metaclass=MetaService): def __init__(self): super().__init__(AppConfig.Connections.url_keycloak_token, 10) - user_config = UserConfig() - if user_config.is_logged_in(): - self.config = user_config - else: - raise Exception('Login session not found, please login first.') + self.config = UserConfig() def update_token(self, access_token, refresh_token): self.config.access_token = access_token diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 2f338f18..375de75c 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -20,6 +20,7 @@ from app.models.item import ItemStatus from app.models.item import ItemType from app.services.clients.base_auth_client import BaseAuthClient +from app.services.clients.base_auth_client import BaseClient from app.services.logger_services.debugging_log import debug_logger from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import SrvErrorHandler @@ -233,13 +234,24 @@ def remove_the_output_file(filepath: str) -> None: def get_latest_cli_version() -> Tuple[Version, str]: + import logging + import time + try: - httpx_client = BaseAuthClient(AppConfig.Connections.url_fileops_greenroom) + start_time = time.time() + httpx_client = BaseClient(AppConfig.Connections.url_fileops_greenroom) + logging.critical(f'http client init time: {time.time() - start_time}') user_config = UserConfig() + logging.critical(f'user config init time: {time.time() - start_time}') + t1 = time.time() if not user_config.is_access_token_exists(): return Version('0.0.0') + logging.critical(f'Check token time: {time.time() - t1}') + t2 = time.time() - response = httpx_client._get('v1/download/cli/presigned') + headers = {'Authorization': 'Bearer'} + response = httpx_client._get('v1/download/cli/presigned', headers=headers) + logging.critical(f'Get latest version time: {time.time() - t2}') result = response.json().get('result', {}) latest_version = result.get('linux', {}).get('version', '0.0.0') download_url = result.get('linux', {}).get('download_url', '') diff --git a/poetry.lock b/poetry.lock index b0917d33..1289856a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -603,13 +603,13 @@ files = [ [[package]] name = "pefile" -version = "2024.8.26" +version = "2023.2.7" description = "Python PE parsing module" optional = false python-versions = ">=3.6.0" files = [ - {file = "pefile-2024.8.26-py3-none-any.whl", hash = "sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f"}, - {file = "pefile-2024.8.26.tar.gz", hash = "sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632"}, + {file = "pefile-2023.2.7-py3-none-any.whl", hash = "sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6"}, + {file = "pefile-2023.2.7.tar.gz", hash = "sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc"}, ] [[package]] @@ -835,46 +835,47 @@ python-dotenv = ">=0.21.0" [[package]] name = "pyinstaller" -version = "5.13.2" +version = "6.11.1" description = "PyInstaller bundles a Python application and all its dependencies into a single package." optional = false -python-versions = "<3.13,>=3.7" +python-versions = "<3.14,>=3.8" files = [ - {file = "pyinstaller-5.13.2-py3-none-macosx_10_13_universal2.whl", hash = "sha256:16cbd66b59a37f4ee59373a003608d15df180a0d9eb1a29ff3bfbfae64b23d0f"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8f6dd0e797ae7efdd79226f78f35eb6a4981db16c13325e962a83395c0ec7420"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_i686.whl", hash = "sha256:65133ed89467edb2862036b35d7c5ebd381670412e1e4361215e289c786dd4e6"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:7d51734423685ab2a4324ab2981d9781b203dcae42839161a9ee98bfeaabdade"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_s390x.whl", hash = "sha256:2c2fe9c52cb4577a3ac39626b84cf16cf30c2792f785502661286184f162ae0d"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:c63ef6133eefe36c4b2f4daf4cfea3d6412ece2ca218f77aaf967e52a95ac9b8"}, - {file = "pyinstaller-5.13.2-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:aadafb6f213549a5906829bb252e586e2cf72a7fbdb5731810695e6516f0ab30"}, - {file = "pyinstaller-5.13.2-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:b2e1c7f5cceb5e9800927ddd51acf9cc78fbaa9e79e822c48b0ee52d9ce3c892"}, - {file = "pyinstaller-5.13.2-py3-none-win32.whl", hash = "sha256:421cd24f26144f19b66d3868b49ed673176765f92fa9f7914cd2158d25b6d17e"}, - {file = "pyinstaller-5.13.2-py3-none-win_amd64.whl", hash = "sha256:ddcc2b36052a70052479a9e5da1af067b4496f43686ca3cdda99f8367d0627e4"}, - {file = "pyinstaller-5.13.2-py3-none-win_arm64.whl", hash = "sha256:27cd64e7cc6b74c5b1066cbf47d75f940b71356166031deb9778a2579bb874c6"}, - {file = "pyinstaller-5.13.2.tar.gz", hash = "sha256:c8e5d3489c3a7cc5f8401c2d1f48a70e588f9967e391c3b06ddac1f685f8d5d2"}, + {file = "pyinstaller-6.11.1-py3-none-macosx_10_13_universal2.whl", hash = "sha256:44e36172de326af6d4e7663b12f71dbd34e2e3e02233e181e457394423daaf03"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:6d12c45a29add78039066a53fb05967afaa09a672426072b13816fe7676abfc4"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_i686.whl", hash = "sha256:ddc0fddd75f07f7e423da1f0822e389a42af011f9589e0269b87e0d89aa48c1f"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:0d6475559c4939f0735122989611d7f739ed3bf02f666ce31022928f7a7e4fda"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_s390x.whl", hash = "sha256:e21c7806e34f40181e7606926a14579f848bfb1dc52cbca7eea66eccccbfe977"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:32c742a24fe65d0702958fadf4040f76de85859c26bec0008766e5dbabc5b68f"}, + {file = "pyinstaller-6.11.1-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:208c0ef6dab0837a0a273ea32d1a3619a208e3d1fe3fec3785eea71a77fd00ce"}, + {file = "pyinstaller-6.11.1-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:ad84abf465bcda363c1d54eafa76745d77b6a8a713778348377dc98d12a452f7"}, + {file = "pyinstaller-6.11.1-py3-none-win32.whl", hash = "sha256:2e8365276c5131c9bef98e358fbc305e4022db8bedc9df479629d6414021956a"}, + {file = "pyinstaller-6.11.1-py3-none-win_amd64.whl", hash = "sha256:7ac83c0dc0e04357dab98c487e74ad2adb30e7eb186b58157a8faf46f1fa796f"}, + {file = "pyinstaller-6.11.1-py3-none-win_arm64.whl", hash = "sha256:35e6b8077d240600bb309ed68bb0b1453fd2b7ab740b66d000db7abae6244423"}, + {file = "pyinstaller-6.11.1.tar.gz", hash = "sha256:491dfb4d9d5d1d9650d9507daec1ff6829527a254d8e396badd60a0affcb72ef"}, ] [package.dependencies] altgraph = "*" macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} -pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""} -pyinstaller-hooks-contrib = ">=2021.4" +packaging = ">=22.0" +pefile = {version = ">=2022.5.30,<2024.8.26 || >2024.8.26", markers = "sys_platform == \"win32\""} +pyinstaller-hooks-contrib = ">=2024.9" pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} setuptools = ">=42.0.0" [package.extras] -encryption = ["tinyaes (>=1.0.0)"] +completion = ["argcomplete"] hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2024.9" +version = "2025.1" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.8" files = [ - {file = "pyinstaller_hooks_contrib-2024.9-py3-none-any.whl", hash = "sha256:1ddf9ba21d586afa84e505bb5c65fca10b22500bf3fdb89ee2965b99da53b891"}, - {file = "pyinstaller_hooks_contrib-2024.9.tar.gz", hash = "sha256:4793869f370d1dc4806c101efd2890e3c3e703467d8d27bb5a3db005ebfb008d"}, + {file = "pyinstaller_hooks_contrib-2025.1-py3-none-any.whl", hash = "sha256:d3c799470cbc0bda60dcc8e6b4ab976777532b77621337f2037f558905e3a8e9"}, + {file = "pyinstaller_hooks_contrib-2025.1.tar.gz", hash = "sha256:130818f9e9a0a7f2261f1fd66054966a3a50c99d000981c5d1db11d3ad0c6ab2"}, ] [package.dependencies] @@ -1236,23 +1237,23 @@ idna2008 = ["idna"] [[package]] name = "setuptools" -version = "75.2.0" +version = "75.8.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "setuptools-75.2.0-py3-none-any.whl", hash = "sha256:a7fcb66f68b4d9e8e66b42f9876150a3371558f98fa32222ffaa5bced76406f8"}, - {file = "setuptools-75.2.0.tar.gz", hash = "sha256:753bb6ebf1f465a1912e19ed1d41f403a79173a9acf66a42e7e6aec45c3c16ec"}, + {file = "setuptools-75.8.0-py3-none-any.whl", hash = "sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3"}, + {file = "setuptools-75.8.0.tar.gz", hash = "sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] +core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.11.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "six" @@ -1378,4 +1379,4 @@ windows = ["pywin32"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "4e0e8a667efa7cca18858b04d0f2658bb054ee3189a7457fafba1ca98e14f38f" +content-hash = "103a2f6b95b848dc2e257c111a915811517ccc325ba54178311d1aaec0b4873b" diff --git a/pyproject.toml b/pyproject.toml index 1962efcb..9c1db4e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "3.13.0" +version = "3.14.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] @@ -14,7 +14,6 @@ pre-commit = "^2.19.0" httpx = "^0.23.0" qrcode = "^7.4.2" pytest-click = "^1.1.0" -pyinstaller = "^5.13.0" pywin32 = { version = "^306", optional = true } pyjwt = "^2.9.0" requests = "^2.32.3" @@ -22,6 +21,7 @@ click = "^8.1.8" urllib3 = "^2.3.0" cffi = "^1.17.1" charset-normalizer = "3.4.1" +pyinstaller = "^6.11.1" [tool.poetry.extras] diff --git a/tests/app/configs/test_user_config.py b/tests/app/configs/test_user_config.py index 67ae0cb3..c55254c3 100644 --- a/tests/app/configs/test_user_config.py +++ b/tests/app/configs/test_user_config.py @@ -18,10 +18,11 @@ def error_log(mocker): class TestUserConfig: - def test__init__creates_config_folder_with_0700_and_file_with_0600_access_modes(self, tmp_path, fake): + def test__init__creates_config_folder_with_0700_and_file_with_0600_access_modes(self, tmp_path, fake, mocker): config_folder = tmp_path / fake.pystr() file_name = fake.pystr() + mocker.patch('app.configs.user_config.decryption', return_value='') UserConfig(config_folder, file_name) config_folder_mode = stat.S_IMODE(config_folder.stat().st_mode) @@ -77,15 +78,16 @@ def test__init__exits_with_error_when_config_file_does_not_have_expected_access_ error_log.assert_called_with(expected_message) def test__init__does_not_exit_with_error_when_config_folder_has_invalid_access_mode_and_is_cloud_mode_set_to_true( - self, tmp_path, fake + self, tmp_path, fake, mocker ): config_folder = tmp_path / fake.pystr() config_folder.mkdir(mode=0o0755) + mocker.patch('app.configs.user_config.decryption', return_value='') UserConfig(config_folder, is_cloud_mode=True) def test__init__does_not_exit_with_error_when_config_file_has_invalid_access_mode_and_is_cloud_mode_set_to_true( - self, tmp_path, fake + self, tmp_path, fake, mocker ): config_folder = tmp_path / fake.pystr() file_name = fake.pystr() @@ -93,17 +95,19 @@ def test__init__does_not_exit_with_error_when_config_file_has_invalid_access_mod config_folder.mkdir(mode=0o0700) config_file.touch(mode=0o0644) + mocker.patch('app.configs.user_config.decryption', return_value='') UserConfig(config_folder, file_name, is_cloud_mode=True) - def test__init__sets_is_cloud_mode_to_false_by_default(self, tmp_path, fake): + def test__init__sets_is_cloud_mode_to_false_by_default(self, tmp_path, fake, mocker): config_folder = tmp_path / fake.pystr() + mocker.patch('app.configs.user_config.decryption', return_value='') user_config = UserConfig(config_folder) assert user_config.is_cloud_mode is False def test__init__sets_is_cloud_mode_to_true_when_pyinstaller_bundle_params_are_set_and_cloud_mode_file_is_present( - self, tmp_path, monkeypatch + self, tmp_path, monkeypatch, mocker ): monkeypatch.setattr(sys, 'frozen', True, raising=False) monkeypatch.setattr(sys, '_MEIPASS', str(tmp_path), raising=False) @@ -111,6 +115,7 @@ def test__init__sets_is_cloud_mode_to_true_when_pyinstaller_bundle_params_are_se cloud_mode_file = tmp_path / 'ENABLE_CLOUD_MODE' cloud_mode_file.touch() + mocker.patch('app.configs.user_config.decryption', return_value='') user_config = UserConfig() assert user_config.is_cloud_mode is True diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py index 2114b51c..a72cc769 100644 --- a/tests/app/services/file_manager/file_upload/test_upload_client.py +++ b/tests/app/services/file_manager/file_upload/test_upload_client.py @@ -7,9 +7,7 @@ import math import re from functools import wraps -from multiprocessing import TimeoutError from multiprocessing.pool import ThreadPool -from time import sleep import click import pytest @@ -228,32 +226,6 @@ def test_stream_upload_failed_with_etag_mismatch(mocker): pool.join() -def test_token_refresh_auto(mocker): - AppConfig.Env.token_refresh_interval = 1 - - token_refresh_mock = mocker.patch( - 'app.services.user_authentication.token_manager.SrvTokenManager.refresh', return_value=None - ) - - upload_client = UploadClient('project_code', 'parent_folder_id') - pool = ThreadPool(2) - async_fun = pool.apply_async(upload_client.upload_token_refresh) - sleep(3) - upload_client.set_finish_upload() - - # add the timeout to avoid the test stuck - try: - async_fun.get(timeout=5) - except TimeoutError: - raise AssertionError('token refresh failed') - - pool.close() - pool.join() - - # make sure the token refresh function is called - token_refresh_mock.assert_called_once() - - def test_resumable_pre_upload_success(httpx_mock, mocker): mocker.patch( 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', diff --git a/tests/conftest.py b/tests/conftest.py index 69953fb2..9072d179 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,7 +14,9 @@ @pytest.fixture(autouse=True) -def reset_singletons(): +def reset_singletons(mocker): + mocker.patch('app.configs.user_config.decryption', return_value='') + Singleton._instances = {} @@ -30,7 +32,6 @@ def mock_settings(monkeypatch, mocker): monkeypatch.setattr(AppConfig.Connections, 'url_keycloak', 'http://url_keycloak') monkeypatch.setattr(AppConfig.Connections, 'url_portal', 'http://bff_cli') monkeypatch.setattr(UserConfig, 'username', 'test-user') - monkeypatch.setattr(UserConfig, 'password', 'test-password') monkeypatch.setattr(UserConfig, 'api_key', 'test-api-key') monkeypatch.setattr(UserConfig, 'access_token', 'test-access-token') monkeypatch.setattr(UserConfig, 'refresh_token', 'test-refresh-token')