From 78c40a7662dd4165d7b271d6a3c63a94ec21346b Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 5 Feb 2025 22:55:08 -0500 Subject: [PATCH 01/11] testing --- README.md | 2 +- app/configs/user_config.py | 27 ++++++-- app/pilotcli.py | 6 ++ app/services/clients/base_client.py | 1 + .../user_authentication/token_manager.py | 5 ++ app/utils/aggregated.py | 10 +++ poetry.lock | 65 ++++++++++--------- pyproject.toml | 4 +- 8 files changed, 81 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 919bb5ae..cacbe8c1 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Command line tool that allows the user to execute data operations on the platfor Linux example for each environment: - pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.8/site-packages ./app/pilotcli.py -n + pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.10/site-packages ./app/pilotcli.py -n Note: Building for ARM Mac may require a newer version of `pyinstaller`. diff --git a/app/configs/user_config.py b/app/configs/user_config.py index 2a868e68..4121aca3 100644 --- a/app/configs/user_config.py +++ b/app/configs/user_config.py @@ -3,6 +3,7 @@ # Contact Indoc Systems for any questions regarding the use of this source code. import configparser +import logging import platform import sys import time @@ -53,7 +54,6 @@ def __init__( This adjustment is made to prevent complications with mounted NFS volumes where all files have root ownership. """ - if config_path is None: config_path = ConfigClass.config_path if config_filename is None: @@ -106,6 +106,8 @@ def __init__( } self.save() + # print(f'save config time: {time.time() - t2}') + def _check_user_permissions(self, path: Path, expected_bits: Iterable[int]) -> Union[str, None]: """Check if file or folder is owned by the user and has proper access mode.""" @@ -163,7 +165,12 @@ def password(self, val): @property def api_key(self): - return decryption(self.config['USER']['api_key'], self.secret) + import time + + start_time = time.time() + api_key = decryption(self.config['USER']['api_key'], self.secret) + logging.info(f'api_key decryption time: {time.time() - start_time}') + return api_key @api_key.setter def api_key(self, val): @@ -171,7 +178,13 @@ def api_key(self, val): @property def access_token(self): - return decryption(self.config['USER'].get('access_token', ''), self.secret) + # return decryption(self.config['USER'].get('access_token', ''), self.secret) + import time + + start_time = time.time() + access_token = decryption(self.config['USER']['access_token'], self.secret) + logging.info(f'access_token decryption time: {time.time() - start_time}') + return access_token @access_token.setter def access_token(self, val): @@ -179,7 +192,13 @@ def access_token(self, val): @property def refresh_token(self): - return decryption(self.config['USER']['refresh_token'], self.secret) + # return decryption(self.config['USER']['refresh_token'], self.secret) + import time + + start_time = time.time() + refresh_token = decryption(self.config['USER']['refresh_token'], self.secret) + logging.info(f'refresh_token decryption time: {time.time() - start_time}') + return refresh_token @refresh_token.setter def refresh_token(self, val): diff --git a/app/pilotcli.py b/app/pilotcli.py index cccee02c..1ae659ee 100644 --- a/app/pilotcli.py +++ b/app/pilotcli.py @@ -52,10 +52,16 @@ def write_dl(self, rows, col_max=80, col_spacing=2): class ComplexCLI(click.MultiCommand): def format_help_text(self, ctx, formatter): + import logging + import time + + start = time.time() latest_version, download_url = get_latest_cli_version() if Version(pkg_resources.get_distribution('app').version) < latest_version: mhandler.SrvOutPutHandler.newer_version_available(latest_version, download_url) + logging.critical(f'Elapsed time: {time.time() - start}') + click.MultiCommand.format_help_text(self, ctx, formatter) def list_commands(self, ctx): diff --git a/app/services/clients/base_client.py b/app/services/clients/base_client.py index b8423d52..9ddc289d 100644 --- a/app/services/clients/base_client.py +++ b/app/services/clients/base_client.py @@ -44,6 +44,7 @@ def _request( if response.status_code not in self.retry_status: response.raise_for_status() return response + time.sleep(self.retry_interval) logger.debug(f'failed with over {self.retry_count} retries.') diff --git a/app/services/user_authentication/token_manager.py b/app/services/user_authentication/token_manager.py index 6200e81d..a2390497 100644 --- a/app/services/user_authentication/token_manager.py +++ b/app/services/user_authentication/token_manager.py @@ -2,6 +2,7 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +import logging import time import jwt @@ -20,11 +21,15 @@ class SrvTokenManager(BaseClient, metaclass=MetaService): def __init__(self): super().__init__(AppConfig.Connections.url_keycloak_token, 10) + import time + + start = time.time() user_config = UserConfig() if user_config.is_logged_in(): self.config = user_config else: raise Exception('Login session not found, please login first.') + logging.critical(f'Token Manager Init Time: {time.time() - start}') def update_token(self, access_token, refresh_token): self.config.access_token = access_token diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 2f338f18..41e06932 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -233,13 +233,23 @@ def remove_the_output_file(filepath: str) -> None: def get_latest_cli_version() -> Tuple[Version, str]: + import logging + import time + try: + start_time = time.time() httpx_client = BaseAuthClient(AppConfig.Connections.url_fileops_greenroom) + logging.critical(f'http client init time: {time.time() - start_time}') user_config = UserConfig() + logging.critical(f'user config init time: {time.time() - start_time}') + t1 = time.time() if not user_config.is_access_token_exists(): return Version('0.0.0') + logging.critical(f'Check token time: {time.time() - t1}') + t2 = time.time() response = httpx_client._get('v1/download/cli/presigned') + logging.critical(f'Get latest version time: {time.time() - t2}') result = response.json().get('result', {}) latest_version = result.get('linux', {}).get('version', '0.0.0') download_url = result.get('linux', {}).get('download_url', '') diff --git a/poetry.lock b/poetry.lock index b0917d33..1289856a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -603,13 +603,13 @@ files = [ [[package]] name = "pefile" -version = "2024.8.26" +version = "2023.2.7" description = "Python PE parsing module" optional = false python-versions = ">=3.6.0" files = [ - {file = "pefile-2024.8.26-py3-none-any.whl", hash = "sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f"}, - {file = "pefile-2024.8.26.tar.gz", hash = "sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632"}, + {file = "pefile-2023.2.7-py3-none-any.whl", hash = "sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6"}, + {file = "pefile-2023.2.7.tar.gz", hash = "sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc"}, ] [[package]] @@ -835,46 +835,47 @@ python-dotenv = ">=0.21.0" [[package]] name = "pyinstaller" -version = "5.13.2" +version = "6.11.1" description = "PyInstaller bundles a Python application and all its dependencies into a single package." optional = false -python-versions = "<3.13,>=3.7" +python-versions = "<3.14,>=3.8" files = [ - {file = "pyinstaller-5.13.2-py3-none-macosx_10_13_universal2.whl", hash = "sha256:16cbd66b59a37f4ee59373a003608d15df180a0d9eb1a29ff3bfbfae64b23d0f"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8f6dd0e797ae7efdd79226f78f35eb6a4981db16c13325e962a83395c0ec7420"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_i686.whl", hash = "sha256:65133ed89467edb2862036b35d7c5ebd381670412e1e4361215e289c786dd4e6"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:7d51734423685ab2a4324ab2981d9781b203dcae42839161a9ee98bfeaabdade"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_s390x.whl", hash = "sha256:2c2fe9c52cb4577a3ac39626b84cf16cf30c2792f785502661286184f162ae0d"}, - {file = "pyinstaller-5.13.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:c63ef6133eefe36c4b2f4daf4cfea3d6412ece2ca218f77aaf967e52a95ac9b8"}, - {file = "pyinstaller-5.13.2-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:aadafb6f213549a5906829bb252e586e2cf72a7fbdb5731810695e6516f0ab30"}, - {file = "pyinstaller-5.13.2-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:b2e1c7f5cceb5e9800927ddd51acf9cc78fbaa9e79e822c48b0ee52d9ce3c892"}, - {file = "pyinstaller-5.13.2-py3-none-win32.whl", hash = "sha256:421cd24f26144f19b66d3868b49ed673176765f92fa9f7914cd2158d25b6d17e"}, - {file = "pyinstaller-5.13.2-py3-none-win_amd64.whl", hash = "sha256:ddcc2b36052a70052479a9e5da1af067b4496f43686ca3cdda99f8367d0627e4"}, - {file = "pyinstaller-5.13.2-py3-none-win_arm64.whl", hash = "sha256:27cd64e7cc6b74c5b1066cbf47d75f940b71356166031deb9778a2579bb874c6"}, - {file = "pyinstaller-5.13.2.tar.gz", hash = "sha256:c8e5d3489c3a7cc5f8401c2d1f48a70e588f9967e391c3b06ddac1f685f8d5d2"}, + {file = "pyinstaller-6.11.1-py3-none-macosx_10_13_universal2.whl", hash = "sha256:44e36172de326af6d4e7663b12f71dbd34e2e3e02233e181e457394423daaf03"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:6d12c45a29add78039066a53fb05967afaa09a672426072b13816fe7676abfc4"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_i686.whl", hash = "sha256:ddc0fddd75f07f7e423da1f0822e389a42af011f9589e0269b87e0d89aa48c1f"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:0d6475559c4939f0735122989611d7f739ed3bf02f666ce31022928f7a7e4fda"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_s390x.whl", hash = "sha256:e21c7806e34f40181e7606926a14579f848bfb1dc52cbca7eea66eccccbfe977"}, + {file = "pyinstaller-6.11.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:32c742a24fe65d0702958fadf4040f76de85859c26bec0008766e5dbabc5b68f"}, + {file = "pyinstaller-6.11.1-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:208c0ef6dab0837a0a273ea32d1a3619a208e3d1fe3fec3785eea71a77fd00ce"}, + {file = "pyinstaller-6.11.1-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:ad84abf465bcda363c1d54eafa76745d77b6a8a713778348377dc98d12a452f7"}, + {file = "pyinstaller-6.11.1-py3-none-win32.whl", hash = "sha256:2e8365276c5131c9bef98e358fbc305e4022db8bedc9df479629d6414021956a"}, + {file = "pyinstaller-6.11.1-py3-none-win_amd64.whl", hash = "sha256:7ac83c0dc0e04357dab98c487e74ad2adb30e7eb186b58157a8faf46f1fa796f"}, + {file = "pyinstaller-6.11.1-py3-none-win_arm64.whl", hash = "sha256:35e6b8077d240600bb309ed68bb0b1453fd2b7ab740b66d000db7abae6244423"}, + {file = "pyinstaller-6.11.1.tar.gz", hash = "sha256:491dfb4d9d5d1d9650d9507daec1ff6829527a254d8e396badd60a0affcb72ef"}, ] [package.dependencies] altgraph = "*" macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} -pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""} -pyinstaller-hooks-contrib = ">=2021.4" +packaging = ">=22.0" +pefile = {version = ">=2022.5.30,<2024.8.26 || >2024.8.26", markers = "sys_platform == \"win32\""} +pyinstaller-hooks-contrib = ">=2024.9" pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} setuptools = ">=42.0.0" [package.extras] -encryption = ["tinyaes (>=1.0.0)"] +completion = ["argcomplete"] hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2024.9" +version = "2025.1" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.8" files = [ - {file = "pyinstaller_hooks_contrib-2024.9-py3-none-any.whl", hash = "sha256:1ddf9ba21d586afa84e505bb5c65fca10b22500bf3fdb89ee2965b99da53b891"}, - {file = "pyinstaller_hooks_contrib-2024.9.tar.gz", hash = "sha256:4793869f370d1dc4806c101efd2890e3c3e703467d8d27bb5a3db005ebfb008d"}, + {file = "pyinstaller_hooks_contrib-2025.1-py3-none-any.whl", hash = "sha256:d3c799470cbc0bda60dcc8e6b4ab976777532b77621337f2037f558905e3a8e9"}, + {file = "pyinstaller_hooks_contrib-2025.1.tar.gz", hash = "sha256:130818f9e9a0a7f2261f1fd66054966a3a50c99d000981c5d1db11d3ad0c6ab2"}, ] [package.dependencies] @@ -1236,23 +1237,23 @@ idna2008 = ["idna"] [[package]] name = "setuptools" -version = "75.2.0" +version = "75.8.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "setuptools-75.2.0-py3-none-any.whl", hash = "sha256:a7fcb66f68b4d9e8e66b42f9876150a3371558f98fa32222ffaa5bced76406f8"}, - {file = "setuptools-75.2.0.tar.gz", hash = "sha256:753bb6ebf1f465a1912e19ed1d41f403a79173a9acf66a42e7e6aec45c3c16ec"}, + {file = "setuptools-75.8.0-py3-none-any.whl", hash = "sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3"}, + {file = "setuptools-75.8.0.tar.gz", hash = "sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] +core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.11.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "six" @@ -1378,4 +1379,4 @@ windows = ["pywin32"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "4e0e8a667efa7cca18858b04d0f2658bb054ee3189a7457fafba1ca98e14f38f" +content-hash = "103a2f6b95b848dc2e257c111a915811517ccc325ba54178311d1aaec0b4873b" diff --git a/pyproject.toml b/pyproject.toml index 55fda464..c8c980a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "3.12.3" +version = "3.12.4" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] @@ -14,7 +14,6 @@ pre-commit = "^2.19.0" httpx = "^0.23.0" qrcode = "^7.4.2" pytest-click = "^1.1.0" -pyinstaller = "^5.13.0" pywin32 = { version = "^306", optional = true } pyjwt = "^2.9.0" requests = "^2.32.3" @@ -22,6 +21,7 @@ click = "^8.1.8" urllib3 = "^2.3.0" cffi = "^1.17.1" charset-normalizer = "3.4.1" +pyinstaller = "^6.11.1" [tool.poetry.extras] From c57936676ace04a42833852c0de952db24db45b7 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 5 Feb 2025 22:56:43 -0500 Subject: [PATCH 02/11] updating pipeline --- .github/workflows/build-and-publish.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index fca31514..786f8b68 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -10,6 +10,7 @@ on: push: branches: - main + - test20250205 jobs: extract-branch-name: @@ -24,7 +25,7 @@ jobs: push-binary-linux: needs: [ extract-branch-name ] - if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop'}} + if: ${{ needs.extract-branch-name.outputs.branch == 'test20250205' || needs.extract-branch-name.outputs.branch == 'develop'}} runs-on: ubuntu-20.04 outputs: upload_url: ${{steps.create_release.outputs.upload_url}} @@ -97,7 +98,7 @@ jobs: push-binary-macos: needs: [ push-binary-linux ] - if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop'}} + if: ${{ needs.extract-branch-name.outputs.branch == 'test20250205' || needs.extract-branch-name.outputs.branch == 'develop'}} runs-on: macos-13 steps: - name: Checkout repository @@ -148,7 +149,7 @@ jobs: push-binary-windows: needs: [ push-binary-linux ] - if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop'}} + if: ${{ needs.extract-branch-name.outputs.branch == 'test20250205' || needs.extract-branch-name.outputs.branch == 'develop'}} runs-on: windows-2022 steps: - name: Checkout repository From 8a6c09b91dc6e8b916109afe9925199157e635ed Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 5 Feb 2025 23:27:39 -0500 Subject: [PATCH 03/11] update logging level --- app/configs/user_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/configs/user_config.py b/app/configs/user_config.py index 4121aca3..1daa2347 100644 --- a/app/configs/user_config.py +++ b/app/configs/user_config.py @@ -169,7 +169,7 @@ def api_key(self): start_time = time.time() api_key = decryption(self.config['USER']['api_key'], self.secret) - logging.info(f'api_key decryption time: {time.time() - start_time}') + logging.critical(f'api_key decryption time: {time.time() - start_time}') return api_key @api_key.setter @@ -183,7 +183,7 @@ def access_token(self): start_time = time.time() access_token = decryption(self.config['USER']['access_token'], self.secret) - logging.info(f'access_token decryption time: {time.time() - start_time}') + logging.critical(f'access_token decryption time: {time.time() - start_time}') return access_token @access_token.setter @@ -197,7 +197,7 @@ def refresh_token(self): start_time = time.time() refresh_token = decryption(self.config['USER']['refresh_token'], self.secret) - logging.info(f'refresh_token decryption time: {time.time() - start_time}') + logging.critical(f'refresh_token decryption time: {time.time() - start_time}') return refresh_token @refresh_token.setter From 2547db4814e4a2a659f3335883f0df79ef34307d Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 6 Feb 2025 11:25:21 -0500 Subject: [PATCH 04/11] optimize token logic --- app/configs/user_config.py | 52 ++++++++----------- .../user_authentication/token_manager.py | 11 +--- app/utils/aggregated.py | 6 ++- 3 files changed, 28 insertions(+), 41 deletions(-) diff --git a/app/configs/user_config.py b/app/configs/user_config.py index 1daa2347..2ab15027 100644 --- a/app/configs/user_config.py +++ b/app/configs/user_config.py @@ -3,7 +3,6 @@ # Contact Indoc Systems for any questions regarding the use of this source code. import configparser -import logging import platform import sys import time @@ -43,6 +42,11 @@ class UserConfig(metaclass=Singleton): This user config is global. """ + _api_key: str + _access_token: str + _refresh_token: str + _username: str + def __init__( self, config_path: Union[str, Path, None] = None, @@ -106,7 +110,11 @@ def __init__( } self.save() - # print(f'save config time: {time.time() - t2}') + # load all config into memory + self._api_key = decryption(self.config['USER']['api_key'], self.secret) + self._access_token = decryption(self.config['USER']['access_token'], self.secret) + self._refresh_token = decryption(self.config['USER']['refresh_token'], self.secret) + self._username = decryption(self.config['USER']['username'], self.secret) def _check_user_permissions(self, path: Path, expected_bits: Iterable[int]) -> Union[str, None]: """Check if file or folder is owned by the user and has proper access mode.""" @@ -149,59 +157,45 @@ def is_access_token_exists(self) -> bool: @property def username(self): - return decryption(self.config['USER']['username'], self.secret) + return self._username @username.setter def username(self, val): self.config['USER']['username'] = encryption(val, self.secret) - @property - def password(self): - return decryption(self.config['USER']['password'], self.secret) + # @property + # def password(self): + # return decryption(self.config['USER']['password'], self.secret) - @password.setter - def password(self, val): - self.config['USER']['password'] = encryption(val, self.secret) + # @password.setter + # def password(self, val): + # self.config['USER']['password'] = encryption(val, self.secret) @property def api_key(self): - import time - - start_time = time.time() - api_key = decryption(self.config['USER']['api_key'], self.secret) - logging.critical(f'api_key decryption time: {time.time() - start_time}') - return api_key + return self._api_key @api_key.setter def api_key(self, val): + self._api_key = val self.config['USER']['api_key'] = encryption(val, self.secret) @property def access_token(self): - # return decryption(self.config['USER'].get('access_token', ''), self.secret) - import time - - start_time = time.time() - access_token = decryption(self.config['USER']['access_token'], self.secret) - logging.critical(f'access_token decryption time: {time.time() - start_time}') - return access_token + return self._access_token @access_token.setter def access_token(self, val): + self._access_token = val self.config['USER']['access_token'] = encryption(val, self.secret) @property def refresh_token(self): - # return decryption(self.config['USER']['refresh_token'], self.secret) - import time - - start_time = time.time() - refresh_token = decryption(self.config['USER']['refresh_token'], self.secret) - logging.critical(f'refresh_token decryption time: {time.time() - start_time}') - return refresh_token + return self._refresh_token @refresh_token.setter def refresh_token(self, val): + self._refresh_token = val self.config['USER']['refresh_token'] = encryption(val, self.secret) @property diff --git a/app/services/user_authentication/token_manager.py b/app/services/user_authentication/token_manager.py index a2390497..8340456c 100644 --- a/app/services/user_authentication/token_manager.py +++ b/app/services/user_authentication/token_manager.py @@ -2,7 +2,6 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. -import logging import time import jwt @@ -21,15 +20,7 @@ class SrvTokenManager(BaseClient, metaclass=MetaService): def __init__(self): super().__init__(AppConfig.Connections.url_keycloak_token, 10) - import time - - start = time.time() - user_config = UserConfig() - if user_config.is_logged_in(): - self.config = user_config - else: - raise Exception('Login session not found, please login first.') - logging.critical(f'Token Manager Init Time: {time.time() - start}') + self.config = UserConfig() def update_token(self, access_token, refresh_token): self.config.access_token = access_token diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 41e06932..375de75c 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -20,6 +20,7 @@ from app.models.item import ItemStatus from app.models.item import ItemType from app.services.clients.base_auth_client import BaseAuthClient +from app.services.clients.base_auth_client import BaseClient from app.services.logger_services.debugging_log import debug_logger from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import SrvErrorHandler @@ -238,7 +239,7 @@ def get_latest_cli_version() -> Tuple[Version, str]: try: start_time = time.time() - httpx_client = BaseAuthClient(AppConfig.Connections.url_fileops_greenroom) + httpx_client = BaseClient(AppConfig.Connections.url_fileops_greenroom) logging.critical(f'http client init time: {time.time() - start_time}') user_config = UserConfig() logging.critical(f'user config init time: {time.time() - start_time}') @@ -248,7 +249,8 @@ def get_latest_cli_version() -> Tuple[Version, str]: logging.critical(f'Check token time: {time.time() - t1}') t2 = time.time() - response = httpx_client._get('v1/download/cli/presigned') + headers = {'Authorization': 'Bearer'} + response = httpx_client._get('v1/download/cli/presigned', headers=headers) logging.critical(f'Get latest version time: {time.time() - t2}') result = response.json().get('result', {}) latest_version = result.get('linux', {}).get('version', '0.0.0') From 06621e7f26d21b030958465765d08c43f8f005f5 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 20 Feb 2025 11:41:21 -0500 Subject: [PATCH 05/11] remove the thread for token refresh --- app/services/file_manager/file_upload/file_upload.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 33ab4551..4d49fa28 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -237,8 +237,8 @@ def simple_upload( # noqa: C901 # thread number +1 reserve one thread to refresh token # and remove the token decorator in functions - pool = ThreadPool(num_of_thread + 1) - pool.apply_async(upload_client.upload_token_refresh) + pool = ThreadPool(num_of_thread) + # pool.apply_async(upload_client.upload_token_refresh) on_success_res = [] file_object: FileObject @@ -341,8 +341,8 @@ def resume_upload( # thread number +1 reserve one thread to refresh token # and remove the token decorator in functions - pool = ThreadPool(num_of_thread + 1) - pool.apply_async(upload_client.upload_token_refresh) + pool = ThreadPool(num_of_thread) + # pool.apply_async(upload_client.upload_token_refresh) on_success_res = [] for file_object in unfinished_items: upload_client.stream_upload(file_object, pool) From 8659685c7e2f6b86eff4c68af2d284fe4cbdc12f Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 20 Feb 2025 11:46:05 -0500 Subject: [PATCH 06/11] replace hardcoded thread with dynamic setup which will always be number of thread +1 --- app/configs/app_config.py | 2 +- app/configs/config.py | 2 +- app/services/file_manager/file_upload/upload_client.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/configs/app_config.py b/app/configs/app_config.py index 7c902ef6..2b0f2f42 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -31,7 +31,7 @@ class Env: # set hard limit for pending jobs, otherwise cli will consume all memory # to cache jobs. If later on the speed of chunk deliver become faster, we # can increase the concurrency number. - num_of_jobs = ConfigClass.concurrent_job_limit + # num_of_jobs = ConfigClass.concurrent_job_limit github_url = 'PilotDataPlatform/cli' diff --git a/app/configs/config.py b/app/configs/config.py index 977bc270..3cddc461 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -37,7 +37,7 @@ class Settings(BaseSettings): apikey_endpoint: str = 'api-key' upload_batch_size: int = 100 - concurrent_job_limit: int = 10 + # concurrent_job_limit: int = 10 upload_chunk_size: int = 1024 * 1024 * 20 # 20MB def __init__(self, **data): diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index 6e4b28de..1c5e8ea8 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -309,7 +309,7 @@ def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> None: been uploaded. """ count = 0 - semaphore = threading.Semaphore(AppConfig.Env.num_of_jobs) + semaphore = threading.Semaphore(pool._processes + 1) def on_complete(result): semaphore.release() From 4ab025a1b76ac68145ac740a00c0adf908b0efe9 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 20 Feb 2025 12:03:58 -0500 Subject: [PATCH 07/11] optimize the long pulling when checking chunk uploading --- .../file_manager/file_upload/upload_client.py | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index 1c5e8ea8..b0370579 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -8,7 +8,6 @@ import math import os import threading -import time from logging import getLogger from multiprocessing.pool import ThreadPool from typing import Any @@ -22,7 +21,6 @@ import app.services.output_manager.message_handler as mhandler from app.configs.app_config import AppConfig -from app.configs.config import ConfigClass from app.configs.user_config import UserConfig from app.models.upload_form import generate_on_success_form from app.services.clients.base_auth_client import BaseAuthClient @@ -31,7 +29,6 @@ from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import SrvErrorHandler from app.services.user_authentication.decorator import require_valid_token -from app.services.user_authentication.token_manager import SrvTokenManager from app.utils.aggregated import get_file_info_by_geid from .exception import INVALID_CHUNK_ETAG @@ -92,7 +89,7 @@ def __init__( # for tracking the multi-threading chunk upload self.active_jobs = 0 self.lock = threading.Lock() - self.chunk_upload_done = threading.Event() + # self.chunk_upload_done = threading.Event() def generate_meta(self, local_path: str) -> Tuple[int, int]: """ @@ -310,13 +307,14 @@ def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> None: """ count = 0 semaphore = threading.Semaphore(pool._processes + 1) + chunk_upload_done = threading.Event() def on_complete(result): semaphore.release() with self.lock: self.active_jobs -= 1 if self.active_jobs == 0: - self.chunk_upload_done.set() + chunk_upload_done.set() # process on the file content f = open(file_object.local_path, 'rb') @@ -355,8 +353,14 @@ def on_complete(result): count += 1 + # for resumable check ONLY if user resume the upload at 100% + # just check if there is any active job, if not, set the event + while not chunk_upload_done.wait(timeout=60): + if self.active_jobs == 0: + chunk_upload_done.set() + logger.warning('Waiting for all the chunks to be uploaded, remaining jobs: %s', file_object.progress) + f.close() - self.chunk_upload_done.wait() def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str, etag: str, chunk_size: int) -> None: """ @@ -466,16 +470,3 @@ def check_status(self, file_object: FileObject) -> bool: def set_finish_upload(self): self.finish_upload = True - - def upload_token_refresh(self, azp: str = ConfigClass.keycloak_device_client_id): - token_manager = SrvTokenManager() - DEFAULT_INTERVAL = 2 # seconds to check if the upload is finished - total_count = 0 # when total_count equals token_refresh_interval, refresh token - while self.finish_upload is not True: - if total_count >= AppConfig.Env.token_refresh_interval: - token_manager.refresh(azp) - total_count = 0 - - # if not then sleep for DEFAULT_INTERVAL seconds - time.sleep(DEFAULT_INTERVAL) - total_count = total_count + DEFAULT_INTERVAL From bd61512040eecee0c50a67c9468914ef2f0c4050 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 20 Feb 2025 16:14:22 -0500 Subject: [PATCH 08/11] fixup the test cases --- .github/workflows/build-and-publish.yml | 1 - app/configs/user_config.py | 8 ------ .../file_manager/file_upload/file_upload.py | 6 ---- pyproject.toml | 2 +- tests/app/configs/test_user_config.py | 15 ++++++---- .../file_upload/test_upload_client.py | 28 ------------------- tests/conftest.py | 1 - 7 files changed, 11 insertions(+), 50 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 786f8b68..d4d45097 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -10,7 +10,6 @@ on: push: branches: - main - - test20250205 jobs: extract-branch-name: diff --git a/app/configs/user_config.py b/app/configs/user_config.py index 2ab15027..86dac7e9 100644 --- a/app/configs/user_config.py +++ b/app/configs/user_config.py @@ -163,14 +163,6 @@ def username(self): def username(self, val): self.config['USER']['username'] = encryption(val, self.secret) - # @property - # def password(self): - # return decryption(self.config['USER']['password'], self.secret) - - # @password.setter - # def password(self, val): - # self.config['USER']['password'] = encryption(val, self.secret) - @property def api_key(self): return self._api_key diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 4d49fa28..03b33ddf 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -233,12 +233,7 @@ def simple_upload( # noqa: C901 # now loop over each file under the folder and start # the chunk upload - - # thread number +1 reserve one thread to refresh token - # and remove the token decorator in functions - pool = ThreadPool(num_of_thread) - # pool.apply_async(upload_client.upload_token_refresh) on_success_res = [] file_object: FileObject @@ -342,7 +337,6 @@ def resume_upload( # and remove the token decorator in functions pool = ThreadPool(num_of_thread) - # pool.apply_async(upload_client.upload_token_refresh) on_success_res = [] for file_object in unfinished_items: upload_client.stream_upload(file_object, pool) diff --git a/pyproject.toml b/pyproject.toml index 25163091..9c1db4e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "3.13.0" +version = "3.14.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/configs/test_user_config.py b/tests/app/configs/test_user_config.py index 67ae0cb3..c55254c3 100644 --- a/tests/app/configs/test_user_config.py +++ b/tests/app/configs/test_user_config.py @@ -18,10 +18,11 @@ def error_log(mocker): class TestUserConfig: - def test__init__creates_config_folder_with_0700_and_file_with_0600_access_modes(self, tmp_path, fake): + def test__init__creates_config_folder_with_0700_and_file_with_0600_access_modes(self, tmp_path, fake, mocker): config_folder = tmp_path / fake.pystr() file_name = fake.pystr() + mocker.patch('app.configs.user_config.decryption', return_value='') UserConfig(config_folder, file_name) config_folder_mode = stat.S_IMODE(config_folder.stat().st_mode) @@ -77,15 +78,16 @@ def test__init__exits_with_error_when_config_file_does_not_have_expected_access_ error_log.assert_called_with(expected_message) def test__init__does_not_exit_with_error_when_config_folder_has_invalid_access_mode_and_is_cloud_mode_set_to_true( - self, tmp_path, fake + self, tmp_path, fake, mocker ): config_folder = tmp_path / fake.pystr() config_folder.mkdir(mode=0o0755) + mocker.patch('app.configs.user_config.decryption', return_value='') UserConfig(config_folder, is_cloud_mode=True) def test__init__does_not_exit_with_error_when_config_file_has_invalid_access_mode_and_is_cloud_mode_set_to_true( - self, tmp_path, fake + self, tmp_path, fake, mocker ): config_folder = tmp_path / fake.pystr() file_name = fake.pystr() @@ -93,17 +95,19 @@ def test__init__does_not_exit_with_error_when_config_file_has_invalid_access_mod config_folder.mkdir(mode=0o0700) config_file.touch(mode=0o0644) + mocker.patch('app.configs.user_config.decryption', return_value='') UserConfig(config_folder, file_name, is_cloud_mode=True) - def test__init__sets_is_cloud_mode_to_false_by_default(self, tmp_path, fake): + def test__init__sets_is_cloud_mode_to_false_by_default(self, tmp_path, fake, mocker): config_folder = tmp_path / fake.pystr() + mocker.patch('app.configs.user_config.decryption', return_value='') user_config = UserConfig(config_folder) assert user_config.is_cloud_mode is False def test__init__sets_is_cloud_mode_to_true_when_pyinstaller_bundle_params_are_set_and_cloud_mode_file_is_present( - self, tmp_path, monkeypatch + self, tmp_path, monkeypatch, mocker ): monkeypatch.setattr(sys, 'frozen', True, raising=False) monkeypatch.setattr(sys, '_MEIPASS', str(tmp_path), raising=False) @@ -111,6 +115,7 @@ def test__init__sets_is_cloud_mode_to_true_when_pyinstaller_bundle_params_are_se cloud_mode_file = tmp_path / 'ENABLE_CLOUD_MODE' cloud_mode_file.touch() + mocker.patch('app.configs.user_config.decryption', return_value='') user_config = UserConfig() assert user_config.is_cloud_mode is True diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py index 2114b51c..a72cc769 100644 --- a/tests/app/services/file_manager/file_upload/test_upload_client.py +++ b/tests/app/services/file_manager/file_upload/test_upload_client.py @@ -7,9 +7,7 @@ import math import re from functools import wraps -from multiprocessing import TimeoutError from multiprocessing.pool import ThreadPool -from time import sleep import click import pytest @@ -228,32 +226,6 @@ def test_stream_upload_failed_with_etag_mismatch(mocker): pool.join() -def test_token_refresh_auto(mocker): - AppConfig.Env.token_refresh_interval = 1 - - token_refresh_mock = mocker.patch( - 'app.services.user_authentication.token_manager.SrvTokenManager.refresh', return_value=None - ) - - upload_client = UploadClient('project_code', 'parent_folder_id') - pool = ThreadPool(2) - async_fun = pool.apply_async(upload_client.upload_token_refresh) - sleep(3) - upload_client.set_finish_upload() - - # add the timeout to avoid the test stuck - try: - async_fun.get(timeout=5) - except TimeoutError: - raise AssertionError('token refresh failed') - - pool.close() - pool.join() - - # make sure the token refresh function is called - token_refresh_mock.assert_called_once() - - def test_resumable_pre_upload_success(httpx_mock, mocker): mocker.patch( 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', diff --git a/tests/conftest.py b/tests/conftest.py index 69953fb2..7e7ad4b7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,7 +30,6 @@ def mock_settings(monkeypatch, mocker): monkeypatch.setattr(AppConfig.Connections, 'url_keycloak', 'http://url_keycloak') monkeypatch.setattr(AppConfig.Connections, 'url_portal', 'http://bff_cli') monkeypatch.setattr(UserConfig, 'username', 'test-user') - monkeypatch.setattr(UserConfig, 'password', 'test-password') monkeypatch.setattr(UserConfig, 'api_key', 'test-api-key') monkeypatch.setattr(UserConfig, 'access_token', 'test-access-token') monkeypatch.setattr(UserConfig, 'refresh_token', 'test-refresh-token') From dafb00fee678bc773b2bbe07a173a92d7805deb9 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 20 Feb 2025 16:16:16 -0500 Subject: [PATCH 09/11] remove testing code --- .github/workflows/build-and-publish.yml | 6 +++--- app/configs/app_config.py | 4 ---- app/configs/config.py | 1 - 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index d4d45097..fca31514 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -24,7 +24,7 @@ jobs: push-binary-linux: needs: [ extract-branch-name ] - if: ${{ needs.extract-branch-name.outputs.branch == 'test20250205' || needs.extract-branch-name.outputs.branch == 'develop'}} + if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop'}} runs-on: ubuntu-20.04 outputs: upload_url: ${{steps.create_release.outputs.upload_url}} @@ -97,7 +97,7 @@ jobs: push-binary-macos: needs: [ push-binary-linux ] - if: ${{ needs.extract-branch-name.outputs.branch == 'test20250205' || needs.extract-branch-name.outputs.branch == 'develop'}} + if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop'}} runs-on: macos-13 steps: - name: Checkout repository @@ -148,7 +148,7 @@ jobs: push-binary-windows: needs: [ push-binary-linux ] - if: ${{ needs.extract-branch-name.outputs.branch == 'test20250205' || needs.extract-branch-name.outputs.branch == 'develop'}} + if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop'}} runs-on: windows-2022 steps: - name: Checkout repository diff --git a/app/configs/app_config.py b/app/configs/app_config.py index 2b0f2f42..54791112 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -28,10 +28,6 @@ class Env: greenroom_bucket_prefix = 'gr' # the number of items to active interative mode interative_threshold = 10 - # set hard limit for pending jobs, otherwise cli will consume all memory - # to cache jobs. If later on the speed of chunk deliver become faster, we - # can increase the concurrency number. - # num_of_jobs = ConfigClass.concurrent_job_limit github_url = 'PilotDataPlatform/cli' diff --git a/app/configs/config.py b/app/configs/config.py index 3cddc461..634d8337 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -37,7 +37,6 @@ class Settings(BaseSettings): apikey_endpoint: str = 'api-key' upload_batch_size: int = 100 - # concurrent_job_limit: int = 10 upload_chunk_size: int = 1024 * 1024 * 20 # 20MB def __init__(self, **data): From 4b7249e724c92a5fc7640630da5a2a891a697c12 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 20 Feb 2025 16:17:46 -0500 Subject: [PATCH 10/11] remove testing code --- app/services/file_manager/file_upload/upload_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index b0370579..6c5ad80f 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -89,7 +89,6 @@ def __init__( # for tracking the multi-threading chunk upload self.active_jobs = 0 self.lock = threading.Lock() - # self.chunk_upload_done = threading.Event() def generate_meta(self, local_path: str) -> Tuple[int, int]: """ From a7e41e96f305b5000a077d8415de097d2746655a Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 20 Feb 2025 16:35:05 -0500 Subject: [PATCH 11/11] move the user_config init in each class into init function instead of directly under the class --- app/pilotcli.py | 6 ------ app/services/clients/base_auth_client.py | 3 ++- app/services/file_manager/file_manifests.py | 4 +++- app/services/file_manager/file_tag.py | 3 ++- app/services/file_manager/file_upload/file_upload.py | 3 --- tests/conftest.py | 4 +++- 6 files changed, 10 insertions(+), 13 deletions(-) diff --git a/app/pilotcli.py b/app/pilotcli.py index 1ae659ee..cccee02c 100644 --- a/app/pilotcli.py +++ b/app/pilotcli.py @@ -52,16 +52,10 @@ def write_dl(self, rows, col_max=80, col_spacing=2): class ComplexCLI(click.MultiCommand): def format_help_text(self, ctx, formatter): - import logging - import time - - start = time.time() latest_version, download_url = get_latest_cli_version() if Version(pkg_resources.get_distribution('app').version) < latest_version: mhandler.SrvOutPutHandler.newer_version_available(latest_version, download_url) - logging.critical(f'Elapsed time: {time.time() - start}') - click.MultiCommand.format_help_text(self, ctx, formatter) def list_commands(self, ctx): diff --git a/app/services/clients/base_auth_client.py b/app/services/clients/base_auth_client.py index 8bcf551f..01c47b68 100644 --- a/app/services/clients/base_auth_client.py +++ b/app/services/clients/base_auth_client.py @@ -17,11 +17,12 @@ class BaseAuthClient(BaseClient): token_manager: SrvTokenManager - user = UserConfig() + user: UserConfig def __init__(self, endpoint: str, timeout: int = 10) -> None: super().__init__(endpoint, timeout) + self.user = UserConfig() self.token_manager = SrvTokenManager() self.headers = { 'Authorization': 'Bearer ' + self.user.access_token, diff --git a/app/services/file_manager/file_manifests.py b/app/services/file_manager/file_manifests.py index 2e304a7d..24bdca53 100644 --- a/app/services/file_manager/file_manifests.py +++ b/app/services/file_manager/file_manifests.py @@ -28,11 +28,13 @@ def dupe_checking_hook(pairs): class SrvFileManifests(BaseAuthClient, metaclass=MetaService): app_config = AppConfig() - user = UserConfig() + user: UserConfig def __init__(self, interactive=True): super().__init__(self.app_config.Connections.url_bff) + self.user = UserConfig() + self.interactive = interactive self.endpoint = self.app_config.Connections.url_bff + '/v1' diff --git a/app/services/file_manager/file_tag.py b/app/services/file_manager/file_tag.py index 5007c266..711dcffd 100644 --- a/app/services/file_manager/file_tag.py +++ b/app/services/file_manager/file_tag.py @@ -17,10 +17,11 @@ class SrvFileTag(metaclass=MetaService): appconfig = AppConfig() - user = UserConfig() + user: UserConfig def __init__(self, interactive=True): self.interactive = interactive + self.user = UserConfig() @staticmethod def validate_tag(tag): diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 03b33ddf..93abdc3d 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -333,9 +333,6 @@ def resume_upload( mhandler.SrvOutPutHandler.resume_check_success() # lastly, start resumable upload for the rest of the chunks - # thread number +1 reserve one thread to refresh token - # and remove the token decorator in functions - pool = ThreadPool(num_of_thread) on_success_res = [] for file_object in unfinished_items: diff --git a/tests/conftest.py b/tests/conftest.py index 7e7ad4b7..9072d179 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,7 +14,9 @@ @pytest.fixture(autouse=True) -def reset_singletons(): +def reset_singletons(mocker): + mocker.patch('app.configs.user_config.decryption', return_value='') + Singleton._instances = {}