From 77840b5af5c1b3ea3d1d267aa6d58f51d7ce3f1d Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Wed, 25 Jun 2025 08:46:56 +0200 Subject: [PATCH 1/9] added support for new workspaces features --- kbcstorage/branches.py | 7 +++++++ kbcstorage/workspaces.py | 39 ++++++++++++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/kbcstorage/branches.py b/kbcstorage/branches.py index 1e4ecd9..48011a0 100644 --- a/kbcstorage/branches.py +++ b/kbcstorage/branches.py @@ -41,3 +41,10 @@ def metadata(self, branch_id="default"): url = f"{self.base_url}branch/{branch_id}/metadata" return self._get(url) + + def branch_detail(self, branch_id="default"): + """ + Get branch details + """ + url = f"{self.base_url}dev-branches/{branch_id}" + return self._get(url) diff --git a/kbcstorage/workspaces.py b/kbcstorage/workspaces.py index f6f3135..018ca9b 100644 --- a/kbcstorage/workspaces.py +++ b/kbcstorage/workspaces.py @@ -72,7 +72,7 @@ def detail(self, workspace_id): url = '{}/{}'.format(self.base_url, workspace_id) return self._get(url) - def create(self, backend=None, timeout=None): + def create(self, backend=None, timeout=None, login_type=None, public_key=None, read_all_objects=False): """ Create a new Workspace and return the credentials. @@ -87,7 +87,10 @@ def create(self, backend=None, timeout=None): """ body = { 'backend': backend, - 'statementTimeoutSeconds': timeout + 'statementTimeoutSeconds': timeout, + 'loginType': login_type, + 'publicKey': public_key, + 'readOnlyStorageAccess': str(read_all_objects).lower() } return self._post(self.base_url, data=body) @@ -122,7 +125,17 @@ def reset_password(self, workspace_id): url = '{}/{}/password'.format(self.base_url, workspace_id) return self._post(url) - def load_tables(self, workspace_id, table_mapping, preserve=None): + def set_public_key(self, workspace_id, public_key): + """ + Set the public key for the workspace. + """ + data = { + 'publicKey': public_key + } + url = '{}/{}/public-key'.format(self.base_url, workspace_id) + return self._post(url, json=data) + + def load_tables(self, workspace_id: int | str, table_mapping: dict | list[dict], preserve=True, load_type='load'): """ Load tabes from storage into a workspace. @@ -140,11 +153,23 @@ def load_tables(self, workspace_id, table_mapping, preserve=None): Todo: * Column data types. """ - body = _make_body(table_mapping) - body['preserve'] = preserve - url = '{}/{}/load'.format(self.base_url, workspace_id) + load_type = load_type.lower() + if load_type not in ['load', 'load-clone']: + raise ValueError("Invalid load_type: {}, supports only load and load-clone".format(load_type)) + + url = '{}/{}/{}'.format(self.base_url, workspace_id, load_type) + + req = None + if isinstance(table_mapping, dict): + body = _make_body(table_mapping) + body['preserve'] = str(preserve).lower() + req = self._post(url, data=body) + elif isinstance(table_mapping, list): + body = {'input': table_mapping} + body['preserve'] = str(preserve).lower() + req = self._post(url, json=body) - return self._post(url, data=body) + return req def load_files(self, workspace_id, file_mapping): """ From 61dd2870573b497eda21ddb7316f52f0321bfe9c Mon Sep 17 00:00:00 2001 From: soustruh Date: Wed, 25 Jun 2025 10:50:42 +0200 Subject: [PATCH 2/9] update deprecated image artifact GitHub action --- .github/workflows/push.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index f29b836..cbb2c5f 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -49,9 +49,9 @@ jobs: echo "is_semantic_tag=$IS_SEMANTIC_TAG" >> $GITHUB_OUTPUT - name: Upload image - uses: ishworkh/docker-image-artifact-upload@v1 + uses: ishworkh/container-image-artifact-upload@v2.0.0 with: - image: "sapi-python-client" + image: ${{ env.APP_IMAGE }} retention_days: "1" tests_aws: @@ -63,9 +63,9 @@ jobs: uses: actions/checkout@v3 - name: Download image - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/container-image-artifact-download@v2.0.0 with: - image: "sapi-python-client" + image: ${{ env.APP_IMAGE }} - name: Run Tests run: | @@ -80,9 +80,9 @@ jobs: uses: actions/checkout@v3 - name: Download image - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/container-image-artifact-download@v2.0.0 with: - image: "sapi-python-client" + image: ${{ env.APP_IMAGE }} - name: Run Tests run: | @@ -97,9 +97,9 @@ jobs: uses: actions/checkout@v3 - name: Download image - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/container-image-artifact-download@v2.0.0 with: - image: "sapi-python-client" + image: ${{ env.APP_IMAGE }} - name: Run Tests run: | From 9163aa36ce2218936e645943a81ca3d7fccf1026 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Wed, 25 Jun 2025 12:14:17 +0200 Subject: [PATCH 3/9] fix typehint --- kbcstorage/workspaces.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kbcstorage/workspaces.py b/kbcstorage/workspaces.py index 018ca9b..a9f0b9b 100644 --- a/kbcstorage/workspaces.py +++ b/kbcstorage/workspaces.py @@ -9,7 +9,7 @@ from kbcstorage.base import Endpoint from kbcstorage.files import Files from kbcstorage.jobs import Jobs - +from typing import List # the legacy Workspaces class below unfortunately defines its own method called list def _make_body(mapping, source_key='source'): """ @@ -135,17 +135,18 @@ def set_public_key(self, workspace_id, public_key): url = '{}/{}/public-key'.format(self.base_url, workspace_id) return self._post(url, json=data) - def load_tables(self, workspace_id: int | str, table_mapping: dict | list[dict], preserve=True, load_type='load'): + def load_tables(self, workspace_id: int | str, table_mapping: dict | List[dict], preserve=True, load_type='load'): """ Load tabes from storage into a workspace. Args: workspace_id (int or str): The id of the workspace to which to load the tables. - table_mapping (:obj:`dict`): Source table names mapped to - destination table names. + table_mapping (:obj:`dict` or :obj:`list`): Source table names mapped to + destination table names. or a list of dicts with detailed tables specification. preserve (bool): If False, drop tables, else keep tables in workspace. + load_type (str): Type of load, either 'load' or 'load-clone'. Defaults to 'load'. Raises: requests.HTTPError: If the API request fails. From d463e2d6f97d2c876590aa8086b2518119b4a747 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Thu, 26 Jun 2025 08:51:07 +0200 Subject: [PATCH 4/9] add list_config_workspaces --- kbcstorage/configurations.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/kbcstorage/configurations.py b/kbcstorage/configurations.py index 0e2e8da..857590d 100644 --- a/kbcstorage/configurations.py +++ b/kbcstorage/configurations.py @@ -79,6 +79,22 @@ def list(self, component_id): url = '{}/{}/configs'.format(self.base_url, component_id) return self._get(url) + def list_config_workspaces(self, component_id, config_id): + """ + Lists workspaces for component configuration. + + Args: + component_id (str): The id of the component. + config_id (str): The id of the configuration. + + Raises: + requests.HTTPError: If the API request fails. + """ + if not isinstance(component_id, str) or component_id == '': + raise ValueError("Invalid component_id '{}'.".format(component_id)) + url = '{}/{}/configs/{}/workspaces'.format(self.base_url, component_id, config_id) + return self._get(url) + def create(self, component_id, name, description='', configuration=None, state=None, change_description='', is_disabled=False, configuration_id=None): """ From 4c1e2cd5467e9a64160b37dae6082978b3b1f324 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Thu, 26 Jun 2025 08:57:53 +0200 Subject: [PATCH 5/9] flake8 --- kbcstorage/workspaces.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kbcstorage/workspaces.py b/kbcstorage/workspaces.py index a9f0b9b..0aebf32 100644 --- a/kbcstorage/workspaces.py +++ b/kbcstorage/workspaces.py @@ -11,6 +11,7 @@ from kbcstorage.jobs import Jobs from typing import List # the legacy Workspaces class below unfortunately defines its own method called list + def _make_body(mapping, source_key='source'): """ Given a dict mapping Keboola tables to aliases, construct the body of From c5f7f71a9bae993372e61d39c7995dad2ce8a0e4 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Thu, 10 Jul 2025 15:58:02 +0200 Subject: [PATCH 6/9] review --- kbcstorage/workspaces.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/kbcstorage/workspaces.py b/kbcstorage/workspaces.py index 0aebf32..7432dcb 100644 --- a/kbcstorage/workspaces.py +++ b/kbcstorage/workspaces.py @@ -12,7 +12,7 @@ from typing import List # the legacy Workspaces class below unfortunately defines its own method called list -def _make_body(mapping, source_key='source'): +def _make_body(mapping, source_key='source', preserve: bool = True): """ Given a dict mapping Keboola tables to aliases, construct the body of the HTTP request to load said tables. @@ -22,7 +22,7 @@ def _make_body(mapping, source_key='source'): be loaded (ie. 'in.c-bucker.table_name') and values contain the aliases to which they will be loaded (ie. 'table_name'). """ - body = {} + body = {'preserve': str(preserve).lower()} template = 'input[{0}][{1}]' for i, (k, v) in enumerate(mapping.items()): body[template.format(i, source_key)] = k @@ -91,7 +91,7 @@ def create(self, backend=None, timeout=None, login_type=None, public_key=None, r 'statementTimeoutSeconds': timeout, 'loginType': login_type, 'publicKey': public_key, - 'readOnlyStorageAccess': str(read_all_objects).lower() + 'readOnlyStorageAccess': str(read_all_objects).lower() # convert bool to lowercase true or false } return self._post(self.base_url, data=body) @@ -159,16 +159,14 @@ def load_tables(self, workspace_id: int | str, table_mapping: dict | List[dict], if load_type not in ['load', 'load-clone']: raise ValueError("Invalid load_type: {}, supports only load and load-clone".format(load_type)) - url = '{}/{}/{}'.format(self.base_url, workspace_id, load_type) + url = "/".join([self.base_url, workspace_id, load_type]) req = None if isinstance(table_mapping, dict): - body = _make_body(table_mapping) - body['preserve'] = str(preserve).lower() + body = _make_body(table_mapping, preserve=preserve) req = self._post(url, data=body) elif isinstance(table_mapping, list): - body = {'input': table_mapping} - body['preserve'] = str(preserve).lower() + body = {'input': table_mapping, 'preserve': str(preserve).lower()} req = self._post(url, json=body) return req From 6d547a0a6098338c061336c38292fe098900b4bc Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Thu, 10 Jul 2025 16:08:31 +0200 Subject: [PATCH 7/9] review --- kbcstorage/configurations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kbcstorage/configurations.py b/kbcstorage/configurations.py index 857590d..23331f2 100644 --- a/kbcstorage/configurations.py +++ b/kbcstorage/configurations.py @@ -92,7 +92,7 @@ def list_config_workspaces(self, component_id, config_id): """ if not isinstance(component_id, str) or component_id == '': raise ValueError("Invalid component_id '{}'.".format(component_id)) - url = '{}/{}/configs/{}/workspaces'.format(self.base_url, component_id, config_id) + url = f'{self.base_url}/{component_id}/configs/{config_id}/workspaces' return self._get(url) def create(self, component_id, name, description='', configuration=None, state=None, change_description='', From 3bc164ea1cfda5b505af660a8eb57a550d258867 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Thu, 10 Jul 2025 16:10:55 +0200 Subject: [PATCH 8/9] review --- kbcstorage/workspaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kbcstorage/workspaces.py b/kbcstorage/workspaces.py index 7432dcb..4638d1b 100644 --- a/kbcstorage/workspaces.py +++ b/kbcstorage/workspaces.py @@ -159,7 +159,7 @@ def load_tables(self, workspace_id: int | str, table_mapping: dict | List[dict], if load_type not in ['load', 'load-clone']: raise ValueError("Invalid load_type: {}, supports only load and load-clone".format(load_type)) - url = "/".join([self.base_url, workspace_id, load_type]) + url = "/".join([self.base_url, str(workspace_id), load_type]) req = None if isinstance(table_mapping, dict): From 22080b5abecf83594355fa68fb5bc0f34486bd14 Mon Sep 17 00:00:00 2001 From: Josef Kudera <46950237+kudj@users.noreply.github.com> Date: Thu, 10 Jul 2025 18:30:35 +0200 Subject: [PATCH 9/9] check branch_id validity --- kbcstorage/branches.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kbcstorage/branches.py b/kbcstorage/branches.py index 48011a0..10c1520 100644 --- a/kbcstorage/branches.py +++ b/kbcstorage/branches.py @@ -46,5 +46,9 @@ def branch_detail(self, branch_id="default"): """ Get branch details """ + + if not isinstance(branch_id, str) or branch_id == "": + raise ValueError(f"Invalid branch_id '{branch_id}'") + url = f"{self.base_url}dev-branches/{branch_id}" return self._get(url)