From 78934bd6330a0d48bf85ee0abea66c09fa0c083b Mon Sep 17 00:00:00 2001 From: Gergely Imreh Date: Tue, 21 Dec 2021 20:57:22 +0800 Subject: [PATCH 1/8] clients: add API client to this SDK --- faculty/clients/__init__.py | 2 + faculty/clients/api.py | 628 ++++++++++++++++++++++++++++++++++++ tests/clients/test_api.py | 115 +++++++ 3 files changed, 745 insertions(+) create mode 100644 faculty/clients/api.py create mode 100644 tests/clients/test_api.py diff --git a/faculty/clients/__init__.py b/faculty/clients/__init__.py index 4f64af6b..b3403229 100644 --- a/faculty/clients/__init__.py +++ b/faculty/clients/__init__.py @@ -14,6 +14,7 @@ from faculty.clients.account import AccountClient +from faculty.clients.api import APIClient from faculty.clients.cluster import ClusterClient from faculty.clients.environment import EnvironmentClient from faculty.clients.experiment import ExperimentClient @@ -32,6 +33,7 @@ CLIENT_FOR_RESOURCE = { "account": AccountClient, + "api": APIClient, "cluster": ClusterClient, "environment": EnvironmentClient, "experiment": ExperimentClient, diff --git a/faculty/clients/api.py b/faculty/clients/api.py new file mode 100644 index 00000000..edfeb692 --- /dev/null +++ b/faculty/clients/api.py @@ -0,0 +1,628 @@ +# Copyright 2018-2021 Faculty Science Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Manage Faculty APIs. +""" +from collections import namedtuple +from enum import Enum + +from marshmallow import fields, post_load +from marshmallow_enum import EnumField + +from faculty.clients.base import BaseClient, BaseSchema +from faculty.clients.server import _ServerSchema as ServerSchema + +Instance = namedtuple("Instance", ["instance", "outdated", "key"]) +InstanceSize = namedtuple("InstanceSize", ["milli_cpus", "memory_mb"]) +DevInstance = namedtuple("DevInstance", ["instance", "key", "instance_id"]) +DevInstanceId = namedtuple("DevInstanceId", ["instance_id"]) +ResourceLimit = namedtuple( + "ResourceLimit", + [ + "allowed_max_milli_cpus", + "allowed_max_memory_mb", + "remaining_milli_cpus", + "remaining_memory_mb", + ], +) +APIInstance = namedtuple( + "APIInstance", + [ + "project_id", + "api_id", + "error_code", + "error", + "remaining_resource_limits", + ], +) + +APIKey = namedtuple("APIKey", ["id", "material", "enabled", "label"]) +ServerType = namedtuple("ServerType", ["instance_size_type", "instance_size"]) +WSGIDefinition = namedtuple( + "CommandDefinition", + [ + "api_type", + "working_directory", + "conda_environment", + "module", + "wsgi_object", + ], +) +PlumberDefinition = namedtuple( + "PlumberDefinition", ["api_type", "working_director", "script_name"] +) +ScriptDefinition = namedtuple( + "PlumberDefinition", ["api_type", "working_director", "script_name"] +) + +APIUpdate = namedtuple( + "APIUpdate", ["definition", "environment_ids", "default_server_size"] +) + +APIDefinition = namedtuple( + "APIDefinition", ["name", "subdomain", "description", "last_updated_at"] +) + +API = namedtuple( + "API", + [ + "author_id", + "created_at", + "default_server_size", + "definition", + "deployment_status", + "description", + "dev_instances", + "environment_ids", + "id", + "last_deployed_at", + "last_deployed_by", + "name", + "prod_instances", + "prod_keys", + "subdomain", + ], +) + + +class _APIKeySchema(BaseSchema): + id = fields.UUID(data_key="keyId", required=True) + material = fields.String(required=True) + # Development instance keys do not have these fields + enabled = fields.Boolean(load_default=None) + label = fields.String(load_default=None) + + @post_load + def make_apikey(self, data, **kwargs): + return APIKey(**data) + + +class RemainingResourceLimitSchema(BaseSchema): + allowed_max_milli_cpus = fields.Integer( + data_key="allowedMaxMilliCpus", required=True + ) + allowed_max_memory_mb = fields.Integer( + data_key="allowedMaxMemoryMb", required=True + ) + remaining_milli_cpus = fields.Integer( + data_key="remainingMilliCpus", required=True + ) + remaining_memory_mb = fields.Integer( + data_key="remaininigMemoryMb", required=True + ) + + +class DevInstanceIdSchema(BaseSchema): + instance_id = fields.UUID(data_key="instanceId") + + +class APIInstanceResponseSchema(BaseSchema): + project_id = fields.UUID(data_key="projectId") + api_id = fields.UUID(data_key="apiId") + error_code = fields.String(data_key="errorCode") + error = fields.String() + remaining_resource_limits = fields.Nested( + RemainingResourceLimitSchema, data_key="remainingResourceLimits" + ) + + +class APIDevInstanceResponseSchema(APIInstanceResponseSchema): + instance = fields.Nested(DevInstanceIdSchema) + key = fields.Nested(_APIKeySchema) + instance_id = fields.UUID(data_key="instanceId") + + +class InstanceSizeSchema(BaseSchema): + milli_cpus = fields.Integer(data_key="milliCpus", required=True) + memory_mb = fields.Integer(data_key="memoryMb", required=True) + + +class APIType(Enum): + WSGI = "wsgi" + SCRIPT = "script" + PLUMBER = "plumber" + + +class DeploymentStatus(Enum): + NOTDEPLOYED = "not-deployed" + STARTING = "starting" + DEPLOYED = "deployed" + ERROR = "error" + + +class InstanceSchema(BaseSchema): + instance = fields.Nested(ServerSchema, required=True) + outdated = fields.Boolean(required=True) + key = fields.Nested( + _APIKeySchema, load_default=None + ) # Production instances do not have this field + + @post_load + def make_instance(self, data, **kwargs): + return Instance(**data) + + +class CommandDefinitionSchema(BaseSchema): + api_type = EnumField( + APIType, by_value=True, required=True, data_key="type" + ) + working_directory = fields.String( + data_key="workingDirectory", + ) + conda_environment = fields.String(data_key="commandEnvironment") + module = fields.String() + wsgi_object = fields.String(data_key="wsgiObject") + script_name = fields.String(data_key="scriptName") + + @post_load + def make_command_definition(self, data, **kwargs): + if data["api_type"] == "wsgi": + return WSGIDefinition(**data) + elif data["api_type"] == "script": + return ScriptDefinition(**data) + elif data["api_type"] == "plumber": + return PlumberDefinition(**data) + + +class ServerTypeSchema(BaseSchema): + instance_size_type = fields.String( + data_key="instanceSizeType", required=True + ) + instance_size = fields.Nested(InstanceSizeSchema, data_key="instanceSize") + + +class _APISchema(BaseSchema): + definition = fields.Nested(CommandDefinitionSchema, data_key="definition") + environment_ids = fields.List(fields.UUID(), data_key="environmentIds") + default_server_size = fields.Nested( + ServerTypeSchema, data_key="defaultServerSize" + ) + name = fields.String(data_key="name") + subdomain = fields.String(data_key="subdomain") + description = fields.String(data_key="description") + last_updated_at = fields.DateTime(data_key="lastUpdatedAt") + + id = fields.UUID(data_key="apiId", required=True) + author_id = fields.UUID(data_key="authorId", required=True) + created_at = fields.DateTime(data_key="createdAt", required=True) + last_deployed_at = fields.DateTime( + data_key="lastDeployedAt", load_default=None + ) + last_deployed_by = fields.UUID( + data_key="lastDeployedBy", load_default=None + ) + # The following fields do not exist in the response if all APIs are listed + # TODO: what would be the best way to handle these missing fields for Marshmallow + deployment_status = EnumField( + DeploymentStatus, + by_value=True, + data_key="deploymentStatus", + load_default=None, + ) + prod_instances = fields.Nested( + InstanceSchema, + many=True, + data_key="prodInstances", + load_default=None, + ) + prod_keys = fields.Nested( + _APIKeySchema, + many=True, + data_key="prodKeys", + load_default=None, + ) + dev_instances = fields.Nested( + InstanceSchema, + many=True, + data_key="devInstances", + load_default=None, + ) + + @post_load + def make_api(self, data, **kwargs): + return API(**data) + + +class ListAllAPISchema(BaseSchema): + apis = fields.Nested(_APISchema(many=True)) + + @post_load + def make_list_apis(self, data, **kwargs): + # Flatten the API response + return data["apis"] + + +class APIClient(BaseClient): + """Client for the Faculty API deployment. + + Either build this client with a session directly, or use the + :func:`faculty.client` helper function: + + >>> client = faculty.client("api") + + Parameters + ---------- + url : str + The URL of the server management service. + session : faculty.session.Session + The session to use to make requests. + """ + + SERVICE_NAME = "aperture" + + def create( + self, + project_id, + api_definition, + subdomain, + name=None, + description=None, + ): + """Create a new API. + + Parameters + ---------- + project_id : uuid.UUID + The project to create the server in. + + Returns + ------- + """ + payload = {"definition": api_definition} + payload["subdomain"] = subdomain + if name: + payload["name"] = name + if description: + payload["description"] = description + else: + description = "" + + payload["environmentIds"] = [] + payload["defaultServerSize"] = { + "instanceSizeType": "custom", + "instanceSize": {"milliCpus": 1000, "memoryMb": 4096}, + } + return self._post( + "/project/{}/api".format(project_id), _APISchema(), json=payload + ) + + @staticmethod + def flask_definition(working_directory, module, wsgi_object): + return WSGIDefinition( + api_type="wsgi", + working_directory=working_directory, + conda_environment="Python3", + module=module, + wsgi_object=wsgi_object, + ) + + @staticmethod + def plumber_definition(working_directory, script_name): + def_dict = { + "api_type": "plumber", + "workingDirectory": working_directory, + "scriptName": script_name, + } + return PlumberDefinition( + api_type="plumber", + working_directory=working_directory, + script_name=script_name, + ) + + @staticmethod + def script_definition(working_directory, script_name): + return ScriptDefinition( + api_type="script", + working_directory=working_directory, + script_name=script_name, + ) + + def _get_current_api_definition(self, project_id, api_id): + self.get(project_id, api_id) + current_api = _APISchema().dump(current_api) + return current_api + + def get(self, project_id, api_id): + return self._get( + "/project/{}/api/{}".format(project_id, api_id), _APISchema() + ) + + def update( + self, + project_id, + api_id, + environment_ids=None, + api_definition=None, + server_type=None, + ): + if not current_api: + current_api = self._get_current_api_definition(project_id, api_id) + payload = {} + payload["defaultServerSize"] = current_api["defaultServerSize"] + payload["environmentIds"] = current_api["environmentIds"] + payload["definition"] = current_api["definition"] + if server_type: + payload["defaultServerSize"] = server_type + if environment_ids: + payload["environmentIds"] = environment_ids + + if api_definition: + payload["definition"] = api_definition + endpoint = "/project/{}/api/{}/definition".format(project_id, api_id) + return self._put(endpoint, _APISchema(), json=payload) + + def delete(self, project_id, api_id): + return self._delete_raw( + "/project/{}/api/{}".format(project_id, api_id) + ) + + def list(self, project_id): + """List APIs in a project. + + Parameters + ---------- + project_id : uuid.UUID + The project to list APIs in. + name : str, optional + If provided, only return APIs with this name. + + Returns + ------- + List[API] + The matching APIs. + """ + endpoint = "/project/{}/api".format(project_id) + return self._get(endpoint, _APISchema(many=True)) + + def create_production_key(self, project_id, api_id, label=None): + """Create a production key for a given API. + + Parameters + ---------- + project_id : uuid.UUID + The project where the API resides. + api_id : uuid.UUID + The API to create a new key in. + label : str, optional + If provided, set this value as the new key's label. + + Returns + ------- + APIKey + The newly created key. + """ + endpoint = "/project/{}/api/{}/key".format(project_id, api_id) + # TODO: maybe more sensible default, the UI doesn't allow empty name but the api accepts this + payload = {"label": label if label else ""} + return self._post(endpoint, _APIKeySchema(), json=payload) + + def list_production_keys(self, project_id, api_id): + """List all production API keys for a given API. + + Parameters + ---------- + project_id : uuid.UUID + The project where the API resides in. + api_id : uuid.UUID + The API where to list the production keys in. + + Returns + ------- + List[APIKey] + The production keys. + """ + endpoint = "/project/{}/api/{}/key".format(project_id, api_id) + return self._get(endpoint, _APIKeySchema(many=True)) + + def _toggle_production_key(self, project_id, api_id, key_id, enabled): + endpoint = "/project/{}/api/{}/key/{}/enabled".format( + project_id, api_id, key_id + ) + payload = {"enabled": enabled} + return self._put(endpoint, _APIKeySchema(), json=payload) + + def enable_production_key(self, project_id, api_id, key_id): + """Enable a specific production API key. + + Parameters + ---------- + project_id : uuid.UUID + The project in which the API resides. + api_id : uuid.UUID + The API in which to the production key resides. + key_id : uuid.UUID + The key to enable. + """ + return self._toggle_production_key(project_id, api_id, key_id, True) + + def disable_production_key(self, project_id, api_id, key_id): + """Disable a specific production API key. + + Parameters + ---------- + project_id : uuid.UUID + The project in which the API resides. + api_id : uuid.UUID + The API in which to the production key resides. + key_id : uuid.UUID + The key to disable. + """ + return self._toggle_production_key(project_id, api_id, key_id, False) + + def update_production_key(self, project_id, api_id, key_id, label): + """Disable a specific production API key. + + Parameters + ---------- + project_id : uuid.UUID + The project in which the API resides. + api_id : uuid.UUID + The API in which to the production key resides. + key_id : uuid.UUID + The key to update. + label : str + The new label to set to the given key. + + Returns + ------- + APIKey + The updated key. + """ + endpoint = "/project/{}/api/{}/key/{}".format( + project_id, api_id, key_id + ) + payload = {"label": label} + return self._put(endpoint, _APIKeySchema(), json=payload) + + def delete_production_key(self, project_id, api_id, key_id): + endpoint = "/project/{}/api/{}/key/{}".format( + project_id, api_id, key_id + ) + self._delete_raw(endpoint) + + def start( + self, + project_id, + api_id, + key_id, + server_type=None, + image_version=None, + current_api=None, + ): + if not current_api: + current_api = self._get_current_api_definition(project_id, api_id) + payload = { + "instanceSizeType": current_api["instanceSizeType"], + "instanceSize": { + "milliCpus": current_api["instnaceSize"]["milliCpus"], + "memoryMb": current_api["instnaceSize"]["memoryMb"], + }, + } + if server_type: + payload["instanceSizeType"] = server_type["instance_size_type"] + if server_type["instance_size"]: + instance_size = server_type["instance_size"] + payload["instanceSize"] = { + "milliCpus": instance_size.milli_cpus, + "memoryMb": instance_size.memory_mb, + } + if image_version: + payload["imageVersion"] = image_version + endpoint = "/project/{}/api/{}/prod/start".format(project_id, api_id) + return self._put(endpoint, APIInstanceResponseSchema(), json=payload) + + def stop(self, project_id, api_id): + endpoint = "/project/{}/api/{}/prod/stop".format(project_id, api_id) + self._put(endpoint, APIInstanceResponseSchema()) + + def restart( + self, + project_id, + api_id, + key_id, + server_type=None, + image_version=None, + current_api=None, + ): + if not current_api: + current_api = self._get_current_api_definition(project_id, api_id) + payload = { + "instanceSizeType": current_api["instanceSizeType"], + "instanceSize": { + "milliCpus": current_api["instnaceSize"]["milliCpus"], + "memoryMb": current_api["instnaceSize"]["memoryMb"], + }, + } + if server_type: + payload["instanceSizeType"] = server_type["instance_size_type"] + if server_type["instance_size"]: + instance_size = server_type["instance_size"] + payload["instanceSize"] = { + "milliCpus": instance_size.milli_cpus, + "memoryMb": instance_size.memory_mb, + } + if image_version: + payload["imageVersion"] = image_version + endpoint = "/project/{}/api/{}/prod/restart".format(project_id, api_id) + return self._put(endpoint, APIInstanceResponseSchema(), json=payload) + + def reload(self, project_id, api_id): + endpoint = "/project/{}/api/{}/prod/reload".format(project_id, api_id) + self._put(endpoint, APIInstanceResponseSchema()) + + def start_dev( + self, project_id, api_id, key_id, server_type=None, image_version=None + ): + + payload = {} + if server_type: + payload["instanceSizeType"] = server_type["instance_size_type"] + if server_type["instance_size"]: + instance_size = server_type["instance_size"] + payload["instanceSize"] = { + "milliCpus": instance_size["milli_cpus"], + "memoryMb": instance_size["memory_mb"], + } + if image_version: + payload["imageVersion"] = image_version + endpoint = "/project/{}/api/{}/dev/start".format(project_id, api_id) + return self._post( + endpoint, APIDevInstanceResponseSchema(), json=payload + ) + + def stop_dev(self, project_id, api_id, dev_instance_id): + endpoint = "/project/{}/api/{}/dev/{}".format( + project_id, api_id, dev_instance_id + ) + self._delete(endpoint, APIDevInstanceResponseSchema()) + + def reload_dev(self, project_id, api_id, dev_instance_id): + endpoint = "/project/{}/api/{}/dev/{}/reload".format( + project_id, api_id, dev_instance_id + ) + self._put(endpoint, APIDevInstanceResponseSchema()) + + def list_all(self): + """List all APIs on the Faculty deployment. + + This method requires administrative privileges not available to most + users. + + Returns + ------- + List[API] + The APIs. + """ + return self._get("/api", ListAllAPISchema()) diff --git a/tests/clients/test_api.py b/tests/clients/test_api.py new file mode 100644 index 00000000..764cd732 --- /dev/null +++ b/tests/clients/test_api.py @@ -0,0 +1,115 @@ +# Copyright 2018-2021 Faculty Science Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import secrets +import uuid +from datetime import datetime + +import pytest +from dateutil.tz import UTC +from marshmallow import ValidationError + +from faculty.clients.api import ( + API, + APIClient, + APIKey, + _APIKeySchema, + _APISchema, +) + +ENVIRONMENT_ID = uuid.uuid4() +OWNER_ID = uuid.uuid4() +PROJECT_ID = uuid.uuid4() +SERVER_ID = uuid.uuid4() +USER_ID = uuid.uuid4() + +# TODO: replace with full API for tests +TEST_API_ID = uuid.uuid4() + +PROD_KEY = APIKey( + id=uuid.uuid4(), + material=secrets.token_hex(16), + label="test_key", + enabled=True, +) + +PROD_KEY_BODY = { + "keyId": PROD_KEY.id, + "material": PROD_KEY.material, + "label": PROD_KEY.label, + "enabled": PROD_KEY.enabled, +} + + +def test_apikey_schema(): + data = _APIKeySchema().load(PROD_KEY_BODY) + assert data == PROD_KEY + + +def test_apikey_schema_invalid(): + with pytest.raises(ValidationError): + _APIKeySchema().load({}) + + +@pytest.mark.parametrize( + "key_starting_enabled", + [True, False], +) +def test_disable_production_key(mocker, key_starting_enabled): + disabled_prod_key = PROD_KEY._replace(enabled=key_starting_enabled) + + mocker.patch.object(APIClient, "_put", return_value=disabled_prod_key) + schema_mock = mocker.patch("faculty.clients.api._APIKeySchema") + + client = APIClient(mocker.Mock(), mocker.Mock()) + assert ( + client.disable_production_key(PROJECT_ID, TEST_API_ID, PROD_KEY.id) + == disabled_prod_key + ) + + schema_mock.assert_called_once_with() + APIClient._put.assert_called_once_with( + "/project/{}/api/{}/key/{}/enabled".format( + PROJECT_ID, TEST_API_ID, PROD_KEY.id + ), + schema_mock.return_value, + json={"enabled": False}, + ) + + +@pytest.mark.parametrize( + "key_starting_enabled", + [True, False], +) +def test_enable_production_key(mocker, key_starting_enabled): + disabled_prod_key = PROD_KEY._replace(enabled=key_starting_enabled) + + mocker.patch.object(APIClient, "_put", return_value=disabled_prod_key) + schema_mock = mocker.patch("faculty.clients.api._APIKeySchema") + + client = APIClient(mocker.Mock(), mocker.Mock()) + assert ( + client.enable_production_key(PROJECT_ID, TEST_API_ID, PROD_KEY.id) + == disabled_prod_key + ) + + schema_mock.assert_called_once_with() + APIClient._put.assert_called_once_with( + "/project/{}/api/{}/key/{}/enabled".format( + PROJECT_ID, TEST_API_ID, PROD_KEY.id + ), + schema_mock.return_value, + json={"enabled": True}, + ) From 9e129936ae3efcd55bb1955868846a26c03f0141 Mon Sep 17 00:00:00 2001 From: Gergely Imreh Date: Wed, 22 Dec 2021 21:11:24 +0800 Subject: [PATCH 2/8] working on the API creation call --- faculty/clients/api.py | 81 +++++++++++++++++++++------------------ tests/clients/test_api.py | 9 +---- 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/faculty/clients/api.py b/faculty/clients/api.py index edfeb692..6bb543a3 100644 --- a/faculty/clients/api.py +++ b/faculty/clients/api.py @@ -51,20 +51,19 @@ APIKey = namedtuple("APIKey", ["id", "material", "enabled", "label"]) ServerType = namedtuple("ServerType", ["instance_size_type", "instance_size"]) WSGIDefinition = namedtuple( - "CommandDefinition", + "WSGIDefinition", [ "api_type", "working_directory", - "conda_environment", "module", "wsgi_object", ], ) PlumberDefinition = namedtuple( - "PlumberDefinition", ["api_type", "working_director", "script_name"] + "PlumberDefinition", ["api_type", "working_directory", "script_name"] ) ScriptDefinition = namedtuple( - "PlumberDefinition", ["api_type", "working_director", "script_name"] + "ScriptDefinition", ["api_type", "working_directory", "script_name"] ) APIUpdate = namedtuple( @@ -149,7 +148,7 @@ class InstanceSizeSchema(BaseSchema): memory_mb = fields.Integer(data_key="memoryMb", required=True) -class APIType(Enum): +class APIType(str, Enum): WSGI = "wsgi" SCRIPT = "script" PLUMBER = "plumber" @@ -181,7 +180,6 @@ class CommandDefinitionSchema(BaseSchema): working_directory = fields.String( data_key="workingDirectory", ) - conda_environment = fields.String(data_key="commandEnvironment") module = fields.String() wsgi_object = fields.String(data_key="wsgiObject") script_name = fields.String(data_key="scriptName") @@ -287,56 +285,64 @@ def create( project_id, api_definition, subdomain, - name=None, - description=None, + name="", + description="", + environment_ids=[], + default_server_size=None, ): """Create a new API. Parameters ---------- project_id : uuid.UUID - The project to create the server in. - - Returns - ------- + The project to create the API in. + api_definition : CommandDefinition + The API's command definition: WSGI(Flask)/Plumber/Script(Custom). + subdomain : str + The subdomain where the API should run. + name : str + The API's name. + description : str + The API's description field. + environment_ids : List[uuid.UUID] + The environments to apply to the API's instances. + default_server_size : ServerType + The default server size to set """ - payload = {"definition": api_definition} - payload["subdomain"] = subdomain - if name: - payload["name"] = name - if description: - payload["description"] = description - else: - description = "" - - payload["environmentIds"] = [] - payload["defaultServerSize"] = { - "instanceSizeType": "custom", - "instanceSize": {"milliCpus": 1000, "memoryMb": 4096}, + payload = { + "definition": CommandDefinitionSchema().dump(api_definition) } - return self._post( - "/project/{}/api".format(project_id), _APISchema(), json=payload + payload["subdomain"] = subdomain + payload["name"] = name + payload["description"] = description + + payload["environmentIds"] = environment_ids + + # TODO: turn these into schema handling as well + if default_server_size is None: + default_server_size = { + "instanceSizeType": "custom", + "instanceSize": {"milliCpus": 1000, "memoryMb": 4096}, + } + payload["defaultServerSize"] = default_server_size + + return self._post_raw( + "/project/{}/api".format(project_id), json=payload ) @staticmethod def flask_definition(working_directory, module, wsgi_object): return WSGIDefinition( - api_type="wsgi", + api_type=APIType.WSGI, working_directory=working_directory, - conda_environment="Python3", module=module, wsgi_object=wsgi_object, ) @staticmethod def plumber_definition(working_directory, script_name): - def_dict = { - "api_type": "plumber", - "workingDirectory": working_directory, - "scriptName": script_name, - } return PlumberDefinition( - api_type="plumber", + api_type=APIType.PLUMBER, working_directory=working_directory, script_name=script_name, ) @@ -344,13 +350,13 @@ def plumber_definition(working_directory, script_name): @staticmethod def script_definition(working_directory, script_name): return ScriptDefinition( - api_type="script", + api_type=APIType.SCRIPT, working_directory=working_directory, script_name=script_name, ) def _get_current_api_definition(self, project_id, api_id): - self.get(project_id, api_id) + current_api = self.get(project_id, api_id) current_api = _APISchema().dump(current_api) return current_api @@ -366,6 +372,7 @@ def update( environment_ids=None, api_definition=None, server_type=None, + current_api=None, ): if not current_api: current_api = self._get_current_api_definition(project_id, api_id) diff --git a/tests/clients/test_api.py b/tests/clients/test_api.py index 764cd732..b6a43e64 100644 --- a/tests/clients/test_api.py +++ b/tests/clients/test_api.py @@ -21,13 +21,8 @@ from dateutil.tz import UTC from marshmallow import ValidationError -from faculty.clients.api import ( - API, - APIClient, - APIKey, - _APIKeySchema, - _APISchema, -) +from faculty.clients.api import (API, APIClient, APIKey, _APIKeySchema, + _APISchema) ENVIRONMENT_ID = uuid.uuid4() OWNER_ID = uuid.uuid4() From 6280e5cfb122ba067bc27d6e3e578ccb791626ac Mon Sep 17 00:00:00 2001 From: Gergely Imreh Date: Thu, 6 Jan 2022 17:53:53 +0800 Subject: [PATCH 3/8] rewrite based on PR feedback --- faculty/clients/api.py | 898 +++++++++++++++++++++----------------- tests/clients/test_api.py | 16 +- 2 files changed, 514 insertions(+), 400 deletions(-) diff --git a/faculty/clients/api.py b/faculty/clients/api.py index 6bb543a3..c36259f6 100644 --- a/faculty/clients/api.py +++ b/faculty/clients/api.py @@ -15,251 +15,118 @@ """ Manage Faculty APIs. """ -from collections import namedtuple +import uuid +from datetime import datetime from enum import Enum +from typing import List, Optional, Union +from attr import define from marshmallow import fields, post_load from marshmallow_enum import EnumField +from faculty._oneofschema import OneOfSchema from faculty.clients.base import BaseClient, BaseSchema -from faculty.clients.server import _ServerSchema as ServerSchema - -Instance = namedtuple("Instance", ["instance", "outdated", "key"]) -InstanceSize = namedtuple("InstanceSize", ["milli_cpus", "memory_mb"]) -DevInstance = namedtuple("DevInstance", ["instance", "key", "instance_id"]) -DevInstanceId = namedtuple("DevInstanceId", ["instance_id"]) -ResourceLimit = namedtuple( - "ResourceLimit", - [ - "allowed_max_milli_cpus", - "allowed_max_memory_mb", - "remaining_milli_cpus", - "remaining_memory_mb", - ], -) -APIInstance = namedtuple( - "APIInstance", - [ - "project_id", - "api_id", - "error_code", - "error", - "remaining_resource_limits", - ], -) - -APIKey = namedtuple("APIKey", ["id", "material", "enabled", "label"]) -ServerType = namedtuple("ServerType", ["instance_size_type", "instance_size"]) -WSGIDefinition = namedtuple( - "WSGIDefinition", - [ - "api_type", - "working_directory", - "module", - "wsgi_object", - ], -) -PlumberDefinition = namedtuple( - "PlumberDefinition", ["api_type", "working_directory", "script_name"] -) -ScriptDefinition = namedtuple( - "ScriptDefinition", ["api_type", "working_directory", "script_name"] -) - -APIUpdate = namedtuple( - "APIUpdate", ["definition", "environment_ids", "default_server_size"] -) - -APIDefinition = namedtuple( - "APIDefinition", ["name", "subdomain", "description", "last_updated_at"] -) - -API = namedtuple( - "API", - [ - "author_id", - "created_at", - "default_server_size", - "definition", - "deployment_status", - "description", - "dev_instances", - "environment_ids", - "id", - "last_deployed_at", - "last_deployed_by", - "name", - "prod_instances", - "prod_keys", - "subdomain", - ], -) +from faculty.clients.job import InstanceSize, _InstanceSizeSchema +from faculty.clients.server import Server, _ServerSchema -class _APIKeySchema(BaseSchema): - id = fields.UUID(data_key="keyId", required=True) - material = fields.String(required=True) - # Development instance keys do not have these fields - enabled = fields.Boolean(load_default=None) - label = fields.String(load_default=None) +class DeploymentStatus(Enum): + """An enumeration of possible API deployment statuses.""" - @post_load - def make_apikey(self, data, **kwargs): - return APIKey(**data) + NOTDEPLOYED = "not-deployed" + STARTING = "starting" + DEPLOYED = "deployed" + ERROR = "error" -class RemainingResourceLimitSchema(BaseSchema): - allowed_max_milli_cpus = fields.Integer( - data_key="allowedMaxMilliCpus", required=True - ) - allowed_max_memory_mb = fields.Integer( - data_key="allowedMaxMemoryMb", required=True - ) - remaining_milli_cpus = fields.Integer( - data_key="remainingMilliCpus", required=True - ) - remaining_memory_mb = fields.Integer( - data_key="remaininigMemoryMb", required=True - ) +@define +class DevInstance: + instance_id: uuid.UUID -class DevInstanceIdSchema(BaseSchema): - instance_id = fields.UUID(data_key="instanceId") +@define +class APIKey: + id: uuid.UUID + material: str + enabled: bool + label: str -class APIInstanceResponseSchema(BaseSchema): - project_id = fields.UUID(data_key="projectId") - api_id = fields.UUID(data_key="apiId") - error_code = fields.String(data_key="errorCode") - error = fields.String() - remaining_resource_limits = fields.Nested( - RemainingResourceLimitSchema, data_key="remainingResourceLimits" - ) +@define +class ServerType: + instance_size_type: str + instance_size: Optional[InstanceSize] = None -class APIDevInstanceResponseSchema(APIInstanceResponseSchema): - instance = fields.Nested(DevInstanceIdSchema) - key = fields.Nested(_APIKeySchema) - instance_id = fields.UUID(data_key="instanceId") +@define +class WSGIDefinition: + working_directory: str + module: str + wsgi_object: str + last_updated_at: Optional[datetime] = None -class InstanceSizeSchema(BaseSchema): - milli_cpus = fields.Integer(data_key="milliCpus", required=True) - memory_mb = fields.Integer(data_key="memoryMb", required=True) +@define +class PlumberDefinition: + working_directory: str + script_name: str + last_updated_at: Optional[datetime] = None -class APIType(str, Enum): - WSGI = "wsgi" - SCRIPT = "script" - PLUMBER = "plumber" +@define +class ScriptDefinition: + working_directory: str + script_name: str + last_updated_at: Optional[datetime] = None -class DeploymentStatus(Enum): - NOTDEPLOYED = "not-deployed" - STARTING = "starting" - DEPLOYED = "deployed" - ERROR = "error" +@define +class Instance: + instance: Server + outdated: bool + key: APIKey -class InstanceSchema(BaseSchema): - instance = fields.Nested(ServerSchema, required=True) - outdated = fields.Boolean(required=True) - key = fields.Nested( - _APIKeySchema, load_default=None - ) # Production instances do not have this field +@define +class ProjectTemplateReference: + template_id: uuid.UUID + version_id: uuid.UUID - @post_load - def make_instance(self, data, **kwargs): - return Instance(**data) +@define +class APIInstance: + api_id: uuid.UUID + instance_id: uuid.UUID + project_id: uuid.UUID -class CommandDefinitionSchema(BaseSchema): - api_type = EnumField( - APIType, by_value=True, required=True, data_key="type" - ) - working_directory = fields.String( - data_key="workingDirectory", - ) - module = fields.String() - wsgi_object = fields.String(data_key="wsgiObject") - script_name = fields.String(data_key="scriptName") - @post_load - def make_command_definition(self, data, **kwargs): - if data["api_type"] == "wsgi": - return WSGIDefinition(**data) - elif data["api_type"] == "script": - return ScriptDefinition(**data) - elif data["api_type"] == "plumber": - return PlumberDefinition(**data) +@define +class APIDevInstance: + api_id: uuid.UUID + instance: DevInstance + project_id: uuid.UUID + key: APIKey -class ServerTypeSchema(BaseSchema): - instance_size_type = fields.String( - data_key="instanceSizeType", required=True - ) - instance_size = fields.Nested(InstanceSizeSchema, data_key="instanceSize") - - -class _APISchema(BaseSchema): - definition = fields.Nested(CommandDefinitionSchema, data_key="definition") - environment_ids = fields.List(fields.UUID(), data_key="environmentIds") - default_server_size = fields.Nested( - ServerTypeSchema, data_key="defaultServerSize" - ) - name = fields.String(data_key="name") - subdomain = fields.String(data_key="subdomain") - description = fields.String(data_key="description") - last_updated_at = fields.DateTime(data_key="lastUpdatedAt") - - id = fields.UUID(data_key="apiId", required=True) - author_id = fields.UUID(data_key="authorId", required=True) - created_at = fields.DateTime(data_key="createdAt", required=True) - last_deployed_at = fields.DateTime( - data_key="lastDeployedAt", load_default=None - ) - last_deployed_by = fields.UUID( - data_key="lastDeployedBy", load_default=None - ) - # The following fields do not exist in the response if all APIs are listed - # TODO: what would be the best way to handle these missing fields for Marshmallow - deployment_status = EnumField( - DeploymentStatus, - by_value=True, - data_key="deploymentStatus", - load_default=None, - ) - prod_instances = fields.Nested( - InstanceSchema, - many=True, - data_key="prodInstances", - load_default=None, - ) - prod_keys = fields.Nested( - _APIKeySchema, - many=True, - data_key="prodKeys", - load_default=None, - ) - dev_instances = fields.Nested( - InstanceSchema, - many=True, - data_key="devInstances", - load_default=None, - ) - - @post_load - def make_api(self, data, **kwargs): - return API(**data) - - -class ListAllAPISchema(BaseSchema): - apis = fields.Nested(_APISchema(many=True)) - - @post_load - def make_list_apis(self, data, **kwargs): - # Flatten the API response - return data["apis"] +@define +class API: + api_id: uuid.UUID + author_id: uuid.UUID + created_at: datetime + created_from_project_template: Optional[ProjectTemplateReference] + default_server_size: ServerType + definition: Union[WSGIDefinition, PlumberDefinition, ScriptDefinition] + deployment_status: DeploymentStatus + description: str + dev_instances: List[Instance] + environment_ids: List[uuid.UUID] + last_deployed_at: Optional[datetime] + last_deployed_by: Optional[uuid.UUID] + name: str + prod_instances: List[Instance] + prod_keys: List[APIKey] + project_id: uuid.UUID + subdomain: str class APIClient(BaseClient): @@ -280,10 +147,58 @@ class APIClient(BaseClient): SERVICE_NAME = "aperture" + def list(self, project_id): + """List the APIs in a project. + + Parameters + ---------- + project_id : uuid.UUID + The ID of the project to list APIs in. + + Returns + ------- + List[API] + The APIs in the project. + """ + endpoint = "/project/{}/api".format(project_id) + return self._get(endpoint, _APISchema(many=True)) + + def list_all(self): + """List all APIs on the Faculty deployment. + + This method requires administrative privileges not available to most + users. + + Returns + ------- + List[API] + The APIs. + """ + return self._get("/api", _ListAPIsResponseSchema()) + + def get(self, project_id, api_id): + """Get an API. + + Parameters + ---------- + project_id : uuid.UUID + The ID of the project containing the API. + api_id : uuid.UUID + The ID of the API to get. + + Returns + ------- + API + The retrieved API. + """ + return self._get( + "/project/{}/api/{}".format(project_id, api_id), _APISchema() + ) + def create( self, project_id, - api_definition, + command_definition, subdomain, name="", description="", @@ -296,7 +211,7 @@ def create( ---------- project_id : uuid.UUID The project to create the API in. - api_definition : CommandDefinition + command_definition : CommandDefinition The API's command definition: WSGI(Flask)/Plumber/Script(Custom). subdomain : str The subdomain where the API should run. @@ -308,112 +223,76 @@ def create( The environments to apply to the API's instances. default_server_size : ServerType The default server size to set - """ - payload = { - "definition": CommandDefinitionSchema().dump(api_definition) - } - payload["subdomain"] = subdomain - payload["name"] = name - payload["description"] = description - payload["environmentIds"] = environment_ids - - # TODO: turn these into schema handling as well + Returns + ------- + API + The newly created API. + """ if default_server_size is None: - default_server_size = { - "instanceSizeType": "custom", - "instanceSize": {"milliCpus": 1000, "memoryMb": 4096}, - } - payload["defaultServerSize"] = default_server_size - - return self._post_raw( - "/project/{}/api".format(project_id), json=payload - ) + default_server_size = ServerType( + instance_size_type="custom", + instance_size=InstanceSize(milli_cpus=1000, memory_mb=4000), + ) - @staticmethod - def flask_definition(working_directory, module, wsgi_object): - return WSGIDefinition( - api_type=APIType.WSGI, - working_directory=working_directory, - module=module, - wsgi_object=wsgi_object, - ) - - @staticmethod - def plumber_definition(working_directory, script_name): - return PlumberDefinition( - api_type=APIType.PLUMBER, - working_directory=working_directory, - script_name=script_name, - ) - - @staticmethod - def script_definition(working_directory, script_name): - return ScriptDefinition( - api_type=APIType.SCRIPT, - working_directory=working_directory, - script_name=script_name, - ) - - def _get_current_api_definition(self, project_id, api_id): - current_api = self.get(project_id, api_id) - current_api = _APISchema().dump(current_api) - return current_api + payload = { + "definition": _CommandDefinitionSchema().dump(command_definition), + "subdomain": subdomain, + "name": name, + "description": description, + "environmentIds": environment_ids, + "defaultServerSize": _ServerTypeSchema().dump(default_server_size), + } - def get(self, project_id, api_id): - return self._get( - "/project/{}/api/{}".format(project_id, api_id), _APISchema() + return self._post( + "/project/{}/api".format(project_id), _APISchema(), json=payload ) - def update( + def update_definition( self, - project_id, - api_id, + api, + command_definition=None, + default_server_size=None, environment_ids=None, - api_definition=None, - server_type=None, - current_api=None, ): - if not current_api: - current_api = self._get_current_api_definition(project_id, api_id) - payload = {} - payload["defaultServerSize"] = current_api["defaultServerSize"] - payload["environmentIds"] = current_api["environmentIds"] - payload["definition"] = current_api["definition"] - if server_type: - payload["defaultServerSize"] = server_type - if environment_ids: - payload["environmentIds"] = environment_ids - - if api_definition: - payload["definition"] = api_definition - endpoint = "/project/{}/api/{}/definition".format(project_id, api_id) - return self._put(endpoint, _APISchema(), json=payload) - - def delete(self, project_id, api_id): - return self._delete_raw( - "/project/{}/api/{}".format(project_id, api_id) - ) - - def list(self, project_id): - """List APIs in a project. + """Update an API's definition. Parameters ---------- - project_id : uuid.UUID - The project to list APIs in. - name : str, optional - If provided, only return APIs with this name. + api : API + The API to update. + command_definition : CommandDefinition + The API's command definition: WSGI(Flask)/Plumber/Script(Custom). + If None then no change. + environment_ids : List[uuid.UUID] + The environments to apply to the API's instances. + If None then no change. + default_server_size : ServerType + The default server size to set. If None then no change. Returns ------- - List[API] - The matching APIs. + API + A slimmed down version of the API definition """ - endpoint = "/project/{}/api".format(project_id) - return self._get(endpoint, _APISchema(many=True)) + if command_definition is None: + command_definition = api.definition + if environment_ids is None: + environment_ids = api.environment_ids + if default_server_size is None: + default_server_size = api.default_server_size - def create_production_key(self, project_id, api_id, label=None): + payload = { + "defaultServerSize": _ServerTypeSchema().dump(default_server_size), + "definition": _CommandDefinitionSchema().dump(command_definition), + "environmentIds": environment_ids, + } + endpoint = "/project/{}/api/{}/definition".format( + api.project_id, api.api_id + ) + return self._put(endpoint, _APISchema(), json=payload) + + def create_production_key(self, project_id, api_id, label): """Create a production key for a given API. Parameters @@ -422,8 +301,8 @@ def create_production_key(self, project_id, api_id, label=None): The project where the API resides. api_id : uuid.UUID The API to create a new key in. - label : str, optional - If provided, set this value as the new key's label. + label : str + Set this value as the new key's label. Returns ------- @@ -431,8 +310,7 @@ def create_production_key(self, project_id, api_id, label=None): The newly created key. """ endpoint = "/project/{}/api/{}/key".format(project_id, api_id) - # TODO: maybe more sensible default, the UI doesn't allow empty name but the api accepts this - payload = {"label": label if label else ""} + payload = {"label": label} return self._post(endpoint, _APIKeySchema(), json=payload) def list_production_keys(self, project_id, api_id): @@ -514,6 +392,17 @@ def update_production_key(self, project_id, api_id, key_id, label): return self._put(endpoint, _APIKeySchema(), json=payload) def delete_production_key(self, project_id, api_id, key_id): + """Delete a specific production API key. + + Parameters + ---------- + project_id : uuid.UUID + The project in which the API resides. + api_id : uuid.UUID + The API in which to the production key resides. + key_id : uuid.UUID + The key to delete. + """ endpoint = "/project/{}/api/{}/key/{}".format( project_id, api_id, key_id ) @@ -523,113 +412,334 @@ def start( self, project_id, api_id, - key_id, - server_type=None, + server_size, image_version=None, - current_api=None, + restart=False, ): - if not current_api: - current_api = self._get_current_api_definition(project_id, api_id) - payload = { - "instanceSizeType": current_api["instanceSizeType"], - "instanceSize": { - "milliCpus": current_api["instnaceSize"]["milliCpus"], - "memoryMb": current_api["instnaceSize"]["memoryMb"], - }, - } - if server_type: - payload["instanceSizeType"] = server_type["instance_size_type"] - if server_type["instance_size"]: - instance_size = server_type["instance_size"] - payload["instanceSize"] = { - "milliCpus": instance_size.milli_cpus, - "memoryMb": instance_size.memory_mb, - } - if image_version: - payload["imageVersion"] = image_version - endpoint = "/project/{}/api/{}/prod/start".format(project_id, api_id) - return self._put(endpoint, APIInstanceResponseSchema(), json=payload) + """Start or restart an API. + + Parameters + ---------- + project_id : uuid.UUID + The ID of the project containing the API. + api : API + The API to start or restart. + server_size : ServerType + The server size to start. + restart : bool, optional + If True, then restart an API rather than start. Default: False + + Returns + ------- + APIInstance + Information on the started instance. + """ + payload = _ServerTypeSchema().dump(server_size) + payload["imageVersion"] = image_version + + action = "restart" if restart else "start" + endpoint = "/project/{}/api/{}/prod/{}".format( + project_id, api_id, action + ) + return self._put(endpoint, _APIInstanceResponseSchema(), json=payload) def stop(self, project_id, api_id): + """Stop an API. + + Parameters + ---------- + project_id : uuid.UUID + The ID of the project containing the API. + api : API + The API to stop. + + Returns + ------- + APIInstance + Information on the stopped instance. + """ endpoint = "/project/{}/api/{}/prod/stop".format(project_id, api_id) - self._put(endpoint, APIInstanceResponseSchema()) + return self._put(endpoint, _APIInstanceResponseSchema()) + + def reload(self, project_id, api_id): + """Reload a deployed API. + + Parameters + ---------- + project_id : uuid.UUID + The ID of the project containing the API. + api : API + The API to reaload. - def restart( + Returns + ------- + APIInstance + Information on the reloaded instance. + """ + endpoint = "/project/{}/api/{}/prod/reload".format(project_id, api_id) + return self._put(endpoint, _APIInstanceResponseSchema()) + + def start_dev( self, project_id, api_id, - key_id, - server_type=None, + server_size, image_version=None, - current_api=None, ): - if not current_api: - current_api = self._get_current_api_definition(project_id, api_id) - payload = { - "instanceSizeType": current_api["instanceSizeType"], - "instanceSize": { - "milliCpus": current_api["instnaceSize"]["milliCpus"], - "memoryMb": current_api["instnaceSize"]["memoryMb"], - }, - } - if server_type: - payload["instanceSizeType"] = server_type["instance_size_type"] - if server_type["instance_size"]: - instance_size = server_type["instance_size"] - payload["instanceSize"] = { - "milliCpus": instance_size.milli_cpus, - "memoryMb": instance_size.memory_mb, - } - if image_version: - payload["imageVersion"] = image_version - endpoint = "/project/{}/api/{}/prod/restart".format(project_id, api_id) - return self._put(endpoint, APIInstanceResponseSchema(), json=payload) + """Start a test/development instance for an API. - def reload(self, project_id, api_id): - endpoint = "/project/{}/api/{}/prod/reload".format(project_id, api_id) - self._put(endpoint, APIInstanceResponseSchema()) + Parameters + ---------- + project_id : uuid.UUID + The ID of the project containing the API. + api : API + The API for which to start the test / development server. + server_size : ServerType + The server size to start. - def start_dev( - self, project_id, api_id, key_id, server_type=None, image_version=None - ): + Returns + ------- + APIDevInstance + Information on the started dev API instance. + """ + payload = _ServerTypeSchema().dump(server_size) + payload["imageVersion"] = image_version - payload = {} - if server_type: - payload["instanceSizeType"] = server_type["instance_size_type"] - if server_type["instance_size"]: - instance_size = server_type["instance_size"] - payload["instanceSize"] = { - "milliCpus": instance_size["milli_cpus"], - "memoryMb": instance_size["memory_mb"], - } - if image_version: - payload["imageVersion"] = image_version - endpoint = "/project/{}/api/{}/dev/start".format(project_id, api_id) + endpoint = "/project/{}/api/{}/dev".format(project_id, api_id) return self._post( - endpoint, APIDevInstanceResponseSchema(), json=payload + endpoint, _APIDevInstanceResponseSchema(), json=payload ) - def stop_dev(self, project_id, api_id, dev_instance_id): - endpoint = "/project/{}/api/{}/dev/{}".format( - project_id, api_id, dev_instance_id - ) - self._delete(endpoint, APIDevInstanceResponseSchema()) + def stop_dev(self, project_id, api_id, instance_id): + """Stop a test/development instance for an API. - def reload_dev(self, project_id, api_id, dev_instance_id): - endpoint = "/project/{}/api/{}/dev/{}/reload".format( - project_id, api_id, dev_instance_id + Parameters + ---------- + project_id : uuid.UUID + The ID of the project containing the API. + api_id : uuid.UUID + The API for which to stop the test / development server. + instance_id : uuid.UUID + The ID of development instance to stop. + + Returns + ------- + APIInstance + Information on the deleted dev API instance. + """ + endpoint = "/project/{}/api/{}/dev/{}".format( + project_id, api_id, instance_id ) - self._put(endpoint, APIDevInstanceResponseSchema()) + return self._delete(endpoint, _APIInstanceResponseSchema()) - def list_all(self): - """List all APIs on the Faculty deployment. + def reload_dev(self, project_id, api_id, instance_id): + """Reload a test/development instance for an API. - This method requires administrative privileges not available to most - users. + Parameters + ---------- + project_id : uuid.UUID + The ID of the project containing the API. + api_id : uuid.UUID + The API for which to reaload the test / development server. + instance_id : uuid.UUID + The ID of development instance to reload. Returns ------- - List[API] - The APIs. + APIInstance + Information on the reloaded dev API instance. """ - return self._get("/api", ListAllAPISchema()) + endpoint = "/project/{}/api/{}/dev/{}/reload".format( + project_id, api_id, instance_id + ) + return self._put(endpoint, _APIInstanceResponseSchema()) + + +class _APIKeySchema(BaseSchema): + id = fields.UUID(data_key="keyId", required=True) + material = fields.String(required=True) + # Development instance keys do not have these fields + enabled = fields.Boolean(missing=None) + label = fields.String(missing=None) + + @post_load + def make_apikey(self, data, **kwargs): + return APIKey(**data) + + +class _DevInstanceSchema(BaseSchema): + instance_id = fields.UUID(data_key="instanceId") + + @post_load + def make_dev_instance(self, data, **kwargs): + return DevInstance(**data) + + +class _APIInstanceResponseSchema(BaseSchema): + api_id = fields.UUID(data_key="apiId") + instance_id = fields.UUID(data_key="instanceId", missing=None) + project_id = fields.UUID(data_key="projectId") + + @post_load + def make_api_instance(self, data, **kwargs): + return APIInstance(**data) + + +class _APIDevInstanceResponseSchema(BaseSchema): + api_id = fields.UUID(data_key="apiId") + instance = fields.Nested(_DevInstanceSchema) + project_id = fields.UUID(data_key="projectId") + key = fields.Nested(_APIKeySchema) + + @post_load + def make_api_instance(self, data, **kwargs): + return APIDevInstance(**data) + + +class _ProjectTemplateReferenceSchema(BaseSchema): + template_id = fields.UUID(data_key="templateId", required=True) + version_id = fields.UUID(data_key="versionId", required=True) + + @post_load + def make_project_template(self, data, **kwargs): + return ProjectTemplateReference(**data) + + +class _InstanceSchema(BaseSchema): + instance = fields.Nested(_ServerSchema, required=True) + outdated = fields.Boolean(required=True) + key = fields.Nested( + _APIKeySchema, load_default=None + ) # Production instances do not have this field + + @post_load + def make_instance(self, data, **kwargs): + return Instance(**data) + + +class _WSGIDefinitionSchema(BaseSchema): + working_directory = fields.String( + data_key="workingDirectory", required=True + ) + module = fields.String(required=True) + wsgi_object = fields.String(data_key="wsgiObject", required=True) + last_updated_at = fields.DateTime(data_key="lastUpdatedAt") + + @post_load + def make_wsgi_command_definition(self, data, **kwargs): + return WSGIDefinition(**data) + + +class _PlumberDefinitionSchema(BaseSchema): + working_directory = fields.String( + data_key="workingDirectory", required=True + ) + script_name = fields.String(data_key="scriptName", required=True) + last_updated_at = fields.DateTime(data_key="lastUpdatedAt") + + @post_load + def make_plumber_command_definition(self, data, **kwargs): + return PlumberDefinition(**data) + + +class _ScriptDefinitionSchema(BaseSchema): + working_directory = fields.String( + data_key="workingDirectory", required=True + ) + script_name = fields.String(data_key="scriptName", required=True) + last_updated_at = fields.DateTime(data_key="lastUpdatedAt") + + @post_load + def make_plumber_command_definition(self, data, **kwargs): + return ScriptDefinition(**data) + + +class _CommandDefinitionSchema(OneOfSchema): + type_field = "type" + type_schemas = { + "wsgi": _WSGIDefinitionSchema, + "plumber": _PlumberDefinitionSchema, + "script": _ScriptDefinitionSchema, + } + + def get_obj_type(self, obj): + if isinstance(obj, WSGIDefinition): + return "wsgi" + elif isinstance(obj, PlumberDefinition): + return "plumber" + elif isinstance(obj, ScriptDefinition): + return "script" + else: + raise Exception("Unknown object type: %s" % repr(obj)) + + +class _ServerTypeSchema(BaseSchema): + instance_size_type = fields.String( + data_key="instanceSizeType", required=True + ) + instance_size = fields.Nested( + _InstanceSizeSchema, data_key="instanceSize", missing=None + ) + + @post_load + def make_server_type(self, data, **kwargs): + return ServerType(**data) + + +class _APISchema(BaseSchema): + definition = fields.Nested(_CommandDefinitionSchema, required=True) + environment_ids = fields.List(fields.UUID(), data_key="environmentIds") + default_server_size = fields.Nested( + _ServerTypeSchema, data_key="defaultServerSize", missing=None + ) + name = fields.String(missing=None) + subdomain = fields.String(missing=None) + description = fields.String(missing=None) + api_id = fields.UUID(data_key="apiId", required=True) + project_id = fields.UUID(data_key="projectId", required=True) + author_id = fields.UUID(data_key="authorId", missing=None) + created_at = fields.DateTime(data_key="createdAt", missing=None) + last_deployed_at = fields.DateTime(data_key="lastDeployedAt", missing=None) + last_deployed_by = fields.UUID(data_key="lastDeployedBy", missing=None) + deployment_status = EnumField( + DeploymentStatus, + by_value=True, + data_key="deploymentStatus", + missing=None, + ) + prod_instances = fields.Nested( + _InstanceSchema, + many=True, + data_key="prodInstances", + missing=None, + ) + prod_keys = fields.Nested( + _APIKeySchema, + many=True, + data_key="prodKeys", + missing=None, + ) + dev_instances = fields.Nested( + _InstanceSchema, + many=True, + data_key="devInstances", + missing=None, + ) + created_from_project_template = fields.Nested( + _ProjectTemplateReferenceSchema, + data_key="createdFromProjectTemplate", + missing=None, + ) + + @post_load + def make_api(self, data, **kwargs): + return API(**data) + + +class _ListAPIsResponseSchema(BaseSchema): + apis = fields.Nested(_APISchema(many=True)) + + @post_load + def make_list_apis(self, data, **kwargs): + # Flatten the API response + return data["apis"] diff --git a/tests/clients/test_api.py b/tests/clients/test_api.py index b6a43e64..ce23bc9e 100644 --- a/tests/clients/test_api.py +++ b/tests/clients/test_api.py @@ -15,14 +15,16 @@ import secrets import uuid -from datetime import datetime +from copy import deepcopy import pytest -from dateutil.tz import UTC from marshmallow import ValidationError -from faculty.clients.api import (API, APIClient, APIKey, _APIKeySchema, - _APISchema) +from faculty.clients.api import ( + APIClient, + APIKey, + _APIKeySchema, +) ENVIRONMENT_ID = uuid.uuid4() OWNER_ID = uuid.uuid4() @@ -63,7 +65,8 @@ def test_apikey_schema_invalid(): [True, False], ) def test_disable_production_key(mocker, key_starting_enabled): - disabled_prod_key = PROD_KEY._replace(enabled=key_starting_enabled) + disabled_prod_key = deepcopy(PROD_KEY) + disabled_prod_key.enabled = key_starting_enabled mocker.patch.object(APIClient, "_put", return_value=disabled_prod_key) schema_mock = mocker.patch("faculty.clients.api._APIKeySchema") @@ -89,7 +92,8 @@ def test_disable_production_key(mocker, key_starting_enabled): [True, False], ) def test_enable_production_key(mocker, key_starting_enabled): - disabled_prod_key = PROD_KEY._replace(enabled=key_starting_enabled) + disabled_prod_key = deepcopy(PROD_KEY) + disabled_prod_key.enabled = key_starting_enabled mocker.patch.object(APIClient, "_put", return_value=disabled_prod_key) schema_mock = mocker.patch("faculty.clients.api._APIKeySchema") From 70af4da9906f88bb9fc85956efcabcd25da6bbe3 Mon Sep 17 00:00:00 2001 From: Gergely Imreh Date: Wed, 23 Feb 2022 14:51:46 +0800 Subject: [PATCH 4/8] switch from attr.define to dataclasses The advantage being standard Python (for Python 3.7+). --- faculty/clients/api.py | 24 ++++++++++++------------ setup.py | 1 + 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/faculty/clients/api.py b/faculty/clients/api.py index c36259f6..d6d22aa7 100644 --- a/faculty/clients/api.py +++ b/faculty/clients/api.py @@ -16,11 +16,11 @@ Manage Faculty APIs. """ import uuid +from dataclasses import dataclass from datetime import datetime from enum import Enum from typing import List, Optional, Union -from attr import define from marshmallow import fields, post_load from marshmallow_enum import EnumField @@ -39,12 +39,12 @@ class DeploymentStatus(Enum): ERROR = "error" -@define +@dataclass class DevInstance: instance_id: uuid.UUID -@define +@dataclass class APIKey: id: uuid.UUID material: str @@ -52,13 +52,13 @@ class APIKey: label: str -@define +@dataclass class ServerType: instance_size_type: str instance_size: Optional[InstanceSize] = None -@define +@dataclass class WSGIDefinition: working_directory: str module: str @@ -66,41 +66,41 @@ class WSGIDefinition: last_updated_at: Optional[datetime] = None -@define +@dataclass class PlumberDefinition: working_directory: str script_name: str last_updated_at: Optional[datetime] = None -@define +@dataclass class ScriptDefinition: working_directory: str script_name: str last_updated_at: Optional[datetime] = None -@define +@dataclass class Instance: instance: Server outdated: bool key: APIKey -@define +@dataclass class ProjectTemplateReference: template_id: uuid.UUID version_id: uuid.UUID -@define +@dataclass class APIInstance: api_id: uuid.UUID instance_id: uuid.UUID project_id: uuid.UUID -@define +@dataclass class APIDevInstance: api_id: uuid.UUID instance: DevInstance @@ -108,7 +108,7 @@ class APIDevInstance: key: APIKey -@define +@dataclass class API: api_id: uuid.UUID author_id: uuid.UUID diff --git a/setup.py b/setup.py index 04b355dc..cd37cb38 100644 --- a/setup.py +++ b/setup.py @@ -43,5 +43,6 @@ def load_readme(): "marshmallow", "marshmallow_enum", "urllib3", + "dataclasses; python_version<'3.7'", ], ) From c00ca94d52a75f92629617356712a5b282accb4a Mon Sep 17 00:00:00 2001 From: Gergely Imreh Date: Wed, 23 Feb 2022 15:15:01 +0800 Subject: [PATCH 5/8] enum value formatting and mixing in str type for potentially better usability --- faculty/clients/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/faculty/clients/api.py b/faculty/clients/api.py index d6d22aa7..8483718a 100644 --- a/faculty/clients/api.py +++ b/faculty/clients/api.py @@ -30,10 +30,10 @@ from faculty.clients.server import Server, _ServerSchema -class DeploymentStatus(Enum): +class DeploymentStatus(str, Enum): """An enumeration of possible API deployment statuses.""" - NOTDEPLOYED = "not-deployed" + NOT_DEPLOYED = "not-deployed" STARTING = "starting" DEPLOYED = "deployed" ERROR = "error" From 48e70dc82996495ac6023979c482ca57fb20607a Mon Sep 17 00:00:00 2001 From: Gergely Imreh Date: Wed, 23 Feb 2022 15:31:23 +0800 Subject: [PATCH 6/8] add the api client to the documentation as well. --- docs/source/api-index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/api-index.rst b/docs/source/api-index.rst index 13987a23..4fd3d5c5 100644 --- a/docs/source/api-index.rst +++ b/docs/source/api-index.rst @@ -19,6 +19,7 @@ Modules implementing clients: :toctree: api faculty.clients.account + faculty.clients.api faculty.clients.cluster faculty.clients.environment faculty.clients.experiment From dd59bcaeb965697d309c9961bba1181cd0ca1645 Mon Sep 17 00:00:00 2001 From: Gergely Imreh Date: Wed, 4 May 2022 19:57:35 +0800 Subject: [PATCH 7/8] docstring updates --- faculty/clients/api.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/faculty/clients/api.py b/faculty/clients/api.py index 8483718a..43bd3cea 100644 --- a/faculty/clients/api.py +++ b/faculty/clients/api.py @@ -259,15 +259,15 @@ def update_definition( Parameters ---------- - api : API + api : Union[API, APIResponse] The API to update. - command_definition : CommandDefinition + command_definition : Optional[CommandDefinition] The API's command definition: WSGI(Flask)/Plumber/Script(Custom). If None then no change. - environment_ids : List[uuid.UUID] + environment_ids : Optional[List[uuid.UUID]] The environments to apply to the API's instances. If None then no change. - default_server_size : ServerType + default_server_size : Optional[ServerType] The default server size to set. If None then no change. Returns @@ -422,8 +422,8 @@ def start( ---------- project_id : uuid.UUID The ID of the project containing the API. - api : API - The API to start or restart. + api_id : uuid.UUID + The ID of the API to start or restart. server_size : ServerType The server size to start. restart : bool, optional @@ -450,8 +450,8 @@ def stop(self, project_id, api_id): ---------- project_id : uuid.UUID The ID of the project containing the API. - api : API - The API to stop. + api_id : uuid.UUID + The ID of the API to stop. Returns ------- @@ -468,8 +468,8 @@ def reload(self, project_id, api_id): ---------- project_id : uuid.UUID The ID of the project containing the API. - api : API - The API to reaload. + api_id : uuid.UUID + The ID of the API to reload. Returns ------- @@ -540,7 +540,7 @@ def reload_dev(self, project_id, api_id, instance_id): project_id : uuid.UUID The ID of the project containing the API. api_id : uuid.UUID - The API for which to reaload the test / development server. + The API for which to reload the test / development server. instance_id : uuid.UUID The ID of development instance to reload. From 817ae79d9729a6e3614e6f7fb919bbbe05292d5f Mon Sep 17 00:00:00 2001 From: Gergely Imreh Date: Mon, 25 Apr 2022 20:17:09 +0800 Subject: [PATCH 8/8] Use derived classes for response items from the Platform API. --- faculty/clients/api.py | 123 ++++++++++++++++++++++++++++++++++------- 1 file changed, 102 insertions(+), 21 deletions(-) diff --git a/faculty/clients/api.py b/faculty/clients/api.py index 43bd3cea..fc683fa7 100644 --- a/faculty/clients/api.py +++ b/faculty/clients/api.py @@ -63,21 +63,33 @@ class WSGIDefinition: working_directory: str module: str wsgi_object: str - last_updated_at: Optional[datetime] = None + + +@dataclass +class WSGIDefinitionResponse(WSGIDefinition): + last_updated_at: datetime @dataclass class PlumberDefinition: working_directory: str script_name: str - last_updated_at: Optional[datetime] = None + + +@dataclass +class PlumberDefinitionResponse(PlumberDefinition): + last_updated_at: datetime @dataclass class ScriptDefinition: working_directory: str script_name: str - last_updated_at: Optional[datetime] = None + + +@dataclass +class ScriptDefinitionResponse(ScriptDefinition): + last_updated_at: datetime @dataclass @@ -110,23 +122,37 @@ class APIDevInstance: @dataclass class API: + name: str + description: str + definition: Union[WSGIDefinition, PlumberDefinition, ScriptDefinition] + environment_ids: List[uuid.UUID] + default_server_size: ServerType + + +@dataclass +class APIResponse(API): + # Override parent classes field + definition: Union[ + WSGIDefinitionResponse, + PlumberDefinitionResponse, + ScriptDefinitionResponse, + ] + # New fields api_id: uuid.UUID author_id: uuid.UUID created_at: datetime created_from_project_template: Optional[ProjectTemplateReference] - default_server_size: ServerType - definition: Union[WSGIDefinition, PlumberDefinition, ScriptDefinition] deployment_status: DeploymentStatus - description: str dev_instances: List[Instance] - environment_ids: List[uuid.UUID] last_deployed_at: Optional[datetime] last_deployed_by: Optional[uuid.UUID] - name: str prod_instances: List[Instance] prod_keys: List[APIKey] project_id: uuid.UUID subdomain: str + created_from_project_template: Optional[ProjectTemplateReference] + last_deployed_at: datetime + last_deployed_by: uuid.UUID class APIClient(BaseClient): @@ -157,11 +183,11 @@ def list(self, project_id): Returns ------- - List[API] + List[APIResponse] The APIs in the project. """ endpoint = "/project/{}/api".format(project_id) - return self._get(endpoint, _APISchema(many=True)) + return self._get(endpoint, _APIResponseSchema(many=True)) def list_all(self): """List all APIs on the Faculty deployment. @@ -171,7 +197,7 @@ def list_all(self): Returns ------- - List[API] + List[APIResponse] The APIs. """ return self._get("/api", _ListAPIsResponseSchema()) @@ -188,11 +214,12 @@ def get(self, project_id, api_id): Returns ------- - API + APIResponse The retrieved API. """ return self._get( - "/project/{}/api/{}".format(project_id, api_id), _APISchema() + "/project/{}/api/{}".format(project_id, api_id), + _APIResponseSchema(), ) def create( @@ -226,7 +253,7 @@ def create( Returns ------- - API + APIResponse The newly created API. """ if default_server_size is None: @@ -245,7 +272,9 @@ def create( } return self._post( - "/project/{}/api".format(project_id), _APISchema(), json=payload + "/project/{}/api".format(project_id), + _APIResponseSchema(), + json=payload, ) def update_definition( @@ -623,13 +652,20 @@ class _WSGIDefinitionSchema(BaseSchema): ) module = fields.String(required=True) wsgi_object = fields.String(data_key="wsgiObject", required=True) - last_updated_at = fields.DateTime(data_key="lastUpdatedAt") @post_load def make_wsgi_command_definition(self, data, **kwargs): return WSGIDefinition(**data) +class _WSGIDefinitionResponseSchema(_WSGIDefinitionSchema): + last_updated_at = fields.DateTime(data_key="lastUpdatedAt", required=True) + + @post_load + def make_wsgi_command_definition(self, data, **kwargs): + return WSGIDefinitionResponse(**data) + + class _PlumberDefinitionSchema(BaseSchema): working_directory = fields.String( data_key="workingDirectory", required=True @@ -642,6 +678,14 @@ def make_plumber_command_definition(self, data, **kwargs): return PlumberDefinition(**data) +class _PlumberDefinitionResponseSchema(_PlumberDefinitionSchema): + last_updated_at = fields.DateTime(data_key="lastUpdatedAt", required=True) + + @post_load + def make_plumber_command_definition(self, data, **kwargs): + return PlumberDefinitionResponse(**data) + + class _ScriptDefinitionSchema(BaseSchema): working_directory = fields.String( data_key="workingDirectory", required=True @@ -654,6 +698,14 @@ def make_plumber_command_definition(self, data, **kwargs): return ScriptDefinition(**data) +class _ScriptDefinitionResponseSchema(_ScriptDefinitionSchema): + last_updated_at = fields.DateTime(data_key="lastUpdatedAt", required=True) + + @post_load + def make_plumber_command_definition(self, data, **kwargs): + return ScriptDefinitionResponse(**data) + + class _CommandDefinitionSchema(OneOfSchema): type_field = "type" type_schemas = { @@ -673,6 +725,25 @@ def get_obj_type(self, obj): raise Exception("Unknown object type: %s" % repr(obj)) +class _CommandDefinitionResponseSchema(OneOfSchema): + type_field = "type" + type_schemas = { + "wsgi": _WSGIDefinitionResponseSchema, + "plumber": _PlumberDefinitionResponseSchema, + "script": _ScriptDefinitionResponseSchema, + } + + def get_obj_type(self, obj): + if isinstance(obj, WSGIDefinitionResponse): + return "wsgi" + elif isinstance(obj, PlumberDefinitionResponse): + return "plumber" + elif isinstance(obj, ScriptDefinitionResponse): + return "script" + else: + raise Exception("Unknown object type: %s" % repr(obj)) + + class _ServerTypeSchema(BaseSchema): instance_size_type = fields.String( data_key="instanceSizeType", required=True @@ -687,20 +758,27 @@ def make_server_type(self, data, **kwargs): class _APISchema(BaseSchema): + name = fields.String(missing=None) + description = fields.String(missing=None) definition = fields.Nested(_CommandDefinitionSchema, required=True) environment_ids = fields.List(fields.UUID(), data_key="environmentIds") default_server_size = fields.Nested( _ServerTypeSchema, data_key="defaultServerSize", missing=None ) - name = fields.String(missing=None) + + @post_load + def make_api(self, data, **kwargs): + return API(**data) + + +class _APIResponseSchema(_APISchema): + definition = fields.Nested(_CommandDefinitionResponseSchema, required=True) subdomain = fields.String(missing=None) description = fields.String(missing=None) api_id = fields.UUID(data_key="apiId", required=True) project_id = fields.UUID(data_key="projectId", required=True) author_id = fields.UUID(data_key="authorId", missing=None) created_at = fields.DateTime(data_key="createdAt", missing=None) - last_deployed_at = fields.DateTime(data_key="lastDeployedAt", missing=None) - last_deployed_by = fields.UUID(data_key="lastDeployedBy", missing=None) deployment_status = EnumField( DeploymentStatus, by_value=True, @@ -731,13 +809,16 @@ class _APISchema(BaseSchema): missing=None, ) + last_deployed_at = fields.DateTime(data_key="lastDeployedAt", missing=None) + last_deployed_by = fields.UUID(data_key="lastDeployedBy", missing=None) + @post_load def make_api(self, data, **kwargs): - return API(**data) + return APIResponse(**data) class _ListAPIsResponseSchema(BaseSchema): - apis = fields.Nested(_APISchema(many=True)) + apis = fields.Nested(_APIResponseSchema(many=True)) @post_load def make_list_apis(self, data, **kwargs):