diff --git a/.anno.json b/.anno.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/.anno.json @@ -0,0 +1 @@ +{} diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cbd1ce3..2266407 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.0.0 +current_version = 0.1.0 commit = True tag = True diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3b61408..28cf284 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,11 @@ Changelog ========= +0.1.0 (2021-10-15) +------------------ + +* First translation of code directly from GO + 0.0.0 (2021-10-13) ------------------ diff --git a/README.rst b/README.rst index c42bf5d..9862a06 100644 --- a/README.rst +++ b/README.rst @@ -51,9 +51,9 @@ Overview :alt: Supported implementations :target: https://pypi.org/project/component-generator -.. |commits-since| image:: https://img.shields.io/github/commits-since/onna/python-component-generator/v0.0.0.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/onna/python-component-generator/v0.1.0.svg :alt: Commits since latest release - :target: https://github.com/onna/python-component-generator/compare/v0.0.0...master + :target: https://github.com/onna/python-component-generator/compare/v0.1.0...master diff --git a/docs/conf.py b/docs/conf.py index f1d541b..b0f86f6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ year = '2021' author = 'Darren Buttigieg' copyright = '{0}, {1}'.format(year, author) -version = release = '0.0.0' +version = release = '0.1.0' pygments_style = 'trac' templates_path = ['.'] diff --git a/setup.cfg b/setup.cfg index 7b60e26..b4ae35a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,8 +2,9 @@ universal = 1 [flake8] -max-line-length = 140 -exclude = .tox,.eggs,ci/templates,build,dist +max-line-length = 110 +extend-ignore = E203 +exclude = .tox,.eggs,ci/templates,build,dist,src/component_generator/component_files [tool:pytest] # If a pytest section is found in one of the possible config files @@ -33,3 +34,7 @@ known_first_party = component_generator default_section = THIRDPARTY forced_separate = test_component_generator skip = .tox,.eggs,ci/templates,build,dist + +[mypy] +exclude = src/component_generator/component_files +ignore_missing_imports = True diff --git a/setup.py b/setup.py index e354ec0..66edbba 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def read(*names, **kwargs): setup( name='component-generator', - version='0.0.0', + version='0.1.0', license='BSD-2-Clause', description='Generate backend components quickly', long_description='%s\n%s' % ( diff --git a/src/component_generator/__init__.py b/src/component_generator/__init__.py index c57bfd5..b794fd4 100644 --- a/src/component_generator/__init__.py +++ b/src/component_generator/__init__.py @@ -1 +1 @@ -__version__ = '0.0.0' +__version__ = '0.1.0' diff --git a/src/component_generator/__main__.py b/src/component_generator/__main__.py index 4cea4ba..79eb559 100644 --- a/src/component_generator/__main__.py +++ b/src/component_generator/__main__.py @@ -8,7 +8,7 @@ - https://docs.python.org/2/using/cmdline.html#cmdoption-m - https://docs.python.org/3/using/cmdline.html#cmdoption-m """ -from component_generator.cli import main +from cli import main if __name__ == "__main__": main() diff --git a/src/component_generator/cli.py b/src/component_generator/cli.py index b5b6aa0..4885c96 100644 --- a/src/component_generator/cli.py +++ b/src/component_generator/cli.py @@ -16,11 +16,21 @@ """ import argparse -parser = argparse.ArgumentParser(description='Command description.') -parser.add_argument('names', metavar='NAME', nargs=argparse.ZERO_OR_MORE, - help="A name of something.") +from generate import generate_component, ComponentType + +parser = argparse.ArgumentParser(description="Component Generator.") +parser.add_argument("--component", help="Component name.") +parser.add_argument("--service", help="Service name.") +parser.add_argument("--consumer", help="Consumer name.") def main(args=None): args = parser.parse_args(args=args) - print(args.names) + + if not hasattr(args, "component"): + parser.error("Please pass component name") + generate_component(ComponentType.COMPONENT, args.component, args.component) + if args.service: + generate_component(ComponentType.SERVICE, args.service, args.component) + if args.consumer: + generate_component(ComponentType.CONSUMER, args.consumer, args.component) diff --git a/src/component_generator/component_files/component.py b/src/component_generator/component_files/component.py new file mode 100644 index 0000000..be027d8 --- /dev/null +++ b/src/component_generator/component_files/component.py @@ -0,0 +1,358 @@ +from info_file import START_BLOCK, END_BLOCK + +COMPONENT_FILES = { + # ROOT level + "${componentName}/Makefile": """ +POETRY ?= poetry + +_default: test + +clean: + find . -name '__pycache__' | xargs rm -rf + find . -type f -name "*.pyc" -delete + +install-dev: + $(POETRY) install + +install: + $(POETRY) install --no-dev + +format: + $(POETRY) run isort . + $(POETRY) run black . + +lint: + $(POETRY) run isort --check-only . + $(POETRY) run black --check . + $(POETRY) run flake8 --config setup.cfg + +mypy: + $(POETRY) run mypy -p ${componentName} + + test: + $(POETRY) run pytest tests + +test-cov: + $(POETRY) run pytest tests --cov=${componentName} + +coverage-report: + $(POETRY) run coverage xml -i -o coverage.xml + $(POETRY) run coverage report + +test-and-report: + # we want to report on success AND failure + make test-cov && make coverage-report || (make coverage-report; exit 1) + +.PHONY: clean install install-dev test coverage-report test-and-report + +""", + "${componentName}/Dockerfile": """FROM gcr.io/atlas-images/python-poetry:3.8.8-buster as base + +COPY --chown=onnauser:onnagroup ./ /app + +FROM base as production +RUN make install +USER root +RUN apt remove -y --purge binutils libc6-dev gcc --allow-remove-essential +USER onnauser + +FROM base as test +RUN make install-dev +USER root +RUN apt remove -y --purge binutils libc6-dev gcc --allow-remove-essential +USER onnauser + +""", + "${componentName}/pyproject.toml": """[tool.poetry] +name = "${componentName}" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.8" +pydantic = "^1.8.1" +prometheus-client = "^0.9.0" +onna-utils = "^0.1.0" +onna-types = "^0.1.0" + +[tool.poetry.dev-dependencies] +pytest = "^6.2.2" +mypy = "^0.812" +pytest-asyncio = "^0.14.0" +flake8 = "^3.8.4" +isort = "^5.7.0" +black = "^20.8b1" +pytest-cov = "^2.11.1" +async-asgi-testclient = "^1.4.6" + + +[[tool.poetry.source]] +name = "onna" +url = "http://onna:atlasense@pypi.intra.onna.internal/simple/" +secondary = true + +[tool.black] +line-length = 100 +target-version = ['py38'] +include = '\.pyi?$' +exclude = ''' + +( + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + )/ + | foo.py # also separately exclude a file named foo.py in + # the root of the project +) +''' + +[tool.poetry.scripts] +""" + + START_BLOCK + + """ + +""" + + END_BLOCK + + """ + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" + """, + "${componentName}/README.md": """# ${componentName} +Description here! + + +## Develop + +``` +make install-dev +``` + +## Run tests + +Run docker compose before running tests: + +``` +docker-compose up -d +``` + +``` +make test +``` + + """, + "${componentName}/Jenkinsfile": """@Library('onna-library') _ + +def MAJOR_VERSION = 0 +def MINOR_VERSION = 1 + +node { + environments.DoCheckout() + environments.SetGCLOUD() + + // environments.GetAppName() does extra stuff we don't care about + def appName = "${componentName}" + def version = "${MAJOR_VERSION}.${MINOR_VERSION}.${env.BUILD_NUMBER}" + if (env.BRANCH_NAME != 'master') { + version = "${MAJOR_VERSION}.${MINOR_VERSION + 1}.0-${env.BRANCH_NAME}${env.BUILD_NUMBER}" + } + println("Building version ${version}") + + def image = dockerbuilds.BuildDockerImage(appName, env.BRANCH_NAME, env.BUILD_NUMBER, version, ".", null, null, "--target=test") + def prefixDockerName = appName + '-' + env.BRANCH_NAME + '-' + env.BUILD_NUMBER + '-test' + + stage ('Start Persistent Layers') { + // none + } + + stage ('Run Pre-checks') { + sh("docker run --rm ${image} /bin/bash -c 'make lint && make mypy'") + } + + def runName = appName + '-' + env.BRANCH_NAME + '-' + env.BUILD_NUMBER + '-test' + stage("Run Tests") { + try { + sh("docker run --name='${runName}' " + + "${image} /bin/bash -c 'make test-and-report'") + sh("docker cp ${runName}:/app/coverage.xml .") + // fix path to source from report + sh("sed -i 's@\\.*@source>${componentName}@g' coverage.xml") + + } catch(Exception e){ + slackSend ( + channel: '#jenkins', + color: 'danger', + message: "Jenkins Job '${env.JOB_NAME}' (${env.BUILD_NUMBER}) failed in the alltests stage with ${e.message}", to: 'dev@atlasense.com', body: "Please go to ${env.BUILD_URL}."); + throw e + } finally { + sh("docker rm -v ${runName} || true") + } + } + + stage("SonarQube") { + sonarqube.scan(version) + } + + image = dockerbuilds.BuildDockerImage(appName, env.BRANCH_NAME, env.BUILD_NUMBER, version, ".", null, null, "--target=production") + image = dockerbuilds.PushDockerImage(appName, env.BRANCH_NAME, env.BUILD_NUMBER, version, image) + + helm.UpdateHelm(appName, env.BRANCH_NAME, env.BUILD_NUMBER, image, version) + helm.UploadHelm(appName, env.BRANCH_NAME, env.BUILD_NUMBER, version) + + deploy.ComponentDeploy(appName, env.BRANCH_NAME, env.BUILD_NUMBER, version) + + notify.JobDone(appName, env.JOB_BASE_NAME, env.BUILD_NUMBER) + +} +""", + "${componentName}/AUTOUPDATE": "", + "${componentName}/setup.cfg": """[flake8] +max-line-length = 120 +exclude = .eggs + +[mypy-prometheus_client.*] +ignore_missing_imports = True + +[mypy-uvicorn.*] +ignore_missing_imports = True + +[mypy-lru.*] +ignore_missing_imports = True + +[isort] +line_length = 100 +include_trailing_comma=True +force_grid_wrap=4 +use_parentheses=True +force_single_line=False +multi_line_output=3 + +[mypy] +plugins = pydantic.mypy +mypy_path = stubs +""", + # + # Tests + # + "${componentName}/tests/__init__.py": "", + "${componentName}/tests/fixtures.py": "", + "${componentName}/tests/acceptance/__init__.py": "", + "${componentName}/tests/acceptance/test_service.py": """ +def test_it(): + assert True +""", + "${componentName}/tests/unit/__init__.py": "", + "${componentName}/tests/unit/test_service.py": """ +def test_it(): + assert True +""", + # + # Python package + # + "${componentName}/${componentName}/__init__.py": "", + "${componentName}/${componentName}/commands.py": """import argparse +import asyncio +import logging + +from .settings import Settings + +logger = logging.getLogger(__name__) + + +parser = argparse.ArgumentParser(description="command runner", add_help=False) +parser.add_argument( + "-e", + "--env-file", + help="Env file", +) + + +def get_settings() -> Settings: + arguments, _ = parser.parse_known_args() + return Settings(_env_file=arguments.env_file) +""" + + START_BLOCK + + """ + +""" + + END_BLOCK + + """ + + """, + "${componentName}/${componentName}/settings.py": """ +from pydantic import BaseSettings + + +class Settings(BaseSettings): + ... + + """, + # + # Chart + # + "charts/${componentName}/Chart.yaml": """apiVersion: v1 +appVersion: 1.4.2 +description: ${componentName} +name: ${componentName} +sources: + - https://github.com/atlasense/${componentName} +maintainers: +- name: Platform + email: platform@onna.com +version: 99999.99999.99999 +""", + "charts/${componentName}/requirements.yaml": """dependencies: +""", + "charts/${componentName}/values.yaml": """image: IMAGE_TO_REPLACE +pullSecrets: [] + + +""" + + START_BLOCK + + """ + +""" + + END_BLOCK + + """ +""", + "charts/${componentName}/templates/_helpers.tpl": """{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. */}} +{{- define "name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{/* Create a default fully qualified app name. We truncate at 63 chars because . . . */}} +{{- define "fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}}""", + "charts/${componentName}/templates/cm.yaml": """apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "name" . }}-config + namespace: {{ .Release.Namespace }} + labels: + service_name: {{ template "name" . }} + version: {{ .Chart.Version }} + tier: "backend" + release: {{ .Release.Name }} +data: +""" + + START_BLOCK + + """ + +""" + + END_BLOCK + + """ +""", +} diff --git a/src/component_generator/component_files/consumer.py b/src/component_generator/component_files/consumer.py new file mode 100644 index 0000000..123a416 --- /dev/null +++ b/src/component_generator/component_files/consumer.py @@ -0,0 +1,187 @@ +CONSUMER_FILES = { + "charts/${componentName}/templates/${consumerName}.deploy.yaml": """kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ template "name" . }}-${consumerName} + namespace: {{ .Release.Namespace }} + labels: + service_name: {{ template "name" . }}-${consumerName} + release: {{ .Release.Name }} + app: {{ template "name" . }}-${consumerName} + version: {{ .Chart.Version }} + tier: "backend" +spec: + replicas: {{ .Values.${consumerName}_hpa.minReplicas }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit | default "3" }} + selector: + matchLabels: + service_name: {{ template "name" . }}-${consumerName} + tier: "backend" + template: + metadata: + name: {{ template "name" . }}-${consumerName} + annotations: + {{- if .Values.${consumerName}_annotations }} + checksum/config: {{ include (print $.Template.BasePath "/cm.yaml") . | sha256sum }} + {{- tpl (toYaml .Values.${consumerName}_annotations) . | nindent 8 }} + {{- end }} + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + labels: + service_name: {{ template "name" . }}-${consumerName} + tier: "backend" + app: {{ template "name" . }}-${consumerName} + version: {{ .Chart.Version }} + spec: + {{- if .Values.affinity }} + affinity: + {{- tpl (toYaml .Values.affinity) . | nindent 8 }} + {{- end }} + {{- if .Values.${consumerName}_tolerations }} + tolerations: + {{- toYaml .Values.${consumerName}_tolerations | nindent 8 }} + {{- end }} + dnsPolicy: ClusterFirst + {{- if .Values.pullSecrets }} + imagePullSecrets: + - name: {{ .Values.pullSecrets }} + {{- end }} + containers: + - name: {{ template "name" . }}-${consumerName} + image: {{ .Values.image }} + command: ["${consumerName}"] + envFrom: + - configMapRef: + name: {{ template "name" . }}-config + {{- if .Values.${consumerName}_resources }} + resources: + {{- toYaml .Values.${consumerName}_resources | nindent 10 }} + {{- end }} + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + readinessProbe: + httpGet: + path: /health + port: 8080 + ports: + - name: api + containerPort: 8080""", + "charts/${componentName}/templates/${consumerName}.hpa.yaml": """apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "name" . }}-${consumerName} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "name" . }}-api + minReplicas: {{ .Values.${consumerName}_hpa.minReplicas }} + maxReplicas: {{ .Values.${consumerName}_hpa.maxReplicas }} + metrics: + {{- toYaml .Values.${consumerName}_hpa.metrics | nindent 4 }} +""", + "+charts/${componentName}/values.yaml": """${consumerName}_replicaCount: 2 +${consumerName}_annotations: +${consumerName}_tolerations: [] +${consumerName}_affinity: {} +${consumerName}_resources: + limits: + cpu: 1000m + memory: 512Mi + requests: + cpu: 500m + memory: 256Mi +${consumerName}_hpa: + minReplicas: 1 + maxReplicas: 2""", + "${componentName}/${consumerName}.py": """import prometheus_client +from fastapi import APIRouter, FastAPI, Request +from onna_utils.middleware.starlette import PrometheusMiddleware +from starlette.responses import JSONResponse, Response +from onna_utils import metrics +import kafkaesk + +router = kafkaesk.Router() + +class ConsumerApplication(kafkaesk.Application): + app_settings: Settings + + def __init__(self, kafka_settings: KafkaSettings, app_settings: Settings): + super().__init__( + kafka_servers=kafka_settings.kafka_servers, + topic_prefix=kafka_settings.kafka_prefix, + kafka_api_version=kafka_settings.kafka_api_version or "auto", + kafka_settings=kafka_settings.kafka_settings, + ) + self.mount(router) + self.app_settings = app_settings + self.on("finalize", self._close) + + async def _close(self): + ... + +@router.subscribe(MY_TOPIC, group=MY_GROUP) +async def my_subscriber(my_ob: ObjecType): + ... + + +http_router = APIRouter() + + +@http_router.get("/health") +async def health(request: Request) -> None: + with metrics.healthy(): + await request.app.consumer.health_check() + + +@http_router.get("/metrics") +def get_metrics(request: Request) -> Response: + output = prometheus_client.exposition.generate_latest() + return Response(output.decode("utf8")) + + +class HTTPApplication(FastAPI): + def __init__(self, consumer, *args, **kwargs): + super().__init__(title="${consumerName}", redoc_url=None, docs_url=None, *args, **kwargs) + self.add_middleware(PrometheusMiddleware) + self.include_router(http_router) + + self.consumer = consumer + + self.add_event_handler("startup", self.initialize) + self.add_event_handler("shutdown", self.finalize) + + async def initialize(self) -> None: + ... + + async def finalize(self) -> None: + ... +""", + "+${componentName}/${componentName}/commands.py": """ +def run_${consumerName}(): + settings = get_settings() + asyncio.run(_run_${consumerName}(settings)) + + +async def _run_${consumerName}(settings: Settings): + from ${componentName} import ${consumerName} + from onna_utils.commands import serve_consumer_app + + consumer_app = ${consumerName}.ConsumerApplication() + http_app = ${consumerName}.HTTPApplication(consumer_app) + await serve_consumer_app(http_app, consumer_app, port=8080) +""", + "+${componentName}/pyproject.toml": """ +${consumerName} = '${componentName}.commands:run_${consumerName}' +""", +} diff --git a/src/component_generator/component_files/service.py b/src/component_generator/component_files/service.py new file mode 100644 index 0000000..11ca30f --- /dev/null +++ b/src/component_generator/component_files/service.py @@ -0,0 +1,217 @@ +SERVICE_FILES = { + "charts/${componentName}/templates/${serviceName}.deploy.yaml": """kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ template "name" . }}-${serviceName} + namespace: {{ .Release.Namespace }} + labels: + service_name: {{ template "name" . }}-${serviceName} + release: {{ .Release.Name }} + app: {{ template "name" . }}-${serviceName} + version: {{ .Chart.Version }} + tier: "backend" +spec: + replicas: {{ .Values.${serviceName}_hpa.minReplicas }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit | default "3" }} + selector: + matchLabels: + service_name: {{ template "name" . }}-${serviceName} + tier: "backend" + template: + metadata: + name: {{ template "name" . }}-${serviceName} + annotations: + {{- if .Values.${serviceName}_annotations }} + checksum/config: {{ include (print $.Template.BasePath "/cm.yaml") . | sha256sum }} + {{- tpl (toYaml .Values.${serviceName}_annotations) . | nindent 8 }} + {{- end }} + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + labels: + service_name: {{ template "name" . }}-${serviceName} + tier: "backend" + app: {{ template "name" . }}-${serviceName} + version: {{ .Chart.Version }} + spec: + {{- if .Values.affinity }} + affinity: + {{- tpl (toYaml .Values.affinity) . | nindent 8 }} + {{- end }} + {{- if .Values.${serviceName}_tolerations }} + tolerations: + {{- toYaml .Values.${serviceName}_tolerations | nindent 8 }} + {{- end }} + dnsPolicy: ClusterFirst + {{- if .Values.pullSecrets }} + imagePullSecrets: + - name: {{ .Values.pullSecrets }} + {{- end }} + containers: + - name: {{ template "name" . }}-${serviceName} + image: {{ .Values.image }} + command: ["${serviceName}"] + envFrom: + - configMapRef: + name: {{ template "name" . }}-config + {{- if .Values.${serviceName}_resources }} + resources: + {{- toYaml .Values.${serviceName}_resources | nindent 10 }} + {{- end }} + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + readinessProbe: + httpGet: + path: /health + port: 8080 + ports: + - name: api + containerPort: 8080""", + "charts/${componentName}/templates/${serviceName}.hpa.yaml": """apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "name" . }}-${serviceName} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "name" . }}-api + minReplicas: {{ .Values.${serviceName}_hpa.minReplicas }} + maxReplicas: {{ .Values.${serviceName}_hpa.maxReplicas }} + metrics: + {{- toYaml .Values.${serviceName}_hpa.metrics | nindent 4 }} +""", + "charts/${componentName}/templates/${serviceName}.svc.yaml": """kind: Service +apiVersion: v1 +metadata: + name: {{ template "name" . }}-${serviceName} + namespace: {{ .Release.Namespace }} + labels: + service_name: {{ template "name" . }}-${serviceName} + app: {{ template "name" . }}-${serviceName} + version: {{ .Chart.Version }} + tier: "backend" + release: {{ .Release.Name }} + annotations: + getambassador.io/config: | + --- + apiVersion: ambassador/v1 + kind: Mapping + name: ambassador_{{ template "name" . }}_${serviceName}_{{ .Release.Namespace | default .Values.app }}_mapping + prefix: /${serviceName}/ + service: {{ template "name" . }}-${serviceName}.{{ .Release.Namespace | default .Values.app }}:8080 + resolver: endpoint + timeout_ms: 600000 + connect_timeout_ms: 30000 + idle_timeout_ms: 60000 + circuit_breakers: + - priority: default + max_connections: 3072 + max_pending_requests: 2048 + max_requests: 4096 + max_retries: 50 + load_balancer: + policy: round_robin + retry_policy: + retry_on: gateway-error + num_retries: 4 +spec: + type: ClusterIP + ports: + - name: http + port: 8080 + targetPort: 8080 + selector: + service_name: {{ template "name" . }}-${serviceName} +""", + "+charts/${componentName}/values.yaml": """${serviceName}_replicaCount: 2 +${serviceName}_annotations: +${serviceName}_tolerations: [] +${serviceName}_affinity: {} +${serviceName}_resources: + limits: + cpu: 1000m + memory: 512Mi + requests: + cpu: 500m + memory: 256Mi""", + "${componentName}/${serviceName}.py": """import prometheus_client +from fastapi import APIRouter, FastAPI, Request +from onna_utils.middleware.starlette import PrometheusMiddleware +from starlette.responses import JSONResponse, Response + +router = APIRouter() + + +@router.get("/health") +async def health(request: Request) -> None: + ... + + +@router.get("/metrics") +def get_metrics(request: Request) -> Response: + output = prometheus_client.exposition.generate_latest() + return Response(output.decode("utf8")) + + +@router.get("/ping") +def get_ping(request: Request) -> Response: + return JSONResponse({"po": "ng"}) + + +@router.get("/foobar") +async def get_foobar(): + return {} + + +class HTTPApplication(FastAPI): + def __init__( + self, + *args, + **kwargs, + ): + super().__init__(title="${serviceName}", redoc_url=None, docs_url=None, *args, **kwargs) + self.add_middleware(PrometheusMiddleware) + self.include_router(router) + + self.add_event_handler("startup", self.initialize) + self.add_event_handler("shutdown", self.finalize) + + async def initialize(self) -> None: + ... + + async def finalize(self) -> None: + ... +""", + "+${componentName}/${componentName}/commands.py": """ +def run_${serviceName}(): + settings = get_settings() + asyncio.run(_run_${serviceName}(settings)) + + +async def _run_${serviceName}(settings: Settings): + from ${componentName} import ${serviceName} + import uvicorn + + http_config = uvicorn.Config( + ${serviceName}.HTTPApplication(), + port=8080, + host="0.0.0.0", + log_level=None, + ) + server = uvicorn.Server(config=http_config) + await server.serve() +""", + "+${componentName}/pyproject.toml": """ +${serviceName} = '${componentName}.commands:run_${serviceName}' +""", +} diff --git a/src/component_generator/constants.py b/src/component_generator/constants.py new file mode 100644 index 0000000..ae772c0 --- /dev/null +++ b/src/component_generator/constants.py @@ -0,0 +1,2 @@ +START_BLOCK = "#" diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py new file mode 100644 index 0000000..e5fc16d --- /dev/null +++ b/src/component_generator/generate.py @@ -0,0 +1,57 @@ +import json +import os +from enum import Enum +from typing import Dict + +from component_files.component import COMPONENT_FILES +from component_files.consumer import CONSUMER_FILES +from component_files.service import SERVICE_FILES +from helpers import populate_setting_values, create_folder_for_filepath +from info_file import append_to_info_file + +SETTINGS_FILENAME = ".anno.json" + + +class ComponentType(Enum): + COMPONENT = "component" + SERVICE = "service" + CONSUMER = "consumer" + + +COMPONENT_FILESTRUCTURE = { + ComponentType.COMPONENT: COMPONENT_FILES, + ComponentType.SERVICE: SERVICE_FILES, + ComponentType.CONSUMER: CONSUMER_FILES, +} + + +def generate_component(component_type: ComponentType, component_name: str, main_component_name: str): + settings = load_settings_from_file() + settings[f"{component_type.value}Name"] = component_name + settings[f"{ComponentType.COMPONENT.value}Name"] = main_component_name + generate(main_component_name, COMPONENT_FILESTRUCTURE[component_type], settings) + + +def generate(component_name: str, structure: Dict[str, str], settings: Dict[str, str]): + for filepath, filedata in structure.items(): + if filepath.startswith("+"): + filepath = f"{component_name}/{filepath[1:]}" + append_to_info_file(filepath, filedata, settings) + else: + filepath = f"component/{filepath}" + generate_file(filepath, filedata, settings) + + +def generate_file(filepath: str, filedata: str, settings: Dict[str, str]): + populated_filepath = populate_setting_values(filepath, settings) + create_folder_for_filepath(populated_filepath) + + if os.path.exists(populated_filepath): + raise FileExistsError + with open(populated_filepath, "w") as file: + file.write(populate_setting_values(filedata, settings)) + + +def load_settings_from_file(settings_filename: str = SETTINGS_FILENAME) -> Dict[str, str]: + with open(settings_filename, "r") as file: + return json.load(file) diff --git a/src/component_generator/helpers.py b/src/component_generator/helpers.py new file mode 100644 index 0000000..e4f91c9 --- /dev/null +++ b/src/component_generator/helpers.py @@ -0,0 +1,16 @@ +import os +from typing import Dict + + +def populate_setting_values(string: str, settings: Dict[str, str]) -> str: + for key, val in settings.items(): + string = string.replace(f"${{{key}}}", val) + return string + + +def create_folder_for_filepath(filepath: str): + try: + folderpath = filepath[: filepath.rindex("/")] + os.makedirs(folderpath) + except (ValueError, FileExistsError): + pass diff --git a/src/component_generator/info_file.py b/src/component_generator/info_file.py new file mode 100644 index 0000000..d998b3e --- /dev/null +++ b/src/component_generator/info_file.py @@ -0,0 +1,35 @@ +from typing import Dict + +from helpers import populate_setting_values + +START_BLOCK = "#" + + +class InfoFileNotUpdatableException(Exception): + def __init__(self, filepath: str): + super().__init__(f"File {filepath} not updatable") + + +def _check_info_file(file_data: str, filepath: str): + if not (START_BLOCK in file_data and END_BLOCK in file_data): + raise InfoFileNotUpdatableException(filepath) + + +def _append_filedata(existing_filedata: str, new_filedata: str, filepath: str) -> str: + _check_info_file(existing_filedata, filepath) + return "\n".join( + ( + existing_filedata[: existing_filedata.index(END_BLOCK)], + new_filedata, + existing_filedata[existing_filedata.index(END_BLOCK) :], + ) + ) + + +def append_to_info_file(filepath: str, filedata_part: str, settings: Dict[str, str]): + populated_filepath = populate_setting_values(filepath, settings) + with open(populated_filepath, "r") as reader: + file_data = reader.read() + with open(populated_filepath, "w") as writer: + writer.write(_append_filedata(file_data, filedata_part, populated_filepath)) diff --git a/tests/test_component_generator.py b/tests/test_component_generator.py index 8f7e2d7..5b47d22 100644 --- a/tests/test_component_generator.py +++ b/tests/test_component_generator.py @@ -1,5 +1,4 @@ - -from component_generator.cli import main +from cli import main def test_main(): diff --git a/tests/test_generate.py b/tests/test_generate.py new file mode 100644 index 0000000..6918980 --- /dev/null +++ b/tests/test_generate.py @@ -0,0 +1,15 @@ +from pytest_mock import MockFixture # type: ignore + +from generate import generate + + +def test_generate_should_call_append_to_info_file_if_filepath_startswith_plus(mocker: MockFixture): + insert_info_spy = mocker.patch("component_generator.generate.append_to_info_file") + generate({"+append_info": "dummy"}, {}) + insert_info_spy.assert_called_once_with("apppend_info", "dummy", {}) + + +def test_generate_should_call_generate_file_if_filepath_not_startswith_plus(mocker: MockFixture): + generate_file_spy = mocker.patch("component_generator.generate.generate_file") + generate({"generate": "dummy"}, {}) + generate_file_spy.assert_called_once_with("generate", "dummy", {}) diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..b0cb2c2 --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,43 @@ +from pytest_mock import MockFixture + +from helpers import create_folder_for_filepath, populate_setting_values + + +def test_create_folder_for_filepath(mocker: MockFixture): + dummy_folder = "dummy_folder" + mkdir_spy = mocker.patch("os.makedirs") + create_folder_for_filepath(f"{dummy_folder}/dummy_file") + mkdir_spy.assert_called_once_with(dummy_folder) + + +def test_create_folder_for_filepath_with_nested_folderpath(mocker: MockFixture): + dummy_folderpath = "dummy/folder/child" + mkdir_spy = mocker.patch("os.makedirs") + create_folder_for_filepath(f"{dummy_folderpath}/dummy_file") + mkdir_spy.assert_called_once_with(dummy_folderpath) + + +def test_create_folder_for_filepath_doesnt_create_anything_if_no_folderpath(mocker: MockFixture): + mkdir_spy = mocker.patch("os.makedirs") + create_folder_for_filepath("dummy_file") + mkdir_spy.assert_not_called() + + +def test_create_folder_for_filepath_if_folder_already_exists(mocker: MockFixture): + mocker.patch("os.makedirs", side_effect=FileExistsError()) + create_folder_for_filepath("already_exists/dummy_file") + + +def test_populate_setting_values(): + assert ( + populate_setting_values( + "${this}_${is} ${a} test_${dummy}", {"this": "these", "is": "are", "a": "", "dummy": "dummies"} + ) + == "these_are test_dummies" + ) + assert ( + populate_setting_values( + "$no} changes\nto{this} ${string", {"no": "none", "this": "these", "string": "strings"} + ) + == "$no} changes\nto{this} ${string" + ) diff --git a/tests/test_info_file.py b/tests/test_info_file.py new file mode 100644 index 0000000..3a23644 --- /dev/null +++ b/tests/test_info_file.py @@ -0,0 +1,40 @@ +from unittest.mock import call + +import pytest # type: ignore +from pytest_mock import MockFixture # type: ignore + +from info_file import ( + _check_info_file, + InfoFileNotUpdatableException, + _append_filedata, + append_to_info_file, +) + + +def test_check_info_file(): + _check_info_file("#", "dummy_filepath") + _check_info_file("#", "dummy_filepath") + + +def test_check_info_file_raises_info_file_not_updatable(): + with pytest.raises(InfoFileNotUpdatableException): + _check_info_file("----ANNO____#ANNO ---->", "dummy_filepath") + with pytest.raises(InfoFileNotUpdatableException): + _check_info_file("#", "new_part", "dummy_filepath") + == "#" + ) + + +def test_append_to_info_file(mocker: MockFixture): + open_spy = mocker.patch("builtins.open") + open_spy.return_value.__enter__.return_value.read.return_value = "#" + append_to_info_file("${test_filepath}.dat", "new_part", {"test_filepath": "actual_filepath"}) + assert call("actual_filepath.dat", "w") in open_spy.call_args_list + open_spy.return_value.__enter__.return_value.write.assert_called_once_with( + "#" + )