From 4432a0b9f425acaf7408399897ff6f52a4a95762 Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Thu, 14 Oct 2021 14:36:46 +0200 Subject: [PATCH 01/18] Parse args --- src/component_generator/__init__.py | 2 +- src/component_generator/cli.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/component_generator/__init__.py b/src/component_generator/__init__.py index c57bfd5..6c8e6b9 100644 --- a/src/component_generator/__init__.py +++ b/src/component_generator/__init__.py @@ -1 +1 @@ -__version__ = '0.0.0' +__version__ = "0.0.0" diff --git a/src/component_generator/cli.py b/src/component_generator/cli.py index b5b6aa0..273cd86 100644 --- a/src/component_generator/cli.py +++ b/src/component_generator/cli.py @@ -16,11 +16,15 @@ """ import argparse -parser = argparse.ArgumentParser(description='Command description.') -parser.add_argument('names', metavar='NAME', nargs=argparse.ZERO_OR_MORE, - help="A name of something.") +parser = argparse.ArgumentParser(description="Component Generator.") +parser.add_argument("--component", help="Component name.") +parser.add_argument("--service", help="Service name.") +parser.add_argument("--consumer", help="Consumer name.") def main(args=None): args = parser.parse_args(args=args) - print(args.names) + parts = args.component or args.service or args.upload + + if not parts: + parser.error("Please pass one of the component, service, or consumer args") From 0b9bd29075ae749740fd23aedebbadd90dcc9dee Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Thu, 14 Oct 2021 18:00:57 +0200 Subject: [PATCH 02/18] Generate function + tests --- src/component_generator/generate.py | 17 +++++++++++++++++ tests/test_component_generator.py | 1 - tests/test_generate.py | 15 +++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 src/component_generator/generate.py create mode 100644 tests/test_generate.py diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py new file mode 100644 index 0000000..b2f3892 --- /dev/null +++ b/src/component_generator/generate.py @@ -0,0 +1,17 @@ +from typing import Dict + + +def generate(structure: Dict[str, str], settings: Dict[str, str]): + for filepath, filedata in structure.items(): + if filepath.startswith("+"): + insert_info_file(filepath[1:], filedata, settings) + else: + generate_file(filepath, filedata, settings) + + +def insert_info_file(filepath: str, filedata_part: str, settings: Dict[str, str]): + pass + + +def generate_file(filepath: str, filedata: str, settings: Dict[str, str]): + pass diff --git a/tests/test_component_generator.py b/tests/test_component_generator.py index 8f7e2d7..f7a9dea 100644 --- a/tests/test_component_generator.py +++ b/tests/test_component_generator.py @@ -1,4 +1,3 @@ - from component_generator.cli import main diff --git a/tests/test_generate.py b/tests/test_generate.py new file mode 100644 index 0000000..06d078f --- /dev/null +++ b/tests/test_generate.py @@ -0,0 +1,15 @@ +from pytest_mock import MockFixture + +from component_generator.generate import generate + + +def test_generate_should_call_insert_info_file_if_filepath_startswith_plus(mocker: MockFixture): + insert_info_spy = mocker.patch("component_generator.generate.insert_info_file") + generate({"+insert_info": "dummy", "generate": "dummy"}, {}) + insert_info_spy.assert_called_once_with("insert_info", "dummy", {}) + + +def test_generate_should_call_generate_file_if_filepath_not_startswith_plus(mocker: MockFixture): + generate_file_spy = mocker.patch("component_generator.generate.generate_file") + generate({"+insert_info": "dummy", "generate": "dummy"}, {}) + generate_file_spy.assert_called_once_with("generate", "dummy", {}) From f4df5eb22eea43ac249e80d521b5f0cc53d83c0c Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Thu, 14 Oct 2021 19:15:32 +0200 Subject: [PATCH 03/18] Function to create folder from filepath + tests --- src/component_generator/generate.py | 11 ++++++++++- tests/test_generate.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py index b2f3892..7c17c1e 100644 --- a/src/component_generator/generate.py +++ b/src/component_generator/generate.py @@ -1,4 +1,5 @@ -from typing import Dict +import os +from typing import Dict, Optional def generate(structure: Dict[str, str], settings: Dict[str, str]): @@ -15,3 +16,11 @@ def insert_info_file(filepath: str, filedata_part: str, settings: Dict[str, str] def generate_file(filepath: str, filedata: str, settings: Dict[str, str]): pass + + +def create_folder_for_filepath(filepath: str): + try: + folderpath = filepath[:filepath.rindex("/")] + os.makedirs(folderpath) + except (ValueError, FileExistsError): + pass diff --git a/tests/test_generate.py b/tests/test_generate.py index 06d078f..b0da8cf 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -1,6 +1,6 @@ -from pytest_mock import MockFixture +from pytest_mock import MockFixture # type: ignore -from component_generator.generate import generate +from component_generator.generate import generate, create_folder_for_filepath def test_generate_should_call_insert_info_file_if_filepath_startswith_plus(mocker: MockFixture): @@ -13,3 +13,28 @@ def test_generate_should_call_generate_file_if_filepath_not_startswith_plus(mock generate_file_spy = mocker.patch("component_generator.generate.generate_file") generate({"+insert_info": "dummy", "generate": "dummy"}, {}) generate_file_spy.assert_called_once_with("generate", "dummy", {}) + + +def test_create_folder_for_filepath(mocker: MockFixture): + dummy_folder = "dummy_folder" + mkdir_spy = mocker.patch("os.makedirs") + create_folder_for_filepath(f"{dummy_folder}/dummy_file") + mkdir_spy.assert_called_once_with(dummy_folder) + + +def test_create_folder_for_filepath_with_nested_folderpath(mocker: MockFixture): + dummy_folderpath = "dummy/folder/child" + mkdir_spy = mocker.patch("os.makedirs") + create_folder_for_filepath(f"{dummy_folderpath}/dummy_file") + mkdir_spy.assert_called_once_with(dummy_folderpath) + + +def test_create_folder_for_filepath_doesnt_create_anything_if_no_folderpath(mocker: MockFixture): + mkdir_spy = mocker.patch("os.makedirs") + create_folder_for_filepath("dummy_file") + mkdir_spy.assert_not_called() + + +def test_create_folder_for_filepath_if_folder_already_exists(mocker: MockFixture): + mocker.patch("os.makedirs", side_effect=FileExistsError()) + create_folder_for_filepath("already_exists/dummy_file") From 6f9a7e5b046df554215493a387e3d69f68e302fd Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Thu, 14 Oct 2021 19:32:39 +0200 Subject: [PATCH 04/18] Function to populate setting values + test --- src/component_generator/generate.py | 10 ++++++++-- tests/test_generate.py | 17 ++++++++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py index 7c17c1e..27458c9 100644 --- a/src/component_generator/generate.py +++ b/src/component_generator/generate.py @@ -1,5 +1,5 @@ import os -from typing import Dict, Optional +from typing import Dict def generate(structure: Dict[str, str], settings: Dict[str, str]): @@ -18,9 +18,15 @@ def generate_file(filepath: str, filedata: str, settings: Dict[str, str]): pass +def populate_setting_values(string: str, settings: Dict[str, str]) -> str: + for key, val in settings.items(): + string = string.replace(f"${{{key}}}", val) + return string + + def create_folder_for_filepath(filepath: str): try: - folderpath = filepath[:filepath.rindex("/")] + folderpath = filepath[: filepath.rindex("/")] os.makedirs(folderpath) except (ValueError, FileExistsError): pass diff --git a/tests/test_generate.py b/tests/test_generate.py index b0da8cf..fea7f0e 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -1,6 +1,6 @@ from pytest_mock import MockFixture # type: ignore -from component_generator.generate import generate, create_folder_for_filepath +from component_generator.generate import generate, create_folder_for_filepath, populate_setting_values def test_generate_should_call_insert_info_file_if_filepath_startswith_plus(mocker: MockFixture): @@ -38,3 +38,18 @@ def test_create_folder_for_filepath_doesnt_create_anything_if_no_folderpath(mock def test_create_folder_for_filepath_if_folder_already_exists(mocker: MockFixture): mocker.patch("os.makedirs", side_effect=FileExistsError()) create_folder_for_filepath("already_exists/dummy_file") + + +def test_populate_setting_values(): + assert ( + populate_setting_values( + "${this}_${is} ${a} test_${dummy}", {"this": "these", "is": "are", "a": "", "dummy": "dummies"} + ) + == "these_are test_dummies" + ) + assert ( + populate_setting_values( + "$no} changes\nto{this} ${string", {"no": "none", "this": "these", "string": "strings"} + ) + == "$no} changes\nto{this} ${string" + ) From f4cec77fe238329fb5415275857f80847259aaec Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 10:20:29 +0200 Subject: [PATCH 05/18] Generate file function (no test) --- src/component_generator/generate.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py index 27458c9..19055c6 100644 --- a/src/component_generator/generate.py +++ b/src/component_generator/generate.py @@ -15,7 +15,13 @@ def insert_info_file(filepath: str, filedata_part: str, settings: Dict[str, str] def generate_file(filepath: str, filedata: str, settings: Dict[str, str]): - pass + populated_filepath = populate_setting_values(filepath, settings) + create_folder_for_filepath(populated_filepath) + + if os.path.exists(populated_filepath): + raise FileExistsError + with open(populated_filepath, "w") as file: + file.write(populate_setting_values(filedata, settings)) def populate_setting_values(string: str, settings: Dict[str, str]) -> str: From 27f52065f7817f8b3a127e51c44d15739a7c5767 Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 10:43:47 +0200 Subject: [PATCH 06/18] Insert info function (no tests) --- src/component_generator/constants.py | 2 ++ src/component_generator/generate.py | 29 +++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 src/component_generator/constants.py diff --git a/src/component_generator/constants.py b/src/component_generator/constants.py new file mode 100644 index 0000000..ae772c0 --- /dev/null +++ b/src/component_generator/constants.py @@ -0,0 +1,2 @@ +START_BLOCK = "#" diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py index 19055c6..f3d4f37 100644 --- a/src/component_generator/generate.py +++ b/src/component_generator/generate.py @@ -1,6 +1,13 @@ import os from typing import Dict +from component_generator.constants import START_BLOCK, END_BLOCK + + +class FileNotUpdatableException(Exception): + def __init__(self, filepath: str): + super().__init__(f"File {filepath} not updatable") + def generate(structure: Dict[str, str], settings: Dict[str, str]): for filepath, filedata in structure.items(): @@ -11,7 +18,27 @@ def generate(structure: Dict[str, str], settings: Dict[str, str]): def insert_info_file(filepath: str, filedata_part: str, settings: Dict[str, str]): - pass + populated_filepath = populate_setting_values(filepath, settings) + with open(populated_filepath, "r") as reader: + file_data = reader.read() + _check_info_file(file_data, populated_filepath) + with open(populated_filepath, "w") as writer: + writer.write(_append_filedata(file_data, filedata_part)) + + +def _check_info_file(file_data: str, filepath: str): + if not (START_BLOCK in file_data and END_BLOCK in file_data): + raise FileNotUpdatableException(filepath) + + +def _append_filedata(existing_filedata: str, new_filedata: str) -> str: + return "\n".join( + ( + existing_filedata[: existing_filedata.index(END_BLOCK)], + new_filedata, + existing_filedata[existing_filedata.index(END_BLOCK) :], + ) + ) def generate_file(filepath: str, filedata: str, settings: Dict[str, str]): From c72c6d8257f0e8d190e33b61be366f1ddf41844e Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 12:27:21 +0200 Subject: [PATCH 07/18] Fix flake8 config --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 7b60e26..226ab3c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,8 @@ universal = 1 [flake8] -max-line-length = 140 +max-line-length = 110 +extend-ignore = E203 exclude = .tox,.eggs,ci/templates,build,dist [tool:pytest] From 6e1b4ad4acf1fb30c214fdb9a4d5059272a61ac7 Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 12:55:29 +0200 Subject: [PATCH 08/18] Refactor helper functions into separate module --- src/component_generator/generate.py | 15 +--------- src/component_generator/helpers.py | 16 +++++++++++ tests/test_generate.py | 42 +--------------------------- tests/test_helpers.py | 43 +++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 55 deletions(-) create mode 100644 src/component_generator/helpers.py create mode 100644 tests/test_helpers.py diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py index f3d4f37..fb6ef44 100644 --- a/src/component_generator/generate.py +++ b/src/component_generator/generate.py @@ -2,6 +2,7 @@ from typing import Dict from component_generator.constants import START_BLOCK, END_BLOCK +from component_generator.helpers import populate_setting_values, create_folder_for_filepath class FileNotUpdatableException(Exception): @@ -49,17 +50,3 @@ def generate_file(filepath: str, filedata: str, settings: Dict[str, str]): raise FileExistsError with open(populated_filepath, "w") as file: file.write(populate_setting_values(filedata, settings)) - - -def populate_setting_values(string: str, settings: Dict[str, str]) -> str: - for key, val in settings.items(): - string = string.replace(f"${{{key}}}", val) - return string - - -def create_folder_for_filepath(filepath: str): - try: - folderpath = filepath[: filepath.rindex("/")] - os.makedirs(folderpath) - except (ValueError, FileExistsError): - pass diff --git a/src/component_generator/helpers.py b/src/component_generator/helpers.py new file mode 100644 index 0000000..e4f91c9 --- /dev/null +++ b/src/component_generator/helpers.py @@ -0,0 +1,16 @@ +import os +from typing import Dict + + +def populate_setting_values(string: str, settings: Dict[str, str]) -> str: + for key, val in settings.items(): + string = string.replace(f"${{{key}}}", val) + return string + + +def create_folder_for_filepath(filepath: str): + try: + folderpath = filepath[: filepath.rindex("/")] + os.makedirs(folderpath) + except (ValueError, FileExistsError): + pass diff --git a/tests/test_generate.py b/tests/test_generate.py index fea7f0e..47f04c4 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -1,6 +1,6 @@ from pytest_mock import MockFixture # type: ignore -from component_generator.generate import generate, create_folder_for_filepath, populate_setting_values +from component_generator.generate import generate def test_generate_should_call_insert_info_file_if_filepath_startswith_plus(mocker: MockFixture): @@ -13,43 +13,3 @@ def test_generate_should_call_generate_file_if_filepath_not_startswith_plus(mock generate_file_spy = mocker.patch("component_generator.generate.generate_file") generate({"+insert_info": "dummy", "generate": "dummy"}, {}) generate_file_spy.assert_called_once_with("generate", "dummy", {}) - - -def test_create_folder_for_filepath(mocker: MockFixture): - dummy_folder = "dummy_folder" - mkdir_spy = mocker.patch("os.makedirs") - create_folder_for_filepath(f"{dummy_folder}/dummy_file") - mkdir_spy.assert_called_once_with(dummy_folder) - - -def test_create_folder_for_filepath_with_nested_folderpath(mocker: MockFixture): - dummy_folderpath = "dummy/folder/child" - mkdir_spy = mocker.patch("os.makedirs") - create_folder_for_filepath(f"{dummy_folderpath}/dummy_file") - mkdir_spy.assert_called_once_with(dummy_folderpath) - - -def test_create_folder_for_filepath_doesnt_create_anything_if_no_folderpath(mocker: MockFixture): - mkdir_spy = mocker.patch("os.makedirs") - create_folder_for_filepath("dummy_file") - mkdir_spy.assert_not_called() - - -def test_create_folder_for_filepath_if_folder_already_exists(mocker: MockFixture): - mocker.patch("os.makedirs", side_effect=FileExistsError()) - create_folder_for_filepath("already_exists/dummy_file") - - -def test_populate_setting_values(): - assert ( - populate_setting_values( - "${this}_${is} ${a} test_${dummy}", {"this": "these", "is": "are", "a": "", "dummy": "dummies"} - ) - == "these_are test_dummies" - ) - assert ( - populate_setting_values( - "$no} changes\nto{this} ${string", {"no": "none", "this": "these", "string": "strings"} - ) - == "$no} changes\nto{this} ${string" - ) diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..f27442b --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,43 @@ +from pytest_mock import MockFixture + +from component_generator.helpers import create_folder_for_filepath, populate_setting_values + + +def test_create_folder_for_filepath(mocker: MockFixture): + dummy_folder = "dummy_folder" + mkdir_spy = mocker.patch("os.makedirs") + create_folder_for_filepath(f"{dummy_folder}/dummy_file") + mkdir_spy.assert_called_once_with(dummy_folder) + + +def test_create_folder_for_filepath_with_nested_folderpath(mocker: MockFixture): + dummy_folderpath = "dummy/folder/child" + mkdir_spy = mocker.patch("os.makedirs") + create_folder_for_filepath(f"{dummy_folderpath}/dummy_file") + mkdir_spy.assert_called_once_with(dummy_folderpath) + + +def test_create_folder_for_filepath_doesnt_create_anything_if_no_folderpath(mocker: MockFixture): + mkdir_spy = mocker.patch("os.makedirs") + create_folder_for_filepath("dummy_file") + mkdir_spy.assert_not_called() + + +def test_create_folder_for_filepath_if_folder_already_exists(mocker: MockFixture): + mocker.patch("os.makedirs", side_effect=FileExistsError()) + create_folder_for_filepath("already_exists/dummy_file") + + +def test_populate_setting_values(): + assert ( + populate_setting_values( + "${this}_${is} ${a} test_${dummy}", {"this": "these", "is": "are", "a": "", "dummy": "dummies"} + ) + == "these_are test_dummies" + ) + assert ( + populate_setting_values( + "$no} changes\nto{this} ${string", {"no": "none", "this": "these", "string": "strings"} + ) + == "$no} changes\nto{this} ${string" + ) From 633dcee76905cfc19f4272110363a987d4879120 Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 12:57:55 +0200 Subject: [PATCH 09/18] Refactor Info File functions into separate module --- src/component_generator/generate.py | 33 ++------------------------ src/component_generator/info_file.py | 35 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 31 deletions(-) create mode 100644 src/component_generator/info_file.py diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py index fb6ef44..2fd94e0 100644 --- a/src/component_generator/generate.py +++ b/src/component_generator/generate.py @@ -1,47 +1,18 @@ import os from typing import Dict -from component_generator.constants import START_BLOCK, END_BLOCK from component_generator.helpers import populate_setting_values, create_folder_for_filepath - - -class FileNotUpdatableException(Exception): - def __init__(self, filepath: str): - super().__init__(f"File {filepath} not updatable") +from component_generator.info_file import append_to_info_file def generate(structure: Dict[str, str], settings: Dict[str, str]): for filepath, filedata in structure.items(): if filepath.startswith("+"): - insert_info_file(filepath[1:], filedata, settings) + append_to_info_file(filepath[1:], filedata, settings) else: generate_file(filepath, filedata, settings) -def insert_info_file(filepath: str, filedata_part: str, settings: Dict[str, str]): - populated_filepath = populate_setting_values(filepath, settings) - with open(populated_filepath, "r") as reader: - file_data = reader.read() - _check_info_file(file_data, populated_filepath) - with open(populated_filepath, "w") as writer: - writer.write(_append_filedata(file_data, filedata_part)) - - -def _check_info_file(file_data: str, filepath: str): - if not (START_BLOCK in file_data and END_BLOCK in file_data): - raise FileNotUpdatableException(filepath) - - -def _append_filedata(existing_filedata: str, new_filedata: str) -> str: - return "\n".join( - ( - existing_filedata[: existing_filedata.index(END_BLOCK)], - new_filedata, - existing_filedata[existing_filedata.index(END_BLOCK) :], - ) - ) - - def generate_file(filepath: str, filedata: str, settings: Dict[str, str]): populated_filepath = populate_setting_values(filepath, settings) create_folder_for_filepath(populated_filepath) diff --git a/src/component_generator/info_file.py b/src/component_generator/info_file.py new file mode 100644 index 0000000..78eb15f --- /dev/null +++ b/src/component_generator/info_file.py @@ -0,0 +1,35 @@ +from typing import Dict + +from component_generator.helpers import populate_setting_values + +START_BLOCK = "#" + + +class InfoFileNotUpdatableException(Exception): + def __init__(self, filepath: str): + super().__init__(f"File {filepath} not updatable") + + +def _check_info_file(file_data: str, filepath: str): + if not (START_BLOCK in file_data and END_BLOCK in file_data): + raise InfoFileNotUpdatableException(filepath) + + +def _append_filedata(existing_filedata: str, new_filedata: str, filepath: str) -> str: + _check_info_file(existing_filedata, filepath) + return "\n".join( + ( + existing_filedata[: existing_filedata.index(END_BLOCK)], + new_filedata, + existing_filedata[existing_filedata.index(END_BLOCK) :], + ) + ) + + +def append_to_info_file(filepath: str, filedata_part: str, settings: Dict[str, str]): + populated_filepath = populate_setting_values(filepath, settings) + with open(populated_filepath, "r") as reader: + file_data = reader.read() + with open(populated_filepath, "w") as writer: + writer.write(_append_filedata(file_data, filedata_part, populated_filepath)) From 88046c57734f721e2d1891d83c2f8989a4464b73 Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 12:58:15 +0200 Subject: [PATCH 10/18] Tests for info file --- tests/test_generate.py | 10 +++++----- tests/test_info_file.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 tests/test_info_file.py diff --git a/tests/test_generate.py b/tests/test_generate.py index 47f04c4..ca9079a 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -3,13 +3,13 @@ from component_generator.generate import generate -def test_generate_should_call_insert_info_file_if_filepath_startswith_plus(mocker: MockFixture): - insert_info_spy = mocker.patch("component_generator.generate.insert_info_file") - generate({"+insert_info": "dummy", "generate": "dummy"}, {}) - insert_info_spy.assert_called_once_with("insert_info", "dummy", {}) +def test_generate_should_call_append_to_info_file_if_filepath_startswith_plus(mocker: MockFixture): + insert_info_spy = mocker.patch("component_generator.generate.append_to_info_file") + generate({"+append_info": "dummy"}, {}) + insert_info_spy.assert_called_once_with("apppend_info", "dummy", {}) def test_generate_should_call_generate_file_if_filepath_not_startswith_plus(mocker: MockFixture): generate_file_spy = mocker.patch("component_generator.generate.generate_file") - generate({"+insert_info": "dummy", "generate": "dummy"}, {}) + generate({"generate": "dummy"}, {}) generate_file_spy.assert_called_once_with("generate", "dummy", {}) diff --git a/tests/test_info_file.py b/tests/test_info_file.py new file mode 100644 index 0000000..64ed689 --- /dev/null +++ b/tests/test_info_file.py @@ -0,0 +1,40 @@ +from unittest.mock import call + +import pytest # type: ignore +from pytest_mock import MockFixture # type: ignore + +from component_generator.info_file import ( + _check_info_file, + InfoFileNotUpdatableException, + _append_filedata, + append_to_info_file, +) + + +def test_check_info_file(): + _check_info_file("#", "dummy_filepath") + _check_info_file("#", "dummy_filepath") + + +def test_check_info_file_raises_info_file_not_updatable(): + with pytest.raises(InfoFileNotUpdatableException): + _check_info_file("----ANNO____#ANNO ---->", "dummy_filepath") + with pytest.raises(InfoFileNotUpdatableException): + _check_info_file("#", "new_part", "dummy_filepath") + == "#" + ) + + +def test_append_to_info_file(mocker: MockFixture): + open_spy = mocker.patch("builtins.open") + open_spy.return_value.__enter__.return_value.read.return_value = "#" + append_to_info_file("${test_filepath}.dat", "new_part", {"test_filepath": "actual_filepath"}) + assert call("actual_filepath.dat", "w") in open_spy.call_args_list + open_spy.return_value.__enter__.return_value.write.assert_called_once_with( + "#" + ) From 8d7511ebf15c2a1531c8029f4c82bfd59261a0ae Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 14:33:57 +0200 Subject: [PATCH 11/18] Component files; exclude from flake8 and mypy --- .anno.json | 1 + setup.cfg | 6 +- .../component_files/component.py | 358 ++++++++++++++++++ 3 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 .anno.json create mode 100644 src/component_generator/component_files/component.py diff --git a/.anno.json b/.anno.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/.anno.json @@ -0,0 +1 @@ +{} diff --git a/setup.cfg b/setup.cfg index 226ab3c..b4ae35a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ universal = 1 [flake8] max-line-length = 110 extend-ignore = E203 -exclude = .tox,.eggs,ci/templates,build,dist +exclude = .tox,.eggs,ci/templates,build,dist,src/component_generator/component_files [tool:pytest] # If a pytest section is found in one of the possible config files @@ -34,3 +34,7 @@ known_first_party = component_generator default_section = THIRDPARTY forced_separate = test_component_generator skip = .tox,.eggs,ci/templates,build,dist + +[mypy] +exclude = src/component_generator/component_files +ignore_missing_imports = True diff --git a/src/component_generator/component_files/component.py b/src/component_generator/component_files/component.py new file mode 100644 index 0000000..f52d931 --- /dev/null +++ b/src/component_generator/component_files/component.py @@ -0,0 +1,358 @@ +from info_file import START_BLOCK, END_BLOCK + +COMPONENT_FILES = { + # ROOT level + "${componentName}/Makefile": """ +POETRY ?= poetry + +_default: test + +clean: + find . -name '__pycache__' | xargs rm -rf + find . -type f -name "*.pyc" -delete + +install-dev: + $(POETRY) install + +install: + $(POETRY) install --no-dev + +format: + $(POETRY) run isort . + $(POETRY) run black . + +lint: + $(POETRY) run isort --check-only . + $(POETRY) run black --check . + $(POETRY) run flake8 --config setup.cfg + +mypy: + $(POETRY) run mypy -p ${componentName} + + test: + $(POETRY) run pytest tests + +test-cov: + $(POETRY) run pytest tests --cov=${componentName} + +coverage-report: + $(POETRY) run coverage xml -i -o coverage.xml + $(POETRY) run coverage report + +test-and-report: + # we want to report on success AND failure + make test-cov && make coverage-report || (make coverage-report; exit 1) + +.PHONY: clean install install-dev test coverage-report test-and-report + +""", + "${componentName}/Dockerfile": """FROM gcr.io/atlas-images/python-poetry:3.8.8-buster as base + +COPY --chown=onnauser:onnagroup ./ /app + +FROM base as production +RUN make install +USER root +RUN apt remove -y --purge binutils libc6-dev gcc --allow-remove-essential +USER onnauser + +FROM base as test +RUN make install-dev +USER root +RUN apt remove -y --purge binutils libc6-dev gcc --allow-remove-essential +USER onnauser + +""", + "${componentName}/pyproject.toml": """[tool.poetry] +name = "${componentName}" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.8" +pydantic = "^1.8.1" +prometheus-client = "^0.9.0" +onna-utils = "^0.1.0" +onna-types = "^0.1.0" + +[tool.poetry.dev-dependencies] +pytest = "^6.2.2" +mypy = "^0.812" +pytest-asyncio = "^0.14.0" +flake8 = "^3.8.4" +isort = "^5.7.0" +black = "^20.8b1" +pytest-cov = "^2.11.1" +async-asgi-testclient = "^1.4.6" + + +[[tool.poetry.source]] +name = "onna" +url = "http://onna:atlasense@pypi.intra.onna.internal/simple/" +secondary = true + +[tool.black] +line-length = 100 +target-version = ['py38'] +include = '\.pyi?$' +exclude = ''' + +( + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + )/ + | foo.py # also separately exclude a file named foo.py in + # the root of the project +) +''' + +[tool.poetry.scripts] +""" + + START_BLOCK + + """ + +""" + + END_BLOCK + + """ + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" + """, + "${componentName}/README.md": """# ${componentName} +Description here! + + +## Develop + +``` +make install-dev +``` + +## Run tests + +Run docker compose before running tests: + +``` +docker-compose up -d +``` + +``` +make test +``` + + """, + "${componentName}/Jenkinsfile": """@Library('onna-library') _ + +def MAJOR_VERSION = 0 +def MINOR_VERSION = 1 + +node { + environments.DoCheckout() + environments.SetGCLOUD() + + // environments.GetAppName() does extra stuff we don't care about + def appName = "${componentName}" + def version = "${MAJOR_VERSION}.${MINOR_VERSION}.${env.BUILD_NUMBER}" + if (env.BRANCH_NAME != 'master') { + version = "${MAJOR_VERSION}.${MINOR_VERSION + 1}.0-${env.BRANCH_NAME}${env.BUILD_NUMBER}" + } + println("Building version ${version}") + + def image = dockerbuilds.BuildDockerImage(appName, env.BRANCH_NAME, env.BUILD_NUMBER, version, ".", null, null, "--target=test") + def prefixDockerName = appName + '-' + env.BRANCH_NAME + '-' + env.BUILD_NUMBER + '-test' + + stage ('Start Persistent Layers') { + // none + } + + stage ('Run Pre-checks') { + sh("docker run --rm ${image} /bin/bash -c 'make lint && make mypy'") + } + + def runName = appName + '-' + env.BRANCH_NAME + '-' + env.BUILD_NUMBER + '-test' + stage("Run Tests") { + try { + sh("docker run --name='${runName}' " + + "${image} /bin/bash -c 'make test-and-report'") + sh("docker cp ${runName}:/app/coverage.xml .") + // fix path to source from report + sh("sed -i 's@\\.*@source>${componentName}@g' coverage.xml") + + } catch(Exception e){ + slackSend ( + channel: '#jenkins', + color: 'danger', + message: "Jenkins Job '${env.JOB_NAME}' (${env.BUILD_NUMBER}) failed in the alltests stage with ${e.message}", to: 'dev@atlasense.com', body: "Please go to ${env.BUILD_URL}."); + throw e + } finally { + sh("docker rm -v ${runName} || true") + } + } + + stage("SonarQube") { + sonarqube.scan(version) + } + + image = dockerbuilds.BuildDockerImage(appName, env.BRANCH_NAME, env.BUILD_NUMBER, version, ".", null, null, "--target=production") + image = dockerbuilds.PushDockerImage(appName, env.BRANCH_NAME, env.BUILD_NUMBER, version, image) + + helm.UpdateHelm(appName, env.BRANCH_NAME, env.BUILD_NUMBER, image, version) + helm.UploadHelm(appName, env.BRANCH_NAME, env.BUILD_NUMBER, version) + + deploy.ComponentDeploy(appName, env.BRANCH_NAME, env.BUILD_NUMBER, version) + + notify.JobDone(appName, env.JOB_BASE_NAME, env.BUILD_NUMBER) + +} +""", + "${componentName}/AUTOUPDATE": "", + "${componentName}/setup.cfg": """[flake8] +max-line-length = 120 +exclude = .eggs + +[mypy-prometheus_client.*] +ignore_missing_imports = True + +[mypy-uvicorn.*] +ignore_missing_imports = True + +[mypy-lru.*] +ignore_missing_imports = True + +[isort] +line_length = 100 +include_trailing_comma=True +force_grid_wrap=4 +use_parentheses=True +force_single_line=False +multi_line_output=3 + +[mypy] +plugins = pydantic.mypy +mypy_path = stubs +""", + # + # Tests + # + "${componentName}/tests/__init__.py": "", + "${componentName}/tests/fixtures.py": "", + "${componentName}/tests/acceptance/__init__.py": "", + "${componentName}/tests/acceptance/test_service.py": """ +def test_it(): + assert True +""", + "${componentName}/tests/unit/__init__.py": "", + "${componentName}/tests/unit/test_service.py": """ +def test_it(): + assert True +""", + # + # Python package + # + "${componentName}/${componentName}/__init__.py": "", + "${componentName}/${componentName}/commands.py": """import argparse +import asyncio +import logging + +from .settings import Settings + +logger = logging.getLogger(__name__) + + +parser = argparse.ArgumentParser(description="command runner", add_help=False) +parser.add_argument( + "-e", + "--env-file", + help="Env file", +) + + +def get_settings() -> Settings: + arguments, _ = parser.parse_known_args() + return Settings(_env_file=arguments.env_file) +""" + + START_BLOCK + + """ + +""" + + END_BLOCK + + """ + + """, + "${componentName}/${componentName}/settings.py": """ +from pydantic import BaseSettings + + +class Settings(BaseSettings): + ... + + """, + # + # Chart + # + "${componentName}/charts/${componentName}/Chart.yaml": """apiVersion: v1 +appVersion: 1.4.2 +description: ${componentName} +name: ${componentName} +sources: + - https://github.com/atlasense/${componentName} +maintainers: +- name: Platform + email: platform@onna.com +version: 99999.99999.99999 +""", + "${componentName}/charts/${componentName}/requirements.yaml": """dependencies: +""", + "${componentName}/charts/${componentName}/values.yaml": """image: IMAGE_TO_REPLACE +pullSecrets: [] + + +""" + + START_BLOCK + + """ + +""" + + END_BLOCK + + """ +""", + "${componentName}/charts/${componentName}/templates/_helpers.tpl": """{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. */}} +{{- define "name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{/* Create a default fully qualified app name. We truncate at 63 chars because . . . */}} +{{- define "fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}}""", + "${componentName}/charts/${componentName}/templates/cm.yaml": """apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "name" . }}-config + namespace: {{ .Release.Namespace }} + labels: + service_name: {{ template "name" . }} + version: {{ .Chart.Version }} + tier: "backend" + release: {{ .Release.Name }} +data: +""" + + START_BLOCK + + """ + +""" + + END_BLOCK + + """ +""", +} From bcbde76c1b88e8aebbcc81e4ae98c18bfc0baf6f Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 14:36:27 +0200 Subject: [PATCH 12/18] Function to generate component (no test) --- src/component_generator/__main__.py | 2 +- src/component_generator/cli.py | 8 ++++++++ src/component_generator/generate.py | 29 ++++++++++++++++++++++++++-- src/component_generator/info_file.py | 2 +- tests/test_component_generator.py | 2 +- tests/test_generate.py | 2 +- tests/test_helpers.py | 2 +- tests/test_info_file.py | 2 +- 8 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/component_generator/__main__.py b/src/component_generator/__main__.py index 4cea4ba..79eb559 100644 --- a/src/component_generator/__main__.py +++ b/src/component_generator/__main__.py @@ -8,7 +8,7 @@ - https://docs.python.org/2/using/cmdline.html#cmdoption-m - https://docs.python.org/3/using/cmdline.html#cmdoption-m """ -from component_generator.cli import main +from cli import main if __name__ == "__main__": main() diff --git a/src/component_generator/cli.py b/src/component_generator/cli.py index 273cd86..d421476 100644 --- a/src/component_generator/cli.py +++ b/src/component_generator/cli.py @@ -16,6 +16,8 @@ """ import argparse +from generate import generate_component, ComponentType + parser = argparse.ArgumentParser(description="Component Generator.") parser.add_argument("--component", help="Component name.") parser.add_argument("--service", help="Service name.") @@ -28,3 +30,9 @@ def main(args=None): if not parts: parser.error("Please pass one of the component, service, or consumer args") + if args.component: + generate_component(ComponentType.COMPONENT, args.component) + if args.service: + generate_component(ComponentType.SERVICE, args.service) + if args.consumer: + generate_component(ComponentType.CONSUMER, args.consumer) diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py index 2fd94e0..c42ac44 100644 --- a/src/component_generator/generate.py +++ b/src/component_generator/generate.py @@ -1,8 +1,28 @@ +import json import os +from enum import Enum from typing import Dict -from component_generator.helpers import populate_setting_values, create_folder_for_filepath -from component_generator.info_file import append_to_info_file +from component_files.component import COMPONENT_FILES +from helpers import populate_setting_values, create_folder_for_filepath +from info_file import append_to_info_file + +SETTINGS_FILENAME = ".anno.json" + + +class ComponentType(Enum): + COMPONENT = "component" + SERVICE = "service" + CONSUMER = "consumer" + + +COMPONENT_FILESTRUCTURE = {ComponentType.COMPONENT: COMPONENT_FILES} + + +def generate_component(component_type: ComponentType, component_name: str): + settings = load_settings_from_file() + settings[f"{component_type.value}Name"] = component_name + generate(COMPONENT_FILESTRUCTURE[component_type], settings) def generate(structure: Dict[str, str], settings: Dict[str, str]): @@ -21,3 +41,8 @@ def generate_file(filepath: str, filedata: str, settings: Dict[str, str]): raise FileExistsError with open(populated_filepath, "w") as file: file.write(populate_setting_values(filedata, settings)) + + +def load_settings_from_file(settings_filename: str = SETTINGS_FILENAME) -> Dict[str, str]: + with open(settings_filename, "r") as file: + return json.load(file) diff --git a/src/component_generator/info_file.py b/src/component_generator/info_file.py index 78eb15f..d998b3e 100644 --- a/src/component_generator/info_file.py +++ b/src/component_generator/info_file.py @@ -1,6 +1,6 @@ from typing import Dict -from component_generator.helpers import populate_setting_values +from helpers import populate_setting_values START_BLOCK = "#" diff --git a/tests/test_component_generator.py b/tests/test_component_generator.py index f7a9dea..5b47d22 100644 --- a/tests/test_component_generator.py +++ b/tests/test_component_generator.py @@ -1,4 +1,4 @@ -from component_generator.cli import main +from cli import main def test_main(): diff --git a/tests/test_generate.py b/tests/test_generate.py index ca9079a..6918980 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -1,6 +1,6 @@ from pytest_mock import MockFixture # type: ignore -from component_generator.generate import generate +from generate import generate def test_generate_should_call_append_to_info_file_if_filepath_startswith_plus(mocker: MockFixture): diff --git a/tests/test_helpers.py b/tests/test_helpers.py index f27442b..b0cb2c2 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,6 +1,6 @@ from pytest_mock import MockFixture -from component_generator.helpers import create_folder_for_filepath, populate_setting_values +from helpers import create_folder_for_filepath, populate_setting_values def test_create_folder_for_filepath(mocker: MockFixture): diff --git a/tests/test_info_file.py b/tests/test_info_file.py index 64ed689..3a23644 100644 --- a/tests/test_info_file.py +++ b/tests/test_info_file.py @@ -3,7 +3,7 @@ import pytest # type: ignore from pytest_mock import MockFixture # type: ignore -from component_generator.info_file import ( +from info_file import ( _check_info_file, InfoFileNotUpdatableException, _append_filedata, From a8bfc640f213cffe9deba7c59ab414553d2a46f5 Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 15:51:43 +0200 Subject: [PATCH 13/18] Service and consumer file structure --- .../component_files/consumer.py | 187 +++++++++++++++ .../component_files/service.py | 217 ++++++++++++++++++ src/component_generator/generate.py | 9 +- 3 files changed, 412 insertions(+), 1 deletion(-) create mode 100644 src/component_generator/component_files/consumer.py create mode 100644 src/component_generator/component_files/service.py diff --git a/src/component_generator/component_files/consumer.py b/src/component_generator/component_files/consumer.py new file mode 100644 index 0000000..a35a39e --- /dev/null +++ b/src/component_generator/component_files/consumer.py @@ -0,0 +1,187 @@ +CONSUMER_FILES = { + "charts/${componentName}/templates/${consumerName}.deploy.yaml": """kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ template "name" . }}-${consumerName} + namespace: {{ .Release.Namespace }} + labels: + service_name: {{ template "name" . }}-${consumerName} + release: {{ .Release.Name }} + app: {{ template "name" . }}-${consumerName} + version: {{ .Chart.Version }} + tier: "backend" +spec: + replicas: {{ .Values.${consumerName}_hpa.minReplicas }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit | default "3" }} + selector: + matchLabels: + service_name: {{ template "name" . }}-${consumerName} + tier: "backend" + template: + metadata: + name: {{ template "name" . }}-${consumerName} + annotations: + {{- if .Values.${consumerName}_annotations }} + checksum/config: {{ include (print $.Template.BasePath "/cm.yaml") . | sha256sum }} + {{- tpl (toYaml .Values.${consumerName}_annotations) . | nindent 8 }} + {{- end }} + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + labels: + service_name: {{ template "name" . }}-${consumerName} + tier: "backend" + app: {{ template "name" . }}-${consumerName} + version: {{ .Chart.Version }} + spec: + {{- if .Values.affinity }} + affinity: + {{- tpl (toYaml .Values.affinity) . | nindent 8 }} + {{- end }} + {{- if .Values.${consumerName}_tolerations }} + tolerations: + {{- toYaml .Values.${consumerName}_tolerations | nindent 8 }} + {{- end }} + dnsPolicy: ClusterFirst + {{- if .Values.pullSecrets }} + imagePullSecrets: + - name: {{ .Values.pullSecrets }} + {{- end }} + containers: + - name: {{ template "name" . }}-${consumerName} + image: {{ .Values.image }} + command: ["${consumerName}"] + envFrom: + - configMapRef: + name: {{ template "name" . }}-config + {{- if .Values.${consumerName}_resources }} + resources: + {{- toYaml .Values.${consumerName}_resources | nindent 10 }} + {{- end }} + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + readinessProbe: + httpGet: + path: /health + port: 8080 + ports: + - name: api + containerPort: 8080""", + "charts/${componentName}/templates/${consumerName}.hpa.yaml": """apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "name" . }}-${consumerName} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "name" . }}-api + minReplicas: {{ .Values.${consumerName}_hpa.minReplicas }} + maxReplicas: {{ .Values.${consumerName}_hpa.maxReplicas }} + metrics: + {{- toYaml .Values.${consumerName}_hpa.metrics | nindent 4 }} +""", + "+charts/${componentName}/values.yaml": """${consumerName}_replicaCount: 2 +${consumerName}_annotations: +${consumerName}_tolerations: [] +${consumerName}_affinity: {} +${consumerName}_resources: + limits: + cpu: 1000m + memory: 512Mi + requests: + cpu: 500m + memory: 256Mi +${consumerName}_hpa: + minReplicas: 1 + maxReplicas: 2""", + "${componentName}/${consumerName}.py": """import prometheus_client +from fastapi import APIRouter, FastAPI, Request +from onna_utils.middleware.starlette import PrometheusMiddleware +from starlette.responses import JSONResponse, Response +from onna_utils import metrics +import kafkaesk + +router = kafkaesk.Router() + +class ConsumerApplication(kafkaesk.Application): + app_settings: Settings + + def __init__(self, kafka_settings: KafkaSettings, app_settings: Settings): + super().__init__( + kafka_servers=kafka_settings.kafka_servers, + topic_prefix=kafka_settings.kafka_prefix, + kafka_api_version=kafka_settings.kafka_api_version or "auto", + kafka_settings=kafka_settings.kafka_settings, + ) + self.mount(router) + self.app_settings = app_settings + self.on("finalize", self._close) + + async def _close(self): + ... + +@router.subscribe(MY_TOPIC, group=MY_GROUP) +async def my_subscriber(my_ob: ObjecType): + ... + + +http_router = APIRouter() + + +@http_router.get("/health") +async def health(request: Request) -> None: + with metrics.healthy(): + await request.app.consumer.health_check() + + +@http_router.get("/metrics") +def get_metrics(request: Request) -> Response: + output = prometheus_client.exposition.generate_latest() + return Response(output.decode("utf8")) + + +class HTTPApplication(FastAPI): + def __init__(self, consumer, *args, **kwargs): + super().__init__(title="${consumerName}", redoc_url=None, docs_url=None, *args, **kwargs) + self.add_middleware(PrometheusMiddleware) + self.include_router(http_router) + + self.consumer = consumer + + self.add_event_handler("startup", self.initialize) + self.add_event_handler("shutdown", self.finalize) + + async def initialize(self) -> None: + ... + + async def finalize(self) -> None: + ... +""", + "+${componentName}/commands.py": """ +def run_${consumerName}(): + settings = get_settings() + asyncio.run(_run_${consumerName}(settings)) + + +async def _run_${consumerName}(settings: Settings): + from ${componentName} import ${consumerName} + from onna_utils.commands import serve_consumer_app + + consumer_app = ${consumerName}.ConsumerApplication() + http_app = ${consumerName}.HTTPApplication(consumer_app) + await serve_consumer_app(http_app, consumer_app, port=8080) +""", + "+pyproject.toml": """ +${consumerName} = '${componentName}.commands:run_${consumerName}' +""", +} diff --git a/src/component_generator/component_files/service.py b/src/component_generator/component_files/service.py new file mode 100644 index 0000000..844fc3d --- /dev/null +++ b/src/component_generator/component_files/service.py @@ -0,0 +1,217 @@ +SERVICE_FILES = { + "charts/${componentName}/templates/${serviceName}.deploy.yaml": """kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ template "name" . }}-${serviceName} + namespace: {{ .Release.Namespace }} + labels: + service_name: {{ template "name" . }}-${serviceName} + release: {{ .Release.Name }} + app: {{ template "name" . }}-${serviceName} + version: {{ .Chart.Version }} + tier: "backend" +spec: + replicas: {{ .Values.${serviceName}_hpa.minReplicas }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit | default "3" }} + selector: + matchLabels: + service_name: {{ template "name" . }}-${serviceName} + tier: "backend" + template: + metadata: + name: {{ template "name" . }}-${serviceName} + annotations: + {{- if .Values.${serviceName}_annotations }} + checksum/config: {{ include (print $.Template.BasePath "/cm.yaml") . | sha256sum }} + {{- tpl (toYaml .Values.${serviceName}_annotations) . | nindent 8 }} + {{- end }} + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + labels: + service_name: {{ template "name" . }}-${serviceName} + tier: "backend" + app: {{ template "name" . }}-${serviceName} + version: {{ .Chart.Version }} + spec: + {{- if .Values.affinity }} + affinity: + {{- tpl (toYaml .Values.affinity) . | nindent 8 }} + {{- end }} + {{- if .Values.${serviceName}_tolerations }} + tolerations: + {{- toYaml .Values.${serviceName}_tolerations | nindent 8 }} + {{- end }} + dnsPolicy: ClusterFirst + {{- if .Values.pullSecrets }} + imagePullSecrets: + - name: {{ .Values.pullSecrets }} + {{- end }} + containers: + - name: {{ template "name" . }}-${serviceName} + image: {{ .Values.image }} + command: ["${serviceName}"] + envFrom: + - configMapRef: + name: {{ template "name" . }}-config + {{- if .Values.${serviceName}_resources }} + resources: + {{- toYaml .Values.${serviceName}_resources | nindent 10 }} + {{- end }} + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + readinessProbe: + httpGet: + path: /health + port: 8080 + ports: + - name: api + containerPort: 8080""", + "charts/${componentName}/templates/${serviceName}.hpa.yaml": """apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "name" . }}-${serviceName} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "name" . }}-api + minReplicas: {{ .Values.${serviceName}_hpa.minReplicas }} + maxReplicas: {{ .Values.${serviceName}_hpa.maxReplicas }} + metrics: + {{- toYaml .Values.${serviceName}_hpa.metrics | nindent 4 }} +""", + "charts/${componentName}/templates/${serviceName}.svc.yaml": """kind: Service +apiVersion: v1 +metadata: + name: {{ template "name" . }}-${serviceName} + namespace: {{ .Release.Namespace }} + labels: + service_name: {{ template "name" . }}-${serviceName} + app: {{ template "name" . }}-${serviceName} + version: {{ .Chart.Version }} + tier: "backend" + release: {{ .Release.Name }} + annotations: + getambassador.io/config: | + --- + apiVersion: ambassador/v1 + kind: Mapping + name: ambassador_{{ template "name" . }}_${serviceName}_{{ .Release.Namespace | default .Values.app }}_mapping + prefix: /${serviceName}/ + service: {{ template "name" . }}-${serviceName}.{{ .Release.Namespace | default .Values.app }}:8080 + resolver: endpoint + timeout_ms: 600000 + connect_timeout_ms: 30000 + idle_timeout_ms: 60000 + circuit_breakers: + - priority: default + max_connections: 3072 + max_pending_requests: 2048 + max_requests: 4096 + max_retries: 50 + load_balancer: + policy: round_robin + retry_policy: + retry_on: gateway-error + num_retries: 4 +spec: + type: ClusterIP + ports: + - name: http + port: 8080 + targetPort: 8080 + selector: + service_name: {{ template "name" . }}-${serviceName} +""", + "+charts/${componentName}/values.yaml": """${serviceName}_replicaCount: 2 +${serviceName}_annotations: +${serviceName}_tolerations: [] +${serviceName}_affinity: {} +${serviceName}_resources: + limits: + cpu: 1000m + memory: 512Mi + requests: + cpu: 500m + memory: 256Mi""", + "${componentName}/${serviceName}.py": """import prometheus_client +from fastapi import APIRouter, FastAPI, Request +from onna_utils.middleware.starlette import PrometheusMiddleware +from starlette.responses import JSONResponse, Response + +router = APIRouter() + + +@router.get("/health") +async def health(request: Request) -> None: + ... + + +@router.get("/metrics") +def get_metrics(request: Request) -> Response: + output = prometheus_client.exposition.generate_latest() + return Response(output.decode("utf8")) + + +@router.get("/ping") +def get_ping(request: Request) -> Response: + return JSONResponse({"po": "ng"}) + + +@router.get("/foobar") +async def get_foobar(): + return {} + + +class HTTPApplication(FastAPI): + def __init__( + self, + *args, + **kwargs, + ): + super().__init__(title="${serviceName}", redoc_url=None, docs_url=None, *args, **kwargs) + self.add_middleware(PrometheusMiddleware) + self.include_router(router) + + self.add_event_handler("startup", self.initialize) + self.add_event_handler("shutdown", self.finalize) + + async def initialize(self) -> None: + ... + + async def finalize(self) -> None: + ... +""", + "+${componentName}/commands.py": """ +def run_${serviceName}(): + settings = get_settings() + asyncio.run(_run_${serviceName}(settings)) + + +async def _run_${serviceName}(settings: Settings): + from ${componentName} import ${serviceName} + import uvicorn + + http_config = uvicorn.Config( + ${serviceName}.HTTPApplication(), + port=8080, + host="0.0.0.0", + log_level=None, + ) + server = uvicorn.Server(config=http_config) + await server.serve() +""", + "+pyproject.toml": """ +${serviceName} = '${componentName}.commands:run_${serviceName}' +""", +} diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py index c42ac44..8537cb4 100644 --- a/src/component_generator/generate.py +++ b/src/component_generator/generate.py @@ -4,6 +4,8 @@ from typing import Dict from component_files.component import COMPONENT_FILES +from component_files.consumer import CONSUMER_FILES +from component_files.service import SERVICE_FILES from helpers import populate_setting_values, create_folder_for_filepath from info_file import append_to_info_file @@ -16,7 +18,11 @@ class ComponentType(Enum): CONSUMER = "consumer" -COMPONENT_FILESTRUCTURE = {ComponentType.COMPONENT: COMPONENT_FILES} +COMPONENT_FILESTRUCTURE = { + ComponentType.COMPONENT: COMPONENT_FILES, + ComponentType.SERVICE: SERVICE_FILES, + ComponentType.CONSUMER: CONSUMER_FILES, +} def generate_component(component_type: ComponentType, component_name: str): @@ -34,6 +40,7 @@ def generate(structure: Dict[str, str], settings: Dict[str, str]): def generate_file(filepath: str, filedata: str, settings: Dict[str, str]): + filepath = f"component/{filepath}" populated_filepath = populate_setting_values(filepath, settings) create_folder_for_filepath(populated_filepath) From 07f49da166734219e3c39e7f80ddc74757b2ba26 Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 16:14:21 +0200 Subject: [PATCH 14/18] Always require component arg --- src/component_generator/cli.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/component_generator/cli.py b/src/component_generator/cli.py index d421476..4885c96 100644 --- a/src/component_generator/cli.py +++ b/src/component_generator/cli.py @@ -26,13 +26,11 @@ def main(args=None): args = parser.parse_args(args=args) - parts = args.component or args.service or args.upload - if not parts: - parser.error("Please pass one of the component, service, or consumer args") - if args.component: - generate_component(ComponentType.COMPONENT, args.component) + if not hasattr(args, "component"): + parser.error("Please pass component name") + generate_component(ComponentType.COMPONENT, args.component, args.component) if args.service: - generate_component(ComponentType.SERVICE, args.service) + generate_component(ComponentType.SERVICE, args.service, args.component) if args.consumer: - generate_component(ComponentType.CONSUMER, args.consumer) + generate_component(ComponentType.CONSUMER, args.consumer, args.component) From 29e07ca3baa928676b49a0745d102bcf23d1439f Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 16:15:19 +0200 Subject: [PATCH 15/18] FIX issues in service and consumer file structure --- src/component_generator/component_files/component.py | 10 +++++----- src/component_generator/component_files/consumer.py | 4 ++-- src/component_generator/component_files/service.py | 4 ++-- src/component_generator/generate.py | 3 ++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/component_generator/component_files/component.py b/src/component_generator/component_files/component.py index f52d931..be027d8 100644 --- a/src/component_generator/component_files/component.py +++ b/src/component_generator/component_files/component.py @@ -301,7 +301,7 @@ class Settings(BaseSettings): # # Chart # - "${componentName}/charts/${componentName}/Chart.yaml": """apiVersion: v1 + "charts/${componentName}/Chart.yaml": """apiVersion: v1 appVersion: 1.4.2 description: ${componentName} name: ${componentName} @@ -312,9 +312,9 @@ class Settings(BaseSettings): email: platform@onna.com version: 99999.99999.99999 """, - "${componentName}/charts/${componentName}/requirements.yaml": """dependencies: + "charts/${componentName}/requirements.yaml": """dependencies: """, - "${componentName}/charts/${componentName}/values.yaml": """image: IMAGE_TO_REPLACE + "charts/${componentName}/values.yaml": """image: IMAGE_TO_REPLACE pullSecrets: [] @@ -326,7 +326,7 @@ class Settings(BaseSettings): + END_BLOCK + """ """, - "${componentName}/charts/${componentName}/templates/_helpers.tpl": """{{/* vim: set filetype=mustache: */}} + "charts/${componentName}/templates/_helpers.tpl": """{{/* vim: set filetype=mustache: */}} {{/* Expand the name of the chart. */}} {{- define "name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} @@ -336,7 +336,7 @@ class Settings(BaseSettings): {{- $name := default .Chart.Name .Values.nameOverride -}} {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}}""", - "${componentName}/charts/${componentName}/templates/cm.yaml": """apiVersion: v1 + "charts/${componentName}/templates/cm.yaml": """apiVersion: v1 kind: ConfigMap metadata: name: {{ template "name" . }}-config diff --git a/src/component_generator/component_files/consumer.py b/src/component_generator/component_files/consumer.py index a35a39e..123a416 100644 --- a/src/component_generator/component_files/consumer.py +++ b/src/component_generator/component_files/consumer.py @@ -167,7 +167,7 @@ async def initialize(self) -> None: async def finalize(self) -> None: ... """, - "+${componentName}/commands.py": """ + "+${componentName}/${componentName}/commands.py": """ def run_${consumerName}(): settings = get_settings() asyncio.run(_run_${consumerName}(settings)) @@ -181,7 +181,7 @@ async def _run_${consumerName}(settings: Settings): http_app = ${consumerName}.HTTPApplication(consumer_app) await serve_consumer_app(http_app, consumer_app, port=8080) """, - "+pyproject.toml": """ + "+${componentName}/pyproject.toml": """ ${consumerName} = '${componentName}.commands:run_${consumerName}' """, } diff --git a/src/component_generator/component_files/service.py b/src/component_generator/component_files/service.py index 844fc3d..11ca30f 100644 --- a/src/component_generator/component_files/service.py +++ b/src/component_generator/component_files/service.py @@ -192,7 +192,7 @@ async def initialize(self) -> None: async def finalize(self) -> None: ... """, - "+${componentName}/commands.py": """ + "+${componentName}/${componentName}/commands.py": """ def run_${serviceName}(): settings = get_settings() asyncio.run(_run_${serviceName}(settings)) @@ -211,7 +211,7 @@ async def _run_${serviceName}(settings: Settings): server = uvicorn.Server(config=http_config) await server.serve() """, - "+pyproject.toml": """ + "+${componentName}/pyproject.toml": """ ${serviceName} = '${componentName}.commands:run_${serviceName}' """, } diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py index 8537cb4..fea9db8 100644 --- a/src/component_generator/generate.py +++ b/src/component_generator/generate.py @@ -25,9 +25,10 @@ class ComponentType(Enum): } -def generate_component(component_type: ComponentType, component_name: str): +def generate_component(component_type: ComponentType, component_name: str, main_component_name: str): settings = load_settings_from_file() settings[f"{component_type.value}Name"] = component_name + settings[f"{ComponentType.COMPONENT.value}Name"] = main_component_name generate(COMPONENT_FILESTRUCTURE[component_type], settings) From 30d850585601ce06d5bc0e727641c1ab361ef2ef Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 16:15:35 +0200 Subject: [PATCH 16/18] Put component in own directory --- src/component_generator/generate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py index fea9db8..53eabbd 100644 --- a/src/component_generator/generate.py +++ b/src/component_generator/generate.py @@ -35,13 +35,14 @@ def generate_component(component_type: ComponentType, component_name: str, main_ def generate(structure: Dict[str, str], settings: Dict[str, str]): for filepath, filedata in structure.items(): if filepath.startswith("+"): - append_to_info_file(filepath[1:], filedata, settings) + filepath = f"component/{filepath[1:]}" + append_to_info_file(filepath, filedata, settings) else: + filepath = f"component/{filepath}" generate_file(filepath, filedata, settings) def generate_file(filepath: str, filedata: str, settings: Dict[str, str]): - filepath = f"component/{filepath}" populated_filepath = populate_setting_values(filepath, settings) create_folder_for_filepath(populated_filepath) From 01b9754a10ea8f47bae6a37f7b0b2baebbeeb1ee Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 16:20:58 +0200 Subject: [PATCH 17/18] Component directory from component name --- src/component_generator/generate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/component_generator/generate.py b/src/component_generator/generate.py index 53eabbd..e5fc16d 100644 --- a/src/component_generator/generate.py +++ b/src/component_generator/generate.py @@ -29,13 +29,13 @@ def generate_component(component_type: ComponentType, component_name: str, main_ settings = load_settings_from_file() settings[f"{component_type.value}Name"] = component_name settings[f"{ComponentType.COMPONENT.value}Name"] = main_component_name - generate(COMPONENT_FILESTRUCTURE[component_type], settings) + generate(main_component_name, COMPONENT_FILESTRUCTURE[component_type], settings) -def generate(structure: Dict[str, str], settings: Dict[str, str]): +def generate(component_name: str, structure: Dict[str, str], settings: Dict[str, str]): for filepath, filedata in structure.items(): if filepath.startswith("+"): - filepath = f"component/{filepath[1:]}" + filepath = f"{component_name}/{filepath[1:]}" append_to_info_file(filepath, filedata, settings) else: filepath = f"component/{filepath}" From 95def4cf2878eb88eedbda839e2864b4cb935120 Mon Sep 17 00:00:00 2001 From: Darren Buttigieg Date: Fri, 15 Oct 2021 16:57:54 +0200 Subject: [PATCH 18/18] =?UTF-8?q?Bump=20version:=200.0.0=20=E2=86=92=200.1?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- CHANGELOG.rst | 5 +++++ README.rst | 4 ++-- docs/conf.py | 2 +- setup.py | 2 +- src/component_generator/__init__.py | 2 +- 6 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cbd1ce3..2266407 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.0.0 +current_version = 0.1.0 commit = True tag = True diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3b61408..28cf284 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,11 @@ Changelog ========= +0.1.0 (2021-10-15) +------------------ + +* First translation of code directly from GO + 0.0.0 (2021-10-13) ------------------ diff --git a/README.rst b/README.rst index c42bf5d..9862a06 100644 --- a/README.rst +++ b/README.rst @@ -51,9 +51,9 @@ Overview :alt: Supported implementations :target: https://pypi.org/project/component-generator -.. |commits-since| image:: https://img.shields.io/github/commits-since/onna/python-component-generator/v0.0.0.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/onna/python-component-generator/v0.1.0.svg :alt: Commits since latest release - :target: https://github.com/onna/python-component-generator/compare/v0.0.0...master + :target: https://github.com/onna/python-component-generator/compare/v0.1.0...master diff --git a/docs/conf.py b/docs/conf.py index f1d541b..b0f86f6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ year = '2021' author = 'Darren Buttigieg' copyright = '{0}, {1}'.format(year, author) -version = release = '0.0.0' +version = release = '0.1.0' pygments_style = 'trac' templates_path = ['.'] diff --git a/setup.py b/setup.py index e354ec0..66edbba 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def read(*names, **kwargs): setup( name='component-generator', - version='0.0.0', + version='0.1.0', license='BSD-2-Clause', description='Generate backend components quickly', long_description='%s\n%s' % ( diff --git a/src/component_generator/__init__.py b/src/component_generator/__init__.py index 6c8e6b9..b794fd4 100644 --- a/src/component_generator/__init__.py +++ b/src/component_generator/__init__.py @@ -1 +1 @@ -__version__ = "0.0.0" +__version__ = '0.1.0'