diff --git a/poetry.lock b/poetry.lock index 93b4a14c22..9ae4fa436c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -73,7 +73,6 @@ files = [ ] [package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} @@ -915,7 +914,7 @@ version = "1.3.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" -groups = ["main", "integration", "unit"] +groups = ["integration", "unit"] markers = "python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, @@ -1817,21 +1816,25 @@ testing = ["coverage", "pytest", "pytest-benchmark"] [[package]] name = "postgresql-charms-single-kernel" -version = "16.1.12" +version = "16.2.1" description = "Shared and reusable code for PostgreSQL-related charms" optional = false python-versions = "<4.0,>=3.8" groups = ["main"] files = [ - {file = "postgresql_charms_single_kernel-16.1.12-py3-none-any.whl", hash = "sha256:229c952c216537382f1842b14a2f0163f4b8ea976d5bc6507f48dbc4f7c90a9a"}, - {file = "postgresql_charms_single_kernel-16.1.12.tar.gz", hash = "sha256:a05f2355916ac91d52f709f52d28731087508d9b6683acfdf220638b6a5a3a20"}, + {file = "postgresql_charms_single_kernel-16.2.1-py3-none-any.whl", hash = "sha256:6936d06e225a9b8f1bc6da1b22b9cdc6f96fa5c4db7c47a3dbdc5ba5f956abad"}, + {file = "postgresql_charms_single_kernel-16.2.1.tar.gz", hash = "sha256:5551abb62e7248218a009a0cb5da171de6d0a2919d349c5133bf4ac26cb6fe3c"}, ] [package.dependencies] +httpx = {version = "*", optional = true, markers = "python_full_version >= \"3.12.0\" and extra == \"postgresql\""} ops = ">=2.0.0" psycopg2 = ">=2.9.10" tenacity = ">=9.0.0" +[package.extras] +postgresql = ["httpx ; python_full_version >= \"3.12.0\""] + [[package]] name = "prompt-toolkit" version = "3.0.52" @@ -3104,4 +3107,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "2b780e63599381bb41f43c0da631815ce0f841400beb9f1a1fe77fbff10d5ffc" +content-hash = "39ad43e91ff98d2831d59cd92f3ac1967437e879a98ece59594ab8ea719b9985" diff --git a/pyproject.toml b/pyproject.toml index bf0ea1fac9..eb6857c552 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,10 +18,9 @@ jinja2 = "^3.1.6" pysyncobj = "^0.3.15" psutil = "^7.2.2" charm-refresh = "^3.1.0.2" -httpx = "^0.28.1" charmlibs-snap = "^1.0.1" charmlibs-interfaces-tls-certificates = "^1.8.1" -postgresql-charms-single-kernel = "16.1.12" +postgresql-charms-single-kernel = {extras = ["postgresql"], version="16.2.1"} [tool.poetry.group.charm-libs.dependencies] # data_platform_libs/v0/data_interfaces.py diff --git a/src/backups.py b/src/backups.py index 457b21c23d..5237ff5706 100644 --- a/src/backups.py +++ b/src/backups.py @@ -27,6 +27,8 @@ from ops.charm import ActionEvent, HookEvent from ops.framework import Object from ops.model import ActiveStatus, MaintenanceStatus +from single_kernel_postgresql.config.literals import Substrates +from single_kernel_postgresql.utils import render_file from tenacity import RetryError, Retrying, stop_after_attempt, wait_fixed from constants import ( @@ -46,7 +48,6 @@ UNIT_SCOPE, ) from relations.async_replication import REPLICATION_CONSUMER_RELATION, REPLICATION_OFFER_RELATION -from utils import render_file logger = logging.getLogger(__name__) @@ -1369,7 +1370,10 @@ def _render_pgbackrest_conf_file(self) -> bool: if self._tls_ca_chain_filename != "": render_file( - self._tls_ca_chain_filename, "\n".join(s3_parameters["tls-ca-chain"]), 0o644 + Substrates.VM, + self._tls_ca_chain_filename, + "\n".join(s3_parameters["tls-ca-chain"]), + 0o644, ) with open("templates/pgbackrest.conf.j2") as file: @@ -1395,12 +1399,14 @@ def _render_pgbackrest_conf_file(self) -> bool: process_max=max(self.charm.cpu_count - 2, 1), ) # Render pgBackRest config file. - render_file(f"{PGBACKREST_CONF_PATH}/pgbackrest.conf", rendered, 0o640) + render_file(Substrates.VM, f"{PGBACKREST_CONF_PATH}/pgbackrest.conf", rendered, 0o640) # Render the logrotate configuration file. with open("templates/pgbackrest.logrotate.j2") as file: template = Template(file.read()) - render_file(PGBACKREST_LOGROTATE_FILE, template.render(), 0o644, change_owner=False) + render_file( + Substrates.VM, PGBACKREST_LOGROTATE_FILE, template.render(), 0o644, change_owner=False + ) return True diff --git a/src/charm.py b/src/charm.py index ee5a69d650..759d0e070f 100755 --- a/src/charm.py +++ b/src/charm.py @@ -69,6 +69,7 @@ Substrates, ) from single_kernel_postgresql.events.tls_transfer import TLSTransfer +from single_kernel_postgresql.utils import label2name, new_password, render_file from single_kernel_postgresql.utils.postgresql import ( ACCESS_GROUP_IDENTITY, ACCESS_GROUPS, @@ -140,7 +141,6 @@ from relations.tls import TLS from relations.watcher import PostgreSQLWatcherRelation from rotate_logs import RotateLogs -from utils import label2name, new_password, render_file logger = logging.getLogger(__name__) logging.getLogger("httpx").setLevel(logging.WARNING) @@ -2290,22 +2290,25 @@ def push_tls_files_to_workload(self) -> bool: """Move TLS files to the PostgreSQL storage path and enable TLS.""" key, ca, cert = self.tls.get_client_tls_files() if key is not None: - render_file(f"{PATRONI_CONF_PATH}/{TLS_KEY_FILE}", key, 0o600) + render_file(Substrates.VM, f"{PATRONI_CONF_PATH}/{TLS_KEY_FILE}", key, 0o600) if ca is not None: - render_file(f"{PATRONI_CONF_PATH}/{TLS_CA_FILE}", ca, 0o600) + render_file(Substrates.VM, f"{PATRONI_CONF_PATH}/{TLS_CA_FILE}", ca, 0o600) if cert is not None: - render_file(f"{PATRONI_CONF_PATH}/{TLS_CERT_FILE}", cert, 0o600) + render_file(Substrates.VM, f"{PATRONI_CONF_PATH}/{TLS_CERT_FILE}", cert, 0o600) key, ca, cert = self.tls.get_peer_tls_files() if key is not None: - render_file(f"{PATRONI_CONF_PATH}/peer_{TLS_KEY_FILE}", key, 0o600) + render_file(Substrates.VM, f"{PATRONI_CONF_PATH}/peer_{TLS_KEY_FILE}", key, 0o600) if ca is not None: - render_file(f"{PATRONI_CONF_PATH}/peer_{TLS_CA_FILE}", ca, 0o600) + render_file(Substrates.VM, f"{PATRONI_CONF_PATH}/peer_{TLS_CA_FILE}", ca, 0o600) if cert is not None: - render_file(f"{PATRONI_CONF_PATH}/peer_{TLS_CERT_FILE}", cert, 0o600) + render_file(Substrates.VM, f"{PATRONI_CONF_PATH}/peer_{TLS_CERT_FILE}", cert, 0o600) render_file( - f"{PATRONI_CONF_PATH}/{TLS_CA_BUNDLE_FILE}", self.tls.get_peer_ca_bundle(), 0o600 + Substrates.VM, + f"{PATRONI_CONF_PATH}/{TLS_CA_BUNDLE_FILE}", + self.tls.get_peer_ca_bundle(), + 0o600, ) try: diff --git a/src/cluster.py b/src/cluster.py index 35196c4469..956139886d 100644 --- a/src/cluster.py +++ b/src/cluster.py @@ -26,10 +26,18 @@ from pysyncobj.utility import TcpUtility, UtilityException from requests.auth import HTTPBasicAuth from single_kernel_postgresql.config.literals import ( + API_REQUEST_TIMEOUT, PEER, POSTGRESQL_STORAGE_PERMISSIONS, REWIND_USER, USER, + Substrates, +) +from single_kernel_postgresql.utils import ( + _change_owner, + label2name, + parallel_patroni_get_request, + render_file, ) from tenacity import ( Future, @@ -44,7 +52,6 @@ ) from constants import ( - API_REQUEST_TIMEOUT, PATRONI_CLUSTER_STATUS_ENDPOINT, PATRONI_CONF_PATH, PATRONI_LOGS_PATH, @@ -57,7 +64,6 @@ RAFT_PORT, TLS_CA_BUNDLE_FILE, ) -from utils import _change_owner, label2name, parallel_patroni_get_request, render_file logger = logging.getLogger(__name__) @@ -221,7 +227,7 @@ def bootstrap_cluster(self) -> bool: def configure_patroni_on_unit(self): """Configure Patroni (configuration files and service) on the unit.""" - _change_owner(POSTGRESQL_DATA_PATH) + _change_owner(Substrates.VM, POSTGRESQL_DATA_PATH) # Create empty base config open(PG_BASE_CONF_PATH, "a").close() @@ -709,7 +715,7 @@ def render_patroni_yml_file( if self.charm.watcher_offer.is_active else None, ) - render_file(f"{PATRONI_CONF_PATH}/patroni.yaml", rendered, 0o600) + render_file(Substrates.VM, f"{PATRONI_CONF_PATH}/patroni.yaml", rendered, 0o600) def start_patroni(self) -> bool: """Start Patroni service using snap. diff --git a/src/constants.py b/src/constants.py index 78c429f733..42538c5bbf 100644 --- a/src/constants.py +++ b/src/constants.py @@ -12,7 +12,6 @@ ALL_CLIENT_RELATIONS = [DATABASE] REPLICATION_CONSUMER_RELATION = "replication" REPLICATION_OFFER_RELATION = "replication-offer" -API_REQUEST_TIMEOUT = 5 PATRONI_CLUSTER_STATUS_ENDPOINT = "cluster" BACKUP_USER = "backup" TLS_KEY_FILE = "key.pem" diff --git a/src/relations/postgresql_provider.py b/src/relations/postgresql_provider.py index 215e0d13e3..b9e5a03354 100644 --- a/src/relations/postgresql_provider.py +++ b/src/relations/postgresql_provider.py @@ -21,6 +21,7 @@ RelationDepartedEvent, ) from single_kernel_postgresql.config.literals import SYSTEM_USERS +from single_kernel_postgresql.utils import label2name, new_password from single_kernel_postgresql.utils.postgresql import ( ACCESS_GROUP_RELATION, ACCESS_GROUPS, @@ -34,7 +35,6 @@ ) from constants import APP_SCOPE, DATABASE_MAPPING_LABEL, DATABASE_PORT, USERNAME_MAPPING_LABEL -from utils import label2name, new_password logger = logging.getLogger(__name__) diff --git a/src/relations/watcher.py b/src/relations/watcher.py index c892d0363c..b8f2c5afa4 100644 --- a/src/relations/watcher.py +++ b/src/relations/watcher.py @@ -27,6 +27,7 @@ SecretNotFoundError, ) from pysyncobj.utility import TcpUtility +from single_kernel_postgresql.utils import new_password from constants import ( RAFT_PARTNER_PREFIX, @@ -39,7 +40,6 @@ WATCHER_SECRET_LABEL, WATCHER_USER, ) -from utils import new_password if TYPE_CHECKING: from charm import PostgresqlOperatorCharm diff --git a/src/utils.py b/src/utils.py deleted file mode 100644 index d97700fda9..0000000000 --- a/src/utils.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. - -"""A collection of utility functions that are used in the charm.""" - -import os -import pwd -import secrets -import string -from asyncio import as_completed, create_task, run, wait -from contextlib import suppress -from ssl import CERT_NONE, create_default_context -from typing import Any - -from httpx import AsyncClient, BasicAuth, HTTPError - -from constants import API_REQUEST_TIMEOUT - - -def new_password() -> str: - """Generate a random password string. - - Returns: - A random password string. - """ - choices = string.ascii_letters + string.digits - password = "".join([secrets.choice(choices) for i in range(16)]) - return password - - -def label2name(label: str) -> str: - """Convert a unit label (with `-`) to a unit name (with `/`). - - Args: - label: The label to convert. - - Returns: - The converted name. - """ - return label.rsplit("-", 1)[0] + "/" + label.rsplit("-", 1)[1] - - -def render_file(path: str, content: str, mode: int, change_owner: bool = True) -> None: - """Write a content rendered from a template to a file. - - Args: - path: the path to the file. - content: the data to be written to the file. - mode: access permission mask applied to the - file using chmod (e.g. 0o640). - change_owner: whether to change the file owner - to the _daemon_ user. - """ - # TODO: keep this method to use it also for generating replication configuration files and - # move it to an utils / helpers file. - # Write the content to the file. - with open(path, "w+") as file: - file.write(content) - # Ensure correct permissions are set on the file. - os.chmod(path, mode) - if change_owner: - _change_owner(path) - - -def create_directory(path: str, mode: int) -> None: - """Creates a directory. - - Args: - path: the path of the directory that should be created. - mode: access permission mask applied to the - directory using chmod (e.g. 0o640). - """ - os.makedirs(path, mode=mode, exist_ok=True) - # Ensure correct permissions are set on the directory. - os.chmod(path, mode) - _change_owner(path) - - -def _change_owner(path: str) -> None: - """Change the ownership of a file or a directory to the postgres user. - - Args: - path: path to a file or directory. - """ - # Get the uid/gid for the _daemon_ user. - user_database = pwd.getpwnam("_daemon_") - # Set the correct ownership for the file or directory. - os.chown(path, uid=user_database.pw_uid, gid=user_database.pw_gid) - - -async def _httpx_get_request( - url: str, cafile: str, auth: BasicAuth | None = None, verify: bool = True -) -> dict[str, Any] | None: - ssl_ctx = create_default_context() - if verify: - with suppress(FileNotFoundError): - ssl_ctx.load_verify_locations(cafile=cafile) - else: - ssl_ctx.check_hostname = False - ssl_ctx.verify_mode = CERT_NONE - async with AsyncClient(auth=auth, timeout=API_REQUEST_TIMEOUT, verify=ssl_ctx) as client: - try: - return (await client.get(url)).raise_for_status().json() - except (HTTPError, ValueError): - return None - - -async def _async_get_request( - uri: str, endpoints: list[str], cafile: str, auth: BasicAuth | None, verify: bool = True -) -> dict[str, Any] | None: - tasks = [ - create_task(_httpx_get_request(f"https://{ip}:8008{uri}", cafile, auth, verify)) - for ip in endpoints - ] - for task in as_completed(tasks): - if result := await task: - for task in tasks: - task.cancel() - await wait(tasks) - return result - - -def parallel_patroni_get_request( - uri: str, - endpoints: list[str], - cafile: str, - auth: BasicAuth | None = None, - verify: bool = True, -) -> dict[str, Any] | None: - """Call all possible patroni endpoints in parallel.""" - return run(_async_get_request(uri, endpoints, cafile, auth, verify)) diff --git a/tests/unit/test_backups.py b/tests/unit/test_backups.py index f1e69f2bc0..5da8df7ec2 100644 --- a/tests/unit/test_backups.py +++ b/tests/unit/test_backups.py @@ -11,6 +11,7 @@ from jinja2 import Template from ops import ActiveStatus, BlockedStatus, MaintenanceStatus, Unit from ops.testing import Harness +from single_kernel_postgresql.config.literals import Substrates from tenacity import RetryError, wait_fixed from backups import ( @@ -1887,11 +1888,13 @@ def test_render_pgbackrest_conf_file(harness, tls_ca_chain_filename): # Ensure the correct rendered template is sent to _render_file method. calls = [ call( + Substrates.VM, "/var/snap/charmed-postgresql/current/etc/pgbackrest/pgbackrest.conf", expected_content, 0o640, ), call( + Substrates.VM, "/etc/logrotate.d/pgbackrest.logrotate", log_rotation_expected_content, 0o644, @@ -1899,7 +1902,7 @@ def test_render_pgbackrest_conf_file(harness, tls_ca_chain_filename): ), ] if tls_ca_chain_filename != "": - calls.insert(0, call(tls_ca_chain_filename, "fake-tls-ca-chain", 0o644)) + calls.insert(0, call(Substrates.VM, tls_ca_chain_filename, "fake-tls-ca-chain", 0o644)) _render_file.assert_has_calls(calls) diff --git a/tests/unit/test_cluster.py b/tests/unit/test_cluster.py index a34d9bae75..07f3823986 100644 --- a/tests/unit/test_cluster.py +++ b/tests/unit/test_cluster.py @@ -10,7 +10,7 @@ from jinja2 import Template from ops.testing import Harness from pysyncobj.utility import UtilityException -from single_kernel_postgresql.config.literals import REWIND_USER +from single_kernel_postgresql.config.literals import REWIND_USER, Substrates from tenacity import ( RetryError, stop_after_delay, @@ -323,6 +323,7 @@ def test_render_patroni_yml_file(peers_ips, patroni): assert mock.call_args_list[0][0] == ("templates/patroni.yml.j2",) # Ensure the correct rendered template is sent to _render_file method. _render_file.assert_called_once_with( + Substrates.VM, "/var/snap/charmed-postgresql/current/etc/patroni/patroni.yaml", expected_content, 0o600, diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py deleted file mode 100644 index e73918a0af..0000000000 --- a/tests/unit/test_utils.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2021 Canonical Ltd. -# See LICENSE file for licensing details. - -import re -from unittest.mock import mock_open, patch - -from utils import new_password, render_file - - -def test_new_password(): - # Test the password generation twice in order to check if we get different passwords and - # that they meet the required criteria. - first_password = new_password() - assert len(first_password) == 16 - assert re.fullmatch("[a-zA-Z0-9\b]{16}$", first_password) is not None - - second_password = new_password() - assert re.fullmatch("[a-zA-Z0-9\b]{16}$", second_password) is not None - assert second_password != first_password - - -def test_render_file(): - with ( - patch("os.chmod") as _chmod, - patch("os.chown") as _chown, - patch("pwd.getpwnam") as _pwnam, - patch("tempfile.NamedTemporaryFile") as _temp_file, - ): - # Set a mocked temporary filename. - filename = "/tmp/temporaryfilename" - _temp_file.return_value.name = filename - # Setup a mock for the `open` method. - mock = mock_open() - # Patch the `open` method with our mock. - with patch("builtins.open", mock, create=True): - # Set the uid/gid return values for lookup of 'postgres' user. - _pwnam.return_value.pw_uid = 35 - _pwnam.return_value.pw_gid = 35 - # Call the method using a temporary configuration file. - render_file(filename, "rendered-content", 0o640) - - # Check the rendered file is opened with "w+" mode. - assert mock.call_args_list[0][0] == (filename, "w+") - # Ensure that the correct user is lookup up. - _pwnam.assert_called_with("_daemon_") - # Ensure the file is chmod'd correctly. - _chmod.assert_called_with(filename, 0o640) - # Ensure the file is chown'd correctly. - _chown.assert_called_with(filename, uid=35, gid=35) - - # Test when it's requested to not change the file owner. - mock.reset_mock() - _pwnam.reset_mock() - _chmod.reset_mock() - _chown.reset_mock() - with patch("builtins.open", mock, create=True): - render_file(filename, "rendered-content", 0o640, change_owner=False) - _pwnam.assert_not_called() - _chmod.assert_called_once_with(filename, 0o640) - _chown.assert_not_called()