diff --git a/charmcraft.yaml b/charmcraft.yaml index df136beb..75d7d368 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -11,8 +11,8 @@ parts: build-packages: [git] platforms: - ubuntu@20.04:amd64: - ubuntu@22.04:amd64: + # ubuntu@20.04:amd64: + # ubuntu@22.04:amd64: ubuntu@24.04:amd64: ubuntu@20.04:arm64: ubuntu@22.04:arm64: @@ -41,3 +41,7 @@ actions: Use the re-detected list of hardware tools as the new enable-list to reconfigure and restart the exporter. default: false + +charm-libs: + - lib: grafana-agent.cos_agent + version: "0" diff --git a/config.yaml b/config.yaml index b2746b0c..62d6cdd7 100644 --- a/config.yaml +++ b/config.yaml @@ -51,7 +51,7 @@ options: description: | Timeout for collectors' shell commands in seconds. Changing this will also change the scrape_timeout config option for prometheus for the cos-agent relation with - grafana-agent. + opentelemetry or grafana-agent. This value is also used for the redfish client's timeout parameter. The value of this timeout should not be greater than prometheus scrape_interval (which is 60 seconds by default), as it greater would cause the scrape_timeout to be diff --git a/dev-environment.md b/dev-environment.md index 968343e0..bd37d2a8 100644 --- a/dev-environment.md +++ b/dev-environment.md @@ -99,14 +99,14 @@ juju add-machine ssh:ubuntu@$BR0_ADDR juju switch hw-obs juju deploy ubuntu --to 0 juju deploy hardware-observer -juju deploy grafana-agent +juju deploy opentelemetry-collector --channel=2/stable ``` Add necessary relations: ``` juju relate hardware-observer ubuntu -juju relate grafana-agent ubuntu -juju relate grafana-agent hardware-observer +juju relate opentelemetry-collector ubuntu +juju relate opentelemetry-collector hardware-observer ``` @@ -181,9 +181,9 @@ juju deploy cos-lite --trust --overlay ./offers-overlay.yaml Switch back to your physical model and add the relations to COS: ``` juju switch hw-obs -juju relate grafana-agent k8s-controller:cos.prometheus-receive-remote-write -juju relate grafana-agent k8s-controller:cos.grafana-dashboards -juju relate grafana-agent k8s-controller:cos.loki-logging +juju relate opentelemetry-collector k8s-controller:cos.prometheus-receive-remote-write +juju relate opentelemetry-collector k8s-controller:cos.grafana-dashboards +juju relate opentelemetry-collector k8s-controller:cos.loki-logging ``` diff --git a/lib/charms/grafana_agent/v0/cos_agent.py b/lib/charms/grafana_agent/v0/cos_agent.py index 7bf3eb1a..d3944207 100644 --- a/lib/charms/grafana_agent/v0/cos_agent.py +++ b/lib/charms/grafana_agent/v0/cos_agent.py @@ -211,7 +211,9 @@ def __init__(self, *args): ``` """ +import copy import enum +import hashlib import json import logging import socket @@ -254,7 +256,7 @@ class _MetricsEndpointDict(TypedDict): LIBID = "dc15fa84cef84ce58155fb84f6c6213a" LIBAPI = 0 -LIBPATCH = 22 +LIBPATCH = 25 PYDEPS = ["cosl >= 0.0.50", "pydantic"] @@ -264,12 +266,6 @@ class _MetricsEndpointDict(TypedDict): logger = logging.getLogger(__name__) SnapEndpoint = namedtuple("SnapEndpoint", "owner, name") -# Note: MutableMapping is imported from the typing module and not collections.abc -# because subscripting collections.abc.MutableMapping was added in python 3.9, but -# most of our charms are based on 20.04, which has python 3.8. - -_RawDatabag = MutableMapping[str, str] - class TransportProtocolType(str, enum.Enum): """Receiver Type.""" @@ -305,6 +301,22 @@ class TransportProtocolType(str, enum.Enum): ReceiverProtocol = Literal["otlp_grpc", "otlp_http", "zipkin", "jaeger_thrift_http", "jaeger_grpc"] +def _dedupe_list(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Deduplicate items in the list via object identity.""" + unique_items = [] + for item in items: + if item not in unique_items: + unique_items.append(item) + return unique_items + + +def _dict_hash_except_key(scrape_config: Dict[str, Any], key: Optional[str]): + """Get a hash of the scrape_config dict, except for the specified key.""" + cfg_for_hash = {k: v for k, v in scrape_config.items() if k != key} + serialized = json.dumps(cfg_for_hash, sort_keys=True) + return hashlib.blake2b(serialized.encode(), digest_size=4).hexdigest() + + class TracingError(Exception): """Base class for custom errors raised by tracing.""" @@ -619,7 +631,8 @@ def __init__( refresh_events: Optional[List] = None, tracing_protocols: Optional[List[str]] = None, *, - scrape_configs: Optional[Union[List[dict], Callable]] = None, + scrape_configs: Optional[Union[List[dict], Callable[[], List[Dict[str, Any]]]]] = None, + extra_alert_groups: Optional[Callable[[], Dict[str, Any]]] = None, ): """Create a COSAgentProvider instance. @@ -640,6 +653,9 @@ def __init__( scrape_configs: List of standard scrape_configs dicts or a callable that returns the list in case the configs need to be generated dynamically. The contents of this list will be merged with the contents of `metrics_endpoints`. + extra_alert_groups: A callable that returns a dict of alert rule groups in case the + alerts need to be generated dynamically. The contents of this dict will be merged + with generic and bundled alert rules. """ super().__init__(charm, relation_name) dashboard_dirs = dashboard_dirs or ["./src/grafana_dashboards"] @@ -648,6 +664,7 @@ def __init__( self._relation_name = relation_name self._metrics_endpoints = metrics_endpoints or [] self._scrape_configs = scrape_configs or [] + self._extra_alert_groups = extra_alert_groups or {} self._metrics_rules = metrics_rules_dir self._logs_rules = logs_rules_dir self._recursive = recurse_rules_dirs @@ -689,12 +706,34 @@ def _on_refresh(self, event): ) as e: logger.error("Invalid relation data provided: %s", e) + def _deterministic_scrape_configs( + self, scrape_configs: List[Dict[str, Any]] + ) -> List[Dict[str, Any]]: + """Get deterministic scrape_configs with stable job names. + + For stability across serializations, compute a short per-config hash + and append it to the existing job name (or 'default'). Keep the app + name as a prefix: __<8hex-hash>. + + Hash the whole scrape_config (except any existing job_name) so the + suffix is sensitive to all stable fields. Use deterministic JSON + serialization. + """ + local_scrape_configs = copy.deepcopy(scrape_configs) + for scrape_config in local_scrape_configs: + name = scrape_config.get("job_name", "default") + short_id = _dict_hash_except_key(scrape_config, "job_name") + scrape_config["job_name"] = f"{self._charm.app.name}_{name}_{short_id}" + + return sorted(local_scrape_configs, key=lambda c: c.get("job_name", "")) + @property def _scrape_jobs(self) -> List[Dict]: - """Return a prometheus_scrape-like data structure for jobs. + """Return a list of scrape_configs. https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config """ + # Optionally allow the charm to set the scrape_configs if callable(self._scrape_configs): scrape_configs = self._scrape_configs() else: @@ -712,26 +751,30 @@ def _scrape_jobs(self) -> List[Dict]: scrape_configs = scrape_configs or [] - # Augment job name to include the app name and a unique id (index) - for idx, scrape_config in enumerate(scrape_configs): - scrape_config["job_name"] = "_".join( - [self._charm.app.name, str(idx), scrape_config.get("job_name", "default")] - ) - - return scrape_configs + return self._deterministic_scrape_configs(scrape_configs) @property def _metrics_alert_rules(self) -> Dict: - """Use (for now) the prometheus_scrape AlertRules to initialize this.""" + """Return a dict of alert rule groups.""" + # Optionally allow the charm to add the metrics_alert_rules + if callable(self._extra_alert_groups): + rules = self._extra_alert_groups() + else: + rules = {"groups": []} + alert_rules = AlertRules( query_type="promql", topology=JujuTopology.from_charm(self._charm) ) alert_rules.add_path(self._metrics_rules, recursive=self._recursive) alert_rules.add( - generic_alert_groups.application_rules, + copy.deepcopy(generic_alert_groups.application_rules), group_name_prefix=JujuTopology.from_charm(self._charm).identifier, ) - return alert_rules.as_dict() + + # NOTE: The charm could supply rules we implement in this method, so we deduplicate + rules["groups"] = _dedupe_list(rules["groups"] + alert_rules.as_dict()["groups"]) + + return rules @property def _log_alert_rules(self) -> Dict: diff --git a/tests/functional/bundle.yaml.j2 b/tests/functional/bundle.yaml.j2 index 7999a286..06c9f533 100644 --- a/tests/functional/bundle.yaml.j2 +++ b/tests/functional/bundle.yaml.j2 @@ -1,7 +1,7 @@ # Test basic deployment: -# ubuntu:juju-info <-> grafana-agent:juju-info +# ubuntu:juju-info <-> opentelemetry-collector:juju-info # ubuntu:juju-info <-> hardware-observer:general-info -# grafana-agent:cos-agent <-> hardware-observer:cos-agent +# opentelemetry-collector:cos-agent <-> hardware-observer:cos-agent default-base: {{ base }} @@ -14,16 +14,16 @@ applications: num_units: 1 to: - "0" - grafana-agent: - charm: grafana-agent - channel: 1/stable + opentelemetry-collector: + charm: opentelemetry-collector + channel: 2/stable hardware-observer: charm: {{ charm }} options: redfish-disable: {{ redfish_disable }} relations: -- - grafana-agent:juju-info +- - opentelemetry-collector:juju-info - ubuntu:juju-info - - hardware-observer:general-info - ubuntu:juju-info diff --git a/tests/functional/test_charm.py b/tests/functional/test_charm.py index d3211e25..9057e948 100644 --- a/tests/functional/test_charm.py +++ b/tests/functional/test_charm.py @@ -34,7 +34,7 @@ METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) APP_NAME = METADATA["name"] PRINCIPAL_APP_NAME = "ubuntu" -GRAFANA_AGENT_APP_NAME = "grafana-agent" +OTEL_APP_NAME = "opentelemetry-collector" TIMEOUT = 600 @@ -85,7 +85,7 @@ async def test_build_and_deploy( # noqa: C901, function is too complex timeout=TIMEOUT, ) await ops_test.model.wait_for_idle( - apps=[GRAFANA_AGENT_APP_NAME], + apps=[OTEL_APP_NAME], status="blocked", timeout=TIMEOUT, ) @@ -101,8 +101,8 @@ async def test_build_and_deploy( # noqa: C901, function is too complex else: assert unit.workload_status_message == AppStatus.MISSING_RELATION - for unit in ops_test.model.applications[GRAFANA_AGENT_APP_NAME].units: - messages = ["Missing", "grafana-cloud-config", "logging-consumer", "send-remote-write"] + for unit in ops_test.model.applications[OTEL_APP_NAME].units: + messages = ["cloud-config", "send-loki-logs", "send-remote-write"] for msg in messages: assert msg in unit.workload_status_message @@ -144,16 +144,14 @@ async def test_required_resources(ops_test: OpsTest, required_resources): @pytest.mark.abort_on_fail async def test_cos_agent_relation(ops_test: OpsTest, provided_collectors): - """Test adding relation with grafana-agent.""" + """Test adding relation with opentelemetry-collector.""" redfish_present = True if "redfish" in provided_collectors else False # Add cos-agent relation logging.info("Adding cos-agent relation.") status = "blocked" if redfish_present else "active" await asyncio.gather( - ops_test.model.add_relation( - f"{APP_NAME}:cos-agent", f"{GRAFANA_AGENT_APP_NAME}:cos-agent" - ), + ops_test.model.add_relation(f"{APP_NAME}:cos-agent", f"{OTEL_APP_NAME}:cos-agent"), ops_test.model.wait_for_idle( apps=[APP_NAME], status=status, diff --git a/tests/integration/test_cos_integration.py b/tests/integration/test_cos_integration.py index da387e29..b260b408 100644 --- a/tests/integration/test_cos_integration.py +++ b/tests/integration/test_cos_integration.py @@ -28,7 +28,7 @@ async def test_setup_and_deploy(base, channel, lxd_ctl, k8s_ctl, lxd_model, k8s_ await _add_cross_controller_relations(k8s_ctl, lxd_ctl, k8s_model, lxd_model) # This verifies that the cross-controller relation with COS is successful - assert lxd_model.applications["grafana-agent"].status == "active" + assert lxd_model.applications["opentelemetry-collector"].status == "active" async def test_alerts(ops_test: OpsTest, lxd_model, k8s_model): @@ -155,19 +155,19 @@ async def _deploy_hardware_observer(base, channel, model): model.deploy("ubuntu", num_units=1, base=base, channel=channel), # Hardware Observer model.deploy("hardware-observer", base=base, num_units=0, channel=channel), - # Grafana Agent - model.deploy("grafana-agent", num_units=0, base=base, channel=channel), + # OpenTelemetry Collector + model.deploy("opentelemetry-collector", num_units=0, base=base, channel=channel), ) await model.add_relation("ubuntu:juju-info", "hardware-observer:general-info") - await model.add_relation("hardware-observer:cos-agent", "grafana-agent:cos-agent") - await model.add_relation("ubuntu:juju-info", "grafana-agent:juju-info") + await model.add_relation("hardware-observer:cos-agent", "opentelemetry-collector:cos-agent") + await model.add_relation("ubuntu:juju-info", "opentelemetry-collector:juju-info") await model.block_until(lambda: model.applications["hardware-observer"].status == "active") async def _add_cross_controller_relations(k8s_ctl, lxd_ctl, k8s_model, lxd_model): - """Add relations between Grafana Agent and COS.""" + """Add relations between OpenTelemetry Collector and COS.""" cos_saas_names = ["prometheus-receive-remote-write", "loki-logging", "grafana-dashboards"] for saas in cos_saas_names: # Using juju cli since Model.consume() from libjuju causes error. @@ -180,7 +180,7 @@ async def _add_cross_controller_relations(k8s_ctl, lxd_ctl, k8s_model, lxd_model f"{k8s_ctl.controller_name}:admin/{k8s_model.name}.{saas}", ] subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - await lxd_model.add_relation("grafana-agent", saas), + await lxd_model.add_relation("opentelemetry-collector", saas), # `idle_period` needs to be greater than the scrape interval to make sure metrics ingested. await asyncio.gather( diff --git a/tests/manual/etc/4_deploy_grafana_agent/main.tf b/tests/manual/etc/4_deploy_grafana_agent/main.tf index 2e5cf6d0..8edb4fbe 100644 --- a/tests/manual/etc/4_deploy_grafana_agent/main.tf +++ b/tests/manual/etc/4_deploy_grafana_agent/main.tf @@ -9,11 +9,11 @@ terraform { provider "juju" {} -module "grafana-agent" { +module "opentelemetry-collector" { source = "git::https://github.com/canonical/snap-openstack.git//sunbeam-python/sunbeam/features/observability/etc/deploy-grafana-agent" - grafana-agent-base = var.grafana_agent_base - grafana-agent-channel = "1/stable" # Can move back to latest/stable when the charm is updated + opentelemetry-collector-base = var.otel_base + opentelemetry-collector-channel = "2/stable" principal-application-model = var.machine_model receive-remote-write-offer-url = var.receive-remote-write-offer-url grafana-dashboard-offer-url = var.grafana-dashboard-offer-url @@ -21,7 +21,7 @@ module "grafana-agent" { } -resource "juju_integration" "ubuntu-to-grafana-agent" { +resource "juju_integration" "ubuntu-to-opentelemetry-collector" { model = var.machine_model application { @@ -30,12 +30,12 @@ resource "juju_integration" "ubuntu-to-grafana-agent" { } application { - name = "grafana-agent" + name = "opentelemetry-collector" endpoint = "juju-info" } } -resource "juju_integration" "hardware-observer-to-grafana-agent" { +resource "juju_integration" "hardware-observer-to-opentelemetry-collector" { model = var.machine_model application { @@ -44,7 +44,7 @@ resource "juju_integration" "hardware-observer-to-grafana-agent" { } application { - name = "grafana-agent" + name = "opentelemetry-collector" endpoint = "cos-agent" } } diff --git a/tests/manual/etc/4_deploy_grafana_agent/terragrunt.hcl b/tests/manual/etc/4_deploy_grafana_agent/terragrunt.hcl index 64625641..40c959bc 100644 --- a/tests/manual/etc/4_deploy_grafana_agent/terragrunt.hcl +++ b/tests/manual/etc/4_deploy_grafana_agent/terragrunt.hcl @@ -18,16 +18,16 @@ dependency "deploy_cos" { } terraform { - after_hook "wait-for-grafana-agent" { + after_hook "wait-for-opentelemetry-collector" { commands = ["apply"] - execute = [find_in_parent_folders("./scripts/wait-for-application.sh"), "hw-obs", "grafana-agent"] + execute = [find_in_parent_folders("./scripts/wait-for-application.sh"), "hw-obs", "opentelemetry-collector"] run_on_error = true } } inputs = { machine_model = "${dependency.add_machine.outputs.machine_model}" - grafana_agent_base = "${dependency.add_machine.outputs.machine_base}" + otel_base = "${dependency.add_machine.outputs.machine_base}" ubuntu_name = "${dependency.add_machine.outputs.ubuntu_name}" hardware_observer_name = "${dependency.add_machine.outputs.hardware_observer_name}" receive-remote-write-offer-url = "${dependency.deploy_cos.outputs.receive-remote-write-offer-url}" diff --git a/tests/manual/etc/4_deploy_grafana_agent/variables.tf b/tests/manual/etc/4_deploy_grafana_agent/variables.tf index 7b9d6368..52ef2117 100644 --- a/tests/manual/etc/4_deploy_grafana_agent/variables.tf +++ b/tests/manual/etc/4_deploy_grafana_agent/variables.tf @@ -2,8 +2,8 @@ variable "machine_model" { description = "The machine model name" } -variable "grafana_agent_base" { - description = "The base for grafana agent" +variable "otel_base" { + description = "The base for opentelemetry-collector charm" } variable "ubuntu_name" { diff --git a/tests/manual/jobs/eevee/README.md b/tests/manual/jobs/eevee/README.md index 5d45bdd5..fa2767f7 100644 --- a/tests/manual/jobs/eevee/README.md +++ b/tests/manual/jobs/eevee/README.md @@ -21,7 +21,7 @@ You can run the functional tests for real hardware by following these steps: ```shell # Adding relation will be tested as part of the test case, so we need to remove it before running the tests -juju remove-relation -m hw-obs hardware-observer grafana-agent +juju remove-relation -m hw-obs hardware-observer opentelemetry-collector # We don't have redfish credential for this machine juju config -m hw-obs hardware-observer redfish-disable=true diff --git a/tests/manual/jobs/kongfu/README.md b/tests/manual/jobs/kongfu/README.md index 0b305ee8..72fb7be3 100644 --- a/tests/manual/jobs/kongfu/README.md +++ b/tests/manual/jobs/kongfu/README.md @@ -31,7 +31,7 @@ You can run the functional tests for real hardware by following these steps: ```shell # Adding relation will be tested as part of the test case, so we need to remove it before running the tests -juju remove-relation -m hw-obs hardware-observer grafana-agent +juju remove-relation -m hw-obs hardware-observer opentelemetry-collector # We don't have redfish credential for this machine juju config -m hw-obs hardware-observer redfish-disable=true diff --git a/tests/manual/jobs/romano/README.md b/tests/manual/jobs/romano/README.md index f10d6181..97b88573 100644 --- a/tests/manual/jobs/romano/README.md +++ b/tests/manual/jobs/romano/README.md @@ -21,7 +21,7 @@ You can run the functional tests for real hardware by following these steps: ```shell # Adding relation will be tested as part of the test case, so we need to remove it before running the tests -juju remove-relation -m hw-obs hardware-observer grafana-agent +juju remove-relation -m hw-obs hardware-observer opentelemetry-collector # We don't have redfish credential for this machine juju config -m hw-obs hardware-observer redfish-disable=true @@ -29,4 +29,3 @@ juju config -m hw-obs hardware-observer redfish-disable=true # Running the tests tox -e func -- -v --realhw --model hw-obs --no-deploy --collectors ipmi_dcmi ipmi_sel ipmi_sensor --keep-models ``` - diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 21177c31..1c184844 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -353,7 +353,7 @@ def test_update_status( # noqa: C901 ), ): if cos_agent_related: - self.harness.add_relation("cos-agent", "grafana-agent") + self.harness.add_relation("cos-agent", "opentelemetry-collector") self.harness.begin() self.harness.charm.model.unit.status = BlockedStatus("Random status") @@ -621,7 +621,7 @@ def test_config_changed_update_alert_rules(self): return_value=mock_stored_tools, ), ): - rid = self.harness.add_relation("cos-agent", "grafana-agent") + rid = self.harness.add_relation("cos-agent", "opentelemetry-collector") self.harness.begin() @@ -630,7 +630,7 @@ def test_config_changed_update_alert_rules(self): self.harness.charm.hw_tool_helper.check_installed.return_value = (True, "") self.harness.charm.on.install.emit() - self.harness.add_relation_unit(rid, "grafana-agent/0") + self.harness.add_relation_unit(rid, "opentelemetry-collector/0") self.harness.charm.on.update_status.emit() self.assertEqual(self.harness.charm.unit.status, ActiveStatus("Unit is ready")) @@ -665,7 +665,7 @@ def test_upgrade_charm_update_alert_rules(self): return_value=mock_exporters, ), ): - rid = self.harness.add_relation("cos-agent", "grafana-agent") + rid = self.harness.add_relation("cos-agent", "opentelemetry-collector") self.harness.begin() @@ -683,7 +683,7 @@ def test_upgrade_charm_update_alert_rules(self): ) self.harness.charm.on.install.emit() - self.harness.add_relation_unit(rid, "grafana-agent/0") + self.harness.add_relation_unit(rid, "opentelemetry-collector/0") self.harness.charm.on.update_status.emit() self.assertEqual(self.harness.charm.unit.status, ActiveStatus("Unit is ready"))