diff --git a/components/src/dynamo/planner/monitoring/traffic_metrics.py b/components/src/dynamo/planner/monitoring/traffic_metrics.py index 80d611fee054..103651d3538e 100644 --- a/components/src/dynamo/planner/monitoring/traffic_metrics.py +++ b/components/src/dynamo/planner/monitoring/traffic_metrics.py @@ -453,16 +453,28 @@ def get_avg_kv_hit_rate(self, interval: str, model_name: str) -> Optional[float] f"{prometheus_names.router.KV_HIT_RATE}" ) try: - ns = self.dynamo_namespace.replace("-", "_") - ns_filter = f'{prometheus_names.labels.NAMESPACE}="{ns}"' - query = ( - f"sum(increase({full_metric_name}_sum{{{ns_filter}}}[{interval}])) / " - f"sum(increase({full_metric_name}_count{{{ns_filter}}}[{interval}]))" - ) - result = self.prom.custom_query(query=query) + # The router emits `dynamo_namespace` sanitized to underscores, but + # some scrape paths relabel it back to the hyphenated k8s form + # (nvidia.com/dynamo-namespace). Try both so the query works under + # either emission style instead of silently returning no data. + candidates = [] + for ns in (self.dynamo_namespace.replace("-", "_"), self.dynamo_namespace): + if ns not in candidates: + candidates.append(ns) + result = [] + for ns in candidates: + ns_filter = f'{prometheus_names.labels.NAMESPACE}="{ns}"' + query = ( + f"sum(increase({full_metric_name}_sum{{{ns_filter}}}[{interval}])) / " + f"sum(increase({full_metric_name}_count{{{ns_filter}}}[{interval}]))" + ) + result = self.prom.custom_query(query=query) + if result: + break if not result: logger.info( - f"No prometheus data for {full_metric_name}, returning None" + f"No prometheus data for {full_metric_name} " + f"(tried dynamo_namespace in {candidates}), returning None" ) return None value = float(result[0]["value"][1]) diff --git a/components/src/dynamo/planner/monitoring/worker_info.py b/components/src/dynamo/planner/monitoring/worker_info.py index 2f7825f59c70..6a4390b2d471 100644 --- a/components/src/dynamo/planner/monitoring/worker_info.py +++ b/components/src/dynamo/planner/monitoring/worker_info.py @@ -163,7 +163,21 @@ def resolve_worker_info( model_name = config_model_name else: mdc_model = decode_info.model_name or prefill_info.model_name - if mdc_model: + # Prefer the operator-supplied name. Discovery-derived MDC names are + # normalized to lowercase, but engine metrics carry the model's real + # casing, so an MDC name silently makes every engine-metric query + # (spec-decode accept_length, kv_hit_rate) return empty. + if config_model_name: + model_name = config_model_name + if mdc_model and mdc_model != config_model_name: + logger.info( + "Using model name from config: %s (MDC reported %s)", + model_name, + mdc_model, + ) + else: + logger.info(f"Using model name from config: {model_name}") + elif mdc_model: model_name = mdc_model logger.info(f"Using model name from MDC: {model_name}") elif can_query_mdc: